From a1485c1d214e1069f98e7bf9aee78756188ea3e5 Mon Sep 17 00:00:00 2001 From: Herve Tribouilloy Date: Tue, 10 Feb 2026 13:27:11 +0000 Subject: [PATCH 1/4] ci(e2e): add Playwright-based E2E workflow for Mage-OS Introduce a reusable E2E GitHub Actions workflow designed for browser-based testing (Playwright) against Mage-OS. The workflow reuses existing Magento CI setup and delegates execution to a configurable test command, supporting a two-phase E2E model (setup + execution). Includes an internal workflow to validate the E2E pipeline and documentation clarifying the E2E execution model. --- .github/workflows/_internal-e2e.yaml | 46 +++++ .github/workflows/e2e-README.md | 92 ++++++++++ .github/workflows/e2e.yaml | 246 +++++++++++++++++++++++++++ 3 files changed, 384 insertions(+) create mode 100644 .github/workflows/_internal-e2e.yaml create mode 100644 .github/workflows/e2e-README.md create mode 100644 .github/workflows/e2e.yaml diff --git a/.github/workflows/_internal-e2e.yaml b/.github/workflows/_internal-e2e.yaml new file mode 100644 index 0000000..fe0d9a1 --- /dev/null +++ b/.github/workflows/_internal-e2e.yaml @@ -0,0 +1,46 @@ +name: E2E Test + +on: + workflow_dispatch: {} + push: + branches: + - main + paths: + - "_test/demo-package/**" + - ".github/workflows/_internal-e2e.yaml" + - ".github/workflows/e2e.yaml" + - "supported-version/**" + - "!(**/*.md)" + pull_request: + branches: + - main + paths: + - "_test/demo-package/**" + - ".github/workflows/_internal-e2e.yaml" + - ".github/workflows/e2e.yaml" + - "supported-version/**" + - "!(**/*.md)" + +jobs: + compute_matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.supported-version.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + - uses: ./supported-version + with: + kind: all + id: supported-version + - run: echo ${{ steps.supported-version.outputs.matrix }} + e2e-workflow: + needs: compute_matrix + uses: ./.github/workflows/e2e.yaml + with: + package_name: mage-os/magento2-demo-package + source_folder: $GITHUB_WORKSPACE/_test/demo-package + matrix: ${{ needs.compute_matrix.outputs.matrix }} + test_command: | + npx playwright test --grep "@setup" --trace on + npx playwright test --grep-invert "@setup" --trace on + fail-fast: false diff --git a/.github/workflows/e2e-README.md b/.github/workflows/e2e-README.md new file mode 100644 index 0000000..c850b60 --- /dev/null +++ b/.github/workflows/e2e-README.md @@ -0,0 +1,92 @@ +# End-to-End (E2E) Tests for Mage-OS / Magento + +A reusable GitHub Actions workflow that installs Magento (Mage-OS) +and runs browser-based End-to-End (E2E) tests against a real storefront +using Playwright or an equivalent E2E runner. + +## Inputs + +See the [e2e.yaml](./e2e.yaml) + +| Input | Description | Required | Default | +| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ----------------------------- | +| matrix | JSON string of [version matrix for Magento](./#matrix-format) | true | NULL | +| fail-fast | Same as Github's [fail-fast](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstrategyfail-fast) | false | true | +| package_name | The name of the package | true | NULL | +| source_folder | The source folder of the package | false | $GITHUB_WORKSPACE | +| magento_directory | The folder where Magento will be installed | false | ../magento2 | +| magento_repository | Where to install Magento from | false | https://mirror.mage-os.org/ | +| test_command | The e2e test command to run | false | "../../../vendor/bin/phpunit" | +| composer_cache_key | A key to version the composer cache. Can be incremented if you need to bust the cache. | false | "" | + +## Secrets +| Input | Description | Required | Default | +| ------------- | --------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------- | +| composer_auth | JSON string of [composer credentials](https://devdocs.magento.com/guides/v2.4/install-gde/prereq/connect-auth.html) | false | NULL | +> The `test_command` is executed from the Magento installation directory. +> E2E tests are expected to live under `dev/tests/e2e`. + +### Matrix Format + +The Magento matrix format outlined by the [supported versions action.](https://github.com/mage-os/github-actions/tree/main/supported-version/supported.json) + + +## E2E execution model + +The E2E test suite is executed in two phases: + +### 1. Setup phase + +The setup phase prepares shared browser state (e.g. authenticated sessions, +cookies, required entities). This phase is marked using the `@setup` tag. + +```bash +npx playwright test --grep "@setup" --trace on +``` + +This step is required once per fresh environment. + +### 2. Test phase + +Once setup has completed successfully, the full E2E test suite can be executed. +Setup tests are skipped using --grep-invert. + +```bash +npx playwright test --grep-invert "@setup" --trace on +``` + +This phase validates real user journeys against a live Magento storefront. + +```yml +name: E2E Test + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + compute_matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.supported-version.outputs.matrix }} + steps: + - uses: actions/checkout@v2 + - uses: mage-os/github-actions/supported-version@main + id: supported-version + - run: echo ${{ steps.supported-version.outputs.matrix }} + e2e-workflow: + needs: compute_matrix + uses: mage-os/github-actions/.github/workflows/e2e.yaml@main + with: + package_name: my-vendor/package + matrix: ${{ needs.compute_matrix.outputs.matrix }} + test_command: | + npx playwright test --grep "@setup" --trace on + npx playwright test --grep-invert "@setup" --trace on + secrets: + composer_auth: ${{ secrets.COMPOSER_AUTH }} +``` diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml new file mode 100644 index 0000000..c27df15 --- /dev/null +++ b/.github/workflows/e2e.yaml @@ -0,0 +1,246 @@ +on: + workflow_call: + inputs: + use_local_source: + type: boolean + required: false + default: true + description: "Whether or not you want to test your local package or not." + + source_folder: + type: string + required: false + default: $GITHUB_WORKSPACE + description: "The source folder of the package" + + package_name: + type: string + required: true + description: "The name of the package" + + magento_directory: + type: string + required: false + default: "../magento2" + description: "The folder where Magento will be installed" + + magento_repository: + type: string + required: false + default: "https://mirror.mage-os.org/" + description: "Where to install Magento from" + + matrix: + type: string + required: true + description: "The matrix of Magento versions to test against" + + fail-fast: + type: boolean + required: false + default: true + + test_command: + type: string + required: false + default: "" + description: "Shell commands used to execute the E2E test phases (e.g. Playwright)" + + composer_cache_key: + type: string + required: false + default: '' + description: A key to version the composer cache. Can be incremented if you need to bust the cache. + + secrets: + composer_auth: + required: false + +jobs: + e2e_test: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: ${{ inputs.fail-fast }} + matrix: ${{ fromJSON(inputs.matrix) }} + services: + elasticsearch: + image: ${{ matrix.elasticsearch || '' }} + env: + # By default, ElasticSearch refuses to spawn in single node configuration, as it expects redundancy. + # This is a dev environment, so redundancy is just wasteful. + discovery.type: single-node + # Disable HTTPS and password authentication + # this is a local dev environment, so the added CA chain complexity is an extreme overkill + xpack.security.enabled: false + xpack.security.http.ssl.enabled: false + xpack.security.transport.ssl.enabled: false + + options: >- + --health-cmd "curl http://localhost:9200/_cluster/health" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + ports: + - 9200:9200 + + opensearch: + image: ${{ matrix.opensearch || '' }} + env: + # By default, ElasticSearch refuses to spawn in single node configuration, as it expects redundancy. + # This is a dev environment, so redundancy is just wasteful. + discovery.type: single-node + # Disable HTTPS and password authentication + DISABLE_INSTALL_DEMO_CONFIG: true + DISABLE_SECURITY_PLUGIN: true + + options: >- + --health-cmd "curl http://localhost:9200/_cluster/health" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + ports: + - 9200:9200 + + mysql: + image: ${{ matrix.mysql }} + env: + MYSQL_DATABASE: magento_e2e_tests + MYSQL_USER: user + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: rootpassword + ports: + - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 + + rabbitmq: + image: ${{ matrix.rabbitmq }} + env: + RABBITMQ_DEFAULT_USER: guest + RABBITMQ_DEFAULT_PASS: guest + ports: + - 5672:5672 + steps: + - uses: actions/checkout@v4 + - name: Set PHP Version + uses: shivammathur/setup-php@v2 + with: + php-version: ${{ matrix.php }} + tools: composer:v${{ matrix.composer }} + coverage: none + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Allow SQL triggers + run: | + mysql --host 127.0.0.1 --port ${{ job.services.mysql.ports['3306'] }} -u root -prootpassword -e "set global log_bin_trust_function_creators=1;" + + - run: composer create-project --repository-url="${{ inputs.magento_repository }}" "${{ matrix.magento }}" ${{ inputs.magento_directory }} --no-install + shell: bash + env: + COMPOSER_AUTH: ${{ secrets.composer_auth }} + name: Create Magento ${{ matrix.magento }} Project + + - uses: mage-os/github-actions/get-magento-version@main + id: magento-version + with: + working-directory: ${{ inputs.magento_directory }} + + - name: Get Composer Cache Directory + shell: bash + working-directory: ${{ inputs.magento_directory }} + id: composer-cache + run: | + echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT + + + - name: "Cache Composer Packages" + uses: actions/cache@v4 + with: + key: "composer | v5 | ${{ inputs.composer_cache_key }} | ${{ hashFiles('composer.lock') }} | ${{ matrix.os }} | ${{ matrix.composer }} | ${{ matrix.php }} | ${{ matrix.magento }}" + path: ${{ steps.composer-cache.outputs.dir }} + + - run: composer config repositories.local path ${{ inputs.source_folder }} + name: Add Github Repo for Testing + working-directory: ${{ inputs.magento_directory }} + shell: bash + if: ${{ inputs.use_local_source == true }} + + - run: composer require monolog/monolog:"<2.7.0" --no-update + name: Fixup Monolog (https://github.com/magento/magento2/pull/35596) + working-directory: ${{ inputs.magento_directory }} + if: | + steps.magento-version.outputs.version == '"2.4.4"' + + - run: composer require "dotmailer/dotmailer-magento2-extension-package:4.6.0-p2 as 4.6.0" --no-update + name: Fixup Dotmailer (https://devdocs.magento.com/guides/v2.4/release-notes/release-notes-2-4-0-commerce.html#dotdigital-1) + working-directory: ${{ inputs.magento_directory }} + if: | + steps.magento-version.outputs.version == '"2.4.0"' + + - run: | + composer config --no-interaction allow-plugins.dealerdirect/phpcodesniffer-composer-installer true + composer config --no-interaction allow-plugins.laminas/laminas-dependency-plugin true + composer config --no-interaction allow-plugins.magento/* true + name: Fixup Composer Plugins + working-directory: ${{ inputs.magento_directory }} + if: ${{ !startsWith(matrix.composer, '1') }} + + - run: | + composer global require hirak/prestissimo + name: Install composer plugin for parallel downloads + working-directory: ${{ inputs.magento_directory }} + if: ${{ startsWith(matrix.composer, '1') }} + + - run: composer require ${{ inputs.package_name }} "@dev" --no-update && composer install + name: Require and attempt install + working-directory: ${{ inputs.magento_directory }} + shell: bash + env: + COMPOSER_CACHE_DIR: ${{ steps.composer-cache.outputs.dir }} + COMPOSER_AUTH: ${{ secrets.composer_auth }} + + - name: Replace Configuration Settings for env + working-directory: ${{ inputs.magento_directory }}/dev/tests/e2e + run: | + sed -i "s/'db-host' => 'localhost'/'db-host' => '127.0.0.1'/" etc/install-config-mysql.php.dist + sed -i "s/'db-user' => 'root'/'db-user' => 'user'/" etc/install-config-mysql.php.dist + sed -i "s/'db-password' => '123123q'/'db-password' => 'password'/" etc/install-config-mysql.php.dist + sed -i "s/'elasticsearch-host' => 'localhost'/'elasticsearch-host' => '127.0.0.1'/" etc/install-config-mysql.php.dist + sed -i "s/'amqp-host' => 'localhost'/'amqp-host' => '127.0.0.1'/" etc/install-config-mysql.php.dist + + # mysql server 5.7 doesn't have the column-statistics expected by mysql client 8 (failing 2.3.7-p* builds) + # ref: https://gist.github.com/tobias-khs/8dcf82f719a2b3a7c3b9604b4df53bbf + - name: Switch from mysql-client 8 to mysql-client 5.7 + if: | + steps.magento-version.outputs.version == '"2.3.7-p3"' || steps.magento-version.outputs.version == '"2.3.7-p4"' + run: | + mkdir -p /tmp/mysql-5.7 + cd /tmp/mysql-5.7 + sudo apt-get purge mysql-server mysql-client mysql-common mysql-server-core-* mysql-client-core-* + sudo rm -rf /etc/mysql /var/lib/mysql + sudo apt-get autoremove + sudo apt-get autoclean + wget --quiet https://downloads.mysql.com/archives/get/p/23/file/mysql-server_5.7.30-1ubuntu18.04_amd64.deb-bundle.tar + tar -xf mysql-server_5.7.30-1ubuntu18.04_amd64.deb-bundle.tar + sudo dpkg -i mysql-common_5.7.30-1ubuntu18.04_amd64.deb + sudo dpkg -i libmysqlclient20_5.7.30-1ubuntu18.04_amd64.deb + sudo dpkg -i mysql-community-client_5.7.30-1ubuntu18.04_amd64.deb + sudo dpkg -i mysql-client_5.7.30-1ubuntu18.04_amd64.deb + sudo dpkg -i libmysqlclient20_5.7.30-1ubuntu18.04_amd64.deb + mysqldump --version + + - run: ${{ inputs.test_command }} + working-directory: ${{ inputs.magento_directory }}/dev/tests/e2e + name: Run Playwright E2E phases + + - name: Upload test sandbox dir + uses: actions/upload-artifact@v4 + if: failure() + with: + name: sandbox-data-${{ steps.magento-version.outputs.version }} + path: /home/runner/work/infrastructure/magento2/dev/tests/e2e/tmp/sandbox-* + retention-days: 3 + From f18ea93e91662841a87c26d99a5051c94b1f0b43 Mon Sep 17 00:00:00 2001 From: Herve Tribouilloy Date: Thu, 5 Mar 2026 12:07:52 +0000 Subject: [PATCH 2/4] Test with dummy test file --- .github/workflows/_internal-e2e.yaml | 1 + _test/demo-package/dummy.spec.ts | 0 2 files changed, 1 insertion(+) create mode 100644 _test/demo-package/dummy.spec.ts diff --git a/.github/workflows/_internal-e2e.yaml b/.github/workflows/_internal-e2e.yaml index fe0d9a1..58cacae 100644 --- a/.github/workflows/_internal-e2e.yaml +++ b/.github/workflows/_internal-e2e.yaml @@ -34,6 +34,7 @@ jobs: id: supported-version - run: echo ${{ steps.supported-version.outputs.matrix }} e2e-workflow: + if: ${{ hashFiles('_test/demo-package/**') != '' }} needs: compute_matrix uses: ./.github/workflows/e2e.yaml with: diff --git a/_test/demo-package/dummy.spec.ts b/_test/demo-package/dummy.spec.ts new file mode 100644 index 0000000..e69de29 From 35164c1e734cdc19b338f28d7c2ceea68a557141 Mon Sep 17 00:00:00 2001 From: Herve Tribouilloy Date: Thu, 5 Mar 2026 12:25:13 +0000 Subject: [PATCH 3/4] Removed dummy file not needed in final PR --- _test/demo-package/dummy.spec.ts | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 _test/demo-package/dummy.spec.ts diff --git a/_test/demo-package/dummy.spec.ts b/_test/demo-package/dummy.spec.ts deleted file mode 100644 index e69de29..0000000 From 5b400e85e04157c34a79ef33d829dba215a40f40 Mon Sep 17 00:00:00 2001 From: Herve Tribouilloy Date: Tue, 14 Apr 2026 12:15:20 +0100 Subject: [PATCH 4/4] Remove use of hashfile in favour of a github flow customised to check the existence of the folder demo-package --- .github/workflows/_internal-e2e.yaml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_internal-e2e.yaml b/.github/workflows/_internal-e2e.yaml index 58cacae..5e01e00 100644 --- a/.github/workflows/_internal-e2e.yaml +++ b/.github/workflows/_internal-e2e.yaml @@ -22,6 +22,24 @@ on: - "!(**/*.md)" jobs: + check-demo-package: + runs-on: ubuntu-latest + outputs: + exists: ${{ steps.check.outputs.exists }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Check if folder exists + id: check + run: | + if [ -d "_test/demo-package" ]; then + echo "exists=true" >> $GITHUB_OUTPUT + else + echo "exists=false" >> $GITHUB_OUTPUT + fi + compute_matrix: runs-on: ubuntu-latest outputs: @@ -34,7 +52,7 @@ jobs: id: supported-version - run: echo ${{ steps.supported-version.outputs.matrix }} e2e-workflow: - if: ${{ hashFiles('_test/demo-package/**') != '' }} + if: needs.check-demo-package.outputs.exists == 'true' needs: compute_matrix uses: ./.github/workflows/e2e.yaml with: