metabase/.github/workflows/embedding-sdk-host-sample-apps-tests.yml

name: "[NOT REQUIRED] SDK Compatibility tests"

on:
  workflow_call:

env:
  DISPLAY: "" # https://github.com/cypress-io/cypress/issues/4034
  TERM: xterm
  TZ: US/Pacific # to make Node match the instance tz
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  HASH: ${{ github.event.pull_request.head.sha || github.sha }}-${{ github.run_attempt }}
  MB_EDITION: ee
jobs:
  get-sample-apps-data:
    runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
    timeout-minutes: 10
    env:
      # we must support the last two stable releases
      SUPPORTED_RELEASES_COUNT: 2
      TEST_SUITES: |
        [
          "metabase-nodejs-react-sdk-embedding-sample-e2e",
          "metabase-nextjs-sdk-embedding-sample-e2e"
        ]
      SAMPLE_APP_MAIN_BRANCH_NAME: "main"
      SAMPLE_APP_STABLE_RELEASE_BRANCH_NAME_SUFFIX: "stable"
    outputs:
      branch_name: ${{ fromJson(steps.build-sample-apps-data.outputs.result).branch_name }}
      matrix: ${{ toJSON(fromJson(steps.build-sample-apps-data.outputs.result).matrix) }}
    steps:
      - uses: actions/checkout@v6

      - name: Prepare release scripts
        uses: ./.github/actions/prepare-release-scripts

      - name: Build sample app compatibility data
        uses: actions/github-script@v9
        id: build-sample-apps-data
        with:
          script: | # js
            const {
              isReleaseBranch,
              getMajorVersionNumberFromReleaseBranch,
            } = require('${{ github.workspace }}/release/dist/index.cjs');

            const testSuites = JSON.parse(process.env.TEST_SUITES);

            const ref = '${{ github.base_ref || github.ref }}';
            const branchName = ref.replace(/^refs\/heads\//, '');

            const isMainBranchRef = branchName === 'master';
            const isReleaseBranchRef = isReleaseBranch(branchName);

            if (isMainBranchRef) {
              return {
                branch_name: '${{ env.SAMPLE_APP_MAIN_BRANCH_NAME }}',
                matrix: testSuites,
              };
            }

            if (isReleaseBranchRef) {
              const supportedReleasesCount = Number(${{ env.SUPPORTED_RELEASES_COUNT }});

              const currentReleaseVersion = Number(${{ vars.CURRENT_VERSION }});
              const currentBranchVersion = Number(getMajorVersionNumberFromReleaseBranch(branchName));

              const sampleAppBranchName = `${currentBranchVersion}-${{ env.SAMPLE_APP_STABLE_RELEASE_BRANCH_NAME_SUFFIX }}`

              const shouldRunTestsForVersion = (currentReleaseVersion - currentBranchVersion) <= (supportedReleasesCount - 1);

              return {
                branch_name: sampleAppBranchName,
                matrix: shouldRunTestsForVersion ? testSuites : [],
              };
            }

            return {
              branch_name: '',
              matrix: [],
            };

  # In order to check compatibility between Sample Apps and Embedding SDK, the job for each sample app:
  # - pulls a Sample App
  # - starts it
  # - runs its Cypress tests against the local metabase.jar and local @metabase/embedding-sdk-react package.
  # The job is not required, though if it's failing better to be sure that it's not caused by a breaking change.
  # For more details see `Sample Apps compatibility with Embedding SDK tests` in `enterprise/frontend/src/embedding-sdk-package/dev.md`
  e2e-sample-apps-tests:
    needs: [get-sample-apps-data]
    strategy:
      fail-fast: false
      matrix:
        test_suite: ${{ fromJSON(needs.get-sample-apps-data.outputs.matrix) }}
        environment: ["production", "development"]
    if: |
      !cancelled() && needs.get-sample-apps-data.result == 'success'
    runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
    timeout-minutes: 45
    name: e2e-sample-app-${{ matrix.test_suite }}-tests-${{ matrix.environment }}-bundle
    env:
      MB_EDITION: ee
      MB_RUN_MODE: e2e
      MB_WAREHOUSE_ALLOWED_NETWORKS: "allow-all"
      METASTORE_DEV_SERVER_URL: ${{ secrets.METASTORE_DEV_SERVER_URL }}
      CYPRESS_MB_ALL_FEATURES_TOKEN: ${{ secrets.STAGING_MB_ALL_FEATURES_TOKEN }}
      SAMPLE_APP_BRANCH_NAME: ${{ needs.get-sample-apps-data.outputs.branch_name }}
      SAMPLE_APP_ENVIRONMENT: ${{ matrix.environment }}
    permissions:
      id-token: write
      contents: read
      actions: read

    steps:
      - uses: actions/checkout@v6

      - uses: ./.github/actions/maximize-disk-space

      # We have to use the latest version that does not have this issue https://github.com/docker/compose/pull/12752
      - name: Set up Docker Compose
        uses: docker/setup-compose-action@8cccb8c14b6500aaffebff1aa49c502c34d2e5e6 # v2.1.0
        with:
          version: latest

      - name: Get Sample App name
        id: get-sample-app-name
        shell: bash
        run: |
          test_suite="${{ matrix.test_suite }}"
          echo "sample_app_name=${test_suite%-e2e}" >> "$GITHUB_OUTPUT"

      - name: Retrieve uberjar artifact for ee
        uses: ./.github/actions/fetch-artifact
        with:
          commit: ${{ github.event.pull_request.head.sha || github.sha }}
          edition: ee
          extract-path: "target/uberjar"
          output-name: "metabase.jar"

      - name: Retrieve Embedding SDK dist artifact
        uses: actions/download-artifact@v8
        with:
          name: embedding-sdk-${{ github.event.pull_request.head.sha || github.sha }}
          path: ${{ github.workspace }}/resources/embedding-sdk

      - name: Prepare front-end environment
        uses: ./.github/actions/prepare-frontend

      - name: Prepare Cypress environment
        id: cypress-prep
        uses: ./.github/actions/prepare-cypress

      - name: Check if sample app branch exists
        id: check-sample-app-branch
        run: |
          TEST_SUITE="${{ matrix.test_suite }}"
          REPO_NAME="${TEST_SUITE%-e2e}"
          REPO_URL="https://github.com/metabase/${REPO_NAME}.git"
          echo "Checking if branch '${SAMPLE_APP_BRANCH_NAME}' exists in ${REPO_URL}..."
          if git ls-remote --exit-code --heads "$REPO_URL" "${SAMPLE_APP_BRANCH_NAME}"; then
            echo "sample_app_branch_exists=true" >> "$GITHUB_OUTPUT"
          else
            echo "sample_app_branch_exists=false" >> "$GITHUB_OUTPUT"
          fi

      - name: Prepare and launch Sample Apps
        if: ${{ steps.check-sample-app-branch.outputs.sample_app_branch_exists == 'true' }}
        run: npx tsx ./e2e/runner/embedding-sdk/sample-apps/start-ci.ts ${{ matrix.test_suite }}

      - name: Prune build cache
        run: docker builder prune -af

      - name: Log free disk space
        run: df -h /

      - uses: ./.github/actions/resolve-job-id

      - name: Run e2e tests for Sample App
        id: run-e2e-tests
        if: ${{ steps.check-sample-app-branch.outputs.sample_app_branch_exists == 'true' }}
        continue-on-error: true
        env:
          CYPRESS_BROWSER: ${{ steps.cypress-prep.outputs.chrome-path }}
        run: node e2e/runner/run_cypress_ci.js ${{ matrix.test_suite }}

      - name: Upload Cypress Artifacts upon failure
        uses: actions/upload-artifact@v7
        if: ${{ steps.check-sample-app-branch.outputs.sample_app_branch_exists == 'true' && steps.run-e2e-tests.outcome != 'success' }}
        with:
          name: cypress-recording-sample-app-${{ matrix.test_suite }}-${{ matrix.environment }}-latest
          path: |
            ./e2e/tmp/${{ steps.get-sample-app-name.outputs.sample_app_name }}/cypress
          if-no-files-found: ignore

      - name: Fail job if tests failed
        if: ${{ steps.check-sample-app-branch.outputs.sample_app_branch_exists == 'true' && steps.run-e2e-tests.outcome != 'success' }}
        run: |
          exit 1

  # Shoppy e2e tests run separately using the shared composite action for setup.
  e2e-shoppy-tests:
    needs: [get-sample-apps-data]
    strategy:
      fail-fast: false
      matrix:
        environment: ["production", "development"]
    if: |
      !cancelled() && needs.get-sample-apps-data.result == 'success' &&
      (github.ref == 'refs/heads/master' || github.base_ref == 'master')
    runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
    timeout-minutes: 45
    name: e2e-sample-app-shoppy-e2e-tests-${{ matrix.environment }}-bundle
    env:
      MB_EDITION: ee
      MB_RUN_MODE: e2e
      MB_WAREHOUSE_ALLOWED_NETWORKS: "allow-all"
      METASTORE_DEV_SERVER_URL: ${{ secrets.METASTORE_DEV_SERVER_URL }}
      CYPRESS_MB_ALL_FEATURES_TOKEN: ${{ secrets.STAGING_MB_ALL_FEATURES_TOKEN }}
      SAMPLE_APP_BRANCH_NAME: ${{ needs.get-sample-apps-data.outputs.branch_name }}
      SAMPLE_APP_ENVIRONMENT: ${{ matrix.environment }}
    permissions:
      id-token: write
      contents: read
      actions: read

    steps:
      - uses: actions/checkout@v6

      - name: Prepare Shoppy E2E environment
        uses: ./.github/actions/prepare-shoppy-e2e
        id: shoppy-prep
        with:
          commit: ${{ github.event.pull_request.head.sha || github.sha }}
          sample-app-branch-name: ${{ needs.get-sample-apps-data.outputs.branch_name }}
          sample-app-environment: ${{ matrix.environment }}
          staging-mb-all-features-token: ${{ secrets.STAGING_MB_ALL_FEATURES_TOKEN }}
          metastore-dev-server-url: ${{ secrets.METASTORE_DEV_SERVER_URL }}
          shoppy-datadog-application-id: ${{ secrets.SHOPPY_METABASE_APP_DATADOG_APPLICATION_ID }}
          shoppy-datadog-client-token: ${{ secrets.SHOPPY_METABASE_APP_DATADOG_CLIENT_TOKEN }}

      - uses: ./.github/actions/resolve-job-id

      - name: Run e2e tests for Shoppy
        id: run-e2e-tests
        if: ${{ steps.shoppy-prep.outputs.sample_app_branch_exists == 'true' }}
        continue-on-error: true
        env:
          CYPRESS_BROWSER: ${{ steps.shoppy-prep.outputs.chrome-path }}
        run: node e2e/runner/run_cypress_ci.js shoppy-e2e

      - name: Upload Cypress Artifacts upon failure
        uses: actions/upload-artifact@v7
        if: ${{ steps.shoppy-prep.outputs.sample_app_branch_exists == 'true' && steps.run-e2e-tests.outcome != 'success' }}
        with:
          name: cypress-recording-sample-app-shoppy-e2e-${{ matrix.environment }}-latest
          path: |
            ./e2e/tmp/shoppy/cypress
          if-no-files-found: ignore

      - name: Fail job if tests failed
        if: ${{ steps.shoppy-prep.outputs.sample_app_branch_exists == 'true' && steps.run-e2e-tests.outcome != 'success' }}
        run: |
          exit 1

  # Synthetic monitoring tests run only on master merges, in parallel with the regular e2e tests.
  # They use the shoppy production environment and include performance measurement tests.
  e2e-shoppy-synthetic-tests:
    needs: [get-sample-apps-data]
    if: |
      !cancelled() && needs.get-sample-apps-data.result == 'success' &&
      github.event_name == 'push' &&
      github.ref == 'refs/heads/master'
    runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
    timeout-minutes: 45
    name: e2e-shoppy-synthetic-monitoring-tests
    env:
      MB_EDITION: ee
      MB_RUN_MODE: e2e
      MB_WAREHOUSE_ALLOWED_NETWORKS: "allow-all"
      METASTORE_DEV_SERVER_URL: ${{ secrets.METASTORE_DEV_SERVER_URL }}
      CYPRESS_MB_ALL_FEATURES_TOKEN: ${{ secrets.STAGING_MB_ALL_FEATURES_TOKEN }}
      SAMPLE_APP_BRANCH_NAME: ${{ needs.get-sample-apps-data.outputs.branch_name }}
      SAMPLE_APP_ENVIRONMENT: production
    permissions:
      id-token: write
      contents: read
      actions: read

    steps:
      - uses: actions/checkout@v6

      - name: Prepare Shoppy E2E environment
        uses: ./.github/actions/prepare-shoppy-e2e
        id: shoppy-prep
        with:
          commit: ${{ github.event.pull_request.head.sha || github.sha }}
          sample-app-branch-name: ${{ needs.get-sample-apps-data.outputs.branch_name }}
          sample-app-environment: production
          staging-mb-all-features-token: ${{ secrets.STAGING_MB_ALL_FEATURES_TOKEN }}
          metastore-dev-server-url: ${{ secrets.METASTORE_DEV_SERVER_URL }}
          shoppy-datadog-application-id: ${{ secrets.SHOPPY_METABASE_APP_DATADOG_APPLICATION_ID }}
          shoppy-datadog-client-token: ${{ secrets.SHOPPY_METABASE_APP_DATADOG_CLIENT_TOKEN }}

      - name: Get merged PR number
        if: ${{ steps.shoppy-prep.outputs.sample_app_branch_exists == 'true' }}
        id: get-merged-pr-number
        run: |
          PR=$(gh api repos/${{ github.repository }}/commits/${{ github.sha }}/pulls --jq '.[0].number' 2>/dev/null || echo "")
          if [ -z "$PR" ] || [ "$PR" = "null" ]; then
            echo "number=" >> "$GITHUB_OUTPUT"
          else
            echo "number=$PR" >> "$GITHUB_OUTPUT"
          fi
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - uses: ./.github/actions/resolve-job-id

      - name: Run synthetic monitoring tests
        id: run-e2e-tests
        if: ${{ steps.shoppy-prep.outputs.sample_app_branch_exists == 'true' }}
        continue-on-error: true
        env:
          CYPRESS_BROWSER: ${{ steps.shoppy-prep.outputs.chrome-path }}
          PR_NUMBER: ${{ steps.get-merged-pr-number.outputs.number }}
          INCLUDE_SYNTHETIC_MONITORING_TESTS: "true"
        run: node e2e/runner/run_cypress_ci.js shoppy-e2e

      - name: Upload Cypress Artifacts upon failure
        uses: actions/upload-artifact@v7
        if: ${{ steps.shoppy-prep.outputs.sample_app_branch_exists == 'true' && steps.run-e2e-tests.outcome != 'success' }}
        with:
          name: cypress-recording-shoppy-synthetic-tests-latest
          path: |
            ./e2e/tmp/shoppy/cypress
          if-no-files-found: ignore

      - name: Fail job if tests failed
        if: ${{ steps.shoppy-prep.outputs.sample_app_branch_exists == 'true' && steps.run-e2e-tests.outcome != 'success' }}
        run: |
          exit 1

  # The job checks integration of the Embedding SDK with consumer's apps that use different frameworks/bundlers,
  # or when we want to test some tricky integration cases like conflicting types.
  # The job
  # - starts an app
  # - runs its Cypress tests against the e2e test metabase instance and local @metabase/embedding-sdk-react package.
  # The job is not required, though if it's failing better to be sure that it's not caused by a breaking change.
  # For more details see `Embedding SDK integration tests with Host Apps` in `enterprise/frontend/src/embedding-sdk-package/dev.md`
  e2e-host-apps-tests:
    strategy:
      fail-fast: false
      matrix:
        test_suite:
          [
            "vite-6-host-app-e2e",
            "vite-6-data-app-host-app-e2e",
            "next-15-app-router-host-app-e2e",
            "next-15-pages-router-host-app-e2e",
            "angular-20-host-app-e2e",
          ]
        environment: ["production", "development"]
    runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
    timeout-minutes: 45
    name: e2e-host-app-${{ matrix.test_suite }}-tests-${{ matrix.environment }}-bundle
    env:
      # Host apps are the only suite here running through Metabase's Cypress config,
      # whose `after:spec` hook reports failures and writes QUARANTINE_FAILURES_FILE.
      # Sample-app suites load their own config (no-op), so ci-conductor is scoped to this job.
      CI_CONDUCTOR_BASE_URL: ${{ vars.CI_CONDUCTOR_BASE_URL }}
      CI_CONDUCTOR_WEBHOOK_SECRET: ${{ secrets.CI_CONDUCTOR_WEBHOOK_SECRET }}
      CI_CONDUCTOR_TEST_SUITE: embedding-sdk-host-app
      REPO_ID: ${{ github.event.repository.id }}
      COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
      TARGET_BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}
      QUARANTINE_FAILURES_FILE: ${{ github.workspace }}/target/quarantine-failures.jsonl
      # `Fail job if tests failed` is the arbiter until ci-conductor takes over.
      # Exactly one of the two gates the job.
      CI_CONDUCTOR_QUARANTINE_ACTIVE: ${{ vars.QUARANTINE_ENGINE == 'ci-conductor' }}
      CYPRESS_MB_ALL_FEATURES_TOKEN: ${{ secrets.STAGING_MB_ALL_FEATURES_TOKEN }}
      HOST_APP_ENVIRONMENT: ${{ matrix.environment }}
    permissions:
      id-token: write
      contents: read
      actions: read

    steps:
      - uses: actions/checkout@v6

      - name: Retrieve uberjar artifact for ee
        uses: ./.github/actions/fetch-artifact
        with:
          commit: ${{ github.event.pull_request.head.sha || github.sha }}
          edition: ee
          extract-path: "target/uberjar"
          output-name: "metabase.jar"

      - name: Retrieve Embedding SDK dist artifact
        uses: actions/download-artifact@v8
        with:
          name: embedding-sdk-${{ github.event.pull_request.head.sha || github.sha }}
          path: ${{ github.workspace }}/resources/embedding-sdk

      - name: Prepare front-end environment
        uses: ./.github/actions/prepare-frontend
      - name: Prepare back-end environment
        uses: ./.github/actions/prepare-backend

      - name: Prepare Cypress environment
        id: cypress-prep
        uses: ./.github/actions/prepare-cypress

      # esbuild resolves every import in the spec bundle's graph, even the ones tree shaking then drops.
      # The cljs imports resolve into target/cljs_dev, so the compiled CLJS has to exist before Cypress runs.
      # run-tests.yml compiles it in parallel with the uberjar build, so downloading is ~10x faster than compiling here.
      - name: Download pre-built CLJS
        id: prebuilt-cljs
        uses: actions/download-artifact@v8
        continue-on-error: true
        with:
          name: e2e-cljs-${{ github.event.pull_request.head.sha || github.sha }}
          path: target/cljs_dev

      - name: Compile CLJS
        if: steps.prebuilt-cljs.outcome != 'success'
        run: bun run build-pure:cljs

      # The data-app suite synchronizes real actions, and Metabase only permits
      # those against a writable warehouse, which comes from the `qa-db`
      # snapshot below. That snapshot creator builds every QA database in one
      # pass — Mongo and MySQL included — so all three have to be up even though
      # only `postgres-writable` is restored. The other host apps read the sample
      # database, so they skip the pull entirely.
      - name: Prepare Docker containers
        if: ${{ matrix.test_suite == 'vite-6-data-app-host-app-e2e' }}
        uses: ./.github/actions/e2e-prepare-containers
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
          postgres: true
          mysql: true
          mongo: true

      - name: Run Metabase
        run: node e2e/runner/run_cypress_ci.js start

      - uses: ./.github/actions/resolve-job-id

      # Without `--spec` the snapshot config picks up `qa-db.cy.snap.js` too,
      # which is what creates the `postgres-writable` snapshot the data-app
      # action tests restore.
      - name: Generate database snapshots
        run: |
          if [ "${{ matrix.test_suite }}" = "vite-6-data-app-host-app-e2e" ]; then
            node e2e/runner/run_cypress_ci.js snapshot
          else
            node e2e/runner/run_cypress_ci.js snapshot \
              --spec "e2e/snapshot-creators/default.cy.snap.js"
          fi

      - name: Prepare and launch Host App
        run: npx tsx ./e2e/runner/embedding-sdk/host-apps/start-ci.ts ${{ matrix.test_suite }}

      - name: Run e2e tests for Host App
        id: run-e2e-tests
        continue-on-error: true
        env:
          CYPRESS_BROWSER: ${{ steps.cypress-prep.outputs.chrome-path }}
        run: node e2e/runner/run_cypress_ci.js ${{ matrix.test_suite }}

      - name: Upload Cypress Artifacts upon failure
        uses: actions/upload-artifact@v7
        if: ${{ !cancelled() && steps.run-e2e-tests.outcome == 'failure' }}
        with:
          name: cypress-recording-host-app-${{ matrix.test_suite }}-${{ matrix.environment }}-latest
          path: |
            ./cypress
          if-no-files-found: ignore

      - name: Quarantine gate
        if: ${{ !cancelled() && steps.run-e2e-tests.outcome == 'failure' }}
        continue-on-error: ${{ env.CI_CONDUCTOR_QUARANTINE_ACTIVE != 'true' }}
        timeout-minutes: 3
        uses: ./.github/actions/quarantine-gate
        with:
          adapter: e2e
          test-suite: ${{ env.CI_CONDUCTOR_TEST_SUITE }}
          enforce-quarantine: ${{ env.CI_CONDUCTOR_QUARANTINE_ACTIVE }}

      - name: Fail job if tests failed
        if: ${{ !cancelled() && steps.run-e2e-tests.outcome == 'failure' && env.CI_CONDUCTOR_QUARANTINE_ACTIVE != 'true' }}
        run: |
          exit 1