see if they fail again with build instead of build-release

.github/actions/prepare-cypress/action.yml

@@ -1,6 +1,12 @@
 name: Prepare cypress environment
 description: Cypress preparation steps
 
+inputs:
+  working-directory:
+    description: "Optional working directory for yarn commands"
+    required: false
+    default: "."
+
 outputs:
   chrome-path:
     description: Path to the custom Chrome executable
@@ -33,3 +39,4 @@ runs:
         yarn cypress cache list
         yarn cypress verify
       shell: bash
+      working-directory: ${{ inputs.working-directory }}

.github/workflows/e2e-cross-version-for-breaking-changes.yml

@@ -13,6 +13,11 @@ on:
         type: string
         required: true
         default: "master"
+      total-shards:
+        description: "Total number of parallel test shards"
+        type: number
+        required: false
+        default: 4
   workflow_call:
     inputs:
       fe-ref:
@@ -23,9 +28,38 @@ on:
         description: "Backend git ref to use"
         type: string
         required: true
+      total-shards:
+        description: "Total number of parallel test shards"
+        type: number
+        required: false
+        default: 4
 
 jobs:
-  cross-version-for-breaking-changes:
+  generate-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix_json: ${{ steps.set_matrix.outputs.matrix_json }}
+      shards_total: ${{ steps.set_matrix.outputs.shards_total }}
+    steps:
+      - name: Set up shard matrix
+        id: set_matrix
+        uses: actions/github-script@v7
+        with:
+          # generate an output like `{"matrixJson": "{\"shard\": [1, 2, 3, 4]}", "shardsTotal": 4}` to be used in the other jobs
+          script: |
+            const totalShards = parseInt(process.env.TOTAL_SHARDS_INPUT, 10);
+
+            const matrix = { shard: Array.from({ length: totalShards }, (_, i) => i + 1) };
+
+            console.log("Generated matrix JSON", matrix);
+            console.log(`Total shards: ${totalShards}`);
+
+            core.setOutput('matrix_json', JSON.stringify(matrix));
+            core.setOutput('shards_total', totalShards);
+        env:
+          TOTAL_SHARDS_INPUT: ${{ inputs.total-shards }}
+
+  build-jar:
     runs-on: ubuntu-22.04
     timeout-minutes: 45
 
@@ -33,8 +67,12 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Prepare frontend environment
-        uses: ./.github/actions/prepare-frontend
+      - name: Prepare Node.js
+        # We're not using `actions/prepare-frontend` because we don't want to install deps from the root repo
+        # nor we want to touch the cache as we'll be potentially be installing different versions of packages
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: ".nvmrc"
 
       - name: Prepare back-end environment
         uses: ./.github/actions/prepare-backend
@@ -50,9 +88,57 @@ jobs:
             ${{ inputs.fe-ref }} ${{ inputs.be-ref }} \
             build
 
+      - name: Upload Uberjar
+        uses: actions/upload-artifact@v4
+        with:
+          name: metabase-jar-cross-version
+          path: .tmp/metabase-be/target/uberjar/metabase.jar
+          if-no-files-found: error
+
+  run-breaking-changes-tests:
+    needs: [build-jar, generate-matrix]
+    runs-on: ubuntu-22.04
+    timeout-minutes: 45
+    env:
+      TOTAL_SHARDS: ${{ needs.generate-matrix.outputs.shards_total }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix_json) }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Prepare Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: ".nvmrc"
+
+      - name: Prepare back-end environment
+        uses: ./.github/actions/prepare-backend
+        with:
+          m2-cache-key: cross-version-uberjar
+
+      - name: Download Uberjar
+        uses: actions/download-artifact@v4
+        with:
+          name: metabase-jar-cross-version
+          path: .tmp/metabase-be/target/uberjar/
+
+      - name: Download FE repo
+        run: |
+          git clone --depth 1 -b ${{ inputs.fe-ref }} https://github.com/metabase/metabase.git .tmp/metabase-fe
+
+      - name: Install FE dependencies
+        working-directory: .tmp/metabase-fe
+        run: |
+          yarn install
+
       - name: Prepare Cypress environment
         id: cypress-prep
         uses: ./.github/actions/prepare-cypress
+        with:
+          working-directory: .tmp/metabase-fe
 
       - name: Run the tests
         env:
@@ -66,4 +152,4 @@ jobs:
             start > /dev/null & \
             node bin/backward-compatibility-test.js \
             ${{ inputs.fe-ref }} ${{ inputs.be-ref }} \
-            test
+            test ${{ matrix.shard}}/${{ env.TOTAL_SHARDS }}

.github/workflows/embedding-sdk.yml

@@ -139,7 +139,7 @@ jobs:
 
   cross-version-for-breaking-changes:
     # run on: every pr that targets a release branch
-    if: ${{ github.event_name == 'pull_request' && startsWith(github.base_ref, 'release-x.') }}
+    # if: ${{ github.event_name == 'pull_request' && startsWith(github.base_ref, 'release-x.') }} # TODO: uncomment this before merging
     uses: ./.github/workflows/e2e-cross-version-for-breaking-changes.yml
     secrets: inherit
     with:

bin/backward-compatibility-test.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 /* eslint-disable no-console */
-/* eslint-disable no-undef -- avoid errors on process */
+/* global process */
 
 const os = require("os");
 const fs = require("fs");
@@ -8,10 +8,10 @@ const path = require("path");
 const { execSync } = require("child_process");
 
 const INSTRUCTIONS =
-  "Usage: node bin/backward-compatibility-test.js <FE_GIT_REF> <BE_GIT_REF> <build|start|test>";
+  "Usage: node bin/backward-compatibility-test.js <FE_GIT_REF> <BE_GIT_REF> <build|start|test> <?optional shard in format shard/total>";
 
 const args = process.argv.slice(2);
-if (args.length !== 3) {
+if (args.length !== 3 && args.length !== 4) {
   console.log(INSTRUCTIONS);
   process.exit(1);
 }
@@ -19,17 +19,31 @@ if (args.length !== 3) {
 const FE_GIT_REF = args[0];
 const BE_GIT_REF = args[1];
 const COMMAND = args[2];
+const SHARD_ARG = args[3];
 
 const TMP_FOLDER = path.join(process.cwd(), ".tmp");
 const FE_FOLDER = path.join(TMP_FOLDER, "metabase-fe");
 const BE_FOLDER = path.join(TMP_FOLDER, "metabase-be");
 const JAR_PATH = path.join(BE_FOLDER, "target/uberjar/metabase.jar");
 
-const TESTS_TO_RUN = [
-  "e2e/test/scenarios/dashboard/dashboard.cy.spec.js",
-  "e2e/test/scenarios/question/caching.cy.spec.js",
+const getTestFiles = () => {
+  const questionFiles = getFilesInsideFolder(
+    path.join(FE_FOLDER, "e2e/test/scenarios/question"),
+  ).map((file) => path.join("e2e/test/scenarios/question", file)); // make sure the path is how we expect, starting with "e2e/test..."
+
+  return [
+    "e2e/test/scenarios/dashboard/dashboard.cy.spec.js",
+    ...questionFiles,
+  ].filter((file) => !SKIPPED_FILES.includes(file));
+};
+
+// those are temporarily ignored because they contain a test that fails
+const SKIPPED_FILES = [
+  // "e2e/test/scenarios/question/native-query-drill.cy.spec.ts",
+  // "e2e/test/scenarios/question/multiple-column-breakouts.cy.spec.ts",
+  // "e2e/test/scenarios/question/notebook-data-source.cy.spec.ts",
+  // "e2e/test/scenarios/question/notebook-link-to-data-source.cy.spec.ts",
 ];
-const COMMA_SEPARATED_TESTS_TO_RUN = TESTS_TO_RUN.join(",");
 
 console.log(`Using frontend from ${FE_GIT_REF}`);
 console.log(`Using backend from ${BE_GIT_REF}`);
@@ -43,7 +57,6 @@ console.log(`TMP_FOLDER: ${TMP_FOLDER}`);
 console.log(`FE_FOLDER: ${FE_FOLDER}`);
 console.log(`BE_FOLDER: ${BE_FOLDER}`);
 console.log(`JAR_PATH: ${JAR_PATH}`);
-console.log(`COMMA_SEPARATED_TESTS_TO_RUN: ${COMMA_SEPARATED_TESTS_TO_RUN}`);
 
 function printStep(message) {
   console.log("---");
@@ -83,7 +96,9 @@ function build() {
 
   console.log("Building frontend...");
   executeCommand("yarn install", FE_FOLDER);
-  executeCommand("yarn build", FE_FOLDER, { MB_EDITION: "ee" });
+  executeCommand("yarn build", FE_FOLDER, {
+    MB_EDITION: "ee",
+  });
 
   printStep("Copying frontend build to backend...");
   const feResourcesClientPath = path.join(
@@ -171,6 +186,14 @@ async function waitForBackend() {
 }
 
 async function test() {
+  const { shard, totalShards, testForThisShard } = getTestShardInfo();
+
+  console.log(
+    `Shard ${shard} of ${totalShards} running tests:\n${testForThisShard.join(
+      "\n",
+    )}`,
+  );
+
   await waitForBackend();
 
   printStep("Creating snapshot...");
@@ -181,10 +204,39 @@ async function test() {
     BACKEND_PORT: "4000",
     TEST_SUITE: "e2e",
   };
-  const cypressCommand = `node e2e/runner/run_cypress_ci.js e2e --env grepTags="--@flaky --@external",grepOmitFiltered=true --spec "${COMMA_SEPARATED_TESTS_TO_RUN}"`;
+
+  const cypressCommand = `node e2e/runner/run_cypress_ci.js e2e --env grepTags="--@flaky --@external",grepOmitFiltered=true --spec "${testForThisShard.join(",")}"`;
   executeCommand(cypressCommand, FE_FOLDER, testEnv);
 }
 
+const getTestShardInfo = () => {
+  const testFiles = getTestFiles();
+
+  if (!SHARD_ARG) {
+    return {
+      shard: 1,
+      totalShards: 1,
+      testForThisShard: testFiles,
+    };
+  }
+
+  const totalShards = SHARD_ARG.split("/")[1];
+  const shard = SHARD_ARG.split("/")[0];
+  const testForThisShard = testFiles.filter(
+    (_test, i) => i % totalShards === shard - 1,
+  );
+
+  return {
+    shard,
+    totalShards,
+    testForThisShard,
+  };
+};
+
+function getFilesInsideFolder(folder) {
+  return fs.readdirSync(folder); //.map((file) => path.join(folder, file));
+}
+
 // main logic
 (async () => {
   try {