test

.github/workflows/e2e-stress-test-flake-fix.yml

@@ -22,14 +22,10 @@ on:
         required: false
         default: "ee"
       qa_db:
-        description: "Select the type of Database you want to run your tests with"
-        type: choice
+        description: "Whether to set up and include external QA databases"
+        type: boolean
         required: true
-        default: "none"
-        options:
-          - none
-          - sql
-          - mongo
+        default: false
       enable_network_throttling:
         description: "Enable network throttling simulation (4g slow like in chrome dev tools)"
         type: boolean
@@ -81,8 +77,6 @@ jobs:
     name: Stress test E2E flake fix
     env:
       DISPLAY: ""
-      QA_DB_ENABLED: ${{ contains(fromJSON('["sql", "mongo"]'), inputs.qa_db) }}
-      CYPRESS_QA_DB_MONGO: ${{ inputs.qa_db == 'mongo' }}
       MB_RUN_MODE: e2e
       METASTORE_DEV_SERVER_URL: ${{ secrets.METASTORE_DEV_SERVER_URL }}
       CYPRESS_MB_ALL_FEATURES_TOKEN: ${{ secrets.STAGING_MB_ALL_FEATURES_TOKEN }}
@@ -109,9 +103,9 @@ jobs:
           maildev: true
           openldap: true
           snowplow: true
-          postgres: ${{ inputs.qa_db == 'sql' }}
-          mysql: ${{ inputs.qa_db == 'sql' }}
-          mongo: ${{ inputs.qa_db == 'mongo' }}
+          postgres: ${{ inputs.qa_db }}
+          mysql: ${{ inputs.qa_db }}
+          mongo: ${{ inputs.qa_db }}
 
       - name: Download Metabase ${{ matrix.edition }} uberjar
         uses: ./.github/actions/e2e-download-uberjar
@@ -140,8 +134,11 @@ jobs:
         uses: ./.github/actions/prepare-cypress
       - name: Run Metabase
         run: node e2e/runner/run_cypress_ci.js start
+
       - name: Generate database snapshots
-        run: node e2e/runner/run_cypress_ci.js snapshot
+        run: |
+          node e2e/runner/run_cypress_ci.js snapshot \
+            --env grepTags="${{inputs.qa_db && '' || '-@external'}}"
 
       # For all options and fine-grained control, take a look at the documentation
       # https://github.com/cypress-io/cypress/tree/develop/npm/grep

.github/workflows/e2e-test.yml

@@ -37,7 +37,6 @@ jobs:
     env:
       MB_EDITION: ${{ inputs.edition }}
       DISPLAY: ""
-      QA_DB_ENABLED: true
       PR_NUMBER: ${{ github.event.pull_request.number || '' }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       HASH: ${{ github.event.pull_request.head.sha || github.sha }}-${{ github.run_attempt }}
@@ -58,7 +57,7 @@ jobs:
       MB_SNOWPLOW_URL: "http://localhost:9090" # Snowplow micro
       TZ: US/Pacific # to make node match the instance tz
       CYPRESS_CI: true
-      CYPRESS_QA_DB_MONGO: ${{ inputs.name == 'mongo' }}
+      MB_INTERNAL_DO_NOT_USE_SAMPLE_DB_DIR: "e2e/tmp/"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -80,9 +79,9 @@ jobs:
           openldap: true
           webhook: true
           snowplow: true
-          postgres: ${{ inputs.name != 'mongo' }}
-          mysql: ${{ inputs.name != 'mongo' }}
-          mongo: ${{ inputs.name == 'mongo' }}
+          postgres: true
+          mysql: true
+          mongo: true
 
       - name: Retrieve uberjar artifact for ${{ inputs.edition }}
         uses: ./.github/actions/fetch-artifact

.github/workflows/embedding-sdk-component-tests.yml

@@ -62,7 +62,10 @@ jobs:
       - name: Make app db snapshot
         env:
           CYPRESS_BROWSER: ${{ steps.cypress-prep.outputs.chrome-path }}
-        run: node e2e/runner/run_cypress_ci.js snapshot
+        # Component tests only need the default snapshot
+        run: |
+          node e2e/runner/run_cypress_ci.js snapshot \
+            --spec "e2e/snapshot-creators/default.cy.snap.js"
 
       # Has to be run after the app db snapshot is made, as it is not compatible with React 19
       - name: Change React version to ${{ matrix.react-version }}

.github/workflows/embedding-sdk-host-sample-apps-tests.yml

@@ -278,7 +278,9 @@ jobs:
         run: node e2e/runner/run_cypress_ci.js start
 
       - name: Generate database snapshots
-        run: node e2e/runner/run_cypress_ci.js snapshot
+        run: |
+          node e2e/runner/run_cypress_ci.js snapshot \
+            --spec "e2e/snapshot-creators/default.cy.snap.js"
 
       - name: Prepare and launch Host App
         run: npx tsx ./e2e/runner/embedding-sdk/host-apps/start-ci.ts ${{ matrix.test_suite }}

.github/workflows/run-tests.yml

@@ -2,9 +2,6 @@ name: Run tests
 
 on:
   push:
-    branches:
-      - "master"
-      - "release-**"
   pull_request:
     types: [opened, synchronize, reopened]
   merge_group:

cypress.env.json.example

@@ -10,5 +10,4 @@
   "MB_PRO_CLOUD_TOKEN": "get it from the password manager",
   "MB_PRO_SELF_HOSTED_TOKEN": "get it from the password manager",
   "MB_EDITION": "ee",
-  "QA_DB_ENABLED": "true"
 }

docs/developers-guide/e2e-tests.md

@@ -122,28 +122,27 @@ One great feature of Cypress is that you can use the Chrome inspector after each
 
 `yarn build` and `yarn build-hot` each overwrite an HTML template to reference the correct JavaScript files. If you run `yarn build` before building an Uberjar for Cypress tests, you won’t see changes to your JavaScript reflected even if you then start `yarn build-hot`.
 
-### Running Cypress on M1 machines
+### Running Cypress on Apple Silicon
+
+You might run into problems when running Cypress on Apple Silicon processors.
 
-You might run into problems when running Cypress on M1 machine.
 This is caused by the `@bahmutov/cypress-esbuild-preprocessor` that is using `esbuild` as a dependency. The error might look [like this](https://github.com/evanw/esbuild/issues/1819#issuecomment-1018771557). [The solution](https://github.com/evanw/esbuild/issues/1819#issuecomment-1080720203) is to install NodeJS using one of the Node version managers like [nvm](https://github.com/nvm-sh/nvm) or [n](https://github.com/tj/n).
 
-Another issue you will almost surely face is the inability to connect to our Mongo QA Database. You can solve it by providing the following env:
+Another issue you will almost surely face is the inability to connect to our Mongo QA Database. The supported Docker image is incompatible (AMD64). You can solve it by providing the following env:
 
 ```shell
 export EXPERIMENTAL_DOCKER_DESKTOP_FORCE_QEMU=1
 ```
 
+Please note that some users experienced Mongo connection timeouts even with this env var set. If that happens, try using OrbStack instead of Docker Desktop.
+
 ### Running tests that depend on Docker images
 
-A subset of our tests depend on the external services that are available through the Docker images. At the time of this writing, those are the three supported external QA databases, Webmail, Snowplow and LDAP servers. The default cypress command will spin up all necessary docker containers for these tests to function properly, but you can toggle them off if you want
-
-```sh
-START_CONTAINERS=false yarn test-cypress
-```
+A large portion of our tests depend on the external services that are available through the Docker images. At the time of this writing, those are the three supported external QA databases, Webmail, Snowplow and LDAP servers. See `e2e/test/scenarios/docker-compose.yml` for up to date information. The default cypress command will spin up all necessary Docker containers for the tests to function properly. You can manually set up the e2e environment without them but be aware that you will run into test failures.
 
 ### Running tests with Snowplow involved
 
-Tests that depend on Snowplow expect a running server. This is enabled by default. You can manually enable them as well by spinning up the snowplow micro docker container and setting the appropriate environment variables:
+Tests that depend on Snowplow expect a running server. This is enabled by default. You can manually enable them as well by spinning up the Snowplow micro Docker container and setting the appropriate environment variables:
 
 ```
 docker-compose -f ./snowplow/docker-compose.yml up -d

e2e/runner/cypress-runner-backend.js

@@ -33,6 +33,11 @@ const CypressBackend = {
         "-Djava.awt.headless=true", // when running on macOS prevent little Java icon from popping up in Dock
         "-Dmail.smtps.ssl.trust=*", // trust self signed certs for testing
         "-Duser.timezone=US/Pacific",
+        // FIXME @nemanjaglumac 2025-11-28
+        // This is a bit convoluted because this particular log4j config outputs all logs to a file (logs/test.log)
+        // See: https://github.com/metabase/metabase/pull/21360
+        // If we want them in the console, we'd have to unset the log4j config, to let stdio inherit the logs.
+        // Override in your .env if you need to see the logs in the console locally.
         process.env.SHOW_BACKEND_LOGS === "true"
           ? null
           : `-Dlog4j.configurationFile=file:${__dirname}/../../frontend/test/__runner__/log4j2.xml`,
@@ -50,6 +55,8 @@ const CypressBackend = {
         MB_CONFIG_FILE_PATH: "__cypress__", // ignore config.yml
         MB_HTTP_CHANNEL_HOST_STRATEGY: "allow-all", // we use a local webhook service for testing
         MB_IS_CYPRESS: "true", // custom flag so we can detect we're running in Cypress, used in tests.
+        MB_SNOWPLOW_AVAILABLE: true,
+        MB_SNOWPLOW_URL: "http://localhost:9090",
       };
 
       this.server.process = spawn("java", [...javaFlags, "-jar", jarPath], {

e2e/runner/run_cypress_ci.js

@@ -46,7 +46,10 @@ if (command === "start") {
 }
 
 if (command === "snapshot") {
-  runTests({ configFile: "e2e/support/cypress-snapshots.config.js" });
+  runTests(
+    { configFile: "e2e/support/cypress-snapshots.config.js" },
+    cliArguments,
+  );
 }
 
 // Metabase component or e2e tests

e2e/runner/run_cypress_local.ts

@@ -1,7 +1,3 @@
-import fs from "fs";
-import os from "os";
-import path from "path";
-
 import { BACKEND_PORT } from "./constants/backend-port";
 import { FAILURE_EXIT_CODE, SUCCESS_EXIT_CODE } from "./constants/exit-code";
 import runCypress from "./cypress-node-js-runner";
@@ -14,103 +10,69 @@ import {
   unBooleanify,
 } from "./cypress-runner-utils";
 
-let tempSampleDBDir: string | null = null;
-
 // if you want to change these, set them as environment variables in your shell
-const userOptions = {
+const options = {
   CYPRESS_TESTING_TYPE: "e2e", // e2e | component
   MB_EDITION: "ee", // ee | oss
-  START_CONTAINERS: true,
-  STOP_CONTAINERS: false,
+  BUILD_BACKEND: true,
   BACKEND_PORT: BACKEND_PORT, // override with MB_JETTY_PORT in your env
   OPEN_UI: true,
-  SHOW_BACKEND_LOGS: false,
   GENERATE_SNAPSHOTS: true,
-  QUIET: false,
-  TZ: "UTC",
   ...booleanify(process.env),
 };
 
-const derivedOptions = {
-  QA_DB_ENABLED: userOptions.START_CONTAINERS,
-  BUILD_JAR: userOptions.BACKEND_PORT === 4000,
-  START_BACKEND: userOptions.BACKEND_PORT === 4000,
-  MB_SNOWPLOW_AVAILABLE: true,
-  MB_SNOWPLOW_URL: "http://localhost:9090",
-};
-
-const options = {
-  ...derivedOptions,
-  ...userOptions,
-};
-
 process.env = unBooleanify(options);
 
 const missingTokens = [
-  "MB_ALL_FEATURES_TOKEN",
-  "MB_STARTER_CLOUD_TOKEN",
-  "MB_PRO_CLOUD_TOKEN",
-  "MB_PRO_SELF_HOSTED_TOKEN",
+  "CYPRESS_MB_ALL_FEATURES_TOKEN",
+  "CYPRESS_MB_STARTER_CLOUD_TOKEN",
+  "CYPRESS_MB_PRO_CLOUD_TOKEN",
+  "CYPRESS_MB_PRO_SELF_HOSTED_TOKEN",
 ].filter((token) => !process.env[token]);
 
 if (options.MB_EDITION === "ee" && missingTokens.length > 0) {
   printBold(
     `⚠️ Missing tokens: ${missingTokens.join(", ")}. Either set them or run with MB_EDITION=oss`,
   );
-  process.exit(FAILURE_EXIT_CODE);
 }
 
 printBold(`Running Cypress with options:
   - CYPRESS_TESTING_TYPE : ${options.CYPRESS_TESTING_TYPE}
   - MB_EDITION           : ${options.MB_EDITION}
-  - START_CONTAINERS     : ${options.START_CONTAINERS}
-  - STOP_CONTAINERS      : ${options.STOP_CONTAINERS}
-  - BUILD_JAR            : ${options.BUILD_JAR}
-  - GENERATE_SNAPSHOTS   : ${options.GENERATE_SNAPSHOTS}
   - BACKEND_PORT         : ${options.BACKEND_PORT}
-  - START_BACKEND        : ${options.START_BACKEND}
+  - BUILD_BACKEND        : ${options.BUILD_BACKEND}
+  - GENERATE_SNAPSHOTS   : ${options.GENERATE_SNAPSHOTS}
   - OPEN_UI              : ${options.OPEN_UI}
-  - SHOW_BACKEND_LOGS    : ${options.SHOW_BACKEND_LOGS}
-  - TZ                   : ${options.TZ}
 `);
 
 const init = async () => {
   const cliArguments = process.argv.slice(2);
   const userOverrides = await parseArguments(cliArguments);
 
-  if (options.START_CONTAINERS) {
-    printBold("⏳ Starting containers");
-    shell("docker compose -f ./e2e/test/scenarios/docker-compose.yml up -d");
-  }
+  printBold("⏳ Starting containers");
+  shell("docker compose -f ./e2e/test/scenarios/docker-compose.yml up -d");
 
-  if (options.BUILD_JAR) {
+  if (options.BUILD_BACKEND) {
     printBold("⏳ Building backend");
     shell("./bin/build-backend-for-test");
 
-    if (options.START_BACKEND) {
-      const isBackendRunning = shell(
-        `lsof -ti:${options.BACKEND_PORT} || echo ""`,
-        { quiet: true },
+    const isBackendRunning = shell(
+      `lsof -ti:${options.BACKEND_PORT} || echo ""`,
+      { quiet: true },
+    );
+    if (isBackendRunning) {
+      printBold(
+        "⚠️ Your backend is already running, you may want to kill pid " +
+          isBackendRunning,
       );
-      if (isBackendRunning) {
-        printBold(
-          "⚠️ Your backend is already running, you may want to kill pid " +
-            isBackendRunning,
-        );
-        process.exit(FAILURE_EXIT_CODE);
-      }
-
-      // Use a temporary copy of the sample db so it won't use and lock the db used for local development
-      tempSampleDBDir = path.join(
-        os.tmpdir(),
-        `metabase-sample-db-e2e-${process.pid}`,
-      );
-      fs.mkdirSync(tempSampleDBDir, { recursive: true });
-      process.env.MB_INTERNAL_DO_NOT_USE_SAMPLE_DB_DIR = tempSampleDBDir;
-
-      printBold("⏳ Starting backend");
-      await CypressBackend.start("target/uberjar/metabase-backend.jar");
+      process.exit(FAILURE_EXIT_CODE);
     }
+
+    // Use a temporary copy of the sample db so it won't use and lock the db used for local development
+    process.env.MB_INTERNAL_DO_NOT_USE_SAMPLE_DB_DIR = "e2e/tmp"; // already .gitignored
+
+    printBold("⏳ Starting backend");
+    await CypressBackend.start("target/uberjar/metabase-backend.jar");
   } else {
     printBold(
       `Not building a jar, expecting metabase to be running on port ${options.BACKEND_PORT}. Make sure your metabase instance is running with an h2 app db and the following environment variables:
@@ -126,7 +88,12 @@ const init = async () => {
 
     printBold("⏳ Generating app db snapshots");
     process.env.OPEN_UI = "false";
-    await runCypress({ configFile: "e2e/support/cypress-snapshots.config.js" });
+    await runCypress({
+      configFile: "e2e/support/cypress-snapshots.config.js",
+      ...(options.CYPRESS_TESTING_TYPE === "component" && {
+        env: { grepTags: "-@external" }, // component tests do not need QA DB snapshots for now
+      }),
+    });
     process.env.OPEN_UI = `${options.OPEN_UI}`;
   } else {
     printBold("Skipping snapshot generation, beware of stale snapshot caches");
@@ -157,29 +124,14 @@ const init = async () => {
 };
 
 const cleanup = async (exitCode: string | number = SUCCESS_EXIT_CODE) => {
-  if (options.BUILD_JAR) {
+  if (options.BUILD_BACKEND) {
     printBold("⏳ Cleaning up...");
     await CypressBackend.stop();
   }
 
-  // Add cleanup for the temporary sample database directory
-  if (tempSampleDBDir) {
-    try {
-      fs.rmSync(tempSampleDBDir, { recursive: true, force: true });
-      printBold(
-        `🗑️ Cleaned up temporary sample database directory: ${tempSampleDBDir}`,
-      );
-    } catch (e) {
-      console.error(
-        `Error cleaning up temporary sample database directory: ${e}`,
-      );
-    }
-  }
-
-  if (options.STOP_CONTAINERS) {
-    printBold("⏳ Stopping containers");
-    shell("docker compose -f ./e2e/test/scenarios/docker-compose.yml down");
-  }
+  printBold(
+    "🧹 Containers are running in background. If you wish to stop them, run:\n`docker compose -f ./e2e/test/scenarios/docker-compose.yml down`",
+  );
 
   typeof exitCode === "number"
     ? process.exit(exitCode)

e2e/snapshot-creators/qa-db.cy.snap.js

@@ -9,29 +9,29 @@ import {
 
 describe("qa databases snapshots", { tags: "@external" }, () => {
   it("creates snapshots for supported qa databases", () => {
-    if (Cypress.env("QA_DB_MONGO") === true) {
-      restoreAndAuthenticate();
-      addMongoDatabase();
-      snapshot("mongo-5");
-    } else {
-      // Postgres
-      restoreAndAuthenticate();
+    // Mongo
+    restoreAndAuthenticate();
 
-      addPostgresDatabase();
-      snapshot("postgres-12");
+    addMongoDatabase();
+    snapshot("mongo-5");
 
-      convertToWritable("postgres");
-      snapshot("postgres-writable");
+    // Postgres
+    restoreAndAuthenticate();
 
-      // MySQL
-      restoreAndAuthenticate();
+    addPostgresDatabase();
+    snapshot("postgres-12");
 
-      addMySQLDatabase({});
-      snapshot("mysql-8");
+    convertToWritable("postgres");
+    snapshot("postgres-writable");
 
-      convertToWritable("mysql");
-      snapshot("mysql-writable");
-    }
+    // MySQL
+    restoreAndAuthenticate();
+
+    addMySQLDatabase({});
+    snapshot("mysql-8");
+
+    convertToWritable("mysql");
+    snapshot("mysql-writable");
 
     restore("blank");
   });

e2e/support/cypress-snapshots.config.js

@@ -2,15 +2,10 @@ const { defineConfig } = require("cypress");
 
 const { defaultConfig } = require("./config");
 
-const isQaDatabase = process.env["QA_DB_ENABLED"] === "true";
-
 module.exports = defineConfig({
   e2e: {
     ...defaultConfig,
     specPattern: "e2e/snapshot-creators/**/*.cy.snap.js",
-    excludeSpecPattern: !isQaDatabase
-      ? "e2e/snapshot-creators/qa-db.cy.snap.js"
-      : undefined,
     video: false,
   },
 });

e2e/support/helpers/e2e-setup-helpers.js

@@ -19,8 +19,6 @@ export function snapshot(name) {
  * } name
  */
 export function restore(name = "default") {
-  cy.skipOn(name.includes("mongo") && Cypress.env("QA_DB_MONGO") !== true);
-
   cy.log("Restore Data Set");
 
   // automatically reset the data db if this is a test that uses a writable db