Cleaning up code

Replace scattered is-test? checks at var definition time with a proper
:mq initialization step in the test startup flow. This mirrors how
production initializes MQ via MqInit in startup/run-startup-logic!.

The :mq step:
- Loads driver.init (so def-listener! namespaces are loaded)
- Calls mq.init/init-for-tests! which sets sync backends, disables
  publish buffering, and registers all listeners

Also:
- Reverts var defaults to production values (appdb backends, 100ms buffer)
- Reverts def-listener! to lazy registration (no eager call)
- Restores with-sync-mq to use fresh *listeners* atom
- Removes all [TZ-DEBUG] diagnostic traces
- :db initialization depends on :mq

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

.claude/skills/add-tracing/SKILL.md

@@ -1,4 +1,4 @@
---b-
+---
 name: add-tracing
 description: Add OpenTelemetry tracing spans to Clojure code following Metabase tracing conventions. Use when instrumenting backend code with trace coverage.
 ---

.clj-kondo/config.edn

@@ -176,8 +176,10 @@
      clojure.core.async/to-chan!
      clojure.core.async/to-chan!!
      metabase.channel.core/send!
+     metabase.warehouses.models.database/assert-not-h2!
      metabase.driver.sql-jdbc.execute/execute-prepared-statement!
      metabase.notification.models/create-notification!
+     metabase.oauth-server.core/reset-provider!
      metabase.pulse.send/send-pulse!
      metabase-enterprise.transforms-python.python-runner-test/execute!
      ;; Workspace isolation functions are safe in parallel tests because they operate on
@@ -552,6 +554,7 @@
    metabase-enterprise.cache.config-test/with-temp-persist-models                                                            hooks.common/with-seven-bindings
    metabase-enterprise.dependencies.api-test/with-dependents-test!                                                           hooks.common/with-two-bindings
    metabase-enterprise.metabot-v3.tools.dependencies-test/with-dependent-transforms!                                         hooks.common/with-two-bindings
+   metabase-enterprise.metabot.tools.dependencies-test/with-dependent-transforms!                                            hooks.common/with-two-bindings
    metabase-enterprise.semantic-search.db.connection/with-migrate-tx                                                         hooks.common/let-one-with-optional-value
    metabase-enterprise.serialization.api-test/with-serialization-test-data!                                                  hooks.common/with-three-bindings
    metabase-enterprise.serialization.test-util/with-temp-dpc                                                                 hooks.toucan2.tools.with-temp/with-temp
@@ -593,6 +596,9 @@
    metabase.model-persistence.api-test/with-persistence-setup!                                                               hooks.common/with-one-top-level-binding
    metabase.model-persistence.api-test/with-setup!                                                                           hooks.common/with-one-top-level-binding
    metabase.model-persistence.test-util/with-persistence-enabled!                                                            hooks.metabase.model-persistence.test-util/with-persistence-enabled!
+   metabase.mq.core/with-queue                                                                                               hooks.common/let-second
+   metabase.mq.core/with-topic                                                                                               hooks.common/let-second
+   metabase.mq.core/def-listener!                                                                                            hooks.metabase.mq.impl/def-listener
    metabase.models.interface/define-batched-hydration-method                                                                 hooks.metabase.models.interface/define-hydration-method
    metabase.models.interface/define-simple-hydration-method                                                                  hooks.metabase.models.interface/define-hydration-method
    metabase.permissions.core/allow-changing-all-users-group-members                                                          hooks.common/do*

.clj-kondo/src/hooks/metabase/mq/impl.clj

@@ -0,0 +1,24 @@
+(ns hooks.metabase.mq.impl
+  (:require
+   [clj-kondo.hooks-api :as hooks]))
+
+(defn def-listener
+  "Kondo hook for `mq/def-listener`. Rewrites to a `fn` form so kondo can analyze bindings.
+
+   Forms:
+     (mq/def-listener! :topic/foo [msg] body...)
+     (mq/def-listener! :queue/bar {:config map} [msgs] body...)"
+  [{:keys [node]}]
+  (let [[_ _channel-name & args] (:children node)
+        ;; skip optional config map
+        args (if (hooks/map-node? (first args))
+               (rest args)
+               args)
+        [bindings & body] args
+        ;; rewrite as: (fn [bindings...] body...)
+        new-node (hooks/list-node
+                  (list*
+                   (hooks/token-node 'fn)
+                   bindings
+                   body))]
+    {:node new-node}))

.cljfmt.edn

@@ -303,7 +303,6 @@
   metabase.util.malli/with                                                                    [[:default]]
   metabase.util.malli/with-api-error-message                                                  [[:default]]
   metabase.util.namespaces/import-fns                                                         [[:default]]
-  metabase.util.queue-test/await-test-while                                                   [[:block 1]]
   metabase.util/auto-retry                                                                    [[:block 1]]
   metabase.util/case-enum                                                                     [[:block 1]]
   metabase.util/for-map                                                                       [[:block 1]]

.github/team.json

@@ -1,13 +1,13 @@
 {
   "teams": [
     {
-      "name": "Querying",
-      "label": ".Team/Querying",
+      "name": "Querying Platform",
+      "label": ".Team/QueryingPlatform",
       "members": [
         "bshepherdson",
         "metamben",
         "rileythomp",
-        "camsual"
+        "camsaul"
       ]
     },
     {

.github/workflows/backend.yml

@@ -37,16 +37,16 @@ jobs:
         name: Run Eastwood linter
 
   be-linter-eastwood-test:
-    if: ${{ !inputs.skip }}
+    if: ${{ !inputs.skip && github.ref_name == 'master' }}
     runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
     timeout-minutes: 30
-    name: Eastwood (test)
+    name: Eastwood (tests only)
     steps:
       - uses: actions/checkout@v4
       - name: Prepare back-end environment
         uses: ./.github/actions/prepare-backend
       - run: clojure -X:dev:ee:ee-dev:drivers:drivers-dev:test:eastwood/test
-        name: Run Eastwood linter on tests
+        name: Run Eastwood linter on our backend tests
 
   be-tests:
     if: ${{ !inputs.skip }}

.github/workflows/claude-code-bug-fix.yml

@@ -1,360 +0,0 @@
-name: "Claude Code Bug Fix"
-
-on:
-  issues:
-    types:
-      - labeled
-  pull_request_review:
-    types:
-      - submitted
-  workflow_dispatch:
-    inputs:
-      issue_number:
-        description: "Issue number to fix"
-        required: true
-        type: string
-      override_employee_check:
-        description: "Override employee check for issue creator"
-        required: false
-        default: false
-        type: boolean
-
-jobs:
-  auto-fix-bug:
-    runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
-    timeout-minutes: 30
-    permissions:
-      contents: write
-      issues: write
-      pull-requests: write
-      id-token: write
-    if: |
-      (github.event_name == 'issues' && github.event.label.name == 'Claude Bug Fix' && github.event.issue.state == 'open') ||
-      (github.event_name == 'workflow_dispatch') ||
-      (github.event_name == 'pull_request_review' && github.event.review.state == 'changes_requested' && startsWith(github.event.pull_request.head.ref, 'claude-fix/'))
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Get issue data and check employee status
-        id: get_issue
-        uses: actions/github-script@v7
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          script: |
-            const { getIssueData } = require('${{ github.workspace }}/.github/scripts/get-issue-data.js');
-
-            try {
-              const result = await getIssueData({
-                context,
-                github,
-                manualIssueNumber: '${{ github.event.inputs.issue_number }}'
-              });
-
-              const { issueNumber, issueData, employeeComments, prReviews, triggerType, isIssueAuthorEmployee } = result;
-
-              // Check employee status and fail if not authorized
-              if (!result.isIssueAuthorEmployee && '${{ github.event.inputs.override_employee_check }}' !== 'true') {
-                  core.setFailed(`User ${result.issueData.user.login} is not a Metabase employee. Skipping bug fix automation.`);
-              }
-
-              core.setOutput('issue_number', issueNumber);
-              core.setOutput('issue_data', JSON.stringify(issueData));
-              core.setOutput('issue_comments', JSON.stringify(employeeComments));
-              core.setOutput('pr_reviews', JSON.stringify(prReviews));
-              core.setOutput('trigger_type', triggerType);
-            } catch (error) {
-              core.setFailed(error.message);
-            }
-
-      - name: Find linked Linear issue
-        id: find_linear
-        continue-on-error: true
-        uses: actions/github-script@v7
-        env:
-          LINEAR_API_KEY: ${{ secrets.LINEAR_API_KEY }}
-          ISSUE_NUMBER: ${{ steps.get_issue.outputs.issue_number }}
-        with:
-          script: |
-            const { findLinearIssue } = require('${{ github.workspace }}/.github/scripts/find-linear-issue.js');
-
-            const githubIssueUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/issues/${process.env.ISSUE_NUMBER}`;
-
-            const result = await findLinearIssue({
-              githubIssueUrl,
-              linearApiKey: process.env.LINEAR_API_KEY
-            });
-
-            if (result.success) {
-              console.log(`Found Linear issue: ${result.issue}`);
-              core.setOutput('linear_issue', result.issue);
-            } else {
-              console.log(`Could not find Linear issue: ${result.message}`);
-            }
-
-      - name: Set up git configuration
-        run: |
-          git config --global user.name "Claude Code[bot]"
-          git config --global user.email "noreply@anthropic.com"
-
-      - name: Prepare issue context and bug fix prompt
-        id: prepare_prompt
-        uses: actions/github-script@v7
-        env:
-          ISSUE_DATA: ${{ steps.get_issue.outputs.issue_data }}
-          ISSUE_COMMENTS: ${{ steps.get_issue.outputs.issue_comments }}
-          PR_REVIEWS: ${{ steps.get_issue.outputs.pr_reviews }}
-          TRIGGER_TYPE: ${{ steps.get_issue.outputs.trigger_type }}
-        with:
-          script: |
-            const issueData = JSON.parse(process.env.ISSUE_DATA);
-            const issueComments = JSON.parse(process.env.ISSUE_COMMENTS);
-            const prReviews = JSON.parse(process.env.PR_REVIEWS);
-            const triggerType = process.env.TRIGGER_TYPE;
-            const issueNumber = '${{ steps.get_issue.outputs.issue_number }}';
-
-            // Convert issue title to kebab case and create branch name
-            const issueTitle = issueData.title || 'untitled';
-            const kebabTitle = issueTitle
-              .toLowerCase()
-              .replace(/[^a-z0-9]+/g, '-')
-              .replace(/^-+|-+$/g, '')
-              .replace(/-+/g, '-')
-              .substring(0, 50);
-            const branchName = `claude-fix/issue-${issueNumber}-${kebabTitle}`;
-
-            // Format data for bash environment
-            const formattedComments = issueComments.length > 0
-              ? issueComments.map(comment =>
-                  `**Comment by @${comment.user.login} on ${comment.created_at}:**\n${comment.body}`
-                ).join('\n\n---\n\n')
-              : '';
-            const formattedReviews = prReviews.length > 0
-              ? prReviews.map(review =>
-                  `**Review by @${review.user.login} on ${review.submitted_at} (Changes Requested):**\n${review.body || 'No review comment provided'}`
-                ).join('\n\n---\n\n')
-              : '';
-
-            core.exportVariable('BRANCH_NAME', branchName);
-            core.exportVariable('ISSUE_NUMBER', issueNumber);
-            core.exportVariable('ISSUE_TITLE', issueData.title);
-            core.exportVariable('ISSUE_AUTHOR', issueData.user.login);
-            core.exportVariable('ISSUE_CREATED', issueData.created_at);
-            core.exportVariable('ISSUE_BODY', issueData.body || 'No description provided');
-            core.exportVariable('ISSUE_COMMENTS_FORMATTED', formattedComments);
-            core.exportVariable('PR_REVIEWS_FORMATTED', formattedReviews);
-            core.exportVariable('TRIGGER_TYPE', triggerType);
-
-      - name: Generate bug fix prompt
-        run: |
-          cat << 'EOF' >> $GITHUB_ENV
-          BUG_FIX_PROMPT<<PROMPT_END
-          You are helping to fix a bug reported in GitHub issue #${{ env.ISSUE_NUMBER }}.
-
-          ## Bug Report Details
-          **Title:** ${{ env.ISSUE_TITLE }}
-          **Reporter:** ${{ env.ISSUE_AUTHOR }}
-          **Created:** ${{ env.ISSUE_CREATED }}
-
-          **Description:**
-          ${{ env.ISSUE_BODY }}
-
-          **Comments:**
-          ${{ env.ISSUE_COMMENTS_FORMATTED }}
-          EOF
-
-          # Add review feedback section only if this is a review trigger
-          if [ "${{ env.TRIGGER_TYPE }}" = "pull_request_review" ]; then
-            cat << 'EOF' >> $GITHUB_ENV
-
-          ## Previous Fix Attempt - Review Feedback (MUST ADDRESS)
-          Your previous automated fix attempt received the following feedback that needs to be addressed:
-
-          ${{ env.PR_REVIEWS_FORMATTED }}
-          EOF
-          fi
-
-          # Add the rest of the prompt (always the same)
-          cat << 'EOF' >> $GITHUB_ENV
-
-          ## Instructions
-          1. **Analyze the issue description and comments** - Understand what the issue is describing
-             a. Pay particular attention to reproduction steps and if there's an agreed upon fix.
-          2. **Search the codebase** - Use Grep and Glob tools to find relevant files
-          3. **Implement a fix** - Use the appropriate write skill based on the file types:
-             - Use clojure-write for Clojure/ClojureScript files
-             - Use typescript-write for TypeScript/JavaScript files
-             - Follow the coding standards and patterns in each skill
-          4. **Test the fix** - Run appropriate tests as specified in the skills
-          5. **Verify the fix** - Ensure tests pass and code follows conventions
-
-          ## Important Guidelines
-          - Use the appropriate write skill for the language you're working in
-          - Write failing tests first, then fix them
-          - Work in small, testable increments
-          - Don't commit changes - leave that for the workflow
-          - IMPORTANT: Focus only on fixing the reported bug, don't make unrelated changes
-
-          ## Scratchpad Directory
-          A `.claude-scratchpad/` directory has been created for your temporary files:
-          - Use this directory for ALL temporary verification files. However, prefer using production test files rather than adhoc ones.
-          - Only modify actual production code in the main repository directories
-          - The scratchpad will be cleaned up automatically - nothing from it will be committed
-
-          Start by understanding the bug and then systematically work toward a solution.
-          PROMPT_END
-          EOF
-
-      - name: Determine branch strategy
-        run: |
-          BRANCH_NAME="${{ env.BRANCH_NAME }}"
-          if git branch -r --list "origin/${BRANCH_NAME}" | grep -q .; then
-            echo "Branch already exists: $BRANCH_NAME"
-            echo "Will push to existing branch and skip PR creation"
-            echo "BRANCH_EXISTS=true" >> $GITHUB_ENV
-          else
-            echo "Creating new branch: $BRANCH_NAME"
-            echo "Will create new PR"
-            echo "BRANCH_EXISTS=false" >> $GITHUB_ENV
-          fi
-
-          # Export the branch name
-          echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
-
-      - name: Create scratchpad directory for Claude
-        run: |
-          # Create a scratchpad directory for Claude to use for temporary files
-          mkdir -p .claude-scratchpad
-          echo "Scratchpad directory created for Claude's temporary verification files"
-
-      - name: Prepare front-end environment
-        uses: ./.github/actions/prepare-frontend
-
-      - name: Prepare back-end environment
-        uses: ./.github/actions/prepare-backend
-
-      - name: Generate GitHub App Token for Claude
-        uses: actions/create-github-app-token@v1
-        id: claude-app-token
-        with:
-          app-id: ${{ secrets.METABASE_BOT_APP_ID }}
-          private-key: ${{ secrets.METABASE_BOT_APP_PRIVATE_KEY }}
-
-      - name: Generate Bug Fix with Claude Code
-        uses: anthropics/claude-code-action@v1
-        with:
-          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
-          github_token: ${{ steps.claude-app-token.outputs.token }}
-          prompt: ${{ env.BUG_FIX_PROMPT }}
-          claude_args: |
-            --allowedTools "Bash(bun run test-unit*),Bash(bun run lint-eslint-pure),Bash(./bin/mage*),Bash(clojure*),Bash(find*),Bash(ls*),Bash(grep*),Write,Read,Glob,Grep,Edit,MultiEdit"
-            --disallowedTools "Bash(git commit*),Bash(git push*),Bash(gh*)"
-
-      - name: Clean up scratchpad directory
-        run: |
-          # Remove the scratchpad directory so its contents don't get committed
-          rm -rf .claude-scratchpad/
-          # Remove output file generated by Claude Code action
-          rm -f output.txt
-          echo "Scratchpad directory and action output files cleaned up"
-
-      - name: Check for changes and analysis
-        id: check_changes
-        run: |
-          # Now check for actual code changes (excluding analysis file)
-          if [ -n "$(git status --porcelain)" ]; then
-            echo "has_changes=true" >> $GITHUB_OUTPUT
-            echo "Changes detected:"
-            git status --short
-          else
-            echo "has_changes=false" >> $GITHUB_OUTPUT
-            echo "No changes made by Claude."
-          fi
-
-      - name: Get issue team labels and prepare reviewers
-        if: steps.check_changes.outputs.has_changes == 'true'
-        id: prepare_reviewers
-        uses: actions/github-script@v7
-        env:
-          ISSUE_DATA: ${{ steps.get_issue.outputs.issue_data }}
-        with:
-          script: |
-            const issueData = JSON.parse(process.env.ISSUE_DATA);
-            const issueLabels = issueData.labels.map(label => label.name);
-            const teamLabels = issueLabels.filter(label => label.startsWith('.Team/'));
-            const issueCreator = issueData.user.login;
-            core.setOutput('team_labels', teamLabels.join(','));
-            core.setOutput('reviewers', issueCreator);
-
-      - name: Create Pull Request
-        id: create_pr
-        if: steps.check_changes.outputs.has_changes == 'true'
-        uses: peter-evans/create-pull-request@v6
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          branch: ${{ env.BRANCH_NAME }}
-          base: master
-          commit-message: |
-            Fix: ${{ env.ISSUE_TITLE }}
-
-            Fixes #${{ env.ISSUE_NUMBER }}
-
-            🤖 Generated with [Claude Code](https://claude.ai/code)
-
-            Co-Authored-By: Claude <noreply@anthropic.com>
-          committer: "Claude Code[bot] <noreply@anthropic.com>"
-          author: "Claude Code[bot] <noreply@anthropic.com>"
-          title: "Fix: ${{ env.ISSUE_TITLE }}"
-          body: |
-            This PR automatically fixes the bug reported in #${{ env.ISSUE_NUMBER }}.
-
-            ${{ steps.find_linear.outputs.linear_issue && format('Closes {0}', steps.find_linear.outputs.linear_issue) || '' }}
-
-            ## Bug Report
-            **Original Issue:** #${{ env.ISSUE_NUMBER }}
-            **Reporter:** @${{ env.ISSUE_AUTHOR }}
-            **Issue Title:** ${{ env.ISSUE_TITLE }}
-
-            ## Automated Fix
-            🤖 This fix was generated automatically by Claude Code based on the issue description.
-
-            **Please review carefully:**
-            - [ ] The fix addresses the reported issue
-            - [ ] Tests pass and code follows conventions
-            - [ ] No unrelated changes were introduced
-            - [ ] The solution is appropriate and maintainable
-
-            ## Next Steps
-            - Review the changes and test thoroughly
-            - If the fix looks good, approve and merge
-            - If issues are found, either fix them or close this PR
-
-            ---
-
-            🤖 Generated with [Claude Code](https://claude.ai/code)
-
-            Co-Authored-By: Claude <noreply@anthropic.com>
-          labels: |
-            ${{ steps.prepare_reviewers.outputs.team_labels }}
-            automated-fix
-            ci skip
-          reviewers: ${{ steps.prepare_reviewers.outputs.reviewers }}
-          assignees: ${{ github.actor }}
-
-      - name: Log existing branch update
-        if: steps.check_changes.outputs.has_changes == 'true' && env.BRANCH_EXISTS == 'true'
-        uses: actions/github-script@v7
-        with:
-          script: |
-            console.log('🔄 Updated existing branch: ${{ env.BRANCH_NAME }}');
-            console.log('The changes have been pushed to the existing PR for this issue.');
-
-      - name: Log analysis for no changes
-        if: steps.check_changes.outputs.has_changes == 'false'
-        uses: actions/github-script@v7
-        with:
-          script: |
-            console.log('🤖 No code changes were made by Claude.');

.github/workflows/containerize-uberjar.yml

@@ -76,7 +76,7 @@ jobs:
         with:
           ref: ${{ github.event.inputs.commit || github.event.pull_request.head.sha || github.sha }}
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@0.29.0
+        uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1
         env:
           TRIVY_OFFLINE_SCAN: true
           repo: ${{ fromJson(needs.get-container-info.outputs[matrix.edition]).repo }}
@@ -85,7 +85,6 @@ jobs:
           image-ref: docker.io/${{ env.repo }}:${{ env.tag }}
           format: sarif
           output: trivy-results.sarif
-          version: "v0.57.1"
       - name: Upload Trivy scan results to GitHub Security tab
         uses: github/codeql-action/upload-sarif@v3
         with:

.github/workflows/cross-branch-migrations.yml

@@ -154,7 +154,6 @@ jobs:
     services:
       mariadb:
         image: metabase/mariadb:10.6 # https://github.com/metabase/metabase-mariadb-docker-image
-        image: ${{ matrix.version.image }}
         ports:
           - "3306:3306"
         env:

.github/workflows/cross-version.yml

@@ -61,13 +61,38 @@ jobs:
       - name: Prepare release scripts
         uses: ./.github/actions/prepare-release-scripts
 
+      - name: Prepare front-end environment
+        uses: ./.github/actions/prepare-frontend
+
+      - name: Prepare Cypress environment
+        uses: ./.github/actions/prepare-cypress
+
       - name: Run cross-version test
         env:
           CURRENT_VERSION: ${{ vars.CURRENT_VERSION }}
+          RESULT_FILE: ${{ runner.temp }}/cross-version-result-${{ matrix.source }}-${{ matrix.target }}.json
         run: |
           cd cross-version
           ./test.sh --source "${{ matrix.source }}" --target "${{ matrix.target }}"
 
+      - name: Upload failure result
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: failure-result-${{ matrix.source }}-${{ matrix.target }}
+          path: ${{ runner.temp }}/cross-version-result-${{ matrix.source }}-${{ matrix.target }}.json
+          if-no-files-found: ignore
+
+      - name: Upload Cypress Artifacts upon failure
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: cypress-recording-${{ matrix.source }}-${{ matrix.target }}
+          path: |
+            ./cypress
+            ./logs/test.log
+          if-no-files-found: ignore
+
       - name: Show logs on failure
         if: failure()
         run: |
@@ -75,6 +100,84 @@ jobs:
           docker compose logs metabase || true
           docker compose logs postgres || true
 
+  # Notify Slack on nightly failures
+  notify-on-failure:
+    if: always() && github.event_name == 'schedule' && needs.test-matrix.result == 'failure'
+    needs: [test-matrix]
+    runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            release
+            .github
+
+      - name: Prepare release scripts
+        uses: ./.github/actions/prepare-release-scripts
+
+      - name: Download failure results
+        uses: actions/download-artifact@v4
+        continue-on-error: true
+        with:
+          pattern: failure-result-*
+          path: failure-results
+          merge-multiple: true
+
+      - name: Notify Slack
+        continue-on-error: true
+        uses: actions/github-script@v7
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+        with:
+          script: |
+            const fs = require('fs');
+            const path = require('path');
+            const { buildFailureResults, sendCrossVersionSlackNotification } = require('${{ github.workspace }}/release/dist/index.cjs');
+
+            // Get all failed jobs from the GitHub API
+            const { owner, repo } = context.repo;
+            const jobInfo = await github.rest.actions.listJobsForWorkflowRun({
+              owner,
+              repo,
+              run_id: context.runId,
+              per_page: 100,
+            });
+
+            const failedJobs = jobInfo.data.jobs
+              .filter(job => job.status === 'completed' && job.conclusion === 'failure' && job.name.startsWith('test-matrix'))
+              .map(job => ({ name: job.name, url: job.html_url }));
+
+            if (failedJobs.length === 0) {
+              console.log('No failed test-matrix jobs found');
+              return;
+            }
+
+            // Load artifact data for enrichment (phase)
+            const resultsDir = 'failure-results';
+            const artifactData = new Map();
+            if (fs.existsSync(resultsDir)) {
+              for (const file of fs.readdirSync(resultsDir)) {
+                if (!file.endsWith('.json')) continue;
+                try {
+                  const result = JSON.parse(fs.readFileSync(path.join(resultsDir, file), 'utf8'));
+                  artifactData.set(`${result.source}-${result.target}`, result);
+                } catch (e) {
+                  console.log(`Skipping malformed result: ${file}`);
+                }
+              }
+            }
+
+            const failures = buildFailureResults(failedJobs, artifactData);
+            const runUrl = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${context.runId}`;
+            const slackBotToken = process.env.SLACK_BOT_TOKEN;
+            if (!slackBotToken) {
+              console.log('SLACK_BOT_TOKEN is not set; skipping Slack notification.');
+              return;
+            }
+            await sendCrossVersionSlackNotification(failures, runUrl, slackBotToken);
+            console.log('Slack notification sent');
+
   # Manual test for workflow_dispatch
   test-manual:
     if: github.event_name == 'workflow_dispatch'
@@ -87,6 +190,12 @@ jobs:
       - name: Prepare release scripts
         uses: ./.github/actions/prepare-release-scripts
 
+      - name: Prepare front-end environment
+        uses: ./.github/actions/prepare-frontend
+
+      - name: Prepare Cypress environment
+        uses: ./.github/actions/prepare-cypress
+
       - name: Run cross-version test
         env:
           CURRENT_VERSION: ${{ vars.CURRENT_VERSION }}
@@ -94,6 +203,16 @@ jobs:
           cd cross-version
           ./test.sh --source "${{ inputs.source_version }}" --target "${{ inputs.target_version }}"
 
+      - name: Upload Cypress Artifacts upon failure
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: cypress-recording-${{ inputs.source_version }}-${{ inputs.target_version }}
+          path: |
+            ./cypress
+            ./logs/test.log
+          if-no-files-found: ignore
+
       - name: Show logs on failure
         if: failure()
         run: |

.github/workflows/docs-master-generate.yml

@@ -2,7 +2,7 @@ name: Auto-update documentation (master)
 
 on:
   schedule:
-    - cron: "0 1 * * 1-5" # Every Monday through Friday at 1 AM UTC
+    - cron: "0 21 * * 1-5" # Every Monday through Friday at 9 PM UTC
   workflow_dispatch:
 
 jobs:

.github/workflows/docs-release-generate.yml

@@ -2,7 +2,7 @@ name: Auto-update documentation (release)
 
 on:
   schedule:
-    - cron: "0 2 * * 1-5" # Every Monday through Friday at 2 AM UTC
+    - cron: "0 22 * * 1-5" # Every Monday through Friday at 10 PM UTC
   workflow_dispatch:
     inputs:
       target_branch:

.github/workflows/e2e-stress-test-flake-fix.yml

@@ -31,6 +31,11 @@ on:
         type: boolean
         required: false
         default: true
+      fail_fast:
+        description: "Stop running remaining tests after the first failure"
+        type: boolean
+        required: false
+        default: false
 
 jobs:
   determine-test-type:
@@ -67,6 +72,7 @@ jobs:
           echo '- `grep`: "${{ inputs.grep }}"' >> $GITHUB_STEP_SUMMARY
           echo '- `qa_db`: "${{ inputs.qa_db }}"' >> $GITHUB_STEP_SUMMARY
           echo '- `enable_network_throttling`: "${{ inputs.enable_network_throttling }}"' >> $GITHUB_STEP_SUMMARY
+          echo '- `fail_fast`: "${{ inputs.fail_fast }}"' >> $GITHUB_STEP_SUMMARY
           echo '' >> $GITHUB_STEP_SUMMARY
           echo 'triggered by: @${{ github.event.sender.login }}' >> $GITHUB_STEP_SUMMARY
 
@@ -144,6 +150,7 @@ jobs:
         env:
           GREP: ${{ github.event.inputs.grep }}
           CYPRESS_BROWSER: ${{ steps.cypress-prep.outputs.chrome-path }}
+          FAIL_FAST: ${{ inputs.fail_fast }}
         run: |
           node e2e/runner/run_cypress_ci.js ${{ needs.determine-test-type.outputs.type }} \
           --spec '${{ github.event.inputs.spec }}' \

.github/workflows/loki-stress-test-flake-fix.yml

@@ -88,22 +88,30 @@ jobs:
     timeout-minutes: 5
     steps:
       - name: Generate workflow summary
+        env:
+          BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+          STORY_FILES: ${{ needs.detect-changed-stories.outputs.story_files }}
+          BURN_IN: ${{ inputs.burn_in || 50 }}
+          EVENT_NAME: ${{ github.event_name }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          PR_ACTOR: ${{ github.event.pull_request.user.login }}
+          ACTOR: ${{ github.actor }}
         run: |
           echo '## Loki Stress Test' >> $GITHUB_STEP_SUMMARY
           echo '' >> $GITHUB_STEP_SUMMARY
-          echo '**Branch:** `${{ github.head_ref || github.ref_name }}`' >> $GITHUB_STEP_SUMMARY
+          echo "**Branch:** \`${BRANCH_NAME}\`" >> $GITHUB_STEP_SUMMARY
           echo '' >> $GITHUB_STEP_SUMMARY
           echo '**Story files:**' >> $GITHUB_STEP_SUMMARY
-          echo '${{ needs.detect-changed-stories.outputs.story_files }}' | tr ',' '\n' | while read -r file; do
+          echo "${STORY_FILES}" | tr ',' '\n' | while read -r file; do
             echo "- \`$file\`" >> $GITHUB_STEP_SUMMARY
           done
           echo '' >> $GITHUB_STEP_SUMMARY
-          echo '**Burn-in:** ${{ inputs.burn_in || 50 }} runs' >> $GITHUB_STEP_SUMMARY
+          echo "**Burn-in:** ${BURN_IN} runs" >> $GITHUB_STEP_SUMMARY
           echo '' >> $GITHUB_STEP_SUMMARY
-          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
-            echo 'Triggered by PR #${{ github.event.pull_request.number }} from @${{ github.event.pull_request.user.login }}' >> $GITHUB_STEP_SUMMARY
+          if [[ "${EVENT_NAME}" == "pull_request" ]]; then
+            echo "Triggered by PR #${PR_NUMBER} from @${PR_ACTOR}" >> $GITHUB_STEP_SUMMARY
           else
-            echo 'Triggered manually by @${{ github.actor }}' >> $GITHUB_STEP_SUMMARY
+            echo "Triggered manually by @${ACTOR}" >> $GITHUB_STEP_SUMMARY
           fi
 
   stress-test-loki:

.github/workflows/module-boundaries-stats.yml

@@ -20,6 +20,7 @@ jobs:
           NAMED_MODULES=$(node --input-type=module -e "
             import { elements } from './frontend/lint/module-boundaries.mjs';
             const named = elements.filter(e => !['shared/other', 'app/misc', 'other'].includes(e.type));
+            console.log(named.length);
           ")
 
           NAMED_DIRS=$(node --input-type=module -e "
@@ -29,6 +30,7 @@ jobs:
               const m = el.pattern.match(/^frontend\/src\/metabase\/(\w[\w-]*)\//)
               if (m) dirs.add(m[1]);
             }
+            for (const d of dirs) console.log(d);
           ")
           TOTAL_DIRS=$(ls -d frontend/src/metabase/*/ | xargs -I{} basename {} | wc -l | tr -d ' ')
           NAMED_COUNT=$(echo "$NAMED_DIRS" | grep -c . || true)

.github/workflows/querying-team-pagerduty.yml

@@ -13,32 +13,35 @@ jobs:
       (github.event.label.name == 'Priority:P0' || github.event.label.name == 'Priority:P1' || github.event.label.name == '.Escalation') &&
       contains(github.event.issue.labels.*.name, '.Team/Querying')
     steps:
-      - name: Setting title
-        uses: actions/github-script@v7
-        id: vars
-        with:
-          script: |
-            core.setOutput('issue_title', ${{ toJson(github.event.issue.title) }}.replaceAll(/"/g, '\\"'));
       - name: Trigger PagerDuty Incident
+        env:
+          ROUTING_KEY: ${{ secrets.QUERYING_PAGERDUTY_INTEGRATION_KEY }}
+          ISSUE_TITLE: ${{ github.event.issue.title }}
+          LABEL_NAME: ${{ github.event.label.name }}
+          ISSUE_NUMBER: ${{ github.event.issue.number }}
+          ISSUE_URL: ${{ github.event.issue.html_url }}
+          REPO: ${{ github.repository }}
+          ALL_LABELS: ${{ join(github.event.issue.labels.*.name, ', ') }}
         run: |
+          payload=$(jq -n \
+            --arg rk  "$ROUTING_KEY" \
+            --arg dk  "github-issue-$ISSUE_NUMBER" \
+            --arg sum "[$LABEL_NAME] $ISSUE_TITLE" \
+            --arg src "GitHub Issues - $REPO" \
+            --arg lbl "$ALL_LABELS" \
+            --arg url "$ISSUE_URL" \
+            '{
+              routing_key: $rk,
+              event_action: "trigger",
+              dedup_key: $dk,
+              payload: {
+                summary: $sum,
+                source: $src,
+                severity: "critical",
+                custom_details: { Labels: $lbl }
+              },
+              links: [{ href: $url, text: "View GitHub Issue" }]
+            }')
           curl -X POST https://events.pagerduty.com/v2/enqueue \
             -H 'Content-Type: application/json' \
-            -d '{
-              "routing_key": "${{ secrets.QUERYING_PAGERDUTY_INTEGRATION_KEY }}",
-              "event_action": "trigger",
-              "dedup_key": "github-issue-${{ github.event.issue.number }}",
-              "payload": {
-                "summary": "[${{ github.event.label.name }}] ${{ steps.vars.outputs.issue_title }}",
-                "source": "GitHub Issues - ${{ github.repository }}",
-                "severity": "critical",
-                "custom_details": {
-                  "Labels": "${{ join(github.event.issue.labels.*.name, ', ') }}"
-                }
-              },
-              "links": [
-                {
-                  "href": "${{ github.event.issue.html_url }}",
-                  "text": "View GitHub Issue"
-                }
-              ]
-            }'
+            -d "$payload"

.github/workflows/release-custom-viz.yml

@@ -0,0 +1,68 @@
+name: Release Custom Viz Package
+run-name: Release @metabase/custom-viz v${{ inputs.version }} from ${{ inputs.branch }}
+
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: "Branch to release from"
+        type: string
+        required: true
+        default: "custom-viz"
+      version:
+        description: "Version to publish (e.g. 0.1.0)"
+        type: string
+        required: true
+      npm_tag:
+        description: "NPM dist-tag"
+        type: choice
+        required: true
+        default: "latest"
+        options:
+          - latest
+          - next
+          - canary
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
+    defaults:
+      run:
+        working-directory: frontend/src/custom-viz
+    steps:
+      - name: Check out the code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.branch }}
+
+      - name: Prepare front-end environment
+        uses: ./.github/actions/prepare-frontend
+
+      - name: Setup npm registry auth
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: ".nvmrc"
+          registry-url: "https://registry.npmjs.org"
+
+      - name: Set package version
+        run: npm version ${{ inputs.version }} --no-git-tag-version
+
+      - name: Build package
+        run: bun run build
+
+      - name: Publish to NPM
+        run: npm publish --tag ${{ inputs.npm_tag }}
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_RELEASE_TOKEN }}
+
+      - name: Generate workflow summary
+        run: |
+          tee -a $GITHUB_STEP_SUMMARY << EOF
+          ## Published @metabase/custom-viz
+
+          - **Version**: ${{ inputs.version }}
+          - **Tag**: ${{ inputs.npm_tag }}
+          - **Branch**: ${{ inputs.branch }}
+          - **Triggered by**: @${{ github.event.sender.login }}
+          EOF

.github/workflows/release-patch.yml

@@ -13,13 +13,13 @@ on:
         type: string
         required: false
         default: ""
-  # schedule:
-  #   - cron: "0 21 * * 1-5" # every weekday at 4 pm EST/5 pm EDT
+  schedule:
+    - cron: "0 21 * * 1-5" # every weekday at 4 pm EST/5 pm EDT
 
 jobs:
   auto-patch-trigger:
     # don't run this workflow on a cron for forks
-    if: ${{ github.event_name != 'schedule' || github.repository == 'metabase/metabase' }}
+    if: ${{ github.event_name != 'schedule' || (github.repository == 'metabase/metabase' && vars.ENABLE_AUTO_PATCH == 'true') }}
     runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
     timeout-minutes: 5
     steps:

.github/workflows/run-tests.yml

@@ -110,7 +110,7 @@ jobs:
 
   sdk-tests:
     needs: [uberjar, embedding-sdk-package, files-changed]
-    if: always()
+    if: always() && !cancelled()
     uses: ./.github/workflows/embedding-sdk.yml
     secrets: inherit
     with:

.github/workflows/tag-for-release.yml

@@ -192,6 +192,42 @@ jobs:
           ./COMMIT-ID
           ./SHA256.sum
 
+  upload-to-cloud:
+    runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
+    needs: tag-uberjar-for-release
+    timeout-minutes: 15
+    steps:
+    - name: Configure AWS credentials
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        aws-access-key-id: ${{ secrets.AWS_S3_RELEASE_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_S3_RELEASE_SECRET_ACCESS_KEY }}
+        aws-region: ${{ vars.AWS_REGION }}
+    - uses: actions/checkout@v4
+      with:
+        sparse-checkout: .github
+    - name: Retrieve release ee uberjar artifact
+      uses: ./.github/actions/fetch-artifact
+      with:
+        commit: ${{ inputs.commit }}
+        edition: ee
+        release: "true"
+    - name: Determine the upload path
+      uses: actions/github-script@v7
+      id: version_path
+      with:
+        result-encoding: string
+        script: | # js
+          const version = '${{ inputs.version }}';
+          // only EE version
+          const version_path = 'release/' + version.replace(/^v0\./, "v1.");
+
+          console.log("sending to", version_path);
+          return version_path;
+    - name: Upload to S3
+      run: |
+        aws s3 cp ./metabase.jar s3://${{ secrets.AWS_S3_CLOUD_DEPLOY_BUCKET }}/${{ steps.version_path.outputs.result }}/metabase.jar
+
   run-pre-release-tests:
     name: Run pre-release tests for ${{ inputs.version }}
     needs: tag-uberjar-for-release

.github/workflows/team-issues-slack-notification.yml

@@ -19,7 +19,7 @@ jobs:
         id: vars
         with:
           script: |
-            core.setOutput('issue_title', ${{ toJson(github.event.issue.title) }}.replaceAll(/"/g, '\\"'));
+            core.setOutput('issue_title', JSON.stringify(${{ toJson(github.event.issue.title) }} + '\n' + ${{ toJson(github.event.issue.html_url) }}));
       - name: Extracting team name
         id: team_name
         run: |
@@ -39,7 +39,7 @@ jobs:
                   "type": "section",
                   "text": {
                     "type": "mrkdwn",
-                    "text": "${{ steps.vars.outputs.issue_title }}\n${{ github.event.issue.html_url }}"
+                    "text": ${{ steps.vars.outputs.issue_title }}
                   }
                 }
               ]

.gitignore

@@ -96,7 +96,7 @@ coverage-summary.json
 junit.xml
 pom.xml
 pom.xml.asc
-profiles.clj
+/profiles.clj
 target/checksum.txt
 xcshareddata
 xcuserdata
@@ -176,3 +176,4 @@ CLAUDE.local.md
 /resources/python-sources/sqlglot-*.dist-info/
 /resources/python-sources/.lock
 __pycache__/
+.mcp.json

CLAUDE.md

@@ -6,7 +6,7 @@ For detailed guidance on writing and reviewing code and documentation, see the s
 
 ## Clojure
 
-- **[clojure-eval](.claude/skills/clojure-eval/SKILL.md)** - Always use this to evaluate Clojure code, **run tests**, and verify edits/compile. Prefer this over `clojure -X:...` shell commands.
+- **[clojure-eval](.claude/skills/clojure-eval/SKILL.md)** - Always use this to evaluate Clojure code, **run tests**, and verify edits/compile. Prefer this over shell commands.
 - **[clojure-write](.claude/skills/clojure-write/SKILL.md)** - Clojure/ClojureScript development with REPL-driven workflow and coding conventions
 - **[clojure-review](.claude/skills/clojure-review/SKILL.md)** - Clojure/ClojureScript code review guidelines and style enforcement
 

Dockerfile

@@ -42,6 +42,9 @@ FROM eclipse-temurin:21-jre-alpine AS runner
 
 ENV FC_LANG=en-US LC_CTYPE=en_US.UTF-8
 
+# copy certs before the RUN so keytool can import them
+COPY bin/docker/DigiCertGlobalRootG2.crt.pem /app/certs/DigiCertGlobalRootG2.crt.pem
+
 # dependencies
 RUN apk add -U bash fontconfig curl font-noto font-noto-arabic font-noto-hebrew font-noto-cjk java-cacerts && \
     apk upgrade && \
@@ -49,7 +52,6 @@ RUN apk add -U bash fontconfig curl font-noto font-noto-arabic font-noto-hebrew
     mkdir -p /app/certs && \
     curl https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem -o /app/certs/rds-combined-ca-bundle.pem  && \
     /opt/java/openjdk/bin/keytool -noprompt -import -trustcacerts -alias aws-rds -file /app/certs/rds-combined-ca-bundle.pem -keystore /etc/ssl/certs/java/cacerts -keypass changeit -storepass changeit && \
-    curl https://cacerts.digicert.com/DigiCertGlobalRootG2.crt.pem -o /app/certs/DigiCertGlobalRootG2.crt.pem  && \
     /opt/java/openjdk/bin/keytool -noprompt -import -trustcacerts -alias azure-cert -file /app/certs/DigiCertGlobalRootG2.crt.pem -keystore /etc/ssl/certs/java/cacerts -keypass changeit -storepass changeit && \
     mkdir -p /plugins && chmod a+rwx /plugins
 

bb.edn

@@ -303,17 +303,21 @@
    :task (task! (doctor/doctor!))}
 
   ci-report
-  {:doc "Generate a CI failure report for a PR"
-   :examples [["./bin/mage ci-report" "Check current branch, find its PR, and generate report"]
+  {:doc "Generate a CI failure report for a PR, branch, or commit"
+   :examples [["./bin/mage ci-report" "Check current branch (PR or branch CI)"]
               ["./bin/mage ci-report 68292" "Generate report for PR #68292"]
-              ["./bin/mage ci-report --detailed 68292" "Generate detailed report with full FAIL/ERROR blocks"]
+              ["./bin/mage ci-report source-replacement" "Report for a branch by name"]
+              ["./bin/mage ci-report abc123def" "Report for a specific commit SHA"]
+              ["./bin/mage ci-report --detailed 68292" "Detailed report with full FAIL/ERROR blocks"]
               ["./bin/mage ci-report --detailed > ci_report.md"
                "Save detailed report to markdown file and print progress."]]
    :options [["-d" "--detailed" "Include full FAIL/ERROR blocks with expected/actual diffs"]]
    :usage-fn (fn [_]
                (str "Fetches CI check status and failed job logs from GitHub Actions.\n"
                     "Parses Trunk test reports to extract test failure details.\n\n"
-                    "With no arguments, looks up the PR for your current git branch.\n\n"
+                    "Accepts a PR number/URL, branch name, or commit SHA.\n"
+                    "With no arguments, uses the current branch's PR if one exists,\n"
+                    "otherwise falls back to the current branch's latest CI run.\n\n"
                     "Use --detailed to get full test failure output with expected/actual diffs.\n\n"
                     "Requires GitHub CLI (gh) to be installed and authenticated.\n"
                     "If you see 'gh: command not found' or auth errors, run: gh auth login"))

bin/docker/DigiCertGlobalRootG2.crt.pem

@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDjjCCAnagAwIBAgIQAzrx5qcRqaC7KGSxHQn65TANBgkqhkiG9w0BAQsFADBh
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBH
+MjAeFw0xMzA4MDExMjAwMDBaFw0zODAxMTUxMjAwMDBaMGExCzAJBgNVBAYTAlVT
+MRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5j
+b20xIDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IEcyMIIBIjANBgkqhkiG
+9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzfNNNx7a8myaJCtSnX/RrohCgiN9RlUyfuI
+2/Ou8jqJkTx65qsGGmvPrC3oXgkkRLpimn7Wo6h+4FR1IAWsULecYxpsMNzaHxmx
+1x7e/dfgy5SDN67sH0NO3Xss0r0upS/kqbitOtSZpLYl6ZtrAGCSYP9PIUkY92eQ
+q2EGnI/yuum06ZIya7XzV+hdG82MHauVBJVJ8zUtluNJbd134/tJS7SsVQepj5Wz
+tCO7TG1F8PapspUwtP1MVYwnSlcUfIKdzXOS0xZKBgyMUNGPHgm+F6HmIcr9g+UQ
+vIOlCsRnKPZzFBQ9RnbDhxSJITRNrw9FDKZJobq7nMWxM4MphQIDAQABo0IwQDAP
+BgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQUTiJUIBiV
+5uNu5g/6+rkS7QYXjzkwDQYJKoZIhvcNAQELBQADggEBAGBnKJRvDkhj6zHd6mcY
+1Yl9PMWLSn/pvtsrF9+wX3N3KjITOYFnQoQj8kVnNeyIv/iPsGEMNKSuIEyExtv4
+NeF22d+mQrvHRAiGfzZ0JFrabA0UWTW98kndth/Jsw1HKj2ZL7tcu7XUIOGZX1NG
+Fdtom/DzMNU+MeKNhJ7jitralj41E6Vf8PlwUHBHQRFXGU7Aj64GxJUTFy8bJZ91
+8rGOmaFvE7FBcf6IKshPECBV1/MUReXgRPTqh5Uykw7+U0b6LJ3/iyK5S9kJRaTe
+pLiaWN0bfVKfjllDiIGknibVb63dDcY3fe0Dkhvld1927jyNxF1WW6LZZm6zNTfl
+MrY=
+-----END CERTIFICATE-----

bin/docker/Dockerfile

@@ -5,6 +5,9 @@ ENV FC_LANG=en-US LC_CTYPE=en_US.UTF-8
 ARG GIT_COMMIT_SHA
 ENV GIT_COMMIT_SHA=${GIT_COMMIT_SHA}
 
+# copy cert before RUN so keytool can import it
+COPY DigiCertGlobalRootG2.crt.pem /app/certs/DigiCertGlobalRootG2.crt.pem
+
 # dependencies
 RUN apk add -U bash fontconfig curl font-noto font-noto-arabic font-noto-hebrew font-noto-cjk font-noto-thai java-cacerts && \
     apk upgrade && \
@@ -12,7 +15,6 @@ RUN apk add -U bash fontconfig curl font-noto font-noto-arabic font-noto-hebrew
     mkdir -p /app/certs && \
     curl https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem -o /app/certs/rds-combined-ca-bundle.pem  && \
     /opt/java/openjdk/bin/keytool -noprompt -import -trustcacerts -alias aws-rds -file /app/certs/rds-combined-ca-bundle.pem -keystore /etc/ssl/certs/java/cacerts -keypass changeit -storepass changeit && \
-    curl https://cacerts.digicert.com/DigiCertGlobalRootG2.crt.pem -o /app/certs/DigiCertGlobalRootG2.crt.pem  && \
     /opt/java/openjdk/bin/keytool -noprompt -import -trustcacerts -alias azure-cert -file /app/certs/DigiCertGlobalRootG2.crt.pem -keystore /etc/ssl/certs/java/cacerts -keypass changeit -storepass changeit && \
     mkdir -p /plugins && chmod a+rwx /plugins
 

bin/docker/Dockerfile_ubuntu

@@ -5,13 +5,15 @@ ENV FC_LANG=en-US LC_CTYPE=en_US.UTF-8
 ARG GIT_COMMIT_SHA
 ENV GIT_COMMIT_SHA=${GIT_COMMIT_SHA}
 
+# copy cert before RUN so it's available without a network fetch
+COPY DigiCertGlobalRootG2.crt.pem /usr/local/share/ca-certificates/DigiCertGlobalRootG2.crt.pem
+
 # Dependencies
 RUN apt-get update && \
   apt-get upgrade -y && \
   apt-get install -y ca-certificates ca-certificates-java fonts-noto && \
   apt-get clean && \
   curl https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem -o /usr/local/share/ca-certificates/rds-combined-ca-bundle.pem && \
-  curl https://cacerts.digicert.com/DigiCertGlobalRootG2.crt.pem -o /usr/local/share/ca-certificates/DigiCertGlobalRootG2.crt.pem && \
   update-ca-certificates && \
   mkdir -p /plugins && chmod a+rwx /plugins && \
   keytool -list -cacerts

bin/test-agent

@@ -10,15 +10,30 @@
 # Usage:
 #   bin/test-agent :only '[metabase.foo-test]'
 #   bin/test-agent :only '[metabase.foo-test/some-test]'
+#   bin/test-agent --oss :only '[metabase.foo-test]'   # run without EE extensions
 #   bin/test-agent                          # runs all tests (rarely what you want)
 #
-# Any extra arguments are forwarded directly to `clojure -X:dev:ee:ee-dev:test`.
+# Options:
+#   --oss    Run without enterprise extensions (uses :oss:oss-dev instead of :ee:ee-dev)
+#
+# Any extra arguments are forwarded directly to the clojure test command.
 # The :test alias uses port 0 (OS-assigned random port) so parallel runs never collide.
 
 set -euo pipefail
 
 export HAWK_MODE=cli/ci
 
+# Check for --oss flag
+ALIASES=":dev:ee:ee-dev:test"
+args=()
+for arg in "$@"; do
+  if [[ "$arg" == "--oss" ]]; then
+    ALIASES=":dev:ci:test:oss:oss-dev"
+  else
+    args+=("$arg")
+  fi
+done
+
 # Filter noise and strip ANSI escape sequences
 clean_output() {
   grep -v '^Downloading: ' \
@@ -28,4 +43,4 @@ clean_output() {
   | sed $'s/\033[\[(][0-9;]*[A-Za-z]//g; s/\033[^[]*//g'
 }
 
-exec clojure -X:dev:ee:ee-dev:test "$@" 2>&1 | clean_output
+exec clojure -X"${ALIASES}" "${args[@]}" 2>&1 | clean_output

bun.lock

@@ -41,7 +41,7 @@
         "@react-oauth/google": "^0.11.1",
         "@reduxjs/toolkit": "^2.5.0",
         "@snowplow/browser-tracker": "^3.1.6",
-        "@tanstack/react-table": "^8.21.2",
+        "@tanstack/react-table": "^8.21.3",
         "@tanstack/react-virtual": "^3.13.2",
         "@tippyjs/react": "^4.2.6",
         "@tiptap/core": "^3.2.0",
@@ -1510,7 +1510,7 @@
 
     "@szmarczak/http-timer": ["@szmarczak/http-timer@1.1.2", "", { "dependencies": { "defer-to-connect": "^1.0.1" } }, "sha512-XIB2XbzHTN6ieIjfIMV9hlVcfPU26s2vafYWQcZHWXHOxiaRZYEDKEwdl129Zyg50+foYV2jCgtrqSA6qNuNSA=="],
 
-    "@tanstack/react-table": ["@tanstack/react-table@8.21.2", "", { "dependencies": { "@tanstack/table-core": "8.21.2" } }, "sha512-11tNlEDTdIhMJba2RBH+ecJ9l1zgS2kjmexDPAraulc8jeNA4xocSNeyzextT0XJyASil4XsCYlJmf5jEWAtYg=="],
+    "@tanstack/react-table": ["@tanstack/react-table@8.21.3", "", { "dependencies": { "@tanstack/table-core": "8.21.3" }, "peerDependencies": { "react": ">=16.8", "react-dom": ">=16.8" } }, "sha512-5nNMTSETP4ykGegmVkhjcS8tTLW6Vl4axfEGQN3v0zdHYbK4UfoqfPChclTrJ4EoK9QynqAu9oUf8VEmrpZ5Ww=="],
 
     "@tanstack/react-virtual": ["@tanstack/react-virtual@3.13.2", "", { "dependencies": { "@tanstack/virtual-core": "3.13.2" } }, "sha512-LceSUgABBKF6HSsHK2ZqHzQ37IKV/jlaWbHm+NyTa3/WNb/JZVcThDuTainf+PixltOOcFCYXwxbLpOX9sCx+g=="],
 
@@ -5718,8 +5718,6 @@
 
     "@svgr/plugin-svgo/deepmerge": ["deepmerge@4.3.1", "", {}, "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A=="],
 
-    "@tanstack/react-table/@tanstack/table-core": ["@tanstack/table-core@8.21.2", "", {}, "sha512-uvXk/U4cBiFMxt+p9/G7yUWI/UbHYbyghLCjlpWZ3mLeIZiUBSKcUnw9UnKkdRz7Z/N4UBuFLWQdJCjUe7HjvA=="],
-
     "@testing-library/jest-dom/aria-query": ["aria-query@5.0.0", "", {}, "sha512-V+SM7AbUwJ+EBnB8+DXs0hPZHO0W6pqBcc0dW90OwtVG02PswOu/teuARoLQjdDOH+t9pJgGnW5/Qmouf3gPJg=="],
 
     "@testing-library/jest-dom/chalk": ["chalk@3.0.0", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg=="],

cross-version/.gitignore

@@ -0,0 +1 @@
+.xv-*

cross-version/README.md

@@ -1,55 +1,70 @@
 # Cross-Version Migration Testing
 
-Tests Metabase database migrations across version upgrades and downgrades.
+Tests that Metabase data survives database migrations across version upgrades and downgrades.
 
-## Quick Start
+Uses Cypress e2e tests to create data on a **source** version, then verify it's intact on a **target** version after migration. The application database is PostgreSQL (matching production), and state is preserved between versions via a Docker volume.
+
+## Scripts
+
+### `test.sh` — CI / production runs
+
+Runs the full migration test end-to-end. Starts source, runs `@source` specs, stops it, starts target, runs `@target` specs.
 
 ```bash
-# Test upgrade (OSS)
-./test.sh --source v0.58.6 --target v0.58.7
-
-# Test downgrade (OSS)
-./test.sh --source v0.58.7 --target v0.58.6
-
-# Test Enterprise edition (use v1.x.x)
-./test.sh --source v1.58.6 --target v1.58.7
+./test.sh --source v1.57.6 --target v1.58.3       # upgrade
+./test.sh --source v1.58.3 --target v1.57.6       # downgrade
 ```
 
-Edition is inferred from version prefix: `v0.x.x` = OSS, `v1.x.x` = EE.
+We only test Enterprise Edition (`v1.x`) for now. Direction is inferred from version comparison.
 
-## Requirements
+For **downgrades**, the script expects the target to refuse startup, then runs cascading `migrate down` (one step per major version) before retrying.
 
-- Docker
-- Docker Compose v2
-- Bun (for version helpers CLI)
+### `dev.sh` — local development
 
-## How It Works
+Starts a single Metabase version in Docker with H2 and opens Cypress interactively. Snapshot/restore works via `/api/testing/*` endpoints, controlled by the `CROSS_VERSION_DEV_MODE` env var.
 
-### Upgrade Test
-1. Start PostgreSQL container
-2. Start SOURCE Metabase version, wait for healthy
-3. Stop SOURCE container (PostgreSQL persists)
-4. Start TARGET version - migrations run automatically
-5. Verify health check passes
+```bash
+./dev.sh --version v1.57.6
+./dev.sh --version v1.58.3 --port 3001
+```
 
-### Downgrade Test
-1. Start PostgreSQL container
-2. Start SOURCE (newer) Metabase version, wait for healthy
-3. Stop SOURCE container
-4. Start TARGET (older) version - expects refusal
-5. Run `migrate down` command
-6. Start TARGET version again
-7. Verify health check passes
+## E2E test specs
 
-## Environment Variables
+Specs live in `e2e/cross-version/` and are organized by major version:
+
+```
+e2e/cross-version/
+  56/           # specs for v56
+  57/           # specs for v57
+  latest/       # specs for HEAD / newest versions
+```
+
+Each spec file uses `@source` and `@target` tags (via `cypress-grep`) to separate setup from verification. Each folder also contains a `helpers.ts` with version-specific utility functions used by the specs.
+
+### When to create a new versioned folder
+
+The `latest/` folder is the working copy — it targets the current development branch and is used for local dev with `dev.sh`. When a Metabase release introduces a breaking change that makes existing specs fail on older versions (e.g., a renamed UI element, a changed API response), copy `latest/` into a new numbered folder (e.g., `58/`). That snapshot covers all future versions until the next breaking change requires a new folder.
+
+In short: versioned folders are frozen snapshots, `latest/` is the living version.
+
+### Spec resolution
+
+Both `test.sh` and `dev.sh` resolve which spec folder to use for a given version:
+
+1. **Exact match** — `e2e/cross-version/{major}/` if it exists
+2. **Closest newer** — the lowest numbered folder >= the major version
+3. **Fallback** — `e2e/cross-version/latest/`
+
+## Environment variables
 
 | Variable | Default | Description |
 |----------|---------|-------------|
-| `METABASE_PORT` | 3000 | Port to expose Metabase |
-| `HEALTH_TIMEOUT` | 120 | Seconds to wait for health check |
+| `METABASE_PORT` | `3000` (`3077` in dev) | Port to expose Metabase |
+| `HEALTH_TIMEOUT` | `120` | Seconds to wait for health check |
+| `CROSS_VERSION_DEV_MODE` | — | Set automatically by `dev.sh`; enables snapshot/restore helpers in specs |
 
-## CI
+## Requirements
 
-The GitHub workflow (`.github/workflows/cross-version.yml`) supports:
-- **Manual trigger**: Specify source/target versions
-- **Nightly schedule**: Runs predefined version matrix
+- Docker and Docker Compose v2
+- Bun (for `cli.ts` version helpers)
+- Node.js with `bunx` (for Cypress)

cross-version/dev.sh

@@ -0,0 +1,167 @@
+#!/usr/bin/env bash
+#
+# Interactive dev environment for writing cross-version e2e tests.
+#
+# Starts an older Metabase version in Docker with H2 and opens Cypress.
+# Uses H2 with a shared volume so snapshot/restore works via test endpoints.
+#
+# Resolves which spec folder to use:
+#   e2e/cross-version/{major}/ if it exists, otherwise e2e/cross-version/latest/
+#
+# Usage:
+#   ./dev.sh --version v1.57.6
+#   ./dev.sh --version v1.58.7 --port 3001
+#
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+VERSION=""
+PORT="${METABASE_PORT:-3077}"
+
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+NC='\033[0m'
+
+log() { echo -e "${GREEN}[xv-dev]${NC} $*"; }
+error() { echo -e "${RED}[xv-dev]${NC} $*" >&2; }
+
+usage() {
+  cat <<EOF
+Usage: $0 --version VERSION [--port PORT]
+
+Options:
+  --version VERSION   Metabase version to run (e.g., v1.57.6, v1.58.7)
+  --port PORT         Port to expose (default: 3077)
+  --help              Show this help
+
+Examples:
+  $0 --version v1.57.6
+  $0 --version v1.58.7 --port 3001
+EOF
+  exit 1
+}
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --version) VERSION="$2"; shift 2 ;;
+    --port) PORT="$2"; shift 2 ;;
+    --help) usage ;;
+    *) error "Unknown option: $1"; usage ;;
+  esac
+done
+
+if [[ -z "$VERSION" ]]; then
+  error "--version is required"
+  usage
+fi
+
+# CLI wrapper for version helpers
+cli() {
+  bun "$SCRIPT_DIR/cli.ts" "$@"
+}
+
+IMAGE=$(cli image "$VERSION")
+MAJOR=$(cli major "$VERSION")
+H2_DIR="$SCRIPT_DIR/.xv-h2/v${MAJOR}"
+LOG_FILE="$SCRIPT_DIR/.xv-metabase.log"
+
+# Resolve spec folder: exact match, then closest older-version folder, then latest
+SPECS_DIR="$REPO_ROOT/e2e/cross-version/${MAJOR}"
+if [[ ! -d "$SPECS_DIR" ]]; then
+  closest=""
+  closest_dist=999
+  for dir in "$REPO_ROOT"/e2e/cross-version/[0-9]*/; do
+    v=$(basename "$dir")
+    (( v < MAJOR )) && continue
+    dist=$(( v - MAJOR ))
+    if (( dist < closest_dist )); then
+      closest_dist=$dist
+      closest="$dir"
+    fi
+  done
+  if [[ -n "$closest" ]]; then
+    SPECS_DIR="${closest%/}"
+  else
+    SPECS_DIR="$REPO_ROOT/e2e/cross-version/latest"
+  fi
+fi
+
+log "============================================"
+log "Cross-Version Dev Environment"
+log "============================================"
+log "Version: $VERSION"
+log "Image:   $IMAGE"
+log "Port:    $PORT"
+log "H2 dir:  $H2_DIR"
+log "Logs:    $LOG_FILE"
+log "Specs:   $SPECS_DIR"
+log "============================================"
+
+# Clean up previous session
+rm -f "$LOG_FILE"
+
+# Start with a clean app db every time
+# H2 files are owned by the Docker container's user, so use Docker to clean up
+if [[ -d "$H2_DIR" ]]; then
+  docker run --rm -v "$(dirname "$H2_DIR"):/data" alpine rm -rf "/data/$(basename "$H2_DIR")"
+fi
+mkdir -p "$H2_DIR"
+
+cleanup() {
+  log "Stopping Metabase..."
+  docker rm -f xv-dev-metabase 2>/dev/null || true
+}
+
+trap cleanup EXIT
+
+# Remove any leftover container from a previous run (e.g., after SIGKILL)
+docker rm -f xv-dev-metabase 2>/dev/null || true
+
+log "Starting Metabase ${VERSION}..."
+docker run -d \
+  --name xv-dev-metabase \
+  -p "${PORT}:3000" \
+  -v "${H2_DIR}:/metabase.db" \
+  -e MB_ENABLE_TEST_ENDPOINTS=true \
+  -e MB_DANGEROUS_UNSAFE_ENABLE_TESTING_H2_CONNECTIONS_DO_NOT_ENABLE=true \
+  "$IMAGE"
+
+# Stream backend logs to file for debugging (docker logs xv-dev-metabase)
+docker logs -f xv-dev-metabase > "$LOG_FILE" 2>&1 &
+
+# Create snapshots dir inside the container for /api/testing/snapshot and /api/testing/restore
+docker exec xv-dev-metabase sh -c "mkdir -p /e2e/snapshots && chmod 777 /e2e/snapshots"
+
+log "Waiting for Metabase to be ready..."
+TIMEOUT=120
+START=$(date +%s)
+while true; do
+  ELAPSED=$(( $(date +%s) - START ))
+  if (( ELAPSED >= TIMEOUT )); then
+    error "Timed out after ${TIMEOUT}s"
+    docker logs xv-dev-metabase
+    exit 1
+  fi
+  if curl -sf "http://localhost:${PORT}/api/health" 2>/dev/null | grep -q '"ok"'; then
+    break
+  fi
+  sleep 2
+done
+log "Metabase is ready at http://localhost:${PORT}"
+
+log "Saving blank snapshot..."
+curl -sf -X POST "http://localhost:${PORT}/api/testing/snapshot/blank"
+
+log ""
+log "Opening Cypress..."
+cd "$REPO_ROOT"
+
+CROSS_VERSION_DEV_MODE=true \
+CYPRESS_BASE_URL="http://localhost:${PORT}" \
+  bunx cypress open \
+    --e2e \
+    --config-file "e2e/cross-version/cypress.config.js" \
+    --config "specPattern=${SPECS_DIR}/**/*.cy.spec.ts"

cross-version/test.sh

@@ -11,6 +11,7 @@
 set -euo pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
 cd "$SCRIPT_DIR"
 
 # Defaults
@@ -18,6 +19,7 @@ SOURCE_VERSION=""
 TARGET_VERSION=""
 METABASE_PORT="${METABASE_PORT:-3000}"
 HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-120}"
+RESULT_FILE="${RESULT_FILE:-/tmp/cross-version-result.json}"
 
 # Colors for output
 RED='\033[0;31m'
@@ -29,6 +31,17 @@ log() { echo -e "${GREEN}[cross-version]${NC} $*"; }
 warn() { echo -e "${YELLOW}[cross-version]${NC} $*"; }
 error() { echo -e "${RED}[cross-version]${NC} $*" >&2; }
 
+# Write a structured result file on failure so CI can categorize the failure
+write_failure() {
+  local phase="$1"  # "migration" or "e2e"
+  jq -n \
+    --arg phase "$phase" \
+    --arg source "$SOURCE_VERSION" \
+    --arg target "$TARGET_VERSION" \
+    '{phase:$phase, source:$source, target:$target}' > "$RESULT_FILE"
+  log "Wrote failure result to $RESULT_FILE"
+}
+
 usage() {
   cat <<EOF
 Usage: $0 --source VERSION --target VERSION
@@ -66,6 +79,41 @@ cli() {
   bun "$SCRIPT_DIR/cli.ts" "$@"
 }
 
+run_e2e() {
+  local phase="$1"
+  local version="$2"
+  local major specs_dir
+
+  major=$(cli major "$version")
+  specs_dir="$REPO_ROOT/e2e/cross-version/${major}"
+  if [[ ! -d "$specs_dir" ]]; then
+    local closest="" closest_dist=999
+    for dir in "$REPO_ROOT"/e2e/cross-version/[0-9]*/; do
+      local v=$(basename "$dir")
+      (( v < major )) && continue
+      local dist=$(( v - major ))
+      if (( dist < closest_dist )); then
+        closest_dist=$dist
+        closest="$dir"
+      fi
+    done
+    if [[ -n "$closest" ]]; then
+      specs_dir="${closest%/}"
+      log "No exact specs for v${major} — using closest: ${specs_dir##*/}/"
+    else
+      specs_dir="$REPO_ROOT/e2e/cross-version/latest"
+      log "No exact specs for v${major} — falling back to latest/"
+    fi
+  else
+    log "Found exact specs for v${major}"
+  fi
+
+  log "Running @${phase} e2e tests for ${version} from: e2e/cross-version/${specs_dir##*/}/"
+  CYPRESS_SPEC_PATTERN="${specs_dir}/**/*.cy.spec.ts" \
+    "$REPO_ROOT/e2e/cross-version/run.sh" --phase "$phase"
+}
+
+
 wait_for_health() {
   local timeout="$1"
   local start_time=$(date +%s)
@@ -194,6 +242,11 @@ migrate_down_step() {
     error "migrate down encountered errors (check logs above)"
     return 1
   fi
+
+  if echo "$output" | grep -q "not rolled back"; then
+    error "migrate down had changesets that were not rolled back (check logs above)"
+    return 1
+  fi
 }
 
 # Cascading migrate down: rolls back multiple major versions one at a time
@@ -319,17 +372,25 @@ main() {
   if ! wait_for_health "$HEALTH_TIMEOUT"; then
     error "❌ SOURCE version ($SOURCE_VERSION) failed health check"
     docker compose logs metabase
+    write_failure "migration"
     exit 1
   fi
 
   log "✅ SOURCE version ($SOURCE_VERSION) is healthy"
 
   log ""
-  log "Step 2: Stopping SOURCE version ($SOURCE_VERSION)..."
+  log "Step 2: Running e2e tests (@source)..."
+  if ! run_e2e source "$SOURCE_VERSION"; then
+    write_failure "e2e"
+    exit 1
+  fi
+
+  log ""
+  log "Step 3: Stopping SOURCE version ($SOURCE_VERSION)..."
   stop_metabase
 
   log ""
-  log "Step 3: Starting TARGET version ($TARGET_VERSION)..."
+  log "Step 4: Starting TARGET version ($TARGET_VERSION)..."
   start_metabase "$target_image"
 
   if [[ "$direction" == "upgrade" ]]; then
@@ -337,15 +398,24 @@ main() {
     if ! wait_for_health "$HEALTH_TIMEOUT"; then
       error "❌ TARGET version ($TARGET_VERSION) failed health check after upgrade"
       docker compose logs metabase
+      write_failure "migration"
       exit 1
     fi
     log "✅ UPGRADE successful - TARGET version ($TARGET_VERSION) is healthy"
 
+    log ""
+    log "Step 5: Running e2e tests (@target)..."
+    if ! run_e2e target "$TARGET_VERSION"; then
+      write_failure "e2e"
+      exit 1
+    fi
+
   else
     # Downgrade: should refuse to start, then we run migrate down
     if ! check_downgrade_refused; then
       error "❌ TARGET version ($TARGET_VERSION) did not properly detect downgrade"
       docker compose logs metabase
+      write_failure "migration"
       exit 1
     fi
 
@@ -353,25 +423,32 @@ main() {
     stop_metabase
 
     log ""
-    log "============================================"
-    log "Step 4: Rolling back database ($SOURCE_VERSION → $TARGET_VERSION)..."
-    log "============================================"
+    log "Step 5: Rolling back database ($SOURCE_VERSION → $TARGET_VERSION)..."
     if ! cascading_migrate_down "$SOURCE_VERSION" "$TARGET_VERSION"; then
       error "❌ migrate down failed"
+      write_failure "migration"
       exit 1
     fi
 
     # Try starting again
     log ""
-    log "Step 5: Starting TARGET version ($TARGET_VERSION) after migrate down..."
+    log "Step 6: Starting TARGET version ($TARGET_VERSION) after migrate down..."
     start_metabase "$target_image"
 
     if ! wait_for_health "$HEALTH_TIMEOUT"; then
       error "❌ TARGET version ($TARGET_VERSION) failed health check after migrate down"
       docker compose logs metabase
+      write_failure "migration"
       exit 1
     fi
     log "✅ DOWNGRADE successful - TARGET version ($TARGET_VERSION) is healthy after migrate down"
+
+    log ""
+    log "Step 7: Running e2e tests (@target)..."
+    if ! run_e2e target "$TARGET_VERSION"; then
+      write_failure "e2e"
+      exit 1
+    fi
   fi
 
   log ""

deps.edn

@@ -106,6 +106,7 @@
   methodical/methodical                     {:mvn/version "1.0.127"}            ; drop-in replacements for Clojure multimethods and adds several advanced features
   metosin/malli                             {:git/url "https://github.com/metabase/malli" ; Data-driven Schemas for Clojure/Script and babashka
                                              :git/sha "b4a2d78647a049870b316753d7aae7fb4ade3ae5"} ; patched to resolve an issue causing OOM
+  net.carcdr/oidc-provider                  {:mvn/version "0.6.2"}              ; OAuth/OIDC server abstract implementation
   net.cgrand/macrovich                      {:mvn/version "0.2.2"}              ; utils for writing macros for both Clojure & ClojureScript
   net.clojars.wkok/openai-clojure           {:mvn/version "0.23.0"
                                              :exclusions  [lambdaisland/uri]}   ; OpenAI

dev/notes/model-boundary-enforcement.md

@@ -11,11 +11,17 @@ Two optional config keys per module in
 
 | Key              | Default | Meaning |
 |------------------|---------|---------|
-| `:model-exports` | `:any`  | Set of `:model/X` keywords this module allows **others** to use. |
-| `:model-imports` | `:any`  | Set of `:model/X` keywords **this module** may use from other modules. |
+| `:model-exports` | `#{}`   | Set of `:model/X` keywords this module allows **others** to use. |
+| `:model-imports` | `#{}`   | Set of `:model/X` keywords **this module** may use from other modules. |
 
-When omitted (or set to `:any`), no restrictions apply. `#{}` means export
-nothing / import nothing.
+When omitted, no models are exported/imported (closed by default). `:any`
+opens all restrictions.
+
+`:model-imports` also supports `:bypass` for cross-cutting modules that
+reference nearly every model (e.g. `cmd`, `enterprise/serialization`). A
+bypassed module can reference any model — even unexported ones. Models that
+are *only* referenced by bypass modules do not need `:model-exports` entries
+(the staleness test will flag them if added).
 
 ## Why it exists
 
@@ -118,6 +124,24 @@ The module can only reference these models from other modules. Any other
 `:model/X` keyword in the module's source files will cause a test failure.
 Unknown models (not defined anywhere) are also violations.
 
+### Bypassing import restrictions
+
+Some modules are inherently cross-cutting and import nearly every model. Use
+`:bypass` instead of an explicit set:
+
+```clojure
+cmd
+{:team "UX West"
+ :model-imports :bypass
+ ...}
+```
+
+A bypassed module can reference any model, even unexported ones. This means
+bypass modules don't drive `:model-exports` — if a model is *only* used
+outside its home module by bypass modules, it doesn't need to be exported.
+
+Use sparingly. Prefer explicit sets where feasible.
+
 ### Verifying changes
 
 ```bash
@@ -132,8 +156,18 @@ Or at the REPL:
 ```
 
 The test also validates that configured `:model-exports` / `:model-imports`
-only mention known models, and that exported models are actually owned by the
-module exporting them.
+only mention known models, that exported models are actually owned by the
+module exporting them, and that no stale entries exist (exports not referenced
+externally, imports not referenced in the module's source files).
+
+### Computing model boundaries
+
+Use the dev namespace to see what the config should contain:
+
+```clojure
+(dev.model-boundary-config/compute-model-boundaries) ;; returns data
+(dev.model-boundary-config/update-config!)           ;; rewrite config.edn
+```
 
 ## Limitations and future directions
 
@@ -169,12 +203,10 @@ read-only/read-write distinction above.
   (source scanning with rewrite-clj) and avoids adding overhead to normal
   development. The tradeoff: violations are only caught when tests run.
 
-- **Opt-in, not opt-out (for now).** Modules default to `:any` for both
-  exports and imports, so enforcement is incremental — restrict one module at a
-  time without touching anything else. Once enough modules have explicit sets,
-  we might flip the default to `#{}` so that new modules are restricted by
-  default and must explicitly declare their model dependencies. `:any` as the
-  starting default avoids a big-bang migration.
+- **Closed by default.** Both `:model-exports` and `:model-imports` default
+  to `#{}`. New modules must explicitly declare their model dependencies.
+  All existing modules have explicit config entries populated from a scan of
+  actual usage (see `dev.model-boundary-config`).
 
 - **Ownership is derived, not configured.** Instead of manually mapping models
   to modules, ownership is determined by scanning for `t2/table-name` defmethod

dev/src/dev/api/preview.clj

@@ -10,7 +10,7 @@
    [metabase.api.routes.common]
    [metabase.channel.render.core :as channel.render]
    [metabase.channel.urls :as urls]
-   [metabase.query-processor :as qp]
+   [metabase.query-processor.core :as qp]
    [metabase.query-processor.middleware.permissions :as qp.perms]
    [metabase.util.malli.schema :as ms])
   (:import

dev/src/dev/deps_graph.clj

@@ -826,21 +826,50 @@
   #{'metabase.models.resolution})
 
 (defn- model-reference-violations
-  "Return violation types for a single model reference. Pure function — no IO."
+  "Return violation types for a single model reference. Pure function — no IO.
+
+  When `model-imports` is `:bypass`, the using module is exempt from all model boundary checks —
+  it may reference any model regardless of whether it is exported."
   [model defining-mod model-exports model-imports]
-  (cond-> []
-    (nil? defining-mod)
-    (conj :unknown-model)
+  (if (= model-imports :bypass)
+    []
+    (cond-> []
+      (nil? defining-mod)
+      (conj :unknown-model)
 
-    (and defining-mod
-         (not= model-exports :any)
-         (not (contains? model-exports model)))
-    (conj :not-exported)
+      (and defining-mod
+           (not= model-exports :any)
+           (not (contains? model-exports model)))
+      (conj :not-exported)
 
-    (and defining-mod
-         (not= model-imports :any)
-         (not (contains? model-imports model)))
-    (conj :not-imported)))
+      (and defining-mod
+           (not= model-imports :any)
+           (not (contains? model-imports model)))
+      (conj :not-imported))))
+
+(defn model-references-by-module
+  "Scan all source files and build a map of `{module => #{:model/X ...}}` — the set of model keywords
+  referenced in each module's source files. Exempt namespaces (e.g. `metabase.models.resolution`) are excluded.
+  Includes all modules (including bypass modules) — callers filter as needed."
+  []
+  (reduce
+   (fn [acc file]
+     (try
+       (let [ns-symb (-> (ns.file/read-file-ns-decl file)
+                         ns.parse/name-from-ns-decl)
+             mod     (module ns-symb)]
+         (if (and mod (not (contains? model-boundary-exempt-namespaces ns-symb)))
+           (let [models (find-model-keywords file)]
+             (if (seq models)
+               (update acc mod (fnil into (sorted-set)) models)
+               acc))
+           acc))
+       (catch Throwable e
+         (throw (ex-info (format "Error scanning model references in %s" (str file))
+                         {:file file}
+                         e)))))
+   (sorted-map)
+   (find-source-files)))
 
 (defn model-boundary-violations
   "Find all model boundary violations across the codebase.
@@ -849,6 +878,9 @@
   2. The using module's `:model-imports` allows the model (`:any` or set containing it)
   3. The model's definition exists somewhere (`:unknown-model` is always a violation)
 
+  Modules with `:model-imports :bypass` are exempt from all checks — they may reference any model,
+  even unexported ones.
+
   Returns a sequence of violation maps with `:file`, `:module`, `:model`, `:defining-module`, `:violation-type`."
   ([]
    (model-boundary-violations (kondo-config)))
@@ -863,16 +895,15 @@
                                    ns.parse/name-from-ns-decl)
                        mod     (module ns-symb)]
                    (when (and mod
-                              (not (contains? model-boundary-exempt-namespaces ns-symb))
-                              (not= :skip-test (get-in kondo-config [mod :model-imports])))
-                     (let [model-imports (get-in kondo-config [mod :model-imports] :any)
+                              (not (contains? model-boundary-exempt-namespaces ns-symb)))
+                     (let [model-imports (get-in kondo-config [mod :model-imports] #{})
                            models       (find-model-keywords file)
                            rel-path     (file->path-relative-to-project-root file)]
                        (for [model          models
                              :let           [defining-mod  (get ownership model)]
                              :when          (not= defining-mod mod)
                              :let           [model-exports (when defining-mod
-                                                             (get-in kondo-config [defining-mod :model-exports] :any))]
+                                                             (get-in kondo-config [defining-mod :model-exports] #{}))]
                              violation-type (model-reference-violations
                                              model defining-mod model-exports model-imports)]
                          {:file            rel-path

dev/src/dev/model_boundary_config.clj

@@ -0,0 +1,141 @@
+(ns dev.model-boundary-config
+  "Compute `:model-exports` and `:model-imports` for every module.
+
+  Usage at the REPL:
+
+    (dev.model-boundary-config/compute-model-boundaries)
+    ;; => {:model-exports {module => #{:model/X ...}}
+    ;;     :model-imports {module => #{:model/X ...}}}
+
+  To update config.edn with computed model boundaries:
+
+    (dev.model-boundary-config/update-config!)"
+  (:require
+   [clojure.string :as str]
+   [dev.deps-graph :as deps-graph]
+   [rewrite-clj.parser :as r.parser]
+   [rewrite-clj.zip :as z]))
+
+(set! *warn-on-reflection* true)
+
+(defn compute-model-boundaries
+  "Compute the required `:model-exports` and `:model-imports` for every module.
+
+  Returns `{:model-exports {module => sorted-set-of-models}
+            :model-imports {module => sorted-set-of-models}}`
+
+  `:model-exports` for module M = models owned by M that are referenced by at least one non-bypass module.
+  `:model-imports` for module M = models owned by other modules that M references.
+
+  Modules with `:model-imports :bypass` are excluded: they don't need computed imports, and their
+  references don't drive exports (models used only by bypass modules need not be exported)."
+  []
+  (let [config      (deps-graph/kondo-config)
+        ownership   (deps-graph/model-ownership)
+        module-refs (deps-graph/model-references-by-module)
+        all-modules (set (keys config))
+        bypass-modules (into #{}
+                             (keep (fn [[mod mod-config]]
+                                     (when (= (:model-imports mod-config) :bypass)
+                                       mod)))
+                             config)
+        ;; {:model/X => #{modules that reference it}} — inverted index for fast export lookups
+        ;; Only non-bypass modules drive exports.
+        model->referencing-modules
+        (reduce-kv (fn [acc mod models]
+                     (if (contains? bypass-modules mod)
+                       acc
+                       (reduce (fn [acc model]
+                                 (update acc model (fnil conj #{}) mod))
+                               acc
+                               models)))
+                   {}
+                   module-refs)]
+    {:model-exports
+     (into (sorted-map)
+           (for [mod all-modules
+                 :let [owned (into #{} (comp (filter (fn [[_ owner]] (= owner mod))) (map key)) ownership)]
+                 :when (seq owned)]
+             [mod (into (sorted-set)
+                        (for [model owned
+                              :let [refs (get model->referencing-modules model)]
+                              :when (some #(not= % mod) refs)]
+                          model))]))
+     :model-imports
+     (into (sorted-map)
+           (for [mod all-modules
+                 :when (not (contains? bypass-modules mod))
+                 :let [imported (into (sorted-set)
+                                      (for [model (get module-refs mod)
+                                            :let [defining-mod (get ownership model)]
+                                            :when (and defining-mod (not= defining-mod mod))]
+                                        model))]
+                 :when (seq imported)]
+             [mod imported]))}))
+
+(def ^:private config-path ".clj-kondo/config/modules/config.edn")
+
+(def ^:private set-indent
+  "Indentation for multi-line model sets — aligns with existing config.edn style."
+  (apply str (repeat 20 \space)))
+
+(defn- model-set-str
+  "Build a string representation of a sorted set of model keywords.
+  Short sets (<=3 items) stay on one line; longer sets get one item per line."
+  [models]
+  (let [sorted (sort models)]
+    (if (<= (count sorted) 3)
+      (str "#{" (str/join " " sorted) "}")
+      (str "#{" (str/join (str "\n" set-indent) sorted) "}"))))
+
+(defn- find-module-config
+  "Navigate from `root` to the config map for `module-sym`. Returns a zipper or nil."
+  [root module-sym]
+  (let [mods-zloc (-> (z/of-node root)
+                      (z/find-value z/next :metabase/modules)
+                      z/right)]
+    (z/find-value (z/down mods-zloc) z/right module-sym)))
+
+(defn- find-key-value
+  "Find the value zipper for keyword `k` inside a map zipper, or nil."
+  [map-zloc k]
+  (some-> (z/find-value (z/down map-zloc) z/right k)
+          z/right))
+
+(defn update-config!
+  "Compute model boundaries and update config.edn with `:model-exports` and `:model-imports` for all modules.
+
+  Only updates keys that already exist in the config — does not add new keys.
+  Uses rewrite-clj to manipulate the AST directly, preserving formatting."
+  []
+  (let [boundaries    (compute-model-boundaries)
+        root-zloc     (z/of-node (r.parser/parse-file-all config-path))
+        modules-zloc  (-> root-zloc
+                          (z/find-value z/next :metabase/modules)
+                          z/right)
+        module-syms   (loop [zloc (z/down modules-zloc), acc []]
+                        (if-not zloc
+                          acc
+                          (recur (some-> zloc z/right z/right)
+                                 (conj acc (z/sexpr zloc)))))
+        updated-root
+        (reduce
+         (fn [root module-sym]
+           (reduce
+            (fn [root config-key]
+              (let [mod-cfg  (some-> (find-module-config root module-sym) z/right)
+                    val-zloc (when mod-cfg (find-key-value mod-cfg config-key))]
+                ;; Only update keys that already exist and are sets (skip :bypass, :any, etc.)
+                (if (and val-zloc (set? (z/sexpr val-zloc)))
+                  (let [computed (get-in boundaries [config-key module-sym] #{})]
+                    (if (seq computed)
+                      (z/root (z/replace val-zloc (r.parser/parse-string (model-set-str computed))))
+                      ;; Empty — remove the key-value pair
+                      (-> val-zloc z/remove z/remove z/root)))
+                  root)))
+            root
+            [:model-exports :model-imports]))
+         (z/root root-zloc)
+         module-syms)]
+    (spit config-path (z/root-string (z/of-node updated-root)))
+    (println "Updated" config-path "with :model-exports and :model-imports.")))

dev/src/dev/mq_perf.clj

@@ -0,0 +1,532 @@
+(ns dev.mq-perf
+  "Performance benchmarks for the appdb-backed MQ system (queues and topics).
+
+  Measures publishing throughput, end-to-end latency, and sustained throughput
+  under different load levels. Benchmarks exercise the full realistic pipeline:
+  publish buffer → DB → poll thread → worker pool → listener.
+
+  Self-contained — starts the MQ subsystem without the full Metabase server.
+  Supports multi-node testing: run multiple JVM instances against the same DB.
+
+  Example usage:
+    ;; Single-node: start MQ and run all benchmarks
+    (start-mq!)
+    (run-all-benchmarks!)
+
+    ;; Multi-node coordination: start multiple JVMs, each calls:
+    (start-mq!)
+    (run-all-benchmarks-coordinated!)
+    ;; Then from any node (or a separate REPL), publish the go signal:
+    (signal-go!)
+
+    ;; Individual benchmarks
+    (bench-publish-throughput! {:n 1000 :batch-sizes [1 10 100]})
+    (bench-e2e-latency! {:n 50 :rate :medium :transport :queue})
+    (bench-sustained-throughput! {:duration-sec 15 :rate :medium :transport :queue})
+    (bench-batch-listener! {:n 200 :batch-sizes [1 10 50]})
+    (bench-multi-queue! {:queue-counts [1 5 10 20] :msgs-per-queue 50})"
+  (:require
+   [clojure.string :as str]
+   [metabase.mq.core :as mq]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.publish-buffer :as publish-buffer]
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.mq.topic.backend :as topic.backend]
+   [toucan2.core :as t2])
+  (:import
+   (java.util.concurrent CountDownLatch TimeUnit)))
+
+(set! *warn-on-reflection* true)
+
+;;; ------------------------------------------------ Startup ------------------------------------------------
+
+(defonce ^:private started? (atom false))
+
+(defn start-mq!
+  "Minimal startup: connects to app DB and starts the MQ subsystem.
+  No Jetty, no scheduler, no sample data. Idempotent."
+  []
+  (let [setup-db! (requiring-resolve 'metabase.app-db.core/setup-db!)]
+    (setup-db! :create-sample-content? false))
+  (let [setup-prometheus! (requiring-resolve 'metabase.analytics.prometheus/setup!)]
+    (setup-prometheus!))
+  (require 'metabase.mq.init)
+  (let [startup! (requiring-resolve 'metabase.startup.core/def-startup-logic!)]
+    (startup! :metabase.mq.init/MqInit))
+  (reset! started? true)
+  (println "MQ system started."))
+
+(defn- ensure-started!
+  "Starts the MQ system if not yet started, and restarts any dead poll threads."
+  []
+  (when-not @started?
+    (start-mq!))
+  ;; Restart poll threads if they died (e.g., from InterruptedException during a previous benchmark).
+  ;; start! is idempotent and start-polling! detects dead futures.
+  (let [start-transports! (requiring-resolve 'metabase.mq.impl/start-transports)]
+    (start-transports!)))
+
+;;; ------------------------------------------------ Utilities ------------------------------------------------
+
+(def ^:private shared-queues
+  "Fixed set of 10 shared queue names used across all nodes for cross-node contention testing."
+  (mapv #(keyword "queue" (str "bench-shared-" %)) (range 10)))
+
+(def ^:private shared-topic :topic/bench-shared)
+
+(defn- run-id
+  "Generates an 8-character unique identifier for a benchmark run."
+  []
+  (subs (str (random-uuid)) 0 8))
+
+(defn- make-channel
+  "Creates a unique channel keyword for benchmarking."
+  [transport id suffix]
+  (keyword (name transport) (str "bench-" id "-" suffix)))
+
+(defn- random-shared-queue
+  "Returns a random shared queue name."
+  []
+  (nth shared-queues (rand-int (count shared-queues))))
+
+(defn- percentile
+  "Returns the p-th percentile from a sorted vector (p in [0,1])."
+  [sorted-vec p]
+  (if (empty? sorted-vec)
+    0.0
+    (let [idx (min (dec (count sorted-vec))
+                   (int (* p (count sorted-vec))))]
+      (double (nth sorted-vec idx)))))
+
+(defn- collect-stats
+  "Computes summary statistics from a collection of numeric values.
+  Returns {:count :min :max :avg :p50 :p95 :p99}."
+  [values]
+  (if (empty? values)
+    {:count 0 :min 0.0 :max 0.0 :avg 0.0 :p50 0.0 :p95 0.0 :p99 0.0}
+    (let [sorted (vec (sort values))
+          n      (count sorted)]
+      {:count n
+       :min   (double (first sorted))
+       :max   (double (last sorted))
+       :avg   (/ (reduce + 0.0 sorted) n)
+       :p50   (percentile sorted 0.50)
+       :p95   (percentile sorted 0.95)
+       :p99   (percentile sorted 0.99)})))
+
+(defn- fmt
+  "Formats a number for display."
+  [x]
+  (if (integer? x)
+    (str x)
+    (format "%.2f" (double x))))
+
+(defn- print-table!
+  "Prints a formatted ASCII table."
+  [title headers rows]
+  (let [col-widths (mapv (fn [i]
+                           (apply max
+                                  (count (nth headers i))
+                                  (map #(count (str (nth % i))) rows)))
+                         (range (count headers)))
+        separator  (str "+-" (str/join "-+-" (map #(apply str (repeat % \-)) col-widths)) "-+")
+        fmt-row    (fn [vals]
+                     (str "| "
+                          (str/join " | "
+                                    (map-indexed (fn [i v]
+                                                   (format (str "%-" (nth col-widths i) "s") (str v)))
+                                                 vals))
+                          " |"))]
+    (println)
+    (println title)
+    (println separator)
+    (println (fmt-row headers))
+    (println separator)
+    (doseq [row rows]
+      (println (fmt-row row)))
+    (println separator)
+    (println)))
+
+(defn- cleanup!
+  "Removes all benchmark data for the given run-id."
+  [id]
+  (t2/delete! :queue_message_batch :queue_name [:like (str "bench-" id "%")])
+  (t2/delete! :topic_message_batch :topic_name [:like (str "bench-" id "%")])
+  (doseq [[ch _] @listener/*listeners*]
+    (when (str/includes? (name ch) (str "bench-" id))
+      (mq/unlisten! ch))))
+
+(defn- publish-at-rate!
+  "Publishes `n` messages at the given rate (msgs/sec). `publish-fn` is called with each seq num.
+  If rate is nil, publishes as fast as possible (burst mode)."
+  [n rate publish-fn]
+  (if (nil? rate)
+    (dotimes [i n]
+      (publish-fn i))
+    (let [interval-ms (long (/ 1000 rate))]
+      (dotimes [i n]
+        (let [start (System/currentTimeMillis)]
+          (publish-fn i)
+          (let [elapsed (- (System/currentTimeMillis) start)
+                sleep-ms (- interval-ms elapsed)]
+            (when (pos? sleep-ms)
+              (Thread/sleep (long sleep-ms)))))))))
+
+(def ^:private rate-map
+  {:low    10
+   :medium 100
+   :high   1000
+   :burst  nil})
+
+(defn- rate->msgs-sec [rate]
+  (get rate-map rate rate))
+
+;;; ---------------------------------------- Benchmark 1: Publishing Throughput ----------------------------------------
+
+(defn bench-publish-throughput!
+  "Measures raw publishing speed for different batch sizes.
+  This benchmark bypasses the publish buffer to measure direct DB insert speed.
+  Options:
+    :n           - total messages to publish per batch-size test (default 1000)
+    :batch-sizes - vector of batch sizes to test (default [1 10 100])
+    :transport   - :queue or :topic (default :queue)"
+  ([] (bench-publish-throughput! {}))
+  ([{:keys [n batch-sizes transport]
+     :or   {n 1000 batch-sizes [1 10 100] transport :queue}}]
+   (ensure-started!)
+   (let [id      (run-id)
+         results (atom [])]
+     (try
+       (doseq [bs batch-sizes]
+         (let [channel    (make-channel transport id (str "pub-" bs))
+               iterations (/ n bs)
+               start      (System/nanoTime)]
+           (binding [publish-buffer/*publish-buffer-ms* 0]
+             (dotimes [i iterations]
+               (let [msgs (vec (for [j (range bs)]
+                                 {:seq (+ (* i bs) j) :ts (System/currentTimeMillis)}))]
+                 (if (= transport :queue)
+                   (q.backend/publish! :queue.backend/appdb channel msgs)
+                   (topic.backend/publish! :topic.backend/appdb channel msgs)))))
+           (let [elapsed-ms (/ (- (System/nanoTime) start) 1e6)]
+             (swap! results conj
+                    [(str bs) (fmt (/ n (/ elapsed-ms 1000.0))) (fmt elapsed-ms)]))))
+       (print-table!
+        (str "Publishing Throughput (" (name transport) ", n=" n ")")
+        ["Batch Size" "Msgs/sec" "Total ms"]
+        @results)
+       (finally
+         (cleanup! id))))))
+
+;;; ---------------------------------------- Benchmark 2: End-to-End Latency ----------------------------------------
+
+(defn bench-e2e-latency!
+  "Measures end-to-end latency from publish to listener receipt via the full async pipeline.
+  Uses the real publish API including the publish buffer.
+  Publishes randomly to the 10 shared queues (or the shared topic) so multiple nodes
+  compete for the same messages.
+  Options:
+    :n         - number of messages to publish (default 50)
+    :rate      - :low, :medium, :high, or :burst (default :medium)
+    :transport - :queue or :topic (default :queue)"
+  ([] (bench-e2e-latency! {}))
+  ([{:keys [n rate transport]
+     :or   {n 50 rate :medium transport :queue}}]
+   (ensure-started!)
+   (let [latencies (atom [])
+         received  (atom 0)
+         channels  (if (= transport :queue) shared-queues [shared-topic])]
+     (try
+       ;; Register listeners on all shared channels
+       (doseq [ch channels]
+         (when-not (listener/get-listener ch)
+           (mq/listen! ch {}
+                       (fn [msg]
+                         (let [receive-ns  (System/nanoTime)
+                               publish-ns  (get msg "publish-ns")]
+                           (when publish-ns
+                             (swap! latencies conj (/ (- receive-ns (double publish-ns)) 1e6))))
+                         (swap! received inc)))))
+       (let [msgs-sec (rate->msgs-sec rate)
+             start    (System/nanoTime)]
+         ;; Publish to random shared channels
+         (publish-at-rate!
+          n msgs-sec
+          (fn [i]
+            (let [ch (if (= transport :queue) (random-shared-queue) shared-topic)]
+              (if (= transport :queue)
+                (mq/with-queue ch [q]
+                  (mq/put q {:seq i :publish-ns (System/nanoTime)}))
+                (mq/with-topic ch [t]
+                  (mq/put t {:seq i :publish-ns (System/nanoTime)}))))))
+         ;; Force flush the publish buffer then wait for delivery
+         (publish-buffer/flush-publish-buffer!)
+         ;; Wait until we've received at least n messages or timeout
+         (let [deadline (+ (System/currentTimeMillis)
+                           (long (* 1000 (+ 30 (/ n (max 1 (or msgs-sec 1000)))))))]
+           (while (and (< @received n)
+                       (< (System/currentTimeMillis) deadline))
+             (Thread/sleep (long 100))))
+         (let [elapsed-ms (/ (- (System/nanoTime) start) 1e6)
+               stats      (collect-stats @latencies)]
+           (print-table!
+            (str "End-to-End Latency (" (name transport) ", rate=" (name rate) ", n=" n ")")
+            ["Metric" "Value"]
+            [["Published" (str n)]
+             ["Received" (str @received)]
+             ["Other nodes" (str (max 0 (- n @received)))]
+             ["From others" (str (max 0 (- @received n)))]
+             ["Total ms" (fmt elapsed-ms)]
+             ["P50 ms" (fmt (:p50 stats))]
+             ["P95 ms" (fmt (:p95 stats))]
+             ["P99 ms" (fmt (:p99 stats))]
+             ["Min ms" (fmt (:min stats))]
+             ["Max ms" (fmt (:max stats))]
+             ["Avg ms" (fmt (:avg stats))]])))
+       (finally
+         (doseq [ch channels]
+           (when (listener/get-listener ch)
+             (mq/unlisten! ch))))))))
+
+;;; ---------------------------------------- Benchmark 3: Sustained Throughput ----------------------------------------
+
+(defn bench-sustained-throughput!
+  "Publishes continuously while consuming for a given duration via the full async pipeline.
+  Publishes randomly to the 10 shared queues (or shared topic).
+  Options:
+    :duration-sec - how long to publish (default 15)
+    :rate         - :low, :medium, :high, or :burst (default :medium)
+    :transport    - :queue or :topic (default :queue)"
+  ([] (bench-sustained-throughput! {}))
+  ([{:keys [duration-sec rate transport]
+     :or   {duration-sec 15 rate :medium transport :queue}}]
+   (ensure-started!)
+   (let [channels  (if (= transport :queue) shared-queues [shared-topic])
+         received  (atom 0)
+         latencies (atom [])
+         published (atom 0)
+         stop?     (atom false)]
+     (try
+       ;; Register listeners on all shared channels
+       (doseq [ch channels]
+         (when-not (listener/get-listener ch)
+           (mq/listen! ch {}
+                       (fn [msg]
+                         (let [receive-ns  (System/nanoTime)
+                               publish-ns  (get msg "publish-ns")]
+                           (when publish-ns
+                             (swap! latencies conj (/ (- receive-ns (double publish-ns)) 1e6)))
+                           (swap! received inc))))))
+       (let [msgs-sec   (rate->msgs-sec rate)
+             start      (System/nanoTime)
+             pub-thread (Thread.
+                         (fn []
+                           (loop []
+                             (when-not @stop?
+                               (try
+                                 (let [ch (if (= transport :queue) (random-shared-queue) shared-topic)]
+                                   (if (= transport :queue)
+                                     (mq/with-queue ch [q]
+                                       (mq/put q {:seq @published :publish-ns (System/nanoTime)}))
+                                     (mq/with-topic ch [t]
+                                       (mq/put t {:seq @published :publish-ns (System/nanoTime)}))))
+                                 (swap! published inc)
+                                 (catch Exception _))
+                               (when msgs-sec
+                                 (Thread/sleep (long (/ 1000 msgs-sec))))
+                               (recur)))))]
+         (.start pub-thread)
+         (Thread/sleep (long (* duration-sec 1000)))
+         (reset! stop? true)
+         (.join pub-thread 5000)
+         ;; Flush publish buffer and wait for remaining messages to drain
+         (publish-buffer/flush-publish-buffer!)
+         (let [drain-start (System/currentTimeMillis)]
+           (while (and (< @received @published)
+                       (< (- (System/currentTimeMillis) drain-start) 30000))
+             (publish-buffer/flush-publish-buffer!)
+             (Thread/sleep (long 500))))
+         (let [elapsed-ms  (/ (- (System/nanoTime) start) 1e6)
+               stats       (collect-stats @latencies)]
+           (print-table!
+            (str "Sustained Throughput (" (name transport) ", rate=" (name rate)
+                 ", duration=" duration-sec "s)")
+            ["Metric" "Value"]
+            [["Published" (str @published)]
+             ["Received" (str @received)]
+             ["Other nodes" (str (max 0 (- @published @received)))]
+             ["From others" (str (max 0 (- @received @published)))]
+             ["Duration ms" (fmt elapsed-ms)]
+             ["Actual pub/sec" (fmt (/ @published (/ elapsed-ms 1000.0)))]
+             ["Actual recv/sec" (fmt (/ @received (/ elapsed-ms 1000.0)))]
+             ["P50 latency ms" (fmt (:p50 stats))]
+             ["P95 latency ms" (fmt (:p95 stats))]
+             ["P99 latency ms" (fmt (:p99 stats))]
+             ["Max latency ms" (fmt (:max stats))]])))
+       (finally
+         (doseq [ch channels]
+           (when (listener/get-listener ch)
+             (mq/unlisten! ch))))))))
+
+;;; ---------------------------------------- Benchmark 4: Batch Listener ----------------------------------------
+
+(defn bench-batch-listener!
+  "Compares single-message vs batch listener processing rates.
+  Uses the shared queues so multiple nodes compete.
+  Options:
+    :n           - total messages to publish (default 200)
+    :batch-sizes - vector of max-batch-messages values to test (default [1 10 50])"
+  ([] (bench-batch-listener! {}))
+  ([{:keys [n batch-sizes]
+     :or   {n 200 batch-sizes [1 10 50]}}]
+   (ensure-started!)
+   (let [results (atom [])]
+     (doseq [bs batch-sizes]
+       (let [received (atom 0)]
+         (try
+           ;; Register listeners on all shared queues
+           (doseq [ch shared-queues]
+             (when-not (listener/get-listener ch)
+               (if (= bs 1)
+                 (mq/listen! ch {}
+                             (fn [_msg]
+                               (swap! received inc)))
+                 (mq/batch-listen! ch
+                                   (fn [msgs]
+                                     (swap! received + (count msgs)))
+                                   {:max-batch-messages bs}))))
+           ;; Publish all messages randomly to shared queues
+           (dotimes [i n]
+             (mq/with-queue (random-shared-queue) [q]
+               (mq/put q {:seq i})))
+           (publish-buffer/flush-publish-buffer!)
+           ;; Wait for messages to be consumed
+           (let [start   (System/nanoTime)
+                 deadline (+ (System/currentTimeMillis) 30000)]
+             (while (and (< @received n)
+                         (< (System/currentTimeMillis) deadline))
+               (Thread/sleep (long 100)))
+             (let [elapsed-ms (/ (- (System/nanoTime) start) 1e6)]
+               (swap! results conj
+                      [(str bs) (str @received) (fmt elapsed-ms)
+                       (fmt (/ @received (/ elapsed-ms 1000.0)))])))
+           (finally
+             (doseq [ch shared-queues]
+               (when (listener/get-listener ch)
+                 (mq/unlisten! ch)))))))
+     (print-table!
+      (str "Batch Listener Comparison (n=" n ")")
+      ["Max Batch" "Received" "Process ms" "Msgs/sec"]
+      @results))))
+
+;;; ---------------------------------------- Benchmark 5: Multiple Queues ----------------------------------------
+
+(defn bench-multi-queue!
+  "Measures performance with varying numbers of the shared queues.
+  Tests with subsets of the 10 shared queues to show scaling.
+  Options:
+    :queue-counts   - vector of queue counts to test (default [1 5 10])
+    :msgs-per-queue - messages per queue (default 50)"
+  ([] (bench-multi-queue! {}))
+  ([{:keys [queue-counts msgs-per-queue]
+     :or   {queue-counts [1 5 10] msgs-per-queue 50}}]
+   (ensure-started!)
+   (let [results (atom [])]
+     (doseq [qc queue-counts]
+       (let [channels (subvec shared-queues 0 (min qc (count shared-queues)))
+             total    (* (count channels) msgs-per-queue)
+             received (atom 0)]
+         (try
+           ;; Register listeners
+           (doseq [ch channels]
+             (when-not (listener/get-listener ch)
+               (mq/listen! ch {}
+                           (fn [_msg]
+                             (swap! received inc)))))
+           ;; Publish to the queues
+           (let [start (System/nanoTime)]
+             (doseq [ch channels]
+               (dotimes [i msgs-per-queue]
+                 (mq/with-queue ch [q]
+                   (mq/put q {:seq i}))))
+             (publish-buffer/flush-publish-buffer!)
+             ;; Wait for messages
+             (let [deadline (+ (System/currentTimeMillis) 60000)]
+               (while (and (< @received total)
+                           (< (System/currentTimeMillis) deadline))
+                 (Thread/sleep (long 100))))
+             (let [elapsed-ms (/ (- (System/nanoTime) start) 1e6)]
+               (swap! results conj
+                      [(str (count channels)) (str total) (str @received) (fmt elapsed-ms)
+                       (fmt (/ @received (/ elapsed-ms 1000.0)))])))
+           (finally
+             (doseq [ch channels]
+               (when (listener/get-listener ch)
+                 (mq/unlisten! ch)))))))
+     (print-table!
+      (str "Multiple Queues (msgs-per-queue=" msgs-per-queue ")")
+      ["Queues" "Total Msgs" "Received" "Total ms" "Msgs/sec"]
+      @results))))
+
+;;; ---------------------------------------- Run All ----------------------------------------
+
+(defn run-all-benchmarks!
+  "Runs all benchmark scenarios with default parameters."
+  []
+  (println "=== MQ Performance Benchmarks ===")
+  (println)
+
+  (println "--- 1/5: Publishing Throughput (Queue) ---")
+  (bench-publish-throughput! {:n 1000 :transport :queue})
+
+  (println "--- 1b/5: Publishing Throughput (Topic) ---")
+  (bench-publish-throughput! {:n 1000 :transport :topic})
+
+  (println "--- 2/5: End-to-End Latency ---")
+  (doseq [rate [:low :medium :high :burst]]
+    (bench-e2e-latency! {:n 50 :rate rate :transport :queue}))
+  (doseq [rate [:low :medium :burst]]
+    (bench-e2e-latency! {:n 50 :rate rate :transport :topic}))
+
+  (println "--- 3/5: Sustained Throughput ---")
+  (bench-sustained-throughput! {:duration-sec 15 :rate :medium :transport :queue})
+  (bench-sustained-throughput! {:duration-sec 15 :rate :medium :transport :topic})
+
+  (println "--- 4/5: Batch Listener Comparison ---")
+  (bench-batch-listener! {:n 200})
+
+  (println "--- 5/5: Multiple Queues ---")
+  (bench-multi-queue! {:queue-counts [1 5 10 20] :msgs-per-queue 50})
+
+  (println "=== All benchmarks complete ==="))
+
+;;; ---------------------------------------- Multi-Node Coordination ----------------------------------------
+
+(defn run-all-benchmarks-coordinated!
+  "Waits for a 'go' signal on :topic/bench-coordination, then runs all benchmarks.
+  Use with multiple JVM instances to start benchmarks simultaneously.
+
+  Usage:
+    ;; On each node:
+    (start-mq!)
+    (run-all-benchmarks-coordinated!)
+
+    ;; Then from any node or separate REPL:
+    (signal-go!)"
+  []
+  (let [latch (CountDownLatch. 1)]
+    (mq/listen! :topic/bench-coordination {}
+                (fn [_msg]
+                  (.countDown latch)))
+    (println "Waiting for go signal... (call (signal-go!) from any node)")
+    (.await latch 300 TimeUnit/SECONDS)
+    (mq/unlisten! :topic/bench-coordination)
+    (println "Go signal received! Starting benchmarks...")
+    (run-all-benchmarks!)))
+
+(defn signal-go!
+  "Sends the go signal to all waiting nodes."
+  []
+  (mq/with-topic :topic/bench-coordination [t]
+    (mq/put t {:go true :time (System/currentTimeMillis)}))
+  (println "Go signal sent."))

dev/src/dev/mq_perf_multi.sh

@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+# Runs multiple MQ benchmark nodes against the same database.
+#
+# Usage:
+#   ./dev/src/dev/mq_perf_multi.sh start       # Start 2 nodes (default)
+#   ./dev/src/dev/mq_perf_multi.sh start 4     # Start 4 nodes
+#   ./dev/src/dev/mq_perf_multi.sh go          # Send go signal to all waiting nodes
+#   ./dev/src/dev/mq_perf_multi.sh stop        # Stop all running nodes
+#   ./dev/src/dev/mq_perf_multi.sh logs        # Tail all node logs
+#
+# Each node starts a lightweight JVM with just DB + MQ (no server),
+# waits for a coordinated go signal, then runs all benchmarks.
+
+set -euo pipefail
+cd "$(git rev-parse --show-toplevel)"
+
+LOG_DIR="target/mq-perf-logs"
+PID_DIR="target/mq-perf-pids"
+
+cmd_start() {
+  local num_nodes="${1:-2}"
+  mkdir -p "$LOG_DIR" "$PID_DIR"
+
+  # Clean up stale pids
+  rm -f "$PID_DIR"/*.pid
+
+  echo "Starting $num_nodes MQ benchmark nodes..."
+  echo "Logs in $LOG_DIR/"
+  echo
+
+  for i in $(seq 1 "$num_nodes"); do
+    local log_file="$LOG_DIR/node-$i.log"
+    clojure -M:dev:ee:ee-dev -e "
+      (require 'dev.mq-perf)
+      (dev.mq-perf/start-mq!)
+      (dev.mq-perf/run-all-benchmarks-coordinated!)
+    " > "$log_file" 2>&1 &
+    local pid=$!
+    echo "$pid" > "$PID_DIR/node-$i.pid"
+    echo "  Node $i started (PID $pid, log: $log_file)"
+  done
+
+  echo
+  echo "Nodes are initializing. Once ready, run:"
+  echo "  $0 go"
+}
+
+cmd_go() {
+  echo "Sending go signal..."
+  clojure -M:dev:ee:ee-dev -e "
+    (require 'dev.mq-perf)
+    (dev.mq-perf/start-mq!)
+    (dev.mq-perf/signal-go!)
+  "
+  echo
+  echo "Benchmarks started. Watch progress with:"
+  echo "  $0 logs"
+}
+
+cmd_stop() {
+  if [ ! -d "$PID_DIR" ]; then
+    echo "No nodes running."
+    return
+  fi
+
+  for pid_file in "$PID_DIR"/*.pid; do
+    [ -f "$pid_file" ] || continue
+    local pid
+    pid=$(cat "$pid_file")
+    local name
+    name=$(basename "$pid_file" .pid)
+    if kill -0 "$pid" 2>/dev/null; then
+      kill "$pid"
+      echo "Stopped $name (PID $pid)"
+    else
+      echo "$name (PID $pid) already exited"
+    fi
+    rm -f "$pid_file"
+  done
+}
+
+cmd_logs() {
+  if [ ! -d "$LOG_DIR" ]; then
+    echo "No logs found."
+    return
+  fi
+  tail -f "$LOG_DIR"/node-*.log
+}
+
+case "${1:-}" in
+  start) cmd_start "${2:-2}" ;;
+  go)    cmd_go ;;
+  stop)  cmd_stop ;;
+  logs)  cmd_logs ;;
+  *)
+    echo "Usage: $0 {start [N]|go|stop|logs}"
+    echo
+    echo "  start [N]  Start N benchmark nodes (default 2)"
+    echo "  go         Send go signal to start benchmarks"
+    echo "  stop       Kill all running nodes"
+    echo "  logs       Tail all node logs"
+    exit 1
+    ;;
+esac