WIP runner

* Make with-index! use new, clean, separate database per test

* Add with-test-db! setup modes

* Comment

* Adjust with-test-db! and related

* Drop with-index! in favor of modal with-test-db! in first test

* semantic.tu/db removal

* Drop use of semantic.tu/db in favor of get-pgvector-datasource!

* Drop semantic.tu/db  from tu

* Remove once-fixture

* Remove ensure-no-migration-table-fixture

* Comment

* Comment

* Cleanup

* Make the rest of the query-test ns use with-test-db!

* Remove redundant test

* Remove remaining usage of with-index!

* Drop remnants of with-index! and make linter happy

* Cleanup

* Comment

* Remove with-mock-index-metadata! in favor of with-temp-db!

* Adjust with-index-contents!

* Fixture to avoid running app-db tests

* Fixture

* Require

* whitespace

.clj-kondo/config/modules/config.edn

@@ -574,7 +574,6 @@
    :api  #{metabase.legacy-mbql.normalize
            metabase.legacy-mbql.predicates
            metabase.legacy-mbql.schema
-           metabase.legacy-mbql.schema.helpers
            metabase.legacy-mbql.util}
    :uses #{lib
            models
@@ -591,8 +590,7 @@
            metabase.lib.field
            metabase.lib.field.resolution
            metabase.lib.filter
-           metabase.lib.filter.desugar
-           metabase.lib.filter.desugar.jvm
+           metabase.lib.filter.desugar.jvm ; only so [[metabase.legacy-mbql.jvm-util]] can use defined regexes
            metabase.lib.init
            metabase.lib.join.util
            metabase.lib.metadata
@@ -718,6 +716,7 @@
            metabase.native-query-snippets.core}
    :uses #{api
            collections
+           lib
            models
            permissions
            premium-features
@@ -1984,6 +1983,15 @@
            upload
            util}}
 
+  enterprise/python-transform
+  {:team "Transform"
+   :api  #{metabase-enterprise.python-transform.api
+           metabase-enterprise.python-transform.core
+           metabase-enterprise.python-transform.init}
+   :uses #{api
+           models
+           util}}
+
   enterprise/upload-management
   {:team "UX West"
    :api  #{metabase-enterprise.upload-management.api}

.clj-kondo/src/hooks/clojure/core/require.clj

@@ -52,7 +52,13 @@
 
 (defn- lint-require* [node current-ns config]
   (let [[_require & args]   (:children node)
-        required-namespaces (keep unwrap-require args)]
+        required-namespaces (->> (keep unwrap-require args)
+                                 ;; `require` runs through `cc/load-libs` which allows
+                                 ;; #{:as :reload :reload-all :require :use :verbose :refer :as-alias}
+                                 ;; as args. If you use a REPL tool that applies clj-kondo linting (e.g., Clojure MCP)
+                                 ;; you may wish to `:reload` or `:reload-all` a namespace. So we shouldn't assume all
+                                 ;; args are keywords:
+                                 (remove keyword?))]
     (lint* node current-ns config required-namespaces)))
 
 (defn- lint-requiring-resolve* [node current-ns config]

.github/actions/await-port/action.yml

@@ -11,6 +11,18 @@ runs:
     - name: Wait for port ${{ inputs.port }} to be ready
       shell: bash
       run: |
+        echo "Waiting for port ${{ inputs.port }} to be ready..."
+        timeout=300  # 5 minutes timeout
+        elapsed=0
         while ! nc -z localhost ${{ inputs.port }}; do
-          sleep 0.1;
+          if [ $elapsed -ge $timeout ]; then
+            echo "❌ Timeout waiting for port ${{ inputs.port }} after ${timeout} seconds"
+            exit 1
+          fi
+          sleep 1
+          elapsed=$((elapsed + 1))
+          if [ $((elapsed % 10)) -eq 0 ]; then
+            echo "  Still waiting for port ${{ inputs.port }}... ($elapsed seconds)"
+          fi
         done
+        echo "✅ Port ${{ inputs.port }} is ready!"

.github/file-paths.yaml

@@ -41,10 +41,17 @@ frontend_sources: &frontend_sources
   - "enterprise/frontend/!(test)**/**/{!(*.spec),}.{js,jsx,ts,tsx,css}"
   - "frontend/!(test)**/**/{!(*.spec),}.{js,jsx,ts,tsx,css}"
 
+frontend_embedding_sources:
+  - "enterprise/frontend/src/{embedding,embedding-*}/**/{!(*.spec),}.{js,jsx,ts,tsx,css}"
+  - "enterprise/frontend/src/metabase-enterprise/{embedding,embedding-*}**/{!(*.spec),}.{js,jsx,ts,tsx,css}"
+
 frontend_embedding_sdk_sources:
   - *frontend_configs
   - "enterprise/frontend/!(test)**/**/{!(*.spec),}.{js,jsx,ts,tsx,css}"
 
+frontend_embedding_sdk_package_sources:
+  - "enterprise/frontend/src/embedding-sdk-package/**/!(*.spec).{js,jsx,ts,tsx,css}"
+
 frontend_specs: &frontend_specs
   - *shared_specs
   - "frontend/test/**"

.github/scripts/conventional-commit-pr-title-reminder/utils.js

@@ -1,24 +1,66 @@
-exports.updateComment = updateComment;
+const TYPES = new Set([
+  "feat",
+  "fix",
+  "perf",
+  "docs",
+  "style",
+  "refactor",
+  "test",
+  "build",
+  "ci",
+]);
+
+/**
+ * @param title {string}
+ */
+function isConventionalTitle(title) {
+  const match = /^(?<type>\w+)\((?<rawScopes>[^)]+)\):/.exec(title);
+
+  if (!match) {
+    return false;
+  }
+
+  const [, type, rawScopes] = match;
+  const scopes = rawScopes
+    .split(",")
+    .map((scope) => scope.trim())
+    .filter(Boolean);
+
+  return TYPES.has(type) && scopes.length > 0;
+}
 
 /** @param {import('@types/github-script').AsyncFunctionArguments} AsyncFunctionArguments */
-async function updateComment({ github, context }) {
+async function updateCommentForEmbedding({ github, context }) {
   const githubUsername = context.payload.sender.login;
-  const comment = `${CONVENTIONAL_COMMIT_REMINDER_COMMENT_IDENTIFIER}
-@${githubUsername} You have modified embedding SDK code. Please make sure the PR title follows [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) style.
-Here are all the supported types that will be shown in the changelog:
+
+  const comment = `${CONVENTIONAL_COMMENT_IDENTIFIER}
+@${githubUsername} You have modified embedding code. Please make sure the PR title follows [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) style.
+Here are all the supported types:
 - \`feat\`
 - \`fix\`
 - \`perf\`
+- \`docs\`
+- \`style\`
+- \`refactor\`
+- \`test\`
+- \`build\`
+- \`ci\`
 
-These types \`docs\`, \`style\`, \`refactor\`, \`test\`, \`build\`, and \`ci\` are also encouraged for non-changelog related tasks.
-
-Please also make sure to include \`sdk\` scope, otherwise, the changelog will not include this task.
+Please also make sure to include a scope.
 
 For example, these are valid PR titles:
-- \`feat(sdk): Add interactive dashboard component\`
-- \`feat(sdk): Support theming pie chart\`
-- \`fix(sdk): Fix static dashboard crash\``;
+- \`feat(sdk-bundle): Add interactive dashboard component\`
+- \`fix(sdk-package): Fixed MetabaseProviderPropsStore logic\`
+- \`fix(sdk-package, embed-js): Fix API\``;
 
+  return updateComment(comment, { github, context });
+}
+
+/**
+ * @param comment {string}
+ * @param {import('@types/github-script').AsyncFunctionArguments} AsyncFunctionArguments
+ */
+async function updateComment(comment, { github, context }) {
   const owner = context.repo.owner;
   const repo = context.repo.repo;
   const pullRequestNumber = context.issue.number;
@@ -49,20 +91,19 @@ For example, these are valid PR titles:
   }
 }
 
-const CONVENTIONAL_COMMIT_REMINDER_COMMENT_IDENTIFIER =
-  "<!---conventional commit reminder comment ID-->";
+const CONVENTIONAL_COMMENT_IDENTIFIER =
+  "<!---conventional comment identifier-->";
+
 /**
- * @typedef Comment
- * @property {string} body
- *
- * @param {Comment[]} comments
+ * @param comments {{body: string}[]}
  */
 function getExistingConventionalCommitReminderComment(comments) {
   for (const comment of comments) {
-    if (
-      comment.body.startsWith(CONVENTIONAL_COMMIT_REMINDER_COMMENT_IDENTIFIER)
-    ) {
+    if (comment.body.startsWith(CONVENTIONAL_COMMENT_IDENTIFIER)) {
       return comment;
     }
   }
 }
+
+exports.isConventionalTitle = isConventionalTitle;
+exports.updateCommentForEmbedding = updateCommentForEmbedding;

.github/scripts/conventional-commit-pr-title-reminder/utils.unit.spec.js

@@ -0,0 +1,45 @@
+const { isConventionalTitle } = require("./utils");
+
+describe("isConventionalTitle", () => {
+  it("returns true for a simple conventional commit", () => {
+    expect(isConventionalTitle("feat: add new feature")).toBe(false);
+  });
+
+  it("returns true for a commit with a scope", () => {
+    expect(isConventionalTitle("fix(api): correct API bug")).toBe(true);
+  });
+
+  it("returns true for a commit with multiple scopes, comma separated", () => {
+    expect(isConventionalTitle("feat(build,deps): update dependencies")).toBe(
+      true,
+    );
+  });
+
+  it("returns true for a commit with multiple scopes, comma separated with space", () => {
+    expect(isConventionalTitle("feat(build, deps): update dependencies")).toBe(
+      true,
+    );
+  });
+
+  it("returns false for a commit with round brackets in a message", () => {
+    expect(isConventionalTitle("feat: (something) update dependencies")).toBe(
+      false,
+    );
+  });
+
+  it("returns false for missing colon", () => {
+    expect(isConventionalTitle("feat add feature")).toBe(false);
+  });
+
+  it("returns false for invalid type", () => {
+    expect(isConventionalTitle("invalidType: some message")).toBe(false);
+  });
+
+  it("returns false for empty message", () => {
+    expect(isConventionalTitle("feat:")).toBe(false);
+  });
+
+  it("returns false for empty input", () => {
+    expect(isConventionalTitle("")).toBe(false);
+  });
+});

.github/scripts/get-issue-data.js

@@ -0,0 +1,84 @@
+const githubSlackMap = require("../../release/github-slack-map.json");
+
+async function getIssueData({ context, github, manualIssueNumber }) {
+  let issueNumber;
+  let issueData;
+  let prReviews = [];
+  const triggerType = context.eventName;
+
+  if (context.eventName === "issues") {
+    issueNumber = context.payload.issue.number;
+    issueData = context.payload.issue;
+  } else if (context.eventName === "workflow_dispatch") {
+    issueNumber = parseInt(manualIssueNumber);
+    const { data } = await github.rest.issues.get({
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      issue_number: issueNumber,
+    });
+    issueData = data;
+  } else if (context.eventName === "pull_request_review") {
+    const prNumber = context.payload.pull_request.number;
+    const { data: prData } = await github.rest.pulls.get({
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      pull_number: prNumber,
+    });
+
+    // Extract issue number from branch name (e.g., "claude-fix/issue-12345-some-title")
+    const branchMatch = prData.head.ref.match(/claude-fix\/issue-(\d+)/);
+    if (!branchMatch) {
+      throw new Error(
+        `Could not extract issue number from branch name: ${prData.head.ref}`,
+      );
+    }
+
+    issueNumber = parseInt(branchMatch[1]);
+
+    const { data } = await github.rest.issues.get({
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      issue_number: issueNumber,
+    });
+    issueData = data;
+
+    const { data: reviews } = await github.rest.pulls.listReviews({
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      pull_number: prNumber,
+    });
+
+    prReviews = reviews.filter(
+      (review) =>
+        review.state === "CHANGES_REQUESTED" &&
+        isMetabaseMember(review.user.login),
+    );
+  }
+
+  const { data: comments } = await github.rest.issues.listComments({
+    owner: context.repo.owner,
+    repo: context.repo.repo,
+    issue_number: issueNumber,
+  });
+
+  const employeeComments = comments.filter((comment) =>
+    isMetabaseMember(comment.user.login),
+  );
+
+  return {
+    issueNumber,
+    issueData,
+    employeeComments,
+    prReviews,
+    triggerType,
+    isIssueAuthorEmployee: isMetabaseMember(issueData.user.login),
+  };
+}
+
+function isMetabaseMember(username) {
+  return username in githubSlackMap;
+}
+
+module.exports = {
+  getIssueData,
+};

.github/workflows/backend.yml

@@ -195,46 +195,6 @@ jobs:
       - name: Run cljfmt
         run: ./bin/mage cljfmt-all --force-check
 
-  be-python-runner-tests:
-    if: ${{ !inputs.skip }}
-    runs-on: ubuntu-22.04
-    timeout-minutes: 30
-    name: Python Runner Tests
-    env:
-      CI: 'true'
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-          token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
-      - name: Prepare back-end environment
-        uses: ./.github/actions/prepare-backend
-        with:
-          m2-cache-key: be-python-runner-tests
-      - name: Start python-runner using Makefile
-        run: |
-          echo "🚀 Starting python-runner using make run..."
-          cd python-runner
-
-          # Use the official make run command
-          make run
-
-          echo "✅ Python runner started successfully!"
-
-      - name: Run Python Runner tests
-        run: >-
-          clojure -X:dev:ee:ee-dev:driver:test:test/python-transform
-      - name: Upload Test Results
-        uses: ./.github/actions/upload-test-results
-        if: always()
-        with:
-          input-path: ./target/junit/
-          output-name: be-python-runner-tests
-          bucket: ${{ vars.AWS_S3_TEST_RESULTS_BUCKET }}
-          aws-access-key-id: ${{ secrets.AWS_TEST_RESULTS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}
-          aws-region: ${{ vars.AWS_REGION }}
-          trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
 
   be-tests-result:
     needs:
@@ -243,7 +203,6 @@ jobs:
       - be-cljfmt
       - be-check
       - be-tests
-      - be-python-runner-tests
     if: always() && !cancelled()
     runs-on: ubuntu-latest
     timeout-minutes: 5

.github/workflows/claude-code-bug-fix.yml

@@ -4,97 +4,22 @@ on:
   issues:
     types:
       - labeled
+  pull_request_review:
+    types:
+      - submitted
   workflow_dispatch:
     inputs:
       issue_number:
         description: "Issue number to fix"
         required: true
         type: string
+      override_employee_check:
+        description: "Override employee check for issue creator"
+        required: false
+        default: false
+        type: boolean
 
 jobs:
-  check-employee-and-issue:
-    runs-on: ubuntu-latest
-    timeout-minutes: 2
-    permissions:
-      contents: read
-      issues: read
-    if: |
-      (github.event_name == 'issues' && github.event.label.name == 'Claude Bug Fix' && github.event.issue.state == 'open') ||
-      (github.event_name == 'workflow_dispatch')
-    outputs:
-      is_employee: ${{ steps.check_employee.outputs.is_employee }}
-      issue_number: ${{ steps.get_issue.outputs.issue_number }}
-      issue_data: ${{ steps.get_issue.outputs.issue_data }}
-    steps:
-      - name: Generate GitHub App Token
-        uses: actions/create-github-app-token@v1
-        id: app-token
-        with:
-          app-id: ${{ secrets.METABASE_BOT_APP_ID }}
-          private-key: ${{ secrets.METABASE_BOT_APP_PRIVATE_KEY }}
-
-      - name: Get issue data
-        id: get_issue
-        uses: actions/github-script@v7
-        with:
-          script: |
-            let issueNumber, issueData;
-
-            if (context.eventName === 'issues') {
-              // Original issue label event
-              issueNumber = context.payload.issue.number;
-              issueData = context.payload.issue;
-            } else if (context.eventName === 'workflow_dispatch') {
-              // Manual trigger with issue number input
-              issueNumber = parseInt('${{ github.event.inputs.issue_number }}');
-              const { data } = await github.rest.issues.get({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                issue_number: issueNumber
-              });
-              issueData = data;
-            }
-
-            core.setOutput('issue_number', issueNumber);
-            core.setOutput('issue_data', JSON.stringify(issueData));
-
-            console.log(`Issue #${issueNumber}: ${issueData.title}`);
-            console.log(`Issue author: ${issueData.user.login}`);
-
-      - name: Check if issue creator is a Metabase employee
-        id: check_employee
-        uses: actions/github-script@v7
-        env:
-          ISSUE_DATA: ${{ steps.get_issue.outputs.issue_data }}
-        with:
-          github-token: ${{ steps.app-token.outputs.token }}
-          script: |
-            const issueData = JSON.parse(process.env.ISSUE_DATA);
-            const issueAuthor = issueData.user.login;
-
-            try {
-              // Check if user is a member of the metabase organization (including private members)
-              const { data: membership } = await github.rest.orgs.getMembershipForUser({
-                org: 'metabase',
-                username: issueAuthor
-              });
-
-              if (membership.state === 'active') {
-                console.log(`User ${issueAuthor} is an active member of the Metabase organization. Proceeding with bug fix automation.`);
-                core.setOutput('is_employee', 'true');
-              } else {
-                console.log(`User ${issueAuthor} is not an active member of the Metabase organization. Skipping bug fix automation.`);
-                core.setOutput('is_employee', 'false');
-              }
-            } catch (error) {
-              if (error.status === 404) {
-                console.log(`User ${issueAuthor} is not a member of the Metabase organization. Skipping bug fix automation.`);
-              } else {
-                console.error('Error checking organization membership:', error);
-              }
-              core.setOutput('is_employee', 'false');
-            }
-
   auto-fix-bug:
     runs-on: ubuntu-latest
     timeout-minutes: 30
@@ -103,8 +28,10 @@ jobs:
       issues: write
       pull-requests: write
       id-token: write
-    needs: check-employee-and-issue
-    if: needs.check-employee-and-issue.outputs.is_employee == 'true'
+    if: |
+      (github.event_name == 'issues' && github.event.label.name == 'Claude Bug Fix' && github.event.issue.state == 'open') ||
+      (github.event_name == 'workflow_dispatch') ||
+      (github.event_name == 'pull_request_review' && github.event.review.state == 'changes_requested' && startsWith(github.event.pull_request.head.ref, 'claude-fix/'))
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -112,67 +39,95 @@ jobs:
           fetch-depth: 0
           token: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Get issue data and check employee status
+        id: get_issue
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const { getIssueData } = require('${{ github.workspace }}/.github/scripts/get-issue-data.js');
+
+            try {
+              const result = await getIssueData({
+                context,
+                github,
+                manualIssueNumber: '${{ github.event.inputs.issue_number }}'
+              });
+
+              const { issueNumber, issueData, employeeComments, prReviews, triggerType, isIssueAuthorEmployee } = result;
+
+              // Check employee status and fail if not authorized
+              if (!result.isIssueAuthorEmployee && '${{ github.event.inputs.override_employee_check }}' !== 'true') {
+                  core.setFailed(`User ${result.issueData.user.login} is not a Metabase employee. Skipping bug fix automation.`);
+              }
+
+              core.setOutput('issue_number', issueNumber);
+              core.setOutput('issue_data', JSON.stringify(issueData));
+              core.setOutput('issue_comments', JSON.stringify(employeeComments));
+              core.setOutput('pr_reviews', JSON.stringify(prReviews));
+              core.setOutput('trigger_type', triggerType);
+            } catch (error) {
+              core.setFailed(error.message);
+            }
+
       - name: Set up git configuration
         run: |
           git config --global user.name "Claude Code[bot]"
           git config --global user.email "noreply@anthropic.com"
 
-      - name: Prepare dynamic issue info
-        id: prepare_issue_info
+      - name: Prepare issue context and bug fix prompt
+        id: prepare_prompt
         uses: actions/github-script@v7
         env:
-          ISSUE_DATA: ${{ needs.check-employee-and-issue.outputs.issue_data }}
+          ISSUE_DATA: ${{ steps.get_issue.outputs.issue_data }}
+          ISSUE_COMMENTS: ${{ steps.get_issue.outputs.issue_comments }}
+          PR_REVIEWS: ${{ steps.get_issue.outputs.pr_reviews }}
+          TRIGGER_TYPE: ${{ steps.get_issue.outputs.trigger_type }}
         with:
           script: |
             const issueData = JSON.parse(process.env.ISSUE_DATA);
-            const issueNumber = '${{ needs.check-employee-and-issue.outputs.issue_number }}';
+            const issueComments = JSON.parse(process.env.ISSUE_COMMENTS);
+            const prReviews = JSON.parse(process.env.PR_REVIEWS);
+            const triggerType = process.env.TRIGGER_TYPE;
+            const issueNumber = '${{ steps.get_issue.outputs.issue_number }}';
 
-            // Convert issue title to kebab case
+            // Convert issue title to kebab case and create branch name
             const issueTitle = issueData.title || 'untitled';
             const kebabTitle = issueTitle
               .toLowerCase()
               .replace(/[^a-z0-9]+/g, '-')
               .replace(/^-+|-+$/g, '')
               .replace(/-+/g, '-')
-              .substring(0, 50); // Limit length
+              .substring(0, 50);
+            const branchName = `claude-fix/issue-${issueNumber}-${kebabTitle}`;
 
-            // Create base branch name
-            const baseBranchName = `claude-fix/issue-${issueNumber}-${kebabTitle}`;
-            core.exportVariable('BASE_BRANCH_NAME', baseBranchName);
+            // Format data for bash environment
+            const formattedComments = issueComments.length > 0
+              ? issueComments.map(comment =>
+                  `**Comment by @${comment.user.login} on ${comment.created_at}:**\n${comment.body}`
+                ).join('\n\n---\n\n')
+              : '';
+            const formattedReviews = prReviews.length > 0
+              ? prReviews.map(review =>
+                  `**Review by @${review.user.login} on ${review.submitted_at} (Changes Requested):**\n${review.body || 'No review comment provided'}`
+                ).join('\n\n---\n\n')
+              : '';
 
-            // Export issue info for the prompt
+            core.exportVariable('BRANCH_NAME', branchName);
             core.exportVariable('ISSUE_NUMBER', issueNumber);
             core.exportVariable('ISSUE_TITLE', issueData.title);
             core.exportVariable('ISSUE_AUTHOR', issueData.user.login);
             core.exportVariable('ISSUE_CREATED', issueData.created_at);
             core.exportVariable('ISSUE_BODY', issueData.body || 'No description provided');
+            core.exportVariable('ISSUE_COMMENTS_FORMATTED', formattedComments);
+            core.exportVariable('PR_REVIEWS_FORMATTED', formattedReviews);
+            core.exportVariable('TRIGGER_TYPE', triggerType);
 
-      - name: Determine branch strategy
+      - name: Generate bug fix prompt
         run: |
-          # Use the base branch name (no counters)
-          BRANCH_NAME="${{ env.BASE_BRANCH_NAME }}"
-
-          # Check if this exact branch already exists
-          if git branch -r --list "origin/${BRANCH_NAME}" | grep -q .; then
-            echo "Branch already exists: $BRANCH_NAME"
-            echo "Will push to existing branch and skip PR creation"
-            echo "BRANCH_EXISTS=true" >> $GITHUB_ENV
-          else
-            echo "Creating new branch: $BRANCH_NAME"
-            echo "Will create new PR"
-            echo "BRANCH_EXISTS=false" >> $GITHUB_ENV
-          fi
-
-          # Export the branch name
-          echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
-
-      - name: Prepare bug fix prompt
-        id: prepare_prompt
-        run: |
-          # Prepare the prompt for Claude
           cat << 'EOF' >> $GITHUB_ENV
           BUG_FIX_PROMPT<<PROMPT_END
-          You are helping to automatically fix a bug reported in GitHub issue #${{ env.ISSUE_NUMBER }}.
+          You are helping to fix a bug reported in GitHub issue #${{ env.ISSUE_NUMBER }}.
 
           ## Bug Report Details
           **Title:** ${{ env.ISSUE_TITLE }}
@@ -182,8 +137,27 @@ jobs:
           **Description:**
           ${{ env.ISSUE_BODY }}
 
+          **Comments:**
+          ${{ env.ISSUE_COMMENTS_FORMATTED }}
+          EOF
+
+          # Add review feedback section only if this is a review trigger
+          if [ "${{ env.TRIGGER_TYPE }}" = "pull_request_review" ]; then
+            cat << 'EOF' >> $GITHUB_ENV
+
+          ## Previous Fix Attempt - Review Feedback (MUST ADDRESS)
+          Your previous automated fix attempt received the following feedback that needs to be addressed:
+
+          ${{ env.PR_REVIEWS_FORMATTED }}
+          EOF
+          fi
+
+          # Add the rest of the prompt (always the same)
+          cat << 'EOF' >> $GITHUB_ENV
+
           ## Instructions
-          1. **Analyze the bug report** - Understand what the issue is describing
+          1. **Analyze the issue description and comments** - Understand what the issue is describing
+             a. Pay particular attention to reproduction steps and if there's an agreed upon fix.
           2. **Search the codebase** - Use Grep and Glob tools to find relevant files
           3. **Implement a fix** - Follow existing code patterns and conventions
           4. **Test the fix** - Run appropriate tests using the commands in CLAUDE.md:
@@ -205,15 +179,26 @@ jobs:
           - Only modify actual production code in the main repository directories
           - The scratchpad will be cleaned up automatically - nothing from it will be committed
 
-          ## If You Cannot Fix The Bug
-          If you determine that you cannot create a definitive fix, create a file called `FAILED_FIX_ANALYSIS.md` with:
-          1. **Why No Fix** - Specific reason why you couldn't implement a fix
-          2. **Tool Limitations** - If you needed additional tools/access that weren't available
-
           Start by understanding the bug and then systematically work toward a solution.
           PROMPT_END
           EOF
 
+      - name: Determine branch strategy
+        run: |
+          BRANCH_NAME="${{ env.BRANCH_NAME }}"
+          if git branch -r --list "origin/${BRANCH_NAME}" | grep -q .; then
+            echo "Branch already exists: $BRANCH_NAME"
+            echo "Will push to existing branch and skip PR creation"
+            echo "BRANCH_EXISTS=true" >> $GITHUB_ENV
+          else
+            echo "Creating new branch: $BRANCH_NAME"
+            echo "Will create new PR"
+            echo "BRANCH_EXISTS=false" >> $GITHUB_ENV
+          fi
+
+          # Export the branch name
+          echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
+
       - name: Create scratchpad directory for Claude
         run: |
           # Create a scratchpad directory for Claude to use for temporary files
@@ -249,20 +234,6 @@ jobs:
       - name: Check for changes and analysis
         id: check_changes
         run: |
-          # Check if Claude created an analysis file first and store its content
-          if [ -f "FAILED_FIX_ANALYSIS.md" ]; then
-            echo "has_analysis=true" >> $GITHUB_OUTPUT
-            echo "Analysis file found"
-            # Store analysis content in a GitHub output (base64 encoded to handle multiline)
-            ANALYSIS_CONTENT=$(base64 -w 0 FAILED_FIX_ANALYSIS.md)
-            echo "analysis_content=$ANALYSIS_CONTENT" >> $GITHUB_OUTPUT
-            # Remove analysis file so it doesn't count as a code change
-            rm -f FAILED_FIX_ANALYSIS.md
-          else
-            echo "has_analysis=false" >> $GITHUB_OUTPUT
-            echo "No analysis file found"
-          fi
-
           # Now check for actual code changes (excluding analysis file)
           if [ -n "$(git status --porcelain)" ]; then
             echo "has_changes=true" >> $GITHUB_OUTPUT
@@ -278,25 +249,16 @@ jobs:
         id: prepare_reviewers
         uses: actions/github-script@v7
         env:
-          ISSUE_DATA: ${{ needs.check-employee-and-issue.outputs.issue_data }}
+          ISSUE_DATA: ${{ steps.get_issue.outputs.issue_data }}
         with:
           script: |
-            // Get existing team labels from the issue
             const issueData = JSON.parse(process.env.ISSUE_DATA);
             const issueLabels = issueData.labels.map(label => label.name);
             const teamLabels = issueLabels.filter(label => label.startsWith('.Team/'));
-
-            console.log('Issue labels:', issueLabels);
-            console.log('Team labels found:', teamLabels);
-
-            // Add issue creator as reviewer
             const issueCreator = issueData.user.login;
-
             core.setOutput('team_labels', teamLabels.join(','));
             core.setOutput('reviewers', issueCreator);
 
-            console.log('Reviewer:', issueCreator);
-
       - name: Create Pull Request
         if: steps.check_changes.outputs.has_changes == 'true'
         uses: peter-evans/create-pull-request@v6
@@ -363,25 +325,3 @@ jobs:
         with:
           script: |
             console.log('🤖 No code changes were made by Claude.');
-
-            // Check if Claude provided a detailed analysis
-            if ('${{ steps.check_changes.outputs.has_analysis }}' === 'true') {
-              try {
-                // Decode the base64-encoded analysis content
-                const analysisBase64 = '${{ steps.check_changes.outputs.analysis_content }}';
-                const analysisContent = Buffer.from(analysisBase64, 'base64').toString('utf8');
-                console.log('📋 Claude provided detailed analysis:');
-                console.log('='.repeat(50));
-                console.log(analysisContent);
-                console.log('='.repeat(50));
-              } catch (error) {
-                console.log('❌ Could not decode analysis content:', error);
-              }
-            } else {
-              console.log('ℹ️  No detailed analysis file was created by Claude.');
-              console.log('This could be because:');
-              console.log('- The issue requires more context or clarification');
-              console.log('- The fix is complex and requires human judgment');
-              console.log('- The issue is not a straightforward code bug');
-              console.log('- Claude couldn\'t locate the relevant code to modify');
-            }

.github/workflows/conventional-commit-pr-title-reminder.yml

@@ -4,39 +4,53 @@ on:
   pull_request:
     branches:
       - master
-    paths:
-      - enterprise/frontend/src/embedding-sdk-bundle/**
 
 jobs:
+  files-changed:
+    name: Check which files changed
+    runs-on: ubuntu-22.04
+    timeout-minutes: 3
+    outputs:
+      frontend_embedding_sources: ${{ steps.changes.outputs.frontend_embedding_sources }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Test which files changed
+        uses: dorny/paths-filter@v3.0.0
+        id: changes
+        with:
+          token: ${{ github.token }}
+          filters: .github/file-paths.yaml
+
   pr-title-reminder:
     runs-on: ubuntu-22.04
     timeout-minutes: 5
     permissions:
       pull-requests: write
+    needs: [ files-changed ]
+    if: |
+      !cancelled() &&
+      needs.files-changed.outputs.frontend_embedding_sources == 'true'
     steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+
       - name: Check if the PR title is already following Conventional Commits style
         uses: actions/github-script@v7
         id: is-conventional-commit
         with:
           result-encoding: string
           script: | # js
+            const { isConventionalTitle } = require(".github/scripts/conventional-commit-pr-title-reminder/utils.js")
             const pullRequestTitle = context.payload.pull_request.title
-            const conventionalCommitRegex = /^(?<type>\w+)\((?<scope>\w+)\):/
-            const match = pullRequestTitle.match(conventionalCommitRegex)
-            if (match) {
-              const { type, scope } = match.groups
-              return ['feat', 'fix', 'perf', 'docs', 'style', 'refactor', 'test', 'build', 'ci'].includes(type) && scope === 'sdk'
-            }
 
-      - name: Checkout repository so `gh` CLI works
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.head_ref }}
+            return isConventionalTitle(pullRequestTitle)
 
       - name: Remind PR authors to use conventional commit in the title
         if: ${{ steps.is-conventional-commit.outputs.result != 'true' }}
         uses: actions/github-script@v7
         with:
-          script: |
-            const { updateComment } = require(".github/scripts/conventional-commit-pr-title-reminder/utils.js")
-            await updateComment({github, context})
+          script: | #js
+            const { updateCommentForEmbedding } = require(".github/scripts/conventional-commit-pr-title-reminder/utils.js")
+
+            await updateCommentForEmbedding({github, context})

.github/workflows/drivers.yml

@@ -338,6 +338,12 @@ jobs:
     name: H2
     steps:
     - uses: actions/checkout@v4
+      with:
+        submodules: true
+        token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+    - name: Start python-runner
+      working-directory: python-runner
+      run: make run
     - name: Test H2 driver
       uses: ./.github/actions/test-driver
       with:
@@ -442,6 +448,12 @@ jobs:
     name: "${{ matrix.version.name }} ${{ matrix.job.name }}"
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
+          token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      - name: Start python-runner
+        working-directory: python-runner
+        run: make run
       - name: Test ${{ matrix.version.name }}
         uses: ./.github/actions/test-driver
         with:
@@ -745,6 +757,12 @@ jobs:
           POSTGRES_HOST_AUTH_METHOD: trust
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
+          token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      - name: Start python-runner
+        working-directory: python-runner
+        run: make run
       - name: Test ${{ matrix.version.name }} (${{ matrix.job.name }})
         uses: ./.github/actions/test-driver
         with:

.github/workflows/e2e-component-tests-embedding-sdk-backward-compatibility.yml

@@ -140,7 +140,10 @@ jobs:
         run: node e2e/runner/run_cypress_ci.js snapshot --browser ${{ steps.cypress-prep.outputs.chrome-path }}
 
       - name: Run component tests for Embedding SDK
-        run: node e2e/runner/run_cypress_ci.js component --browser ${{ steps.cypress-prep.outputs.chrome-path }}
+        run: |
+          node e2e/runner/run_cypress_ci.js component \
+            --browser ${{ steps.cypress-prep.outputs.chrome-path }} \
+            --env grepTags="-@skip-backward-compatibility",grepOmitFiltered=true
 
       - name: Upload Test Results
         uses: ./.github/actions/upload-test-results

.github/workflows/embedding-sdk-labels-reminder.yml

@@ -0,0 +1,49 @@
+name: Embedding SDK Labels Reminder
+
+on:
+  pull_request:
+    types: [auto_merge_enabled, review_requested, labeled, unlabeled, synchronize]
+    branches:
+      - "master"
+  merge_group:
+
+jobs:
+  files-changed:
+    name: Check which files changed
+    runs-on: ubuntu-22.04
+    timeout-minutes: 3
+    outputs:
+      frontend_embedding_sdk_package_sources: ${{ steps.changes.outputs.frontend_embedding_sdk_package_sources }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Test which files changed
+        uses: dorny/paths-filter@v3.0.0
+        id: changes
+        with:
+          token: ${{ github.token }}
+          filters: .github/file-paths.yaml
+
+  embedding-sdk-package-no-release-notes-reminder:
+    needs: [ files-changed ]
+    if: |
+      !cancelled() &&
+      (needs.files-changed.outputs.frontend_embedding_sdk_package_sources == 'true')
+    name: Embedding SDK package release-notes/no-release-notes label reminder
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v4
+      - uses: jesusvasquez333/verify-pr-label-action@v1.4.0
+        id: verify-release-notes-labels
+        with:
+          github-token: '${{ secrets.GITHUB_TOKEN }}'
+          valid-labels: 'release-notes, no-release-notes'
+          disable-reviews: true
+      - name: Comment on PR if label should be added
+        if: ${{ always() }}
+        uses: ./.github/actions/notify-pull-request
+        with:
+          remove-previous-comment: true
+          add-new-comment: ${{ steps.verify-release-notes-labels.outcome == 'failure' }}
+          message: "⚠️ Changes were made to the <b>Embedding SDK Package</b>. Review this PR and apply one of the following labels.<br><br>- <code>release-notes</code> — select if this PR must be included in the main Metabase changelog release notes.<br>- <code>no-release-notes</code> — select if this PR should not be included in the main Metabase changelog release notes."
+          include-log: true

.github/workflows/perf-test-labeled.yml

@@ -9,9 +9,13 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  uberjar:
+    if: ${{ github.event.label.name == 'Run-perf' }}
+    uses: ./.github/workflows/uberjar.yml
+    secrets: inherit
+
   run-perf-test:
+    needs: [uberjar]
     if: ${{ github.event.label.name == 'Run-perf' }}
     uses: ./.github/workflows/perf-test.yml
-    with:
-      wait_for_uberjar: true
     secrets: inherit

.github/workflows/perf-test.yml

@@ -3,11 +3,6 @@ run-name: Run perf test for ${{ github.ref_name }} by @${{ github.actor }}
 
 on:
   workflow_call:
-    inputs:
-      wait_for_uberjar:
-        description: "Wait for uberjar build"
-        required: false
-        type: boolean
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -31,63 +26,6 @@ jobs:
         uses: aws-actions/amazon-ecr-login@v2
         with:
           registries: "${{ secrets.PR_ENV_AWS_ACCOUNT_ID }}"
-      - name: Wait for uberjar
-        id: wait_for_uberjar
-        if: ${{ inputs.wait_for_uberjar == true }}
-        run: |
-          ## Get workflow run id for uberjar build
-          curl -Ls --output e2e-tests.json \
-          -H "Accept: application/vnd.github+json" \
-          -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
-          -H "X-GitHub-Api-Version: 2022-11-28" \
-          https://api.github.com/repos/${{ github.repository }}/actions/workflows/run-tests.yml/runs?head_sha=${{ github.event.pull_request.head.sha || github.sha }}
-          ID=$(jq -r '.workflow_runs[0].id' e2e-tests.json)
-          echo "Run ID: ${ID}"
-
-          ## Get workflow run id for uberjar build
-          while [ -z "$JOB_ID" ]; do
-            NEXT_URL="https://api.github.com/repos/${{ github.repository }}/actions/runs/${ID}/jobs?filter=latest"
-
-            # Search for build job
-            echo "Searching for build job..."
-            while [ -z "$JOB_ID" ] && [ -n "$NEXT_URL" ]; do
-              # Get one page of jobs
-              RESPONSE=$(curl -s -i \
-                -H "Accept: application/vnd.github+json" \
-                -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
-                -H "X-GitHub-Api-Version: 2022-11-28" \
-                "$NEXT_URL")
-
-              # Extract headers and body
-              HEADERS=$(echo "$RESPONSE" | sed -n '1,/^\r$/p')
-              BODY=$(echo "$RESPONSE" | sed '1,/^\r$/d')
-              LINK_HEADER=$(echo "$HEADERS" | grep -i '^link:' | sed 's/^link: //')
-
-              # Check for next page in Link header
-              if echo "$LINK_HEADER" | grep -q 'rel="next"'; then
-                NEXT_URL=$(echo "$LINK_HEADER" | sed -E 's/.*<([^>]+)>; rel="next".*/\1/')
-              else
-                NEXT_URL=""
-              fi
-
-              # Check for build job
-              JOB_ID=$(echo "$BODY" | jq -r '.jobs[] | select(.name | contains("build (ee)")) | .id')
-            done
-          done
-          echo "Job ID: ${JOB_ID}"
-
-          ## Wait for uberjar build to complete
-          while [ true ]; do
-            curl -Ls --output uberjar.json \
-            -H "Accept: application/vnd.github+json" \
-            -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
-            -H "X-GitHub-Api-Version: 2022-11-28" \
-            https://api.github.com/repos/${{ github.repository }}/actions/jobs/${JOB_ID}
-            jq -r '.steps[] | select(.name == "Prepare uberjar artifact") | .status' uberjar.json | grep -q "completed" && break
-            echo "Waiting for uberjar build..."
-            sleep 10
-          done
-          echo "run_id=$(jq -r '.workflow_runs[0].id' e2e-tests.json)" >> $GITHUB_OUTPUT
       - name: Retrieve uberjar artifact for ee
         uses: ./.github/actions/fetch-artifact
         with:

.github/workflows/release-embedding-sdk.yml

@@ -285,10 +285,6 @@ jobs:
         run: |
           ./bin/embedding-sdk/release_utils.bash update_package_json_template ${{ needs.determine-sdk-version.outputs.sdk_version }}
 
-      - name: Update changelog
-        run: |
-          yarn embedding-sdk:generate-changelog -o changelog-diff
-
       - name: Build SDK package
         run: yarn run build-embedding-sdk-package
 
@@ -301,12 +297,6 @@ jobs:
           name: metabase-sdk
           path: ./resources/embedding-sdk
 
-      - name: Upload changelog diff
-        uses: actions/upload-artifact@v4
-        with:
-          name: sdk-changelog-diff
-          path: ./changelog-diff
-
   publish-npm:
     needs: [build-sdk, determine-sdk-version]
     runs-on: ubuntu-22.04
@@ -335,19 +325,7 @@ jobs:
         run: |
           bash ./bin/embedding-sdk/release_utils.bash update_package_json_template ${{ env.sdk_version }}
 
-      - name: Retrieve SDK changelog diff
-        if: inputs.branch != 'master'
-        uses: actions/download-artifact@v4
-        with:
-          name: sdk-changelog-diff
-
-      - name: Update changelog
-        if: inputs.branch != 'master'
-        run: |
-          cat changelog-diff enterprise/frontend/src/embedding-sdk-bundle/CHANGELOG.md > new-changelog
-          mv new-changelog enterprise/frontend/src/embedding-sdk-bundle/CHANGELOG.md
-
-      - name: Create a PR updating readme + published version, and changelog (using GitHub Metabase Automation account)
+      - name: Create a PR updating readme + published version (using GitHub Metabase Automation account)
         run: |
           git checkout -b update-sdk-version-${{ env.sdk_version }}
 

bin/embedding-sdk/generate-sdk-package-files.js

@@ -89,6 +89,6 @@ generateSdkPackage();
 copyFileToOutput("LICENSE.txt");
 copyFileToOutput("frontend/src/embedding-sdk-package/README.md", "README.md");
 copyFileToOutput(
-  "frontend/src/embedding-sdk-bundle/CHANGELOG.md",
+  "frontend/src/embedding-sdk-package/CHANGELOG.md",
   "CHANGELOG.md",
 );

deps.edn

@@ -118,7 +118,7 @@
   net.sf.cssbox/cssbox                      {:mvn/version "5.0.2"               ; HTML / CSS rendering
                                              :exclusions  [org.slf4j/slf4j-api
                                                            junit/junit]}
-  net.thisptr/jackson-jq                    {:mvn/version "1.4.0"}              ; Java implementation of the JQ json query language
+  net.thisptr/jackson-jq                    {:mvn/version "1.6.0"}              ; Java implementation of the JQ json query language
   org.apache.commons/commons-compress       {:mvn/version "1.28.0"}             ; compression utils
   org.apache.commons/commons-lang3          {:mvn/version "3.18.0"}             ; helper methods for working with java.lang stuff
   org.apache.logging.log4j/log4j-1.2-api    {:mvn/version "2.25.1"}             ; apache logging framework
@@ -135,7 +135,7 @@
                                              :exclusions  [org.bouncycastle/bcpkix-jdk15on
                                                            org.bouncycastle/bcprov-jdk15on]}
   org.apache.poi/poi-ooxml-full             {:mvn/version "5.4.1"}
-  org.apache.sshd/sshd-core                 {:mvn/version "2.15.0"              ; ssh tunneling and test server
+  org.apache.sshd/sshd-core                 {:mvn/version "2.16.0"              ; ssh tunneling and test server
                                              :exclusions  [org.slf4j/slf4j-api
                                                            org.slf4j/jcl-over-slf4j]}
   org.apache.tika/tika-core                 {:mvn/version "3.2.2"}
@@ -147,7 +147,7 @@
   org.clj-commons/hickory                   {:mvn/version "0.7.7"               ; Parse HTML into Clojure data structures
                                              :exclusions [org.jsoup/jsoup]}
   org.clojars.pntblnk/clj-ldap              {:mvn/version "0.0.17"}             ; LDAP client
-  org.clojure/clojure                       {:mvn/version "1.12.1"}
+  org.clojure/clojure                       {:mvn/version "1.12.2"}
   org.clojure/core.async                    {:mvn/version "1.7.701"
                                              :exclusions  [org.clojure/tools.reader]}
   org.clojure/core.logic                    {:mvn/version "1.1.0"}              ; optimized pattern matching library for Clojure
@@ -255,7 +255,7 @@
     {:mvn/version "0.4.0"}                     ; Enables easy memory measurement
     com.gfredericks/test.chuck   {:mvn/version "0.2.15"}                    ; generating strings from regexes (useful with malli)
     criterium/criterium          {:mvn/version "0.4.6"}                     ; benchmarking library
-    djblue/portal                {:mvn/version "0.60.2"}                    ; ui for inspecting values
+    djblue/portal                {:mvn/version "0.61.0"}                    ; ui for inspecting values
     io.github.camsaul/humane-are {:mvn/version "1.0.2"}                     ; cleaner test output
     io.github.metabase/hawk      {:mvn/version "1.0.11"}                    ; test runner
     jonase/eastwood              {:mvn/version "1.4.3"                      ; inspects namespaces and reports possible problems using tools.analyzer
@@ -842,7 +842,6 @@
   ;;    clj -X:dev:ee:ee-dev:driver:test:test/python-transform
   :test/python-transform
   {:exec-args {:only [metabase-enterprise.transforms.api-test
-                      metabase.driver.table-creation-test
                       "enterprise/backend/test/metabase_enterprise/transforms"]}}
 
 ;; test the Query Processor

docs/ai/metabot.md

@@ -68,7 +68,9 @@ You can also ask Metabot to tell you about specific tables in your database.
 
 ## How Metabot uses the query builder
 
-When you ask Metabot to create a chart from natural language, it first looks for existing questions that might answer your request. If it finds a relevant question, it'll point you to that instead of creating something new. Otherwise, Metabot will use the [query builder](../questions/query-builder/editor.md) to create a new chart for you. Keep in mind that Metabot can only handle basic query builder operations. It's limited to single-level aggregation and grouping, so if you need more complex analysis, you can take over and refine the query yourself, or switch to the native editor.
+When you ask Metabot to create a chart from natural language, it first looks for existing questions that might answer your request. If it finds a relevant question, it'll point you to that instead of creating something new. Otherwise, Metabot will use the [query builder](../questions/query-builder/editor.md) to create a new chart for you. 
+
+Keep in mind that Metabot is still learning the query builder. Metabot can only handle basic query builder operations, and it lacks access to the library of [custom expressions](../questions/query-builder/expressions-list.md). Metabot is also limited to single-level aggregation and grouping, so if you need more complex analysis, you can take over and refine the query yourself, or switch to the SQL editor.
 
 ## Metabot in the native editor
 

docs/api.html

@@ -12771,7 +12771,7 @@ redirect_from:
                       },
                       "schema-filters-type" : {
                         "type" : "string",
-                        "enum" : [ "include", "exclude", "all" ]
+                        "enum" : [ "inclusion", "exclusion", "all" ]
                       }
                     },
                     "required" : [ "schema-filters-type", "schema-filters-patterns" ]
@@ -12849,7 +12849,7 @@ redirect_from:
                       },
                       "schema-filters-type" : {
                         "type" : "string",
-                        "enum" : [ "include", "exclude", "all" ]
+                        "enum" : [ "inclusion", "exclusion", "all" ]
                       }
                     },
                     "required" : [ "schema-filters-type", "schema-filters-patterns" ]
@@ -15864,7 +15864,35 @@ redirect_from:
       "get" : {
         "summary" : "GET /api/ee/transform-job/",
         "description" : "Get all transform jobs.",
-        "parameters" : [ ],
+        "parameters" : [ {
+          "in" : "query",
+          "name" : "last_run_start_time",
+          "required" : false,
+          "schema" : {
+            "type" : "string",
+            "minLength" : 1
+          }
+        }, {
+          "in" : "query",
+          "name" : "next_run_start_time",
+          "required" : false,
+          "schema" : {
+            "type" : "string",
+            "minLength" : 1
+          }
+        }, {
+          "in" : "query",
+          "name" : "transform_tag_ids",
+          "required" : false,
+          "schema" : {
+            "type" : "array",
+            "items" : {
+              "description" : "value must be an integer greater or equal to than zero.",
+              "type" : "integer",
+              "minimum" : 0
+            }
+          }
+        } ],
         "responses" : {
           "2XX" : {
             "description" : "Successful response"
@@ -16018,6 +16046,34 @@ redirect_from:
         "tags" : [ "/api/ee/transform-job" ]
       }
     },
+    "/api/ee/transform-job/{job-id}/transforms" : {
+      "get" : {
+        "summary" : "GET /api/ee/transform-job/{job-id}/transforms",
+        "description" : "Get the transforms of job specified by the job's ID.",
+        "parameters" : [ {
+          "in" : "path",
+          "name" : "job-id",
+          "required" : true,
+          "schema" : {
+            "type" : "integer",
+            "minimum" : 1
+          },
+          "description" : "value must be an integer greater than zero."
+        } ],
+        "responses" : {
+          "2XX" : {
+            "description" : "Successful response"
+          },
+          "4XX" : {
+            "description" : "Client error response"
+          },
+          "5XX" : {
+            "description" : "Server error response"
+          }
+        },
+        "tags" : [ "/api/ee/transform-job" ]
+      }
+    },
     "/api/ee/transform-tag/" : {
       "post" : {
         "summary" : "POST /api/ee/transform-tag/",
@@ -16448,6 +16504,34 @@ redirect_from:
         "tags" : [ "/api/ee/transform" ]
       }
     },
+    "/api/ee/transform/{id}/dependencies" : {
+      "get" : {
+        "summary" : "GET /api/ee/transform/{id}/dependencies",
+        "description" : "Get the dependencies of a specific transform.",
+        "parameters" : [ {
+          "in" : "path",
+          "name" : "id",
+          "required" : true,
+          "schema" : {
+            "type" : "integer",
+            "minimum" : 1
+          },
+          "description" : "value must be an integer greater than zero."
+        } ],
+        "responses" : {
+          "2XX" : {
+            "description" : "Successful response"
+          },
+          "4XX" : {
+            "description" : "Client error response"
+          },
+          "5XX" : {
+            "description" : "Server error response"
+          }
+        },
+        "tags" : [ "/api/ee/transform" ]
+      }
+    },
     "/api/ee/transform/{id}/run" : {
       "post" : {
         "summary" : "POST /api/ee/transform/{id}/run",

docs/configuring-metabase/config-template.md

@@ -117,6 +117,7 @@ config:
     db-connection-timeout-ms: 10000
     db-query-timeout-minutes: 20
     default-maps-enabled: true
+    disable-cors-on-localhost: false
     download-row-limit: null
     email-from-address: notifications@metabase.com
     email-from-address-override: notifications@metabase.com
@@ -134,7 +135,7 @@ config:
     email-smtp-username: null
     email-smtp-username-override: null
     embedding-app-origins-interactive: null
-    embedding-app-origins-sdk: ""
+    embedding-app-origins-sdk: ''
     embedding-homepage: hidden
     embedding-secret-key: null
     enable-embedding-interactive: false

docs/configuring-metabase/environment-variables.md

@@ -393,6 +393,15 @@ Timeout in minutes for databases query execution, both Metabase application data
 
 Whether or not the default GeoJSON maps are enabled.
 
+### `MB_DISABLE_CORS_ON_LOCALHOST`
+
+- Type: boolean
+- Default: `false`
+- [Exported as](../installation-and-operation/serialization.md): `disable-cors-on-localhost`.
+- [Configuration file name](./config-file.md): `disable-cors-on-localhost`
+
+Prevents the server from sending CORS headers for requests originating from localhost.
+
 ### `MB_DOWNLOAD_ROW_LIMIT`
 
 - Type: positive-integer
@@ -562,7 +571,7 @@ Allow these space delimited origins to embed Metabase interactive.
 > Only available on Metabase [Pro](https://www.metabase.com/product/pro) and [Enterprise](https://www.metabase.com/product/enterprise) plans.
 
 - Type: string
-- Default: `""`
+- Default: ``
 - [Configuration file name](./config-file.md): `embedding-app-origins-sdk`
 
 Allow Metabase SDK access to these space delimited origins.

docs/developers-guide/driver-changelog.md

@@ -14,10 +14,18 @@ title: Driver interface changelog
 
 - Added metabase.driver/compile-transform, metabase.driver/compile-drop-table, metabase.driver/execute-raw-queries!,
   metabase.driver/run-transform!, metabase.driver/drop-transform-target!, metabase.driver/native-query-deps,
-  metabase.driver/connection-details, metabase.driver/table-exists?, metabase.driver.sql/normalize-name,
-  metabase.driver.sql/default-schema, and metabase.driver.sql/find-table to implement sql transforms.
+  metabase.driver/connection-spec, metabase.driver/table-exists?, metabase.driver.sql/normalize-name,
+  and metabase.driver.sql/default-schema to implement sql transforms.
 
-- `driver/humanize-connection-error-message` now takes a list of all messages in the exception chain,
+- Added the driver multi-methods `metabase.driver/schema-exists?` and `metabase.driver/create-schema-if-needed!` which should be implemented by drivers that support `:schemas` and `:transforms/table`.
+
+- Added `metabase.driver/type->database-type` multimethod that returns the database type for a given Metabase type (from the type hierarchy) as a HoneySQL spec. Unlike `upload-type->database-type` which is specific to upload types, this method handles general Metabase base types.
+
+- Added `metabase.driver/rename-table!` multimethod to rename a table from one name to another. Both table names may be qualified by schema (e.g., `schema.table`). This is currently used for transform operations and uploads.
+
+- Added `metabase.driver/insert-from-source!` multimethod that abstracts data insertion from various sources into existing tables. This multimethod dispatches on both the driver and the data source type (`:rows` or `:csv-file`). It allows drivers to optimize based on the data source type and returns the number of rows inserted. Default implementations are provided for both `:rows` and `:csv-file` source types.
+
+- `metabase.driver/humanize-connection-error-message` now takes a list of all messages in the exception chain,
   instead of just the top-level message as a string.
 
 - Removed `driver/set-database-used`. Drivers should set default databases in their connection specs instead.

docs/embedding/sdk/next-js.md

@@ -14,11 +14,11 @@ See a [sample Next.js app that uses the SDK](https://github.com/metabase/metabas
 
 ## SDK components with Server Side Rendering (SSR) or React Server Components
 
-As of Embedding Analytics SDK v57, SDK components automatically skip server-side rendering (SSR) and render only on the client.
+As of Embedded Analytics SDK v57, SDK components automatically skip server-side rendering (SSR) and render only on the client.
 
 ### Compatibility layer for Server Side Rendering (SSR) (DEPRECATED)
 
-As of Embedding analytics SDK 57, the compatibility layer for server-side rendering (SSR) is deprecated and no longer required. If you use the compatibility layer, change your imports from `@metabase/embedding-sdk-react/next` to `@metabase/embedding-sdk-react`.
+As of Embedded Analytics SDK 57, the compatibility layer for server-side rendering (SSR) is deprecated and no longer required. If you use the compatibility layer, change your imports from `@metabase/embedding-sdk-react/next` to `@metabase/embedding-sdk-react`.
 
 ## Handling authentication
 

docs/embedding/sdk/upgrade.md

@@ -10,8 +10,8 @@ Here's a basic overview of the steps you'll want to take when upgrading your SDK
 ## 1. Read the release post and changelog for Metabase and the Embedded analytics SDK
 
 - [Release posts](https://www.metabase.com/releases) give a good overview of what's in each release, and call out breaking changes (which are rare).
-- [Metabase changelogs](https://www.metabase.com/changelog) list Metabase changes.
-- [Embedded analytics SDK changelogs](https://www.metabase.com/changelog/56) list changes specific to the SDK Package.
+- [Metabase + Embedding analytics SDK changelogs](https://www.metabase.com/changelog) list Metabase and Embedding analytics SDK changes.
+- [Embedded analytics SDK package changelog](https://github.com/metabase/metabase/blob/release-x.56.x/enterprise/frontend/src/embedding-sdk-bundle/CHANGELOG.md) list changes specific to the SDK lightweight `@metabase/embedding-sdk-react` package.
 
 Check for any relevant changes, especially deprecations or breaking changes that require you to update your application's code. If there are deprecation changes, we'll have docs that'll walk you through what changes you'll need to make and why.
 

docs/installation-and-operation/development-instance.md

@@ -11,7 +11,7 @@ summary: Create development instances of Metabase for testing without paying per
 
 On Pro and Enterprise plans, you can purchase a development instance or development token as an add-on to your plan.
 
-- **Self-hosted**: you can purchase a dev token that you can use for one or more self-hosted development instances (meaning: you can reuse the same token for multiple dev instances at no additional cost). The development token will have access to the same set of paid features as your production token.
+- **Self-hosted**: you can purchase a dev token so you don't get billed for every test user acccount. You can use the same dev token for one or more self-hosted development instances (meaning: you can reuse the same token for multiple dev instances at no additional cost). The development token will have access to the same set of paid features as your production token.
 - **Cloud**: you can purchase a development instance hosted on [Metabase Cloud](https://www.metabase.com/cloud/). The development instance will have access to the same set of paid features as your production token.
 
 Dev instances are only available to Pro and Enterprise plans; they're not included in the base plans. Dev instances must be purchased as an add-on to your plan, after you've completed your trial and made your first payment.
@@ -22,6 +22,10 @@ That's the basic value proposition. Unlike production instances where you pay pe
 
 Once you're happy with the changes in your dev instance, you can [export your changes and import them](./serialization.md) into your production Metabase.
 
+## Even if you self-host, dev instance tokens can save you money
+
+While it's true that you can use your production token to test multiple Metabases when self-hosting, you'll still get charged for test user accounts in each of those Metabases. In that case, it might make sense to get a dev instance token so you're only charged a flat rate for as many dev instances as you like, with as many test users as you need.
+
 ## You can't use development instances in production
 
 The UI for development instances, including the UI for all its exports and notifications, is watermarked.

e2e/runner/cypress-runner-run-tests.js

@@ -127,7 +127,10 @@ const configs = {
       openMode: args["--open"] || process.env.OPEN_UI === "true",
     };
 
-    return sdkComponentConfig;
+    const userArgs = await parseArguments(args);
+
+    const finalConfig = Object.assign({}, sdkComponentConfig, userArgs);
+    return finalConfig;
   },
 };
 

e2e/support/helpers/api/createSnippet.ts

@@ -1,12 +1,27 @@
-export function createSnippet({
-  name,
-  content,
-}: {
-  name: string;
+import type {
+  NativeQuerySnippet,
+  RegularCollectionId,
+} from "metabase-types/api";
+
+export type NativeQuerySnippetDetails = {
+  name?: string;
+  description?: string | null;
   content: string;
-}): Cypress.Chainable<Cypress.Response<NativeQuerySnippet>> {
-  cy.request("POST", "/api/native-query-snippet", {
+  collection_id?: RegularCollectionId | null;
+};
+
+export function createSnippet({
+  name = "Test snippet",
+  description = null,
+  content,
+  collection_id = null,
+}: NativeQuerySnippetDetails): Cypress.Chainable<
+  Cypress.Response<NativeQuerySnippet>
+> {
+  return cy.request("POST", "/api/native-query-snippet", {
     name,
+    description,
     content,
+    collection_id,
   });
 }

e2e/support/helpers/e2e-misc-helpers.js

@@ -281,6 +281,7 @@ export function interceptIfNotPreviouslyDefined({ method, url, alias } = {}) {
  * @param {boolean=} [options.addToDashboard]
  * @param {boolean=} [options.wrapId]
  * @param {string=} [options.idAlias]
+ * @param {Object=} [pickEntityOptions]
  */
 export function saveQuestion(
   name,

e2e/test-component/scenarios/embedding-sdk/metabot-question.cy.spec.tsx

@@ -115,6 +115,32 @@ describe("scenarios > embedding-sdk > metabot-question", () => {
     });
   });
 
+  it("should start a new conversation when clicking a `start new conversation` button", () => {
+    setup(metabotResponse);
+
+    mountSdkContent(<MetabotQuestion />);
+
+    getSdkRoot().within(() => {
+      cy.findByTestId("metabot-chat-input").type("Show orders {enter}");
+
+      H.mockMetabotResponse({
+        statusCode: 200,
+        body: metabotRetryResponse,
+      });
+
+      cy.findAllByTestId("metabot-chat-message").should("have.length", 2);
+
+      cy.findByTestId("metabot-new-conversation").click();
+
+      cy.findAllByTestId("metabot-chat-message").should("not.exist");
+      cy.findByTestId("metabot-chat-input").should("have.value", "");
+
+      cy.findByTestId("metabot-chat-input").type("Show orders {enter}");
+
+      cy.findAllByTestId("metabot-chat-message").should("have.length", 2);
+    });
+  });
+
   it("should set correct metabot_id both for the a new message and when retrying", () => {
     setup(metabotResponse);
 

e2e/test-component/scenarios/embedding-sdk/sdk-bundle.cy.spec.tsx

@@ -26,14 +26,13 @@ const sdkBundleCleanup = () => {
 
 describe(
   "scenarios > embedding-sdk > sdk-bundle",
-  // These test in some cases load a new SDK Bundle that in combination with the Component Testing is memory-consuming
-  { numTestsKeptInMemory: 1 },
+  {
+    tags: ["@skip-backward-compatibility"],
+    // These test in some cases load a new SDK Bundle that in combination with the Component Testing is memory-consuming
+    numTestsKeptInMemory: 1,
+  },
   () => {
     beforeEach(() => {
-      cy.intercept("POST", "/api/dashboard/*/dashcard/*/card/*/query").as(
-        "dashcardQuery",
-      );
-
       signInAsAdminAndEnableEmbeddingSdk();
 
       cy.signOut();
@@ -44,10 +43,6 @@ describe(
     [{ strictMode: false }, { strictMode: true }].forEach(({ strictMode }) => {
       describe(`Common cases ${strictMode ? "with" : "without"} strict mode`, () => {
         it("should display an SDK question", () => {
-          cy.window().then((win) => {
-            cy.spy(win.console, "warn").as("consoleWarn");
-          });
-
           mountSdkContent(
             <InteractiveQuestion questionId={ORDERS_QUESTION_ID} />,
             { strictMode },
@@ -199,6 +194,13 @@ describe(
         it("should show a custom loader when the SDK bundle is loading", () => {
           sdkBundleCleanup();
 
+          cy.intercept("GET", "/api/card/*", (request) => {
+            // Delay request for 500ms to avoid flakiness
+            request.continue(
+              () => new Promise((resolve) => setTimeout(resolve, 500)),
+            );
+          });
+
           mountSdkContent(
             <InteractiveQuestion questionId={ORDERS_QUESTION_ID} />,
             {
@@ -285,7 +287,7 @@ describe(
     describe("Components", () => {
       it("should display an SDK question with custom layout components", () => {
         cy.window().then((win) => {
-          cy.spy(win.console, "warn").as("consoleWarn");
+          cy.spy(win.console, "error").as("consoleError");
         });
 
         mountSdkContent(
@@ -303,15 +305,13 @@ describe(
 
           cy.findByTestId("visualization-root").should("be.visible");
         });
+
+        cy.get("@consoleError").should("not.be.called");
       });
 
       it("should show an error on a component level if SDK components are not wrapped within the MetabaseProvider", () => {
         sdkBundleCleanup();
 
-        cy.window().then((win) => {
-          cy.spy(win.console, "warn").as("consoleWarn");
-        });
-
         cy.mount(<InteractiveQuestion questionId={ORDERS_QUESTION_ID} />);
 
         getSdkRoot().within(() => {
@@ -320,6 +320,41 @@ describe(
           ).should("exist");
         });
       });
+
+      it("should show a console error if SDK Package component uses a prop that is not yet available in SDK bundle", () => {
+        sdkBundleCleanup();
+
+        cy.window().then((win) => {
+          cy.spy(win.console, "error").as("consoleError");
+        });
+
+        mountSdkContent(
+          <InteractiveQuestion
+            questionId={ORDERS_QUESTION_ID}
+            {...{ foo: "bar" }}
+          />,
+        );
+
+        cy.get("@consoleError").should(
+          "be.calledWithMatch",
+          "this property is not recognized by the component",
+        );
+      });
+
+      it("should show a console error if SDK Package component does not use a prop that is still expected by SDK bundle", () => {
+        sdkBundleCleanup();
+
+        cy.window().then((win) => {
+          cy.spy(win.console, "error").as("consoleError");
+        });
+
+        mountSdkContent(<InteractiveQuestion />);
+
+        cy.get("@consoleError").should(
+          "be.calledWithMatch",
+          "this property is required by the component",
+        );
+      });
     });
 
     describe("Error handling", { retries: 3 }, () => {
@@ -344,7 +379,7 @@ describe(
 
           cy.findByTestId("sdk-error-container").should(
             "contain.text",
-            "Error loading the Embedding Analytics SDK",
+            "Error loading the Embedded Analytics SDK",
           );
         });
 
@@ -363,7 +398,7 @@ describe(
 
           cy.findByTestId("sdk-error-container").should(
             "contain.text",
-            "Custom error: Error loading the Embedding Analytics SDK",
+            "Custom error: Error loading the Embedded Analytics SDK",
           );
         });
       });

e2e/test/scenarios/admin-2/people.cy.spec.js

@@ -767,7 +767,7 @@ describe("scenarios > admin > people > group managers", () => {
       cy.url().should("match", /\/$/);
     });
 
-    it("can manage members from the people page", () => {
+    it("can manage members from the people page", { tags: "@flaky" }, () => {
       // Open membership select for a user
       // eslint-disable-next-line no-unscoped-text-selectors -- deprecated usage
       cy.findByText(noCollectionUserName)

e2e/test/scenarios/admin/i18n/content-translation/dashboards.cy.spec.ts

@@ -7,6 +7,10 @@ import type {
   DashboardDetails,
   StructuredQuestionDetails,
 } from "e2e/support/helpers";
+import {
+  ORDERS_COUNT_BY_CREATED_AT_AND_PRODUCT_CATEGORY,
+  PRODUCTS_COUNT_BY_CATEGORY_PIE,
+} from "e2e/support/test-visualizer-data";
 import type { DictionaryArray } from "metabase-types/api";
 const { ORDERS, ORDERS_ID } = SAMPLE_DATABASE;
 
@@ -23,6 +27,169 @@ import { uploadTranslationDictionaryViaAPI } from "./helpers/e2e-content-transla
 const { H } = cy;
 
 describe("scenarios > content translation > static embedding > dashboards", () => {
+  describe("values translation", () => {
+    beforeEach(() => {
+      H.restore();
+      cy.signInAsAdmin();
+      H.activateToken("bleeding-edge");
+
+      uploadTranslationDictionaryViaAPI([
+        { locale: "fr", msgid: "Gadget", msgstr: "Le gadget" },
+        { locale: "fr", msgid: "Doohickey", msgstr: "Le doohickey" },
+        { locale: "fr", msgid: "Gizmo", msgstr: "Le gizmo" },
+        { locale: "fr", msgid: "Widget", msgstr: "Le widget" },
+      ]);
+
+      cy.intercept("POST", "/api/card/*/query").as("cardQuery");
+      cy.intercept("GET", "/api/embed/dashboard/*").as("dashboard");
+
+      cy.signInAsAdmin();
+
+      H.createQuestion(ORDERS_COUNT_BY_CREATED_AT_AND_PRODUCT_CATEGORY, {
+        idAlias: "productsCountByCreatedAtQuestionId",
+        wrapId: true,
+      });
+
+      H.createQuestion(
+        {
+          ...PRODUCTS_COUNT_BY_CATEGORY_PIE,
+          visualization_settings: {
+            "pie.rows": [
+              ["Widget", "#69C8C8"],
+              ["Gadget", "#C7EAEA"],
+              ["Gizmo", "#98D9D9"],
+              ["Doohickey", "#F3F3F4"],
+            ].map(([key, color]) => ({
+              key,
+              name: key,
+              originalName: key,
+              color,
+              defaultColor: false,
+              enabled: true,
+              hidden: false,
+              isOther: false,
+            })),
+          },
+        },
+        {
+          idAlias: "productsCountByCategoryPieQuestionId",
+          wrapId: true,
+        },
+      );
+    });
+
+    it("should assign the proper colors to a pie", () => {
+      H.createDashboard({
+        name: "the_dashboard",
+      }).then(({ body: { id: dashboardId } }) => {
+        H.visitDashboard(dashboardId);
+        H.editDashboard();
+        H.openQuestionsSidebar();
+
+        H.sidebar().findByText(PRODUCTS_COUNT_BY_CATEGORY_PIE.name).click();
+        H.saveDashboard();
+
+        H.openStaticEmbeddingModal({
+          acceptTerms: false,
+        });
+        H.publishChanges("dashboard", () => {});
+
+        H.visitEmbeddedPage(
+          {
+            resource: { dashboard: dashboardId as number },
+            params: {},
+          },
+          {
+            additionalHashOptions: {
+              locale: "fr",
+            },
+          },
+        );
+
+        cy.wait("@dashboard");
+        cy.wait("@cardQuery");
+
+        H.getDashboardCard(0).within(() => {
+          cy.findAllByText("Le gadget").should("exist");
+          cy.findAllByText("Le doohickey").should("exist");
+          cy.findAllByText("Le gizmo").should("exist");
+          cy.findAllByText("Le widget").should("exist");
+
+          // Verify colors
+          cy.findByTestId("chart-legend").within(() => {
+            cy.get("button [color]").then(($elements) => {
+              const actualColors = Array.from($elements).map((el) =>
+                el.getAttribute("color"),
+              );
+              expect(actualColors).to.deep.equal([
+                "#69C8C8",
+                "#C7EAEA",
+                "#98D9D9",
+                "#F3F3F4",
+              ]);
+            });
+          });
+        });
+      });
+    });
+
+    it("should translate static embedding dashboard values on visualizer cards (metabase#62373)", () => {
+      H.visitDashboard(ORDERS_DASHBOARD_ID);
+
+      H.editDashboard();
+      H.removeDashboardCard(0);
+      H.openQuestionsSidebar();
+
+      // Add the regular question
+      H.sidebar()
+        .findByText(ORDERS_COUNT_BY_CREATED_AT_AND_PRODUCT_CATEGORY.name)
+        .click();
+
+      // Add the visualizer question
+      H.clickVisualizeAnotherWay(
+        ORDERS_COUNT_BY_CREATED_AT_AND_PRODUCT_CATEGORY.name,
+      );
+      H.selectVisualization("bar");
+      H.saveDashcardVisualizerModal({ mode: "create" });
+      H.saveDashboard();
+
+      H.openStaticEmbeddingModal({
+        acceptTerms: false,
+      });
+      H.publishChanges("dashboard", () => {});
+
+      H.visitEmbeddedPage(
+        {
+          resource: { dashboard: ORDERS_DASHBOARD_ID as number },
+          params: {},
+        },
+        {
+          additionalHashOptions: {
+            locale: "fr",
+          },
+        },
+      );
+
+      cy.wait("@dashboard");
+      cy.wait("@cardQuery");
+      cy.wait("@cardQuery");
+
+      H.getDashboardCard(0).within(() => {
+        cy.findAllByText("Le gadget").should("exist");
+        cy.findAllByText("Le doohickey").should("exist");
+        cy.findAllByText("Le gizmo").should("exist");
+        cy.findAllByText("Le widget").should("exist");
+      });
+
+      H.getDashboardCard(1).within(() => {
+        cy.findAllByText("Le gadget").should("exist");
+        cy.findAllByText("Le doohickey").should("exist");
+        cy.findAllByText("Le gizmo").should("exist");
+        cy.findAllByText("Le widget").should("exist");
+      });
+    });
+  });
+
   describe("measure names", () => {
     before(() => {
       cy.intercept("POST", "/api/ee/content-translation/upload-dictionary").as(

e2e/test/scenarios/admin/transforms.cy.spec.ts

@@ -16,6 +16,7 @@ const TARGET_TABLE = "transform_table";
 const TARGET_TABLE_2 = "transform_table_2";
 const TARGET_SCHEMA = "Schema A";
 const TARGET_SCHEMA_2 = "Schema B";
+const CUSTOM_SCHEMA = "custom_schema";
 
 H.describeWithSnowplowEE("scenarios > admin > transforms", () => {
   beforeEach(() => {
@@ -33,6 +34,9 @@ H.describeWithSnowplowEE("scenarios > admin > transforms", () => {
     cy.intercept("DELETE", "/api/ee/transform/*/table").as(
       "deleteTransformTable",
     );
+    cy.intercept("GET", "/api/ee/transform/*/dependencies").as(
+      "transformDependencies",
+    );
     cy.intercept("POST", "/api/ee/transform-tag").as("createTag");
     cy.intercept("PUT", "/api/ee/transform-tag/*").as("updateTag");
     cy.intercept("DELETE", "/api/ee/transform-tag/*").as("deleteTag");
@@ -208,6 +212,90 @@ H.describeWithSnowplowEE("scenarios > admin > transforms", () => {
       });
     });
 
+    it("should be able to create a new schema when saving a transform", () => {
+      visitTransformListPage();
+      getTransformListPage().button("Create a transform").click();
+      H.popover().findByText("Query builder").click();
+
+      H.entityPickerModal().within(() => {
+        cy.findByText(DB_NAME).click();
+        cy.findByText(SOURCE_TABLE).click();
+      });
+      getQueryEditor().button("Save").click();
+      H.modal().within(() => {
+        cy.findByLabelText("Name").type("MBQL transform");
+        cy.findByLabelText("Table name").type(TARGET_TABLE);
+        cy.findByLabelText("Schema").clear().type(CUSTOM_SCHEMA);
+      });
+      H.popover().findByText("Create new schema").click();
+      H.modal().within(() => {
+        cy.button("Save").click();
+        cy.wait("@createTransform");
+        cy.wait("@transformDependencies");
+      });
+
+      getSchemaLink()
+        .should("have.text", CUSTOM_SCHEMA)
+        .should("have.attr", "aria-disabled", "true")
+        .realHover();
+
+      H.tooltip()
+        .should("be.visible")
+        .should(
+          "have.text",
+          "This schema will be created when the transform runs",
+        );
+
+      getTableLink()
+        .should("have.text", TARGET_TABLE)
+        .should("have.attr", "aria-disabled", "true");
+
+      cy.log("run the transform and verify the table");
+      runTransformAndWaitForSuccess();
+      getSchemaLink()
+        .should("have.text", CUSTOM_SCHEMA)
+        .should("have.attr", "aria-disabled", "false")
+        .realHover();
+
+      getTableLink().click();
+      H.queryBuilderHeader().findByText(CUSTOM_SCHEMA).should("be.visible");
+      H.assertQueryBuilderRowCount(3);
+    });
+
+    it("should be able to create a new table in an existing when saving a transform", () => {
+      visitTransformListPage();
+      getTransformListPage().button("Create a transform").click();
+      H.popover().findByText("Query builder").click();
+
+      H.entityPickerModal().within(() => {
+        cy.findByText(DB_NAME).click();
+        cy.findByText(SOURCE_TABLE).click();
+      });
+      getQueryEditor().button("Save").click();
+      H.modal().within(() => {
+        cy.findByLabelText("Name").type("MBQL transform");
+        cy.findByLabelText("Table name").type(TARGET_TABLE);
+        cy.button("Save").click();
+        cy.wait("@createTransform");
+      });
+
+      getSchemaLink()
+        .should("have.text", TARGET_SCHEMA)
+        .should("have.attr", "aria-disabled", "false");
+
+      getTableLink()
+        .should("have.text", TARGET_TABLE)
+        .should("have.attr", "aria-disabled", "true");
+
+      cy.log("run the transform and verify the table");
+      runTransformAndWaitForSuccess();
+
+      getSchemaLink().should("have.attr", "aria-disabled", "false");
+      getTableLink().should("have.attr", "aria-disabled", "false").click();
+
+      H.assertQueryBuilderRowCount(3);
+    });
+
     it("should not be possible to create an mbql transform from a table from an unsupported database", () => {
       visitTransformListPage();
       getTransformListPage().button("Create a transform").click();
@@ -458,16 +546,71 @@ H.describeWithSnowplowEE("scenarios > admin > transforms", () => {
         cy.button("Change target").click();
         cy.wait("@updateTransform");
       });
-      getTableLink().should("have.text", TARGET_TABLE_2);
-      getSchemaLink().should("have.text", TARGET_SCHEMA_2);
+
+      getSchemaLink()
+        .should("have.text", TARGET_SCHEMA_2)
+        .should("have.attr", "aria-disabled", "false");
+
+      getTableLink()
+        .should("have.text", TARGET_TABLE_2)
+        .should("have.attr", "aria-disabled", "true");
 
       cy.log("run the transform and verify the table");
       runTransformAndWaitForSuccess();
-      getTableLink().click();
+      getSchemaLink()
+        .should("have.text", TARGET_SCHEMA_2)
+        .should("have.attr", "aria-disabled", "false");
+
+      getTableLink()
+        .should("have.text", TARGET_TABLE_2)
+        .should("have.attr", "aria-disabled", "false")
+        .click();
+
       H.queryBuilderHeader().findByText(TARGET_SCHEMA_2).should("be.visible");
       H.assertQueryBuilderRowCount(3);
     });
 
+    it("should be possible to create a new schema", () => {
+      cy.log("create but do not run the transform");
+      createMbqlTransform({ visitTransform: true });
+
+      cy.log("modify the transform before running");
+      getTransformPage().button("Change target").click();
+      H.modal().within(() => {
+        cy.findByLabelText("Table name").should("have.value", TARGET_TABLE);
+        cy.findByLabelText("Schema").should("have.value", TARGET_SCHEMA);
+        cy.findByLabelText("Table name").clear().type(TARGET_TABLE_2);
+        cy.findByLabelText("Schema").clear().type(CUSTOM_SCHEMA);
+      });
+      H.popover().findByText("Create new schema").click();
+      H.modal().within(() => {
+        cy.button("Change target").click();
+        cy.wait("@updateTransform");
+      });
+
+      getSchemaLink()
+        .should("have.text", CUSTOM_SCHEMA)
+        .should("have.attr", "aria-disabled", "true");
+
+      getTableLink()
+        .should("have.text", TARGET_TABLE_2)
+        .should("have.attr", "aria-disabled", "true");
+
+      cy.log("run the transform and verify the table");
+      runTransformAndWaitForSuccess();
+      getSchemaLink()
+        .should("have.text", CUSTOM_SCHEMA)
+        .should("have.attr", "aria-disabled", "false");
+
+      getTableLink()
+        .should("have.text", TARGET_TABLE_2)
+        .should("have.attr", "aria-disabled", "false")
+        .click();
+
+      H.queryBuilderHeader().findByText(CUSTOM_SCHEMA).should("be.visible");
+      H.assertQueryBuilderRowCount(3);
+    });
+
     it("should be able to change the target after running a transform and keep the old target", () => {
       cy.log("create and run a transform");
       createMbqlTransform({ visitTransform: true });
@@ -483,11 +626,25 @@ H.describeWithSnowplowEE("scenarios > admin > transforms", () => {
         cy.button("Change target").click();
         cy.wait("@updateTransform");
       });
-      getTableLink().should("have.text", TARGET_TABLE_2);
+
+      getSchemaLink()
+        .should("have.text", TARGET_SCHEMA)
+        .should("have.attr", "aria-disabled", "false");
+
+      getTableLink()
+        .should("have.text", TARGET_TABLE_2)
+        .should("have.attr", "aria-disabled", "true");
 
       cy.log("run the transform and verify the new table");
       runTransformAndWaitForSuccess();
-      getTableLink().click();
+      getSchemaLink()
+        .should("have.text", TARGET_SCHEMA)
+        .should("have.attr", "aria-disabled", "false");
+
+      getTableLink()
+        .should("have.text", TARGET_TABLE_2)
+        .should("have.attr", "aria-disabled", "false")
+        .click();
       H.queryBuilderHeader()
         .findByText("Transform Table 2")
         .should("be.visible");
@@ -514,11 +671,26 @@ H.describeWithSnowplowEE("scenarios > admin > transforms", () => {
         cy.wait("@deleteTransformTable");
         cy.wait("@updateTransform");
       });
-      getTableLink().should("have.text", TARGET_TABLE_2);
+
+      getSchemaLink()
+        .should("have.text", TARGET_SCHEMA)
+        .should("have.attr", "aria-disabled", "false");
+
+      getTableLink()
+        .should("have.text", TARGET_TABLE_2)
+        .should("have.attr", "aria-disabled", "true");
 
       cy.log("run the transform and verify the new table");
       runTransformAndWaitForSuccess();
-      getTableLink().click();
+
+      getSchemaLink()
+        .should("have.text", TARGET_SCHEMA)
+        .should("have.attr", "aria-disabled", "false");
+
+      getTableLink()
+        .should("have.text", TARGET_TABLE_2)
+        .should("have.attr", "aria-disabled", "false")
+        .click();
       H.queryBuilderHeader()
         .findByText("Transform Table 2")
         .should("be.visible");
@@ -529,37 +701,43 @@ H.describeWithSnowplowEE("scenarios > admin > transforms", () => {
       assertTableDoesNotExistError();
     });
 
-    it("should be able to delete the target and restore the same target back", () => {
-      cy.log("create and run a transform");
-      createMbqlTransform({ visitTransform: true });
-      runTransformAndWaitForSuccess();
+    it(
+      "should be able to delete the target and restore the same target back",
+      { tags: "@flaky" },
+      () => {
+        cy.log("create and run a transform");
+        createMbqlTransform({ visitTransform: true });
+        runTransformAndWaitForSuccess();
 
-      cy.log("delete the old target without creating the new one");
-      getTransformPage().button("Change target").click();
-      H.modal().within(() => {
-        cy.findByLabelText("Table name").clear().type(TARGET_TABLE_2);
-        cy.findByLabelText("Delete transform_table").click();
-        cy.button("Change target and delete the old one").click();
-        cy.wait("@deleteTransformTable");
-        cy.wait("@updateTransform");
-      });
+        cy.log("delete the old target without creating the new one");
+        getTransformPage().button("Change target").click();
+        H.modal().within(() => {
+          cy.findByLabelText("Table name").clear().type(TARGET_TABLE_2);
+          cy.findByLabelText("Delete transform_table").click();
+          cy.button("Change target and delete the old one").click();
+          cy.wait("@deleteTransformTable");
+          cy.wait("@updateTransform");
+        });
 
-      cy.log("change the target back to the original one");
-      getTransformPage().button("Change target").click();
-      H.modal().within(() => {
-        cy.findByLabelText("Table name").clear().type(TARGET_TABLE);
-        cy.button("Change target").click();
-        cy.wait("@updateTransform");
-      });
+        cy.log("change the target back to the original one");
+        getTransformPage().button("Change target").click();
+        H.modal().within(() => {
+          cy.findByLabelText("Table name").clear().type(TARGET_TABLE);
+          cy.button("Change target").click();
+          cy.wait("@updateTransform");
+        });
 
-      cy.log("run the transform to re-create the original target");
-      runTransformAndWaitForSuccess();
+        cy.log("run the transform to re-create the original target");
+        runTransformAndWaitForSuccess();
 
-      cy.log("verify the target is available");
-      getTableLink().click();
-      H.queryBuilderHeader().findByText("Transform Table").should("be.visible");
-      H.assertQueryBuilderRowCount(3);
-    });
+        cy.log("verify the target is available");
+        getTableLink().click();
+        H.queryBuilderHeader()
+          .findByText("Transform Table")
+          .should("be.visible");
+        H.assertQueryBuilderRowCount(3);
+      },
+    );
 
     it("should not allow to overwrite an existing table when changing the target", () => {
       createMbqlTransform({ visitTransform: true });
@@ -969,6 +1147,55 @@ H.describeWithSnowplowEE("scenarios > admin > transforms", () => {
   });
 });
 
+describe("scenarios > admin > transforms > databases without :schemas", () => {
+  const DB_NAME = "QA MySQL8";
+
+  beforeEach(() => {
+    H.restore("mysql-8");
+    cy.signInAsAdmin();
+    H.activateToken("bleeding-edge");
+    H.resyncDatabase({ dbId: WRITABLE_DB_ID });
+
+    cy.intercept("PUT", "/api/field/*").as("updateField");
+    cy.intercept("POST", "/api/ee/transform").as("createTransform");
+    cy.intercept("PUT", "/api/ee/transform/*").as("updateTransform");
+    cy.intercept("DELETE", "/api/ee/transform/*").as("deleteTransform");
+    cy.intercept("DELETE", "/api/ee/transform/*/table").as(
+      "deleteTransformTable",
+    );
+    cy.intercept("POST", "/api/ee/transform-tag").as("createTag");
+    cy.intercept("PUT", "/api/ee/transform-tag/*").as("updateTag");
+    cy.intercept("DELETE", "/api/ee/transform-tag/*").as("deleteTag");
+  });
+
+  it("should be not be possible to create a new schema when updating a transform target", () => {
+    createMbqlTransform({
+      databaseId: WRITABLE_DB_ID,
+      sourceTable: "ORDERS",
+      visitTransform: true,
+      targetSchema: null,
+    });
+
+    getTransformPage().button("Change target").click();
+
+    H.modal().findByLabelText("Schema").should("not.exist");
+  });
+
+  it("should be not be possible to create a new schema when the database does not support schemas", () => {
+    cy.log("create a new transform");
+    visitTransformListPage();
+    getTransformListPage().button("Create a transform").click();
+    H.popover().findByText("Query builder").click();
+
+    H.entityPickerModal().within(() => {
+      cy.findByText(DB_NAME).click();
+      cy.findByText("Orders").click();
+    });
+    getQueryEditor().button("Save").click();
+    H.modal().findByLabelText("Schema").should("not.exist");
+  });
+});
+
 describe("scenarios > admin > transforms > jobs", () => {
   beforeEach(() => {
     H.restore("postgres-writable");
@@ -995,7 +1222,9 @@ describe("scenarios > admin > transforms > jobs", () => {
         cy.findByPlaceholderText("Name").should("have.value", "New job");
         cy.findByPlaceholderText("No description yet").should("have.value", "");
         getCronInput().should("have.value", "0 0 * * ?");
-        cy.findByText("This job will run at 12:00 AM").should("be.visible");
+        cy.findByText("This job will run at 12:00 AM, UTC-07:00").should(
+          "be.visible",
+        );
       });
     });
 
@@ -1023,7 +1252,9 @@ describe("scenarios > admin > transforms > jobs", () => {
           "Description",
         );
         getCronInput().should("have.value", "0 * * * ?");
-        cy.findByText("This job will run every hour").should("be.visible");
+        cy.findByText("This job will run every hour, UTC-07:00").should(
+          "be.visible",
+        );
         cy.findByText("daily").should("be.visible");
       });
     });
@@ -1084,7 +1315,9 @@ describe("scenarios > admin > transforms > jobs", () => {
       H.createTransformJob({ name: "New job" }, { visitTransformJob: true });
       getJobPage().within(() => {
         getCronInput().clear().type("0 * * * ?").blur();
-        cy.findByText("This job will run every hour").should("be.visible");
+        cy.findByText("This job will run every hour, UTC-07:00").should(
+          "be.visible",
+        );
       });
       H.undoToast().findByText("Job schedule updated").should("be.visible");
       getJobPage().within(() => {
@@ -1094,7 +1327,7 @@ describe("scenarios > admin > transforms > jobs", () => {
   });
 
   describe("tags", () => {
-    it("should be able to add and remove tags", () => {
+    it("should be able to add and remove tags", { tags: "@flaky" }, () => {
       H.createTransformJob({ name: "New job" }, { visitTransformJob: true });
       getTagsInput().click();
 
@@ -1194,56 +1427,213 @@ describe("scenarios > admin > transforms > jobs", () => {
     });
   });
 
-  describe("dependencies", () => {
-    it("should render a table of transforms that will be run by a job", () => {
-      H.createTransformTag({ name: "Foo" }).then(({ body: tag }) => {
-        createMbqlTransform({
-          name: "Transform A",
-          tagIds: [],
-          targetTable: "table_a",
-          visitTransform: true,
-        }).then(runTransformAndWaitForSuccess);
+  describe("filtering", () => {
+    it("should be able to filter jobs ", () => {
+      cy.log("run hourly job so know that was recently run");
+      visitJobListPage();
 
-        createMbqlTransform({
-          name: "Transform B",
-          tagIds: [tag.id],
-          sourceTable: "table_a",
-          targetTable: "table_b",
-          visitTransform: true,
-        }).then(runTransformAndWaitForSuccess);
+      getContentTable().findByText("Hourly job").click();
+      runJobAndWaitForSuccess();
 
-        createMbqlTransform({
-          name: "Transform C",
-          tagIds: [tag.id],
-          sourceTable: "table_b",
-          targetTable: "table_c",
-          visitTransform: true,
-        });
+      visitJobListPage();
 
-        H.createTransformJob(
-          { name: "Foo job", tag_ids: [tag.id] },
-          { visitTransformJob: true },
-        );
-
-        H.main().findByText("Transforms").should("be.visible");
+      function testLastRunFilter() {
+        cy.log("no filters");
         getContentTable().within(() => {
-          // 3 transforms plus the header row
-          cy.findAllByRole("row").should("have.length", 4);
-
-          // Check the existence and also their order
-          cy.findAllByRole("row").eq(1).should("contain", "Transform A");
-          cy.findAllByRole("row").eq(2).should("contain", "Transform B");
-          cy.findAllByRole("row").eq(3).should("contain", "Transform C");
+          cy.findByText("Hourly job").should("be.visible");
+          cy.findByText("Daily job").should("be.visible");
+          cy.findByText("Weekly job").should("be.visible");
+          cy.findByText("Monthly job").should("be.visible");
         });
+
+        cy.log("last run at - add a filter");
+        getLastRunAtFilterWidget().click();
+        H.popover().findByText("Today").click();
+
+        getLastRunAtFilterWidget().should("contain", "Today");
+        getContentTable().within(() => {
+          cy.findByText("Hourly job").should("be.visible");
+          cy.findByText("Daily job").should("not.exist");
+          cy.findByText("Weekly job").should("not.exist");
+          cy.findByText("Monthly job").should("not.exist");
+        });
+
+        cy.log("last run at filter - update a filter");
+        getLastRunAtFilterWidget().click();
+        H.popover().findByText("Week").click();
+        getLastRunAtFilterWidget().should("contain", "This week");
+        getContentTable().within(() => {
+          cy.findByText("Hourly job").should("be.visible");
+          cy.findByText("Daily job").should("not.exist");
+          cy.findByText("Weekly job").should("not.exist");
+          cy.findByText("Monthly job").should("not.exist");
+        });
+
+        cy.log("last run at filter - remove filter");
+        getLastRunAtFilterWidget().button("Remove filter").click();
+        getContentTable().within(() => {
+          cy.findByText("Hourly job").should("be.visible");
+          cy.findByText("Daily job").should("be.visible");
+          cy.findByText("Weekly job").should("be.visible");
+          cy.findByText("Monthly job").should("be.visible");
+        });
+      }
+
+      function testNextRunFilter() {
+        cy.log("no filters");
+        getContentTable().within(() => {
+          cy.findByText("Hourly job").should("be.visible");
+          cy.findByText("Daily job").should("be.visible");
+          cy.findByText("Weekly job").should("be.visible");
+          cy.findByText("Monthly job").should("be.visible");
+        });
+
+        cy.log("next run - add a filter");
+        getNextRunFilterWidget().click();
+        H.popover().within(() => {
+          cy.findByText("Relative date range…").click();
+          cy.findByText("Current").click();
+          cy.findByText("Week").click();
+        });
+
+        getNextRunFilterWidget().should("contain", "This week");
+        getContentTable().within(() => {
+          cy.findByText("Hourly job").should("be.visible");
+          cy.findByText("Daily job").should("be.visible");
+          cy.findByText("Weekly job").should("not.exist");
+          cy.findByText("Monthly job").should("not.exist");
+        });
+
+        cy.log("next run filter - update a filter");
+        getNextRunFilterWidget().click();
+        H.popover().within(() => {
+          cy.findByText("Next").click();
+          cy.findByDisplayValue(30).clear().type("2");
+          cy.button("Apply").click();
+        });
+        getNextRunFilterWidget().should("contain", "Next 2 weeks");
+        getContentTable().within(() => {
+          cy.findByText("Hourly job").should("not.exist");
+          cy.findByText("Daily job").should("not.exist");
+          cy.findByText("Weekly job").should("be.visible");
+          cy.findByText("Monthly job").should("not.exist");
+        });
+
+        cy.log("next run filter - remove filter");
+        getNextRunFilterWidget().button("Remove filter").click();
+        getContentTable().within(() => {
+          cy.findByText("Hourly job").should("be.visible");
+          cy.findByText("Daily job").should("be.visible");
+          cy.findByText("Weekly job").should("be.visible");
+          cy.findByText("Monthly job").should("be.visible");
+        });
+      }
+
+      function testTagFilter() {
+        cy.log("no filters");
+        getContentTable().within(() => {
+          cy.findByText("Hourly job").should("be.visible");
+          cy.findByText("Daily job").should("be.visible");
+          cy.findByText("Weekly job").should("be.visible");
+          cy.findByText("Monthly job").should("be.visible");
+        });
+
+        cy.log("tag filter - add a filter");
+        getTagFilterWidget().click();
+        H.popover().within(() => {
+          cy.findByText("hourly").click();
+          cy.button("Add filter").click();
+        });
+        getTagFilterWidget().findByText("hourly").should("be.visible");
+        getContentTable().within(() => {
+          cy.findByText("Hourly job").should("be.visible");
+          cy.findByText("Daily job").should("not.exist");
+          cy.findByText("Weekly job").should("not.exist");
+          cy.findByText("Monthly job").should("not.exist");
+        });
+
+        cy.log("tag filter - update a filter");
+        getTagFilterWidget().click();
+        H.popover().within(() => {
+          cy.findByText("hourly").click();
+          cy.findByText("weekly").click();
+          cy.button("Update filter").click();
+        });
+
+        getTagFilterWidget().findByText("weekly").should("be.visible");
+        getContentTable().within(() => {
+          cy.findByText("Hourly job").should("not.exist");
+          cy.findByText("Daily job").should("not.exist");
+          cy.findByText("Weekly job").should("be.visible");
+          cy.findByText("Monthly job").should("not.exist");
+        });
+
+        cy.log("tag filter - multiple options");
+        getTagFilterWidget().click();
+        H.popover().within(() => {
+          cy.findByText("monthly").click();
+          cy.button("Update filter").click();
+        });
+        getTagFilterWidget().findByText("2 tags").should("be.visible");
+        getContentTable().within(() => {
+          cy.findByText("Hourly job").should("not.exist");
+          cy.findByText("Daily job").should("not.exist");
+          cy.findByText("Weekly job").should("be.visible");
+          cy.findByText("Monthly job").should("be.visible");
+        });
+
+        cy.log("tag filter - remove filter");
+        getTagFilterWidget().button("Remove filter").click();
+        getTagFilterWidget().findByText("2 tags").should("not.exist");
+        getContentTable().within(() => {
+          cy.findByText("Hourly job").should("be.visible");
+          cy.findByText("Daily job").should("be.visible");
+          cy.findByText("Weekly job").should("be.visible");
+          cy.findByText("Monthly job").should("be.visible");
+        });
+      }
+
+      testLastRunFilter();
+      testNextRunFilter();
+      testTagFilter();
+    });
+  });
+
+  describe("dependencies", () => {
+    it("should render a table of dependencies", () => {
+      createMbqlTransform({
+        name: "Transform A",
+        targetTable: "table_a",
+        visitTransform: true,
+      }).then(runTransformAndWaitForSuccess);
+
+      createMbqlTransform({
+        name: "Transform B",
+        sourceTable: "table_a",
+        targetTable: "table_b",
+        visitTransform: true,
+      }).then(runTransformAndWaitForSuccess);
+
+      createMbqlTransform({
+        name: "Transform C",
+        sourceTable: "table_b",
+        targetTable: "table_c",
+        visitTransform: true,
+      });
+
+      H.main().findByText("Dependencies").scrollIntoView().should("be.visible");
+      getContentTable().within(() => {
+        // 1 transform plus the header row
+        cy.findAllByRole("row").should("have.length", 2);
+
+        // Check the existence and also their order
+        cy.findAllByRole("row").eq(1).should("contain", "Transform B");
       });
     });
 
-    it("should render an empty table of transforms if there are no transforms for this job", () => {
-      H.createTransformJob({ name: "Foo job" }, { visitTransformJob: true });
-      getContentTable()
-        .findByText("There are no transforms for this job.")
-        .scrollIntoView()
-        .should("be.visible");
+    it("should no dependencies table if the transform has no dependencies", () => {
+      createMbqlTransform({ name: "Transform A", visitTransform: true });
+      H.main().findByText("Dependencies").should("not.exist");
     });
   });
 });
@@ -1665,6 +2055,14 @@ function getTransformFilterWidget() {
   return cy.findByRole("group", { name: "Transform" });
 }
 
+function getLastRunAtFilterWidget() {
+  return cy.findByRole("group", { name: "Last run at" });
+}
+
+function getNextRunFilterWidget() {
+  return cy.findByRole("group", { name: "Next run" });
+}
+
 function getStatusFilterWidget() {
   return cy.findByRole("group", { name: "Status" });
 }
@@ -1718,17 +2116,19 @@ function createMbqlTransform({
   targetTable = TARGET_TABLE,
   targetSchema = TARGET_SCHEMA,
   tagIds,
+  databaseId,
   name = "MBQL transform",
   visitTransform,
 }: {
   sourceTable?: string;
   targetTable?: string;
-  targetSchema?: string;
+  targetSchema?: string | null;
   tagIds?: TransformTagId[];
   name?: string;
+  databaseId?: number;
   visitTransform?: boolean;
 } = {}) {
-  return H.getTableId({ name: sourceTable }).then((tableId) => {
+  return H.getTableId({ databaseId, name: sourceTable }).then((tableId) => {
     return H.createTransform(
       {
         name,
@@ -1845,8 +2245,8 @@ function assertOptionSelected(name: string) {
 
 function assertOptionNotSelected(name: string) {
   cy.findByRole("option", { name }).should(
-    "not.have.attr",
+    "have.attr",
     "aria-selected",
-    "true",
+    "false",
   );
 }

e2e/test/scenarios/collections/uploads.cy.spec.js

@@ -15,80 +15,84 @@ H.describeWithSnowplow(
       cy.intercept("POST", "/api/table/*/replace-csv").as("replaceCSV");
     });
 
-    it("Can upload a CSV file to an empty postgres schema", () => {
-      const testFile = VALID_CSV_FILES[0];
-      const EMPTY_SCHEMA_NAME = "empty_uploads";
+    it(
+      "Can upload a CSV file to an empty postgres schema",
+      { tags: "@flaky" },
+      () => {
+        const testFile = VALID_CSV_FILES[0];
+        const EMPTY_SCHEMA_NAME = "empty_uploads";
 
-      cy.intercept("PUT", "/api/setting/*").as("saveSettings");
-      cy.intercept("GET", "/api/database").as("databaseList");
+        cy.intercept("PUT", "/api/setting/*").as("saveSettings");
+        cy.intercept("GET", "/api/database").as("databaseList");
 
-      H.restore("postgres-writable");
-      cy.signInAsAdmin();
+        H.restore("postgres-writable");
+        cy.signInAsAdmin();
 
-      H.queryWritableDB(
-        "DROP SCHEMA IF EXISTS empty_uploads CASCADE;",
-        "postgres",
-      );
-      H.queryWritableDB(
-        "CREATE SCHEMA IF NOT EXISTS empty_uploads;",
-        "postgres",
-      );
-
-      cy.request("POST", "/api/collection", {
-        name: "Uploads Collection",
-        parent_id: null,
-      }).then(({ body: { id: collectionId } }) => {
-        cy.wrap(collectionId).as("collectionId");
-      });
-      cy.visit("/admin/settings/uploads");
-
-      cy.findByLabelText("Upload Settings Form")
-        .findByPlaceholderText("Select a database")
-        .click();
-      H.popover().findByText("Writable Postgres12").click();
-      cy.findByLabelText("Upload Settings Form")
-        .findByPlaceholderText("Select a schema")
-        .click();
-
-      H.popover().findByText(EMPTY_SCHEMA_NAME).click();
-
-      cy.findByLabelText("Upload Settings Form")
-        .button("Enable uploads")
-        .click();
-
-      cy.wait(["@saveSettings", "@databaseList"]);
-
-      uploadFileToCollection(testFile);
-
-      const tableQuery = `SELECT * FROM information_schema.tables WHERE table_name LIKE '%${testFile.tableName}_%' ORDER BY table_name DESC LIMIT 1;`;
-
-      H.queryWritableDB(tableQuery, "postgres").then((result) => {
-        expect(result.rows.length).to.equal(1);
-        const tableName = result.rows[0].table_name;
         H.queryWritableDB(
-          `SELECT count(*) FROM ${EMPTY_SCHEMA_NAME}.${tableName};`,
+          "DROP SCHEMA IF EXISTS empty_uploads CASCADE;",
           "postgres",
-        ).then((result) => {
-          expect(Number(result.rows[0].count)).to.equal(testFile.rowCount);
+        );
+        H.queryWritableDB(
+          "CREATE SCHEMA IF NOT EXISTS empty_uploads;",
+          "postgres",
+        );
+
+        cy.request("POST", "/api/collection", {
+          name: "Uploads Collection",
+          parent_id: null,
+        }).then(({ body: { id: collectionId } }) => {
+          cy.wrap(collectionId).as("collectionId");
         });
-      });
+        cy.visit("/admin/settings/uploads");
 
-      cy.log(
-        "Ensure that table is visible in admin without refreshing (metabase#38041)",
-      );
+        cy.findByLabelText("Upload Settings Form")
+          .findByPlaceholderText("Select a database")
+          .click();
+        H.popover().findByText("Writable Postgres12").click();
+        cy.findByLabelText("Upload Settings Form")
+          .findByPlaceholderText("Select a schema")
+          .click();
 
-      cy.findByTestId("app-bar")
-        .findByRole("button", { name: "Settings" })
-        .click();
-      H.popover().findByText("Admin settings").click();
+        H.popover().findByText(EMPTY_SCHEMA_NAME).click();
 
-      cy.findByRole("link", { name: "Table Metadata" }).click();
+        cy.findByLabelText("Upload Settings Form")
+          .button("Enable uploads")
+          .click();
 
-      H.DataModel.TablePicker.getDatabase("Writable Postgres12").click();
-      H.DataModel.TablePicker.getSchema(EMPTY_SCHEMA_NAME).click();
-      H.DataModel.TablePicker.getTables().should("have.length", 1);
-      H.DataModel.TablePicker.getTable("Dog Breeds").should("be.visible");
-    });
+        cy.wait(["@saveSettings", "@databaseList"]);
+
+        uploadFileToCollection(testFile);
+
+        const tableQuery = `SELECT * FROM information_schema.tables WHERE table_name LIKE '%${testFile.tableName}_%' ORDER BY table_name DESC LIMIT 1;`;
+
+        H.queryWritableDB(tableQuery, "postgres").then((result) => {
+          expect(result.rows.length).to.equal(1);
+          const tableName = result.rows[0].table_name;
+          H.queryWritableDB(
+            `SELECT count(*) FROM ${EMPTY_SCHEMA_NAME}.${tableName};`,
+            "postgres",
+          ).then((result) => {
+            expect(Number(result.rows[0].count)).to.equal(testFile.rowCount);
+          });
+        });
+
+        cy.log(
+          "Ensure that table is visible in admin without refreshing (metabase#38041)",
+        );
+
+        cy.findByTestId("app-bar")
+          .findByRole("button", { name: "Settings" })
+          .click();
+        H.popover().findByText("Admin settings").click();
+
+        cy.findByRole("link", { name: "Table Metadata" }).click();
+
+        H.DataModel.TablePicker.getDatabase("Writable Postgres12").click();
+        H.DataModel.TablePicker.getSchema(EMPTY_SCHEMA_NAME).click();
+        H.DataModel.TablePicker.getTables().should("have.length", 1);
+        H.DataModel.TablePicker.getTable("Dog Breeds").should("be.visible");
+      },
+    );
 
     ["postgres", "mysql"].forEach((dialect) => {
       describe(`CSV Uploading (${dialect})`, () => {

e2e/test/scenarios/custom-column/custom-column-reproductions.cy.spec.js

@@ -2495,3 +2495,67 @@ describe("issue 62987", () => {
     });
   });
 });
+
+describe("issue 63180", () => {
+  beforeEach(() => {
+    H.restore();
+    cy.signInAsNormalUser();
+    H.createQuestion(
+      {
+        query: {
+          "source-table": ORDERS_ID,
+          expressions: {
+            Foo: ["+", 1, 2],
+          },
+        },
+      },
+      { visitQuestion: true },
+    );
+
+    H.openNotebook();
+  });
+
+  it("should not be possible to close the custom expression editor when creating a new expression from a combine or extract shortcut (metabase#63180)", () => {
+    function testCombineColumns() {
+      H.getNotebookStep("expression").icon("add").click();
+      H.expressionEditorWidget().within(() => {
+        cy.findByText("Combine columns").click();
+        cy.button("Done").scrollIntoView().click();
+      });
+
+      cy.log("clicking outside the editor should not close it");
+      H.getNotebookStep("data").click();
+      H.expressionEditorWidget().should("be.visible");
+      H.modal().should("not.exist");
+
+      cy.log("clearing the expression should allow clicking outside to work");
+      H.CustomExpressionEditor.clear();
+      H.getNotebookStep("data").click();
+      H.expressionEditorWidget().should("not.exist");
+      H.modal().should("not.exist");
+    }
+
+    function testExtractColumns() {
+      H.getNotebookStep("expression").icon("add").click();
+      H.expressionEditorWidget().within(() => {
+        cy.findByText("Extract columns").click();
+        cy.findByText("Email").click();
+        cy.findByText("Domain").click();
+      });
+
+      cy.log("clicking outside the editor should not close it");
+      H.getNotebookStep("data").click();
+      H.expressionEditorWidget().should("be.visible");
+      H.modal().should("not.exist");
+
+      cy.log("clearing the expression should allow clicking outside to work");
+      H.CustomExpressionEditor.clear();
+      H.getNotebookStep("data").click();
+      H.expressionEditorWidget().should("not.exist");
+      H.modal().should("not.exist");
+    }
+
+    testCombineColumns();
+    testExtractColumns();
+  });
+});

e2e/test/scenarios/dashboard/dashboard-reproductions.cy.spec.js

@@ -2180,3 +2180,28 @@ describe("issue 53370", () => {
     //
   });
 });
+
+describe("issue 63176", () => {
+  beforeEach(() => {
+    H.restore();
+    cy.signInAsNormalUser();
+  });
+
+  it("should not be possible to save a dashboard with an empty name and the correct error should be displayed (metabase#63176)", () => {
+    cy.visit("/");
+    H.newButton().click();
+    H.popover().findByText("Dashboard").click();
+    H.modal().within(() => {
+      cy.findByPlaceholderText("What is the name of your dashboard?").type(" ");
+      cy.button("Create").click();
+
+      cy.findByText("value must be a non-blank string.").should("be.visible");
+      cy.findByPlaceholderText("What is the name of your dashboard?").should(
+        "have.attr",
+        "aria-invalid",
+        "true",
+      );
+      cy.button("Failed").should("be.visible");
+    });
+  });
+});

e2e/test/scenarios/embedding/embedding-reproductions.cy.spec.js

@@ -957,20 +957,24 @@ describe("issue 40660", () => {
     });
   });
 
-  it("static dashboard content shouldn't overflow its container (metabase#40660)", () => {
-    H.openStaticEmbeddingModal({
-      activeTab: "parameters",
-      previewMode: "preview",
-    });
+  it(
+    "static dashboard content shouldn't overflow its container (metabase#40660)",
+    { tags: "@flaky" },
+    () => {
+      H.openStaticEmbeddingModal({
+        activeTab: "parameters",
+        previewMode: "preview",
+      });
 
-    H.getIframeBody().within(() => {
-      cy.findByTestId("embed-frame").scrollTo("bottom");
+      H.getIframeBody().within(() => {
+        cy.findByTestId("embed-frame").scrollTo("bottom");
 
-      cy.findByRole("link", { name: "Powered by Metabase" }).should(
-        "be.visible",
-      );
-    });
-  });
+        cy.findByRole("link", { name: "Powered by Metabase" }).should(
+          "be.visible",
+        );
+      });
+    },
+  );
 });
 
 // Skipped since it does not make sense when CSP is disabled

e2e/test/scenarios/models/models-list-view.cy.spec.js

@@ -123,6 +123,17 @@ describe("scenarios > models list view", () => {
         cy.findByText("2.07").should("not.exist");
       });
 
+      cy.log("Verify that empty column preview displays placeholder value");
+      cy.get("@rightColumns").within(() => {
+        cy.get("input").type("DISC");
+      });
+      H.popover().within(() => {
+        cy.findByText("DISCOUNT").click();
+      });
+      cy.get("@listPreview").within(() => {
+        cy.findByText("$123.46").should("be.visible");
+      });
+
       cy.log("Update list item icon");
       cy.get("@listPreview").within(() => {
         cy.findByRole("img").should("have.attr", "aria-label", "document icon");

e2e/test/scenarios/models/reproductions.cy.spec.ts

@@ -143,31 +143,35 @@ describe("issue 35711", () => {
     cy.signInAsAdmin();
   });
 
-  it("can edit metadata of a model with a custom column (metabase#35711)", () => {
-    H.createQuestion(
-      {
-        type: "model",
-        query: {
-          "source-table": ORDERS_ID,
-          expressions: {
-            "Custom column": ["-", DISCOUNT_FIELD_REF, 1],
+  it(
+    "can edit metadata of a model with a custom column (metabase#35711)",
+    { tags: "@flaky" },
+    () => {
+      H.createQuestion(
+        {
+          type: "model",
+          query: {
+            "source-table": ORDERS_ID,
+            expressions: {
+              "Custom column": ["-", DISCOUNT_FIELD_REF, 1],
+            },
+            limit: 5, // optimization
           },
-          limit: 5, // optimization
         },
-      },
-      { visitQuestion: true },
-    );
+        { visitQuestion: true },
+      );
 
-    H.openQuestionActions();
-    H.popover().findByText("Edit metadata").click();
-    H.waitForLoaderToBeRemoved();
+      H.openQuestionActions();
+      H.popover().findByText("Edit metadata").click();
+      H.waitForLoaderToBeRemoved();
 
-    reorderTaxAndTotalColumns();
-    assertNoError();
+      reorderTaxAndTotalColumns();
+      assertNoError();
 
-    cy.findByTestId("editor-tabs-query-name").click();
-    assertNoError();
-  });
+      cy.findByTestId("editor-tabs-query-name").click();
+      assertNoError();
+    },
+  );
 });
 
 describe("issues 25884 and 34349", () => {
@@ -977,20 +981,26 @@ describe("issue 39993", () => {
     cy.intercept("PUT", "/api/card/*").as("updateModel");
   });
 
-  it("should preserve viz settings for models with custom expressions (metabase#39993)", () => {
-    H.createQuestion(modelDetails).then(({ body: card }) =>
-      H.visitModel(card.id),
-    );
-    H.openQuestionActions();
-    H.popover().findByText("Edit metadata").click();
-    H.waitForLoaderToBeRemoved();
-    cy.log("drag & drop the custom column 100 px to the left");
-    H.moveDnDKitElement(H.tableHeaderColumn(columnName), { horizontal: -100 });
-    cy.button("Save changes").click();
-    cy.wait("@updateModel");
-    cy.findAllByTestId("header-cell").eq(0).should("have.text", "Exp");
-    cy.findAllByTestId("header-cell").eq(1).should("have.text", "ID");
-  });
+  it(
+    "should preserve viz settings for models with custom expressions (metabase#39993)",
+    { tags: "@flaky" },
+    () => {
+      H.createQuestion(modelDetails).then(({ body: card }) =>
+        H.visitModel(card.id),
+      );
+      H.openQuestionActions();
+      H.popover().findByText("Edit metadata").click();
+      H.waitForLoaderToBeRemoved();
+      cy.log("drag & drop the custom column 100 px to the left");
+      H.moveDnDKitElement(H.tableHeaderColumn(columnName), {
+        horizontal: -100,
+      });
+      cy.button("Save changes").click();
+      cy.wait("@updateModel");
+      cy.findAllByTestId("header-cell").eq(0).should("have.text", "Exp");
+      cy.findAllByTestId("header-cell").eq(1).should("have.text", "ID");
+    },
+  );
 });
 
 describe("issue 34574", () => {
@@ -1154,36 +1164,40 @@ describe("issue 36161", () => {
     cy.intercept("POST", "/api/dataset").as("dataset");
   });
 
-  it("should allow to override metadata for custom columns (metabase#36161)", () => {
-    H.visitModel(ORDERS_MODEL_ID);
-    cy.wait("@dataset");
+  it(
+    "should allow to override metadata for custom columns (metabase#36161)",
+    { tags: "@flaky" },
+    () => {
+      H.visitModel(ORDERS_MODEL_ID);
+      cy.wait("@dataset");
 
-    H.openQuestionActions("Edit query definition");
-    H.getNotebookStep("data").button("Pick columns").click();
-    H.popover().findByText("Select all").click();
-    H.getNotebookStep("data").button("Custom column").click();
-    H.enterCustomColumnDetails({ formula: "[ID]", name: "ID2" });
-    H.popover().button("Done").click();
-    H.getNotebookStep("expression").icon("add").click();
-    H.enterCustomColumnDetails({ formula: "[ID]", name: "ID3" });
-    H.popover().button("Done").click();
-    H.runButtonOverlay().click();
-    cy.wait("@dataset");
-    cy.findByTestId("editor-tabs-columns-name").click();
-    H.openColumnOptions("ID2");
-    H.renameColumn("ID2", "ID2 custom");
-    H.openColumnOptions("ID3");
-    H.renameColumn("ID3", "ID3 custom");
-    H.saveMetadataChanges();
+      H.openQuestionActions("Edit query definition");
+      H.getNotebookStep("data").button("Pick columns").click();
+      H.popover().findByText("Select all").click();
+      H.getNotebookStep("data").button("Custom column").click();
+      H.enterCustomColumnDetails({ formula: "[ID]", name: "ID2" });
+      H.popover().button("Done").click();
+      H.getNotebookStep("expression").icon("add").click();
+      H.enterCustomColumnDetails({ formula: "[ID]", name: "ID3" });
+      H.popover().button("Done").click();
+      H.runButtonOverlay().click();
+      cy.wait("@dataset");
+      cy.findByTestId("editor-tabs-columns-name").click();
+      H.openColumnOptions("ID2");
+      H.renameColumn("ID2", "ID2 custom");
+      H.openColumnOptions("ID3");
+      H.renameColumn("ID3", "ID3 custom");
+      H.saveMetadataChanges();
 
-    H.openNotebook();
-    H.getNotebookStep("data").button("Filter").click();
-    H.popover().within(() => {
-      cy.findByText("ID").should("be.visible");
-      cy.findByText("ID2 custom").should("be.visible");
-      cy.findByText("ID3 custom").should("be.visible");
-    });
-  });
+      H.openNotebook();
+      H.getNotebookStep("data").button("Filter").click();
+      H.popover().within(() => {
+        cy.findByText("ID").should("be.visible");
+        cy.findByText("ID2 custom").should("be.visible");
+        cy.findByText("ID3 custom").should("be.visible");
+      });
+    },
+  );
 });
 
 describe("issue 34514", () => {

e2e/test/scenarios/native/snippet-tags.cy.spec.ts

@@ -0,0 +1,404 @@
+const { H } = cy;
+
+import { ORDERS_QUESTION_ID } from "e2e/support/cypress_sample_instance_data";
+
+describe("scenarios > native > snippet tags", () => {
+  beforeEach(() => {
+    H.restore();
+    cy.signInAsAdmin();
+  });
+
+  it("should be able to create a snippet with variable tags", () => {
+    H.startNewNativeQuestion();
+    H.NativeEditor.type("select id from products where ");
+
+    cy.log("create a snippet");
+    getEditorTopBar().icon("snippet").click();
+    getEditorSidebar().findByText("Create snippet").click();
+    H.modal().within(() => {
+      getSnippetContentInput().type("category = {{category}}", {
+        parseSpecialCharSequences: false,
+      });
+      getSnippetNameInput().type("variable-snippet");
+      cy.button("Save").click();
+    });
+
+    cy.log("verify that the snippet was inserted");
+    H.NativeEditor.get().should(
+      "contain",
+      "select id from products where {{snippet: variable-snippet}}",
+    );
+
+    cy.log("verify that the query can be run");
+    getEditorTopBar().findByPlaceholderText("Category").type("Widget");
+    H.runNativeQuery();
+    H.assertQueryBuilderRowCount(54);
+  });
+
+  it("should be able to create a snippet with card tags", () => {
+    H.startNewNativeQuestion();
+    H.NativeEditor.type("select * from ");
+
+    cy.log("create a snippet");
+    getEditorTopBar().icon("snippet").click();
+    getEditorSidebar().findByText("Create snippet").click();
+    H.modal().within(() => {
+      getSnippetContentInput().type(`{{#${ORDERS_QUESTION_ID}}}`, {
+        parseSpecialCharSequences: false,
+      });
+      getSnippetNameInput().type("card-snippet");
+      cy.button("Save").click();
+    });
+
+    cy.log("verify that the snippet was inserted");
+    H.NativeEditor.get().should(
+      "contain",
+      "select * from {{snippet: card-snippet}}",
+    );
+
+    cy.log("verify that the query can be run");
+    H.runNativeQuery();
+    H.tableInteractive().should("be.visible");
+  });
+
+  it("should be able to create a snippet with snippet tags", () => {
+    H.startNewNativeQuestion();
+    H.NativeEditor.type("select * from ");
+
+    cy.log("create a snippet");
+    getEditorTopBar().icon("snippet").click();
+    getEditorSidebar().findByText("Create snippet").click();
+    H.modal().within(() => {
+      getSnippetContentInput().type("category = {{category}}", {
+        parseSpecialCharSequences: false,
+      });
+      getSnippetNameInput().type("snippet1");
+      cy.button("Save").click();
+    });
+
+    cy.log("create a snippet that uses the previous snippet");
+    getEditorTopBar().icon("snippet").click();
+    getEditorSidebar().icon("add").click();
+    H.popover().findByText("New snippet").click();
+    H.modal().within(() => {
+      getSnippetContentInput().type("{{snippet: snippet1}}", {
+        parseSpecialCharSequences: false,
+      });
+      getSnippetNameInput().type("snippet2");
+      cy.button("Save").click();
+    });
+
+    cy.log("verify that the snippet can used");
+    H.NativeEditor.clear();
+    H.NativeEditor.type("select id from products where {{snippet: snippet2}}");
+    getEditorTopBar().findByLabelText("Category").type("Gizmo");
+    H.runNativeQuery();
+    H.tableInteractive().should("be.visible");
+  });
+
+  it("should be able to create a tag with the same name as the inner snippet tag", () => {
+    createQuestionAndSnippet().then(({ card }) => {
+      H.visitQuestion(card.id);
+    });
+
+    cy.log("add a local tag with the same name");
+    getEditorVisibilityToggler().click();
+    H.NativeEditor.type(" and category = {{filter}}");
+
+    cy.log("verify that the query can be run");
+    getEditorTopBar().findByPlaceholderText("Filter").type("Widget");
+    H.runNativeQuery();
+    H.assertQueryBuilderRowCount(54);
+  });
+
+  it("should be able to use snippets where tags overlap", () => {
+    cy.log("create snippets with overlapping tags");
+    H.createSnippet({
+      name: "snippet1",
+      content: "category = {{category1}} or category = {{category2}}",
+    });
+    H.createSnippet({
+      name: "snippet2",
+      content: "category = {{category2}}",
+    });
+
+    cy.log("use both snippets in a query");
+    H.startNewNativeQuestion();
+    H.NativeEditor.type(
+      "select id from products where {{snippet: snippet1}} or {{snippet: snippet2}}",
+    );
+
+    cy.log("verify that the query can be run");
+    getEditorTopBar().findByPlaceholderText("Category1").type("Widget");
+    getEditorTopBar().findByPlaceholderText("Category2").type("Gadget");
+    H.runNativeQuery();
+    H.assertQueryBuilderRowCount(107);
+  });
+
+  it("should be able to update a snippet and change tags", () => {
+    createQuestionAndSnippet().then(({ card }) => {
+      H.visitQuestion(card.id);
+    });
+
+    cy.log("update the snippet");
+    getEditorVisibilityToggler().click();
+    getEditorTopBar().icon("snippet").click();
+    getEditorSidebar().within(() => {
+      cy.icon("chevrondown").click({ force: true });
+      cy.button(/Edit/).click();
+    });
+    H.modal().within(() => {
+      getSnippetContentInput()
+        .clear()
+        .type("ean = {{ean}} or vendor = {{vendor}}", {
+          parseSpecialCharSequences: false,
+        });
+      cy.button("Save").click();
+    });
+
+    cy.log("verify that the tags in the query were updated");
+    getEditorTopBar().within(() => {
+      cy.findByPlaceholderText("Filter").should("not.exist");
+      cy.findByPlaceholderText("Ean").type("1018947080336");
+      cy.findByPlaceholderText("Vendor").type("Balistreri-Ankunding");
+    });
+    H.runNativeQuery();
+    H.assertTableRowsCount(2);
+  });
+
+  it("should be able to change the inner tag type to Number", () => {
+    createQuestionAndSnippet({ snippetContent: "ID = {{filter}}" }).then(
+      ({ card }) => {
+        H.visitQuestion(card.id);
+      },
+    );
+
+    cy.log("change the type");
+    getEditorVisibilityToggler().click();
+    getEditorTopBar().icon("variable").click();
+    getVariableTypeSelect().click();
+    H.popover().findByText("Number").click();
+
+    cy.log("verify that the parameter can be used");
+    getEditorTopBar().findByPlaceholderText("Filter").type("10");
+    H.runNativeQuery();
+    H.assertQueryBuilderRowCount(1);
+  });
+
+  it("should be able to change the inner tag type to Field Filter", () => {
+    createQuestionAndSnippet({ snippetContent: "{{filter}}" }).then(
+      ({ card }) => {
+        H.visitQuestion(card.id);
+      },
+    );
+
+    cy.log("change the type");
+    getEditorVisibilityToggler().click();
+    getEditorTopBar().icon("variable").click();
+    getVariableTypeSelect().click();
+    H.popover().findByText("Field Filter").click();
+    H.popover().findByText("Products").click();
+    H.popover().findByText("Category").click();
+
+    cy.log("verify that the parameter can be used");
+    H.filterWidget().click();
+    H.popover().within(() => {
+      cy.findByText("Gadget").click();
+      cy.button("Add filter").click();
+    });
+    H.runNativeQuery();
+    H.assertQueryBuilderRowCount(53);
+  });
+
+  it("should be able to open a question with an old snippet name", () => {
+    createQuestionAndSnippet().then(({ card, snippet }) => {
+      cy.request("PUT", `/api/native-query-snippet/${snippet.id}`, {
+        name: "category-snippet",
+      });
+      cy.visit(`/question/${card.id}`);
+    });
+
+    cy.log("verify that the parameter can be used");
+    getEditorTopBar().findByPlaceholderText("Filter").type("Widget");
+    H.queryBuilderHeader().icon("play").click();
+    H.assertQueryBuilderRowCount(54);
+  });
+
+  it("should not be able to create card cycles via snippets", () => {
+    cy.log("create a card and a snippet referencing this card");
+    H.createNativeQuestion({
+      native: {
+        query: "select 1",
+      },
+    }).then(({ body: card }) => {
+      H.createSnippet({
+        name: "cycle-snippet",
+        content: `{{#${card.id}}}`,
+      });
+      H.visitQuestion(card.id);
+    });
+
+    cy.log("create a cycle and try to save the card");
+    getEditorVisibilityToggler().click();
+    H.NativeEditor.clear();
+    H.NativeEditor.type("select * from {{snippet: cycle-snippet}}");
+    H.queryBuilderHeader().button("Save").click();
+    H.modal().within(() => {
+      cy.button("Save").click();
+      cy.findByText("Cannot save card with cycles.").should("be.visible");
+    });
+  });
+
+  it("should not be able to create snippet cycles via snippets", () => {
+    cy.log("create two snippets, one using another");
+    H.createSnippet({
+      name: "snippet1",
+      content: "1",
+    });
+    H.createSnippet({
+      name: "snippet2",
+      content: "{{snippet: snippet1}}",
+    });
+
+    cy.log("open the modal for the first snippet");
+    H.startNewNativeQuestion();
+    getEditorTopBar().icon("snippet").click();
+    getEditorSidebar().within(() => {
+      cy.icon("chevrondown")
+        .should("have.length", 2)
+        .first()
+        .click({ force: true });
+      cy.button(/Edit/).click();
+    });
+
+    cy.log("create a cycle and save the first snippet");
+    H.modal().within(() => {
+      getSnippetContentInput()
+        .clear()
+        .type("{{snippet: snippet2}}", { parseSpecialCharSequences: false })
+        .click();
+      cy.button("Save").click();
+    });
+
+    cy.log("run the query and verify that we detect a cycle");
+    H.NativeEditor.type("select {{snippet: snippet1}}");
+    H.runNativeQuery();
+    H.queryBuilderMain()
+      .findByText(
+        'This query has circular referencing sub-queries. The snippet "snippet2" and the snippet "snippet1" seem to be part of the problem.',
+      )
+      .should("be.visible");
+
+    cy.log(
+      "try to save a question with a snippet cycle and verify we detect it",
+    );
+    H.queryBuilderHeader().findByText("Save").click();
+    H.modal().within(() => {
+      cy.findByLabelText("Name").type("SQL");
+      cy.button("Save").click();
+      cy.findByText("Cannot save card with cycles.").should("be.visible");
+      cy.button("Cancel").click();
+    });
+
+    cy.log(
+      "remove a cycle, save the card, add a cycle again and verify that we detect it on update",
+    );
+    H.NativeEditor.clear().type("select 1");
+    H.saveQuestion("SQL", undefined, {
+      tab: "Browse",
+      path: ["Our analytics"],
+    });
+    H.NativeEditor.clear().type("select {{snippet: snippet1}}");
+    H.queryBuilderHeader().button("Save").click();
+    H.modal().within(() => {
+      cy.button("Save").click();
+      cy.findByText("Cannot save card with cycles.").should("be.visible");
+    });
+  });
+
+  it("should work for a public or embedded question", () => {
+    createQuestionAndSnippet().then(({ card }) => {
+      cy.log("public question - no parameter value");
+      H.visitPublicQuestion(card.id);
+      H.assertTableRowsCount(200);
+
+      cy.log("public question - with parameter value");
+      H.filterWidget().findByLabelText("Filter").type("Gadget{enter}");
+      H.assertTableRowsCount(53);
+
+      cy.log("embedded question - no parameter value");
+      H.visitEmbeddedPage({
+        resource: { question: card.id },
+        params: {},
+      });
+      H.assertTableRowsCount(200);
+
+      cy.log("embedded question - with parameter value");
+      H.filterWidget().findByLabelText("Filter").type("Gadget{enter}");
+      H.assertTableRowsCount(53);
+    });
+  });
+});
+
+function getEditorSidebar() {
+  return cy.findByTestId("sidebar-right");
+}
+
+function getEditorTopBar() {
+  return cy.findByTestId("native-query-top-bar");
+}
+
+function getEditorVisibilityToggler() {
+  return cy.findByTestId("visibility-toggler");
+}
+
+function getVariableTypeSelect() {
+  return cy.findByTestId("variable-type-select");
+}
+
+function getSnippetNameInput() {
+  return cy.findByLabelText("Give your snippet a name");
+}
+
+function getSnippetContentInput() {
+  return cy.findByLabelText("Enter some SQL here so you can reuse it later");
+}
+
+function createQuestionAndSnippet({
+  snippetContent = "category = {{filter}}",
+}: {
+  snippetContent?: string;
+} = {}) {
+  return H.createSnippet({
+    name: "filter-snippet",
+    content: snippetContent,
+  }).then(({ body: snippet }) => {
+    return H.createNativeQuestion({
+      native: {
+        query: "select id from products [[where {{snippet: filter-snippet}}]]",
+        "template-tags": {
+          "snippet: filter-snippet": {
+            id: "4b77cc1f-ea70-4ef6-84db-58432fce6928",
+            name: "snippet: filter-snippet",
+            "display-name": "snippet: filter-snippet",
+            type: "snippet",
+            "snippet-id": snippet.id,
+            "snippet-name": snippet.name,
+          },
+          filter: {
+            id: "4b77cc1f-ea70-4ef6-84db-58432fce6929",
+            name: "filter",
+            "display-name": "Filter",
+            type: "text",
+          },
+        },
+      },
+      enable_embedding: true,
+      embedding_params: {
+        filter: "enabled",
+      },
+    }).then(({ body: card }) => {
+      return { card, snippet };
+    });
+  });
+}

e2e/test/scenarios/onboarding/setup/setup-embedding.cy.spec.ts

@@ -75,195 +75,201 @@ H.describeWithSnowplowEE("scenarios > setup embedding (EMB-477)", () => {
       .should("be.visible");
   });
 
-  it("[cloud-hosted] should allow users to go through the embedding setup and onboarding flow", () => {
-    // mock `is-hosted`, we can't set the token before the user is created, we'll set it right after that
-    H.activateToken("pro-cloud");
+  it(
+    "[cloud-hosted] should allow users to go through the embedding setup and onboarding flow",
+    { tags: "@flaky" },
+    () => {
+      // mock `is-hosted`, we can't set the token before the user is created, we'll set it right after that
+      H.activateToken("pro-cloud");
 
-    mockCloudHosted();
+      mockCloudHosted();
 
-    cy.visit(
-      "/setup/embedding?first_name=Firstname&last_name=Lastname&email=testy@metabase.test&site_name=Epic Team",
-    );
-
-    cy.log("0: Welcome step");
-    assertEmbeddingOnboardingPageLoaded();
-
-    H.expectUnstructuredSnowplowEvent({
-      event: "embedding_setup_step_seen",
-      event_detail: "welcome",
-    });
-
-    step().within(() => {
-      cy.findByText(
-        /Let's get you up and running with a starting setup for embedded analytics/,
-      ).should("be.visible");
-      cy.button("Start").should("be.visible").click();
-    });
-
-    cy.log("1: User creation step");
-    sidebar().within(() => {
-      cy.findByRole("listitem", { current: "step" }).should(
-        "have.text",
-        "Set up your account",
-      );
-    });
-    step().within(() => {
-      cy.findByRole("heading", { name: "What should we call you?" }).should(
-        "be.visible",
+      cy.visit(
+        "/setup/embedding?first_name=Firstname&last_name=Lastname&email=testy@metabase.test&site_name=Epic Team",
       );
 
+      cy.log("0: Welcome step");
+      assertEmbeddingOnboardingPageLoaded();
+
       H.expectUnstructuredSnowplowEvent({
         event: "embedding_setup_step_seen",
-        event_detail: "user-creation",
+        event_detail: "welcome",
       });
 
-      cy.findByLabelText("First name").should("have.value", "Firstname");
-      cy.findByLabelText("Last name").should("have.value", "Lastname");
-      cy.findByLabelText("Email").should("have.value", "testy@metabase.test");
-      cy.findByLabelText("Company or team name").should(
-        "have.value",
-        "Epic Team",
-      );
+      step().within(() => {
+        cy.findByText(
+          /Let's get you up and running with a starting setup for embedded analytics/,
+        ).should("be.visible");
+        cy.button("Start").should("be.visible").click();
+      });
 
-      fillOutUserForm();
+      cy.log("1: User creation step");
+      sidebar().within(() => {
+        cy.findByRole("listitem", { current: "step" }).should(
+          "have.text",
+          "Set up your account",
+        );
+      });
+      step().within(() => {
+        cy.findByRole("heading", { name: "What should we call you?" }).should(
+          "be.visible",
+        );
 
-      cy.intercept("POST", "/api/setup").as("setup");
-      cy.button("Next").should("be.enabled").click();
-    });
-
-    cy.log(
-      "Now we have a user we simulate being on cloud by setting a Metatabase Token",
-    );
-    cy.wait("@setup");
-
-    // set a real token so we have real token-features for the rest of the flow
-    H.activateToken("pro-cloud");
-
-    cy.log("2: Data connection step");
-    sidebar().within(() => {
-      cy.findByRole("listitem", { current: "step" }).should(
-        "have.text",
-        "Connect to your data",
-      );
-    });
-    step().within(() => {
-      cy.findByRole("heading", { name: "Connect to your data" }).should(
-        "be.visible",
-      );
-
-      cy.findByRole("option", { name: "PostgreSQL" }).click();
-      fillOutDatabaseForm();
-
-      cy.log("simulate database that takes some time to fully sync");
-      const NO_OF_MIN_RETRY = 4;
-      let currentRetry = 0;
-      cy.intercept("GET", "api/database", (req) => {
-        req.on("response", (res) => {
-          currentRetry++;
-          const [postgresDb] = res.body.data;
-          if (
-            postgresDb.initial_sync_status !== "complete" ||
-            currentRetry < NO_OF_MIN_RETRY
-          ) {
-            // Simulate syncing not complete until we reach a certain number of retries
-            postgresDb.initial_sync_status = "incomplete";
-          }
+        H.expectUnstructuredSnowplowEvent({
+          event: "embedding_setup_step_seen",
+          event_detail: "user-creation",
         });
+
+        cy.findByLabelText("First name").should("have.value", "Firstname");
+        cy.findByLabelText("Last name").should("have.value", "Lastname");
+        cy.findByLabelText("Email").should("have.value", "testy@metabase.test");
+        cy.findByLabelText("Company or team name").should(
+          "have.value",
+          "Epic Team",
+        );
+
+        fillOutUserForm();
+
+        cy.intercept("POST", "/api/setup").as("setup");
+        cy.button("Next").should("be.enabled").click();
       });
-      cy.button("Connect database").should("be.enabled").click();
-    });
 
-    cy.log("3: Content generation step");
-    sidebar().within(() => {
-      cy.findByRole("listitem", { current: "step" }).should(
-        "have.text",
-        "Generate starter content",
+      cy.log(
+        "Now we have a user we simulate being on cloud by setting a Metatabase Token",
       );
-    });
+      cy.wait("@setup");
 
-    const SECOND = 1000;
-    // Waiting for the database sync to complete
-    step()
-      .findByRole("heading", {
-        name: "Select Tables to Embed",
-        timeout: 10 * SECOND,
-      })
-      .should("be.visible");
+      // set a real token so we have real token-features for the rest of the flow
+      H.activateToken("pro-cloud");
 
-    expectNoDatabaseStatus();
-
-    step().within(() => {
-      cy.findByRole("checkbox", { name: "feedback" })
-        .should("be.enabled")
-        .check();
-      cy.findByRole("checkbox", { name: "orders" })
-        .should("be.enabled")
-        .check();
-      cy.findByRole("checkbox", { name: "products" })
-        .should("be.enabled")
-        .check();
-
-      cy.button("Continue").should("be.enabled").click();
-    });
-
-    cy.log("4: Processing step");
-
-    step().within(() => {
-      cy.findByRole("heading", { name: "Setting Up Your Analytics" }).should(
-        "be.visible",
-      );
-      /**
-       * Since this loader will go away automatically it's crucial to wait for it to go away
-       * before start asserting the next steps.
-       */
-      cy.findByText("Creating dashboards...").should("not.exist");
-    });
-
-    cy.log("5: Embed in your app step");
-    sidebar().within(() => {
-      cy.findByRole("listitem", { current: "step" }).should(
-        "have.text",
-        "Add to your app",
-      );
-    });
-
-    step().within(() => {
-      cy.findByRole("heading", { name: "Add to your app" }).should(
-        "be.visible",
-      );
-      cy.findByRole("tab", { name: 'A look at "Feedback"' }).click();
-      H.getIframeBody().within(() => {
-        cy.findByText('A look at "Feedback"').should("be.visible");
+      cy.log("2: Data connection step");
+      sidebar().within(() => {
+        cy.findByRole("listitem", { current: "step" }).should(
+          "have.text",
+          "Connect to your data",
+        );
       });
-      cy.findByRole("code").contains(
-        new RegExp(
-          `<iframe src="${Cypress.config("baseUrl")}/dashboard/\\d+" width="800px" height="500px" />`,
-        ),
+      step().within(() => {
+        cy.findByRole("heading", { name: "Connect to your data" }).should(
+          "be.visible",
+        );
+
+        cy.findByRole("option", { name: "PostgreSQL" }).click();
+        fillOutDatabaseForm();
+
+        cy.log("simulate database that takes some time to fully sync");
+        const NO_OF_MIN_RETRY = 4;
+        let currentRetry = 0;
+        cy.intercept("GET", "api/database", (req) => {
+          req.on("response", (res) => {
+            currentRetry++;
+            const [postgresDb] = res.body.data;
+            if (
+              postgresDb.initial_sync_status !== "complete" ||
+              currentRetry < NO_OF_MIN_RETRY
+            ) {
+              // Simulate syncing not complete until we reach a certain number of retries
+              postgresDb.initial_sync_status = "incomplete";
+            }
+          });
+        });
+        cy.button("Connect database").should("be.enabled").click();
+      });
+
+      cy.log("3: Content generation step");
+      sidebar().within(() => {
+        cy.findByRole("listitem", { current: "step" }).should(
+          "have.text",
+          "Generate starter content",
+        );
+      });
+
+      const SECOND = 1000;
+      // Waiting for the database sync to complete
+      step()
+        .findByRole("heading", {
+          name: "Select Tables to Embed",
+          timeout: 10 * SECOND,
+        })
+        .should("be.visible");
+
+      expectNoDatabaseStatus();
+
+      step().within(() => {
+        cy.findByRole("checkbox", { name: "feedback" })
+          .should("be.enabled")
+          .check();
+        cy.findByRole("checkbox", { name: "orders" })
+          .should("be.enabled")
+          .check();
+        cy.findByRole("checkbox", { name: "products" })
+          .should("be.enabled")
+          .check();
+
+        cy.button("Continue").should("be.enabled").click();
+      });
+
+      cy.log("4: Processing step");
+
+      step().within(() => {
+        cy.findByRole("heading", { name: "Setting Up Your Analytics" }).should(
+          "be.visible",
+        );
+        /**
+         * Since this loader will go away automatically it's crucial to wait for it to go away
+         * before start asserting the next steps.
+         */
+        cy.findByText("Creating dashboards...").should("not.exist");
+      });
+
+      cy.log("5: Embed in your app step");
+      sidebar().within(() => {
+        cy.findByRole("listitem", { current: "step" }).should(
+          "have.text",
+          "Add to your app",
+        );
+      });
+
+      step().within(() => {
+        cy.findByRole("heading", { name: "Add to your app" }).should(
+          "be.visible",
+        );
+        cy.findByRole("tab", { name: 'A look at "Feedback"' }).click();
+        H.getIframeBody().within(() => {
+          cy.findByText('A look at "Feedback"').should("be.visible");
+        });
+        cy.findByRole("code").contains(
+          new RegExp(
+            `<iframe src="${Cypress.config("baseUrl")}/dashboard/\\d+" width="800px" height="500px" />`,
+          ),
+        );
+
+        cy.button("I see Metabase").click();
+
+        cy.findByRole("heading", { name: "You're on your way!" }).should(
+          "be.visible",
+        );
+        cy.findByRole("link", { name: "Take me to Metabase" }).click();
+      });
+
+      cy.log("5: Metabase");
+      cy.button("New").should("be.visible");
+
+      cy.log(
+        "Only one 'Automatically Generated Dashboards' collection should be created",
       );
-
-      cy.button("I see Metabase").click();
-
-      cy.findByRole("heading", { name: "You're on your way!" }).should(
-        "be.visible",
-      );
-      cy.findByRole("link", { name: "Take me to Metabase" }).click();
-    });
-
-    cy.log("5: Metabase");
-    cy.button("New").should("be.visible");
-
-    cy.log(
-      "Only one 'Automatically Generated Dashboards' collection should be created",
-    );
-    sidebar().within(() => {
-      cy.findByText("Automatically Generated Dashboards").should("be.visible");
-      cy.findByText("Automatically Generated Dashboards").should(
-        "have.length",
-        1,
-      );
-    });
-    expectNoDatabaseStatus();
-  });
+      sidebar().within(() => {
+        cy.findByText("Automatically Generated Dashboards").should(
+          "be.visible",
+        );
+        cy.findByText("Automatically Generated Dashboards").should(
+          "have.length",
+          1,
+        );
+      });
+      expectNoDatabaseStatus();
+    },
+  );
 });
 
 /**

e2e/test/scenarios/permissions/admin-permissions.cy.spec.js

@@ -310,7 +310,7 @@ describe("scenarios > admin > permissions", { tags: "@OSS" }, () => {
     });
 
     context("group focused view", () => {
-      it("shows filterable list of groups", () => {
+      it("shows filterable list of groups", { tags: "@flaky" }, () => {
         cy.visit("/admin/permissions");
 
         // no groups selected initially and it shows an empty state

e2e/test/scenarios/permissions/sandboxing/sandboxing-via-ui.cy.spec.ts

@@ -78,12 +78,16 @@ describe(
       H.restore("sandboxing-snapshot" as any);
     });
 
-    it("shows all data before sandboxing policy is applied", () => {
-      signInAs(gizmoViewer);
-      assertNoResultsOrValuesAreSandboxed(dashboard, sandboxableQuestions);
-      signInAs(widgetViewer);
-      assertNoResultsOrValuesAreSandboxed(dashboard, sandboxableQuestions);
-    });
+    it(
+      "shows all data before sandboxing policy is applied",
+      { tags: "@flaky" },
+      () => {
+        signInAs(gizmoViewer);
+        assertNoResultsOrValuesAreSandboxed(dashboard, sandboxableQuestions);
+        signInAs(widgetViewer);
+        assertNoResultsOrValuesAreSandboxed(dashboard, sandboxableQuestions);
+      },
+    );
 
     describe("we can apply a sandbox policy", () => {
       beforeEach(() => {

e2e/test/scenarios/question-reproductions/reproductions-3.cy.spec.js

@@ -1955,7 +1955,7 @@ describe("issue 45063", { tags: "@flaky" }, () => {
       });
     });
 
-    it("should work with models", () => {
+    it("should work with models", { tags: "@flaky" }, () => {
       createGuiModel({ sourceTableId: ORDERS_ID });
       verifyRemappedFilter({
         visitCard: () => cy.get("@modelId").then(H.visitModel),

e2e/test/scenarios/question-reproductions/reproductions.cy.spec.ts

@@ -1202,3 +1202,91 @@ describe("issue 52872", () => {
       });
   });
 });
+
+describe("issue 35561", () => {
+  beforeEach(() => {
+    H.restore();
+    cy.signInAsAdmin();
+  });
+
+  it("should display a functional fk drill for both null and non-null values", () => {
+    H.createNativeQuestion(
+      {
+        name: "Model",
+        native: {
+          query: `SELECT 1 AS ID, CAST(NULL AS INT) AS USER_ID
+UNION ALL
+SELECT 2 AS ID, 3 AS USER_ID
+`,
+        },
+        type: "model",
+      },
+      {},
+    ).then(async ({ body: model }) => {
+      await cy.request("POST", `/api/card/${model.id}/query`);
+      await H.setModelMetadata(model.id, (field) => {
+        if (field.name === "USER_ID") {
+          return {
+            ...field,
+            display_name: "User ID",
+            semantic_type: "type/FK",
+            fk_target_field_id: ORDERS.ID,
+          };
+        }
+        return field;
+      });
+      H.createQuestion(
+        {
+          name: "Question",
+          query: {
+            "source-table": `card__${model.id}`,
+          },
+          display: "table",
+        },
+        { wrapId: true, visitQuestion: true },
+      );
+    });
+
+    cy.findByTestId("table-root")
+      .findAllByRole("row")
+      .first()
+      .within(() => {
+        cy.get("[data-column-id='USER_ID']").click();
+      });
+    H.popover().within(() => {
+      cy.findByText("View Models with no User").should("be.visible").click();
+    });
+    cy.findByTestId("table-body").within(() => {
+      cy.findAllByRole("row").should("have.length", 1);
+      cy.findAllByRole("row")
+        .first()
+        .within(() => {
+          cy.get("[data-column-id='USER_ID']").within(() => {
+            cy.get("[data-testid=cell-data]").should("not.exist");
+          });
+        });
+    });
+
+    H.visitQuestion("@questionId");
+
+    cy.findByTestId("table-root")
+      .findAllByRole("row")
+      .eq(1)
+      .within(() => {
+        cy.get("[data-column-id='USER_ID']").click();
+      });
+    H.popover().within(() => {
+      cy.findByText("View this User's Models").should("be.visible").click();
+    });
+    cy.findByTestId("table-body").within(() => {
+      cy.findAllByRole("row").should("have.length", 1);
+      cy.findAllByRole("row")
+        .first()
+        .within(() => {
+          cy.get("[data-column-id='USER_ID']").within(() => {
+            cy.get("[data-testid=cell-data]").should("have.text", "3");
+          });
+        });
+    });
+  });
+});

e2e/test/scenarios/table-editing/table-editing.cy.spec.ts

@@ -304,15 +304,20 @@ describe("scenarios > table-editing", () => {
           cy.findAllByRole("button").last().click();
         });
 
-        cy.wait("@updateTableData").then(({ response }) => {
+        cy.wait("@updateTableData").then(({ response, request }) => {
           const targetDate = dayjs(
             new Date(2024, 4, day, hour, minute, 0),
           ).format("YYYY-MM-DDTHH:mm:ss");
-          const savedDate = response?.body.outputs[0].row.CREATED_AT;
 
-          expect(response?.body.outputs[0].op).to.equal("updated");
-          // Omit timezone offset
-          expect(savedDate.startsWith(targetDate)).to.be.true;
+          const requestDate = request.body.params.CREATED_AT;
+          const responseDate = response?.body.outputs[0].row.CREATED_AT;
+
+          // Check if request date matches the response date.
+          // In theory FE should preserve the initial input date timezone offset (might be based on the CI environment)
+          expect(requestDate).to.equal(responseDate);
+
+          // Check if date param matches the input date (without timezone offset at offset 19)
+          expect(requestDate.slice(0, 19)).to.equal(targetDate);
         });
 
         H.undoToast().findByText("Successfully updated").should("be.visible");
@@ -447,6 +452,10 @@ describe("scenarios > table-editing", () => {
       cy.findAllByTestId("row-select-checkbox").eq(15).click();
       cy.findAllByTestId("row-select-checkbox").eq(16).click();
 
+      cy.log("should not show edit icon when rows are selected");
+      cy.findByTestId("row-edit-icon").should("not.exist");
+
+      cy.log("should bulk delete rows");
       cy.findByTestId("toast-card").findByText("Delete").click();
 
       H.modal().within(() => {
@@ -462,6 +471,9 @@ describe("scenarios > table-editing", () => {
       cy.findByTestId("toast-card").should("not.exist");
 
       H.undoToast().findByText("Successfully deleted").should("be.visible");
+
+      cy.log("should show edit icon when no rows are selected");
+      cy.findAllByTestId("row-edit-icon").should("exist");
     });
   });
 });

enterprise/backend/src/metabase_enterprise/api/routes.clj

@@ -26,6 +26,7 @@
    [metabase-enterprise.metabot-v3.api]
    [metabase-enterprise.metabot-v3.tools.api]
    [metabase-enterprise.permission-debug.api]
+   [metabase-enterprise.python-transform.api]
    [metabase-enterprise.sandbox.api.routes]
    [metabase-enterprise.scim.routes]
    [metabase-enterprise.semantic-search.api]
@@ -107,6 +108,7 @@
    "/metabot-tools"                metabase-enterprise.metabot-v3.tools.api/routes
    "/metabot-v3"                   (premium-handler metabase-enterprise.metabot-v3.api/routes :metabot-v3)
    "/permission_debug"             (premium-handler metabase-enterprise.permission-debug.api/routes :advanced-permissions)
+   "/python-transform"             (premium-handler metabase-enterprise.python-transform.api/routes :transforms)
    "/scim"                         (premium-handler metabase-enterprise.scim.routes/routes :scim)
    "/semantic-search"              (premium-handler metabase-enterprise.semantic-search.api/routes :semantic-search)
    "/serialization"                (premium-handler metabase-enterprise.serialization.api/routes :serialization)

enterprise/backend/src/metabase_enterprise/harbormaster/client.clj

@@ -2,6 +2,7 @@
   "API client for interfacing with Harbormaster on store-api-url."
   (:require
    [clj-http.client :as http]
+   [clojure.core.memoize :as memoize]
    [clojure.string :as str]
    [martian.clj-http :as martian-http]
    [martian.core :as martian]
@@ -135,7 +136,8 @@
    {:headers {"Authorization" (str "Bearer " api-key)}}))
 
 (def ^:private create-client-memo
-  (memoize create-client))
+  (memoize/ttl create-client
+               :ttl/threshold (m.util/minutes->ms 5)))
 
 (defn- client []
   (let [store-api-url (store-api/store-api-url)

enterprise/backend/src/metabase_enterprise/python_transform/api.clj

@@ -0,0 +1,29 @@
+(ns metabase-enterprise.python-transform.api
+  (:require
+   [metabase-enterprise.python-transform.models.python-library :as python-library]
+   [metabase.api.common :as api]
+   [metabase.api.macros :as api.macros]
+   [metabase.api.routes.common :refer [+auth]]
+   [metabase.api.util.handlers :as handlers]))
+
+(api.macros/defendpoint :get "/library/:path"
+  "Get the Python library for user modules."
+  [{:keys [path]} :- [:map [:path :string]]
+   _query-params]
+  (api/check-superuser)
+  (some-> (python-library/get-python-library-by-path path)
+          (select-keys [:source :path :created_at :updated_at])))
+
+(api.macros/defendpoint :put "/library/:path"
+  "Update the Python library source code for user modules."
+  [{:keys [path]} :- [:map [:path :string]]
+   _query-params
+   body :- [:map {:closed true}
+            [:source :string]]]
+  (api/check-superuser)
+  (python-library/update-python-library-source! path (:source body)))
+
+(def ^{:arglists '([request respond raise])} routes
+  "`/api/ee/python-transform` routes."
+  (handlers/routes
+   (api.macros/ns-handler *ns* +auth)))

enterprise/backend/src/metabase_enterprise/python_transform/core.clj

@@ -0,0 +1,5 @@
+(ns metabase-enterprise.python-transform.core
+  (:require
+   [metabase-enterprise.python-transform.models.python-library]))
+
+(comment metabase-enterprise.python-transform.models.python-library/keep-me)

enterprise/backend/src/metabase_enterprise/python_transform/models/python_library.clj

@@ -0,0 +1,41 @@
+(ns metabase-enterprise.python-transform.models.python-library
+  (:require
+   [metabase.app-db.core :as app-db]
+   [metabase.util.i18n :refer [tru]]
+   [methodical.core :as methodical]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(methodical/defmethod t2/table-name :model/PythonLibrary [_model] :python_library)
+
+(doseq [trait [:metabase/model :hook/timestamped?]]
+  (derive :model/PythonLibrary trait))
+
+(def ^:private allowed-paths
+  "Set of allowed library paths. Currently only 'common' is supported."
+  #{"common"})
+
+(defn- validate-path!
+  "Validates that the given path is allowed. Throws an exception if not."
+  [path]
+  (when-not (contains? allowed-paths path)
+    (throw (ex-info (tru "Invalid library path. Only ''common'' is currently supported.")
+                    {:status-code 400
+                     :path path
+                     :allowed-paths allowed-paths}))))
+
+(defn get-python-library-by-path
+  "Get the Python library by path."
+  [path]
+  (validate-path! path)
+  (t2/select-one :model/PythonLibrary :path path))
+
+(defn update-python-library-source!
+  "Update the Python library source code. Creates a new record if none exists. Returns the updated library."
+  [path source]
+  (validate-path! path)
+  (let [id (app-db/update-or-insert! :model/PythonLibrary
+                                     {:path path}
+                                     (constantly {:path path :source source}))]
+    (t2/select-one :model/PythonLibrary id)))

enterprise/backend/src/metabase_enterprise/semantic_search/core.clj

@@ -12,7 +12,8 @@
    [metabase.analytics.core :as analytics]
    [metabase.premium-features.core :refer [defenterprise]]
    [metabase.search.engine :as search.engine]
-   [metabase.util.log :as log]))
+   [metabase.util.log :as log]
+   [toucan2.realize :as t2.realize]))
 
 (defn- fallback-engine
   "Find the highest priority search engine available for fallback."
@@ -58,7 +59,12 @@
           (analytics/inc! :metabase-search/semantic-fallback-triggered {:fallback-engine fallback})
           (analytics/observe! :metabase-search/semantic-results-before-fallback final-count)
           (let [fallback-results (try
-                                   (search.engine/results (assoc search-ctx :search-engine fallback))
+                                   (cond->> (search.engine/results (assoc search-ctx :search-engine fallback))
+                                     ;; The in-place engine returns a reducible (but not seqable) result that needs to
+                                     ;; be realized before we concat and dedup with the semantic engine results.
+                                     (= :search.engine/in-place fallback)
+                                     (into [] (comp (map t2.realize/realize)
+                                                    (take (- (semantic.settings/semantic-search-results-limit) final-count)))))
                                    (catch Throwable t
                                      (log/warn t "Semantic search fallback errored, ignoring")
                                      []))

enterprise/backend/src/metabase_enterprise/semantic_search/index.clj

@@ -127,7 +127,7 @@
    :creator_id          creator_id
    :database_id         database_id
    :last_editor_id      last_editor_id
-   :name                (:name doc)
+   :name                (or (:name doc) "")
    :content             searchable_text
    :display_type        display_type
    :archived            (some-> archived to-boolean)
@@ -145,18 +145,18 @@
    :text_search_vector  (if (:name doc)
                           [:||
                            (search/weighted-tsvector "A" (:name doc))
-                           (search/weighted-tsvector "B" (:searchable_text doc ""))]
-                          (search/weighted-tsvector "A" (:searchable_text doc "")))
+                           (search/weighted-tsvector "B" (or (:searchable_text doc) ""))]
+                          (search/weighted-tsvector "A" (or (:searchable_text doc) "")))
    :text_search_with_native_query_vector
    (if (:name doc)
      [:||
       (search/weighted-tsvector "A" (:name doc))
       (search/weighted-tsvector "B"
-                                (str/join " " (remove str/blank? [(:searchable_text doc "")
-                                                                  (:native_query doc "")])))]
+                                (str/join " " (remove str/blank? [(or (:searchable_text doc) "")
+                                                                  (or (:native_query doc) "")])))]
      (search/weighted-tsvector "A"
-                               (str/join " " (remove str/blank? [(:searchable_text doc "")
-                                                                 (:native_query doc "")]))))})
+                               (str/join " " (remove str/blank? [(or (:searchable_text doc) "")
+                                                                 (or (:native_query doc) "")]))))})
 
 (defn index-size
   "Fetches the number of documents in the index table."

enterprise/backend/src/metabase_enterprise/semantic_search/repair.clj

@@ -13,7 +13,7 @@
    [medley.core :as m]
    [metabase.util :as u]
    [metabase.util.log :as log]
-   [nano-id.core :as nano-id]
+   [metabase.util.string :as u.str]
    [next.jdbc :as jdbc]
    [next.jdbc.result-set :as jdbc.rs]))
 
@@ -93,7 +93,7 @@
   Format: repair_<millis-since-epoch>_<short-id>"
   []
   (let [millis-since-epoch (t/to-millis-from-epoch (t/instant))
-        short-id           (u/lower-case-en (subs (nano-id/nano-id) 0 6))]
+        short-id           (u/lower-case-en (u.str/random-string 6))]
     (format "repair_%d_%s" millis-since-epoch short-id)))
 
 (defn with-repair-table!

enterprise/backend/src/metabase_enterprise/transforms/api.clj

@@ -1,9 +1,10 @@
 (ns metabase-enterprise.transforms.api
   (:require
    [clojure.core.async :as a]
+   [clojure.string :as str]
    [metabase-enterprise.transforms.api.transform-job]
    [metabase-enterprise.transforms.api.transform-tag]
-   [metabase-enterprise.transforms.canceling :as trasnforms.canceling]
+   [metabase-enterprise.transforms.canceling :as transforms.canceling]
    [metabase-enterprise.transforms.execute :as transforms.execute]
    [metabase-enterprise.transforms.models.transform :as transform.model]
    [metabase-enterprise.transforms.models.transform-run :as transform-run]
@@ -45,7 +46,7 @@
 
 (mr/def ::transform-target
   [:map
-   [:database :int]
+   [:database {:optional true} :int]
    [:type [:enum "table"]]
    [:schema {:optional true} [:or ms/NonBlankString :nil]]
    [:name :string]])
@@ -85,12 +86,6 @@
       (api/check-400 (driver.u/supports? (:engine database) feature database)
                      (deferred-tru "The database does not support the requested transform target type.")))))
 
-(defn- localize-timestamps
-  [run]
-  (-> run
-      (u/update-some :start_time transforms.util/local-timestamp-string)
-      (u/update-some :end_time   transforms.util/local-timestamp-string)))
-
 (api.macros/defendpoint :get "/"
   "Get a list of transforms."
   [_route-params
@@ -98,7 +93,7 @@
   (api/check-superuser)
   (-> (t2/select :model/Transform)
       (t2/hydrate :last_run :transform_tag_ids)
-      (->> (map #(update % :last_run localize-timestamps)))))
+      (->> (map #(update % :last_run transforms.util/localize-run-timestamps)))))
 
 (api.macros/defendpoint :post "/"
   "Create a new transform."
@@ -137,7 +132,7 @@
         target-table (transforms.util/target-table database-id target :active true)]
     (-> transform
         (t2/hydrate :last_run :transform_tag_ids)
-        (u/update-some :last_run localize-timestamps)
+        (u/update-some :last_run transforms.util/localize-run-timestamps)
         (assoc :table target-table))))
 
 (api.macros/defendpoint :get "/:id/dependencies"
@@ -170,7 +165,7 @@
   (-> (transform-run/paged-runs (assoc query-params
                                        :offset (request/offset)
                                        :limit  (request/limit)))
-      (update :data #(map localize-timestamps %))))
+      (update :data #(map transforms.util/localize-run-timestamps %))))
 
 (api.macros/defendpoint :put "/:id"
   "Update a transform."
@@ -235,7 +230,7 @@
         run (api/check-404 (transform-run/running-run-for-transform-id id))]
     (transform-run-cancelation/mark-cancel-started-run! (:id run))
     (when (transforms.util/python-transform? transform)
-      (trasnforms.canceling/cancel-run! (:id run))))
+      (transforms.canceling/cancel-run! (:id run))))
   nil)
 
 (api.macros/defendpoint :post "/:id/run"
@@ -279,25 +274,26 @@
   (api/check-superuser)
   (let [run-id python-test-run-id
         cancel-chan (a/promise-chan)]
-    (trasnforms.canceling/chan-start-run! run-id cancel-chan)
+    (transforms.canceling/chan-start-run! run-id cancel-chan)
     (try
-      (let [{:keys [body status]} (transforms.execute/call-python-runner-api! (:code body) (:tables body) run-id cancel-chan)]
+      (let [{:keys [response output events]} (transforms.execute/test-python-transform! (:code body) (:tables body) run-id cancel-chan)
+            {:keys [body status]} response]
         (if (= status 200)
           (do
             (log/info "Python test execution succeeded")
             (-> (response/response {:message (deferred-tru "Python code executed successfully")
-                                    :result  {:body body}})
+                                    :result  {:body (assoc body :output output)}})
                 (assoc :status 200)))
           (do
             (log/error "Error executing Python test code")
             (-> (response/response {:message   (deferred-tru "Python code execution failed")
-                                    :error     (:error body)
-                                    :stdout    (:stdout body)
-                                    :stderr    (:stderr body)
-                                    :exit_code (:exit-code body)})
+                                    :error     body
+                                    :stdout    (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
+                                    :stderr    (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))
+                                    :exit_code (:exit_code (:exit_code body))})
                 (assoc :status status)))))
       (finally
-        (trasnforms.canceling/chan-end-run! run-id)))))
+        (transforms.canceling/chan-end-run! run-id)))))
 
 (api.macros/defendpoint :post "/test-python/cancel"
   "Cancel the current test-python execution."
@@ -305,7 +301,7 @@
    _query-params]
   (log/info "canceling test python execution")
   (api/check-superuser)
-  (if (trasnforms.canceling/chan-signal-cancel! python-test-run-id)
+  (if (transforms.canceling/chan-signal-cancel! python-test-run-id)
     (-> (response/response {:message (deferred-tru "Python test canceled")})
         (assoc :status 200))
     (-> (response/response {:message (deferred-tru "No running Python test to cancel")})

enterprise/backend/src/metabase_enterprise/transforms/api/transform_job.clj

@@ -4,9 +4,12 @@
    [metabase-enterprise.transforms.jobs :as transforms.jobs]
    [metabase-enterprise.transforms.models.transform-job :as transform-job]
    [metabase-enterprise.transforms.schedule :as transforms.schedule]
+   [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.api.common :as api]
    [metabase.api.macros :as api.macros]
    [metabase.api.routes.common :refer [+auth]]
+   [metabase.driver.common.parameters.dates :as params.dates]
+   [metabase.util.date-2 :as u.date]
    [metabase.util.i18n :refer [deferred-tru]]
    [metabase.util.jvm :as u.jvm]
    [metabase.util.log :as log]
@@ -124,6 +127,29 @@
   (let [job (api/check-404 (t2/select-one :model/TransformJob :id job-id))]
     (t2/hydrate job :tag_ids :last_run)))
 
+(defn- add-next-run
+  [{id :id :as job}]
+  (if-let [start-time (-> id transforms.schedule/existing-trigger :next-fire-time)]
+    (assoc job :next_run {:start_time (str (transforms.util/->instant start-time))})
+    job))
+
+(defn- matching-timestamp?
+  [job field-path {:keys [start end]}]
+  (when-let [field-instant (transforms.util/->instant (get-in job field-path))]
+    (let [start-instant (some-> start u.date/parse transforms.util/->instant)
+          end-instant (some-> end u.date/parse transforms.util/->instant)]
+      (and (or (nil? start)
+               (not (.isBefore field-instant start-instant)))
+           (or (nil? end)
+               (.isAfter end-instant field-instant))))))
+
+(defn- ->date-field-filter-xf
+  [field-path filter-value]
+  (let [range (some-> filter-value (params.dates/date-string->range {:inclusive-end? false}))]
+    (if range
+      (filter #(matching-timestamp? % field-path range))
+      identity)))
+
 (api.macros/defendpoint :get "/:job-id/transforms"
   "Get the transforms of job specified by the job's ID."
   [{:keys [job-id]} :- [:map
@@ -136,11 +162,25 @@
 (api.macros/defendpoint :get "/"
   "Get all transform jobs."
   [_route-params
-   _query-params]
+   {:keys [last_run_start_time next_run_start_time transform_tag_ids]} :-
+   [:map
+    [:last_run_start_time {:optional true} [:maybe ms/NonBlankString]]
+    [:next_run_start_time {:optional true} [:maybe ms/NonBlankString]]
+    [:transform_tag_ids   {:optional true} [:maybe (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]]]]
   (log/info "Getting all transform jobs")
   (api/check-superuser)
-  (let [jobs (t2/select :model/TransformJob {:order-by [[:created_at :desc]]})]
-    (t2/hydrate jobs :tag_ids :last_run)))
+  (let [jobs (t2/select :model/TransformJob {:order-by [[:created_at :desc]]})
+        transform-tag-ids (-> transform_tag_ids set not-empty)]
+    (into []
+          (comp (map add-next-run)
+                (->date-field-filter-xf [:last_run :start_time] last_run_start_time)
+                (->date-field-filter-xf [:next_run :start_time] next_run_start_time)
+                (if transform-tag-ids
+                  (filter #(some transform-tag-ids (:tag_ids %)))
+                  identity)
+                (map #(update % :last_run transforms.util/localize-run-timestamps))
+                (map #(update % :next_run transforms.util/localize-run-timestamps)))
+          (t2/hydrate jobs :tag_ids :last_run))))
 
 (def ^{:arglists '([request respond raise])} routes
   "`/api/ee/transform-job` routes."

enterprise/backend/src/metabase_enterprise/transforms/execute.clj

@@ -14,13 +14,11 @@
    [metabase.lib.schema.common :as schema.common]
    [metabase.query-processor.pipeline :as qp.pipeline]
    [metabase.util :as u]
-   [metabase.util.json :as json]
    [metabase.util.jvm :as u.jvm]
    [metabase.util.log :as log]
    [metabase.util.malli.registry :as mr]
    [toucan2.core :as t2])
   (:import
-   (clojure.lang IDeref)
    (java.io Closeable File)
    (java.time Duration)))
 
@@ -29,7 +27,7 @@
 (mr/def ::transform-details
   [:map
    [:transform-type [:enum {:decode/normalize schema.common/normalize-keyword} :table]]
-   [:connection-details :any]
+   [:conn-spec :any]
    [:query :string]
    [:output-table [:keyword {:decode/normalize schema.common/normalize-keyword}]]])
 
@@ -37,24 +35,6 @@
   [:map
    [:overwrite? :boolean]])
 
-(defmacro with-transform-lifecycle
-  "Macro to manage transform run lifecycle. Starts a run, executes body,
-   and handles success/failure appropriately.
-
-   Usage:
-   (with-transform-lifecycle [run-id transform-id opts]
-     ;; body that returns result on success or throws on failure
-     )"
-  [[run-id-sym [transform-id opts]] & body]
-  `(let [{~run-id-sym :id} (transform-run/start-run! ~transform-id ~opts)]
-     (try
-       (let [result# (do ~@body)]
-         (transform-run/succeed-started-run! ~run-id-sym)
-         result#)
-       (catch Throwable t#
-         (transform-run/fail-started-run! ~run-id-sym {:message (.getMessage t#)})
-         (throw t#)))))
-
 (defn- sync-target!
   ([transform-id run-id]
    (let [{:keys [source target]} (t2/select-one :model/Transform transform-id)
@@ -68,9 +48,12 @@
 
 (defn run-cancelable-transform!
   "Run a compiled transform"
-  [run-id run-transform!]
+  [run-id driver {:keys [db conn-spec output-schema]} run-transform!]
   ;; local run is responsible for status, using canceling lifecycle
   (try
+    (when (driver.u/supports? driver :schemas db)
+      (when-not (driver/schema-exists? driver (:id db) output-schema)
+        (driver/create-schema-if-needed! driver conn-spec output-schema)))
     (canceling/chan-start-timeout-vthread! run-id (transforms.settings/transform-timeout))
     (let [cancel-chan (a/promise-chan)
           ret (binding [qp.pipeline/*canceled-chan* cancel-chan]
@@ -107,9 +90,11 @@
      (let [db (get-in source [:query :database])
            {driver :engine :as database} (t2/select-one :model/Database db)
            feature (transforms.util/required-database-feature transform)
-           transform-details {:transform-type (keyword (:type target))
-                              :connection-details (driver/connection-details driver database)
+           transform-details {:db db
+                              :transform-type (keyword (:type target))
+                              :conn-spec (driver/connection-spec driver database)
                               :query (transforms.util/compile-source source)
+                              :output-schema (:schema target)
                               :output-table (transforms.util/qualified-table-name driver target)}
            opts {:overwrite? true}]
        (when-not (driver.u/supports? driver feature database)
@@ -120,7 +105,7 @@
          (when start-promise
            (deliver start-promise [:started run-id]))
          (log/info "Executing transform" id "with target" (pr-str target))
-         (run-cancelable-transform! run-id (fn [_cancel-chan] (driver/run-transform! driver transform-details opts)))
+         (run-cancelable-transform! run-id driver transform-details (fn [_cancel-chan] (driver/run-transform! driver transform-details opts)))
          (transforms.instrumentation/with-stage-timing [run-id :sync :table-sync]
            (sync-target! target database run-id))))
      (catch Throwable t
@@ -182,33 +167,16 @@
       (catch Throwable e
         (log/errorf e "An exception was caught during msg update loop, run-id: %s" run-id)))))
 
-(defn- open-log-thread! ^Closeable [run-id message-log]
-  (let [log-thread-promise (promise)
-        cleanup (fn []
-                  (let [^Thread log-thread (deref log-thread-promise 1000 nil)]
-                    (when-not log-thread (log/fatalf "Log thread reference not bound, run-id: %s" run-id))
-                    (when (.isAlive log-thread)
-                      (.interrupt log-thread)
-                      (when-not (.join log-thread (Duration/ofSeconds 10))
-                        (log/fatalf "Log thread could not be interrupted, run-id: %s" run-id)))))]
-    (u.jvm/in-virtual-thread*
-     (deliver log-thread-promise (Thread/currentThread))
-     (log-loop! run-id message-log))
-    (when-not (deref log-thread-promise 1000 nil) (log/fatalf "Log thread reference not bound, run-id: %s" run-id))
-    (reify
-      IDeref
-      (deref [_] @log-thread-promise)
-      Closeable
-      (close [_] (cleanup)))))
-
-(defn call-python-runner-api!
-  "Call the Python runner API endpoint to execute Python code.
-   Returns the result map or throws on error."
-  [code table-name->id run-id cancel-chan]
-  ;; TODO probably don't need this hack anymore, double check
-  (transforms.instrumentation/with-python-api-timing [run-id]
-    (update (python-runner/execute-python-code run-id code table-name->id cancel-chan)
-            :body #(if (string? %) json/decode+kw %))))
+(defn- open-log-future! ^Closeable [run-id message-log]
+  (let [cleanup (fn [fut]
+                  (future-cancel fut)
+                  (if (= ::timeout (try (deref fut 10000 ::timeout) (catch Throwable _)))
+                    (log/fatalf "Log polling task did not respond to interrupt, run-id: %s" run-id)
+                    (log/debugf "Log polling task done, run-id: %s")))
+        fut     (u.jvm/in-virtual-thread*
+                 (log-loop! run-id message-log))]
+    (reify Closeable
+      (close [_] (cleanup fut)))))
 
 (defn- debug-info-str [{:keys [exit-code events]}]
   (str/join "\n"
@@ -222,31 +190,78 @@
              "======"
              (format "exit code %d" exit-code)]))
 
-(defn- transfer-file-to-db [driver db {:keys [target] :as transform} metadata temp-file]
-  (let [table-name (transforms.util/qualified-table-name driver target)
-        table-schema {:name table-name
-                      :columns (mapv (fn [{:keys [name dtype]}]
+(defn- create-table-and-insert-data!
+  "Create a table from metadata and insert data from source."
+  [driver db-id table-name metadata data-source]
+  (let [table-schema {:name (if (keyword? table-name) table-name (keyword table-name))
+                      :columns (mapv (fn [{:keys [name dtype base_type database_type]}]
                                        {:name name
-                                        :type (transforms.util/dtype->base-type dtype)
+                                        :type (or (some->> base_type (keyword "type"))
+                                                  (transforms.util/dtype->base-type dtype))
+                                        :database-type database_type
                                         :nullable? true})
                                      (:fields metadata))}
+        data-source (assoc data-source :table-schema table-schema)]
+    (transforms.util/create-table-from-schema! driver db-id table-schema)
+    (driver/insert-from-source! driver db-id table-name (mapv :name (:columns table-schema)) data-source)))
+
+(defn- transfer-file-to-db [driver db {:keys [target] :as transform} metadata temp-file]
+  (let [table-name (transforms.util/qualified-table-name driver target)
+        table-exists? (transforms.util/target-table-exists? transform)
         data-source {:type :csv-file
                      :file temp-file}]
-    ;; TODO: should be transactional, perharps go through driver/run-transform!
-    (transforms.util/delete-target-table! transform)
-    (transforms.util/create-table-from-schema! driver (:id db) table-schema)
-    (driver/insert-from-source! driver (:id db) table-name (mapv :name (:columns table-schema)) data-source)))
+    (if table-exists?
+      ;; Table exists - use temp table + atomic swap pattern
+      (let [temp-table-name (keyword (str (u/qualified-name table-name) "_temp_" (System/currentTimeMillis)))]
+        (log/info "Existing table deletected, Create then swap")
+        (try
+          (create-table-and-insert-data! driver (:id db) temp-table-name metadata data-source)
+          ;; TODO: These operations should be within a transaction for atomicity
+          (transforms.util/drop-table! driver (:id db) table-name)
+          (transforms.util/rename-table! driver (:id db) temp-table-name table-name)
+          (catch Exception e
+            (log/error e "Failed to transfer data to table")
+            (try
+              (transforms.util/drop-table! driver (:id db) temp-table-name)
+              (catch Exception _))
+            (throw e))))
+      ;; Table doesn't exist - create directly with target name
+      (do
+        (log/info "New table")
+        (create-table-and-insert-data! driver (:id db) table-name metadata data-source)))))
+
+(defn test-python-transform!
+  "Execute a transform in test mode (does not write result into a table)."
+  [code tables->id run-id cancel-chan]
+  (with-open [shared-storage-ref (python-runner/open-s3-shared-storage! tables->id)]
+    (let [server-url              (transforms.settings/python-execution-server-url)
+          _                       (python-runner/copy-tables-to-s3! {:run-id         run-id
+                                                                     :shared-storage @shared-storage-ref
+                                                                     :table-name->id tables->id
+                                                                     :cancel-chan    cancel-chan})
+          _                       (python-runner/open-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan
+          response
+          (python-runner/execute-python-code-http-call!
+           {:server-url     server-url
+            :code           code
+            :run-id         run-id
+            :table-name->id tables->id
+            :shared-storage @shared-storage-ref})
+          {:keys [output events]} (python-runner/read-output-objects @shared-storage-ref)]
+      {:response response
+       :events   events
+       :output   output})))
 
 (defn- run-python-transform! [{:keys [source] :as transform} db run-id cancel-chan message-log]
-  (with-open [log-thread-ref     (open-log-thread! run-id message-log)
+  (with-open [log-thread-ref     (open-log-future! run-id message-log)
               shared-storage-ref (python-runner/open-s3-shared-storage! (:source-tables source))]
-    (let [driver                        (:engine db)
-          server-url                    (transforms.settings/python-execution-server-url)
-          _                             (python-runner/copy-tables-to-s3! {:run-id                  run-id
-                                                                           :shared-storage @shared-storage-ref
-                                                                           :table-name->id (:source-tables source)
-                                                                           :cancel-chan    cancel-chan})
-          _                             (python-runner/open-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan
+    (let [driver     (:engine db)
+          server-url (transforms.settings/python-execution-server-url)
+          _          (python-runner/copy-tables-to-s3! {:run-id                  run-id
+                                                        :shared-storage @shared-storage-ref
+                                                        :table-name->id (:source-tables source)
+                                                        :cancel-chan    cancel-chan})
+          _          (python-runner/open-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan
           {:keys [status body] :as response}
           (python-runner/execute-python-code-http-call!
            {:server-url     server-url
@@ -302,13 +317,19 @@
                                                :python      nil
                                                :post-python []}))
             {:keys [target] transform-id :id} transform
-            db (t2/select-one :model/Database (:database target))
+            {driver :engine :as db} (t2/select-one :model/Database (:database target))
             {run-id :id} (try-start-unless-already-running transform-id run-method)]
         (some-> start-promise (deliver [:started run-id]))
         (log! message-log "Executing Python transform")
         (log/info "Executing Python transform" transform-id "with target" (pr-str target))
         (let [start-ms (u/start-timer)
-              result   (run-cancelable-transform! run-id (fn [cancel-chan] (run-python-transform! transform db run-id cancel-chan message-log)))]
+              transform-details
+              {:db db
+               :transform-type (keyword (:type target))
+               :conn-spec (driver/connection-spec driver db)
+               :output-schema (:schema target)
+               :output-table (transforms.util/qualified-table-name driver target)}
+              result   (run-cancelable-transform! run-id driver transform-details (fn [cancel-chan] (run-python-transform! transform db run-id cancel-chan message-log)))]
           (transforms.instrumentation/with-stage-timing [run-id :sync :table-sync]
             (sync-target! target db run-id))
           (log! message-log (format "Python execution finished in %s" (Duration/ofMillis (u/since-ms start-ms))))

enterprise/backend/src/metabase_enterprise/transforms/ordering.clj

@@ -16,23 +16,29 @@
 (set! *warn-on-reflection* true)
 
 (defn- transform-deps [transform]
-  (let [query (-> (get-in transform [:source :query])
-                  transforms.util/massage-sql-query
-                  qp.preprocess/preprocess
-                  ;; legacy usage -- don't do things like this going forward
-                  #_{:clj-kondo/ignore [:discouraged-var]}
-                  lib/->legacy-MBQL)]
-    (case (:type query)
-      :native (driver/native-query-deps (-> (qp.store/metadata-provider)
-                                            lib.metadata/database
-                                            :engine)
-                                        (get-in query [:native :query]))
-      :query (into #{}
-                   (keep #(clojure.core.match/match %
-                            [:source-table source] (when (int? source)
-                                                     source)
-                            _ nil))
-                   (tree-seq coll? seq query)))))
+  (if (transforms.util/python-transform? transform)
+    ;; Python transforms have dependencies via source-tables mapping
+    (into #{}
+          (map #(hash-map :table %))
+          (vals (get-in transform [:source :source-tables])))
+    ;; Query transforms have dependencies via SQL query analysis
+    (let [query (-> (get-in transform [:source :query])
+                    transforms.util/massage-sql-query
+                    qp.preprocess/preprocess
+                    ;; legacy usage -- don't do things like this going forward
+                    #_{:clj-kondo/ignore [:discouraged-var]}
+                    lib/->legacy-MBQL)]
+      (case (:type query)
+        :native (driver/native-query-deps (-> (qp.store/metadata-provider)
+                                              lib.metadata/database
+                                              :engine)
+                                          (get-in query [:native :query]))
+        :query (into #{}
+                     (keep #(clojure.core.match/match %
+                              [:source-table source] (when (int? source)
+                                                       {:table source})
+                              _ nil))
+                     (tree-seq coll? seq query))))))
 
 (defn- dependency-map [transforms]
   (into {}
@@ -56,28 +62,32 @@
 
   The result is a map of transform id -> #{transform ids the transform depends on}. Dependencies are limited to just
   the transforms in the original list -- if a transform depends on some transform not in the list, the 'extra'
-  dependency is ignored. Python transforms have no dependencies and run sequentially."
+  dependency is ignored. Both query and Python transforms can have dependencies on tables produced by other transforms."
   [transforms]
-  (let [{python-transforms true
-         sql-transforms false} (group-by transforms.util/python-transform? transforms)
 
-        ;; TODO: figure out python transforms deps
-        python-deps (into {} (map (fn [transform] [(:id transform) #{}])) python-transforms)
+  (let [;; Group all transforms by their database
+        transforms-by-db (->> transforms
+                              (map (fn [transform]
+                                     (let [db-id (if (transforms.util/python-transform? transform)
+                                                   (get-in transform [:target :database])
+                                                   (get-in transform [:source :query :database]))]
+                                       {db-id [transform]})))
+                              (apply merge-with into))
 
-        sql-transforms-by-db (->> sql-transforms
-                                  (map (fn [transform]
-                                         {(get-in transform [:source :query :database]) [transform]}))
-                                  (apply merge-with into))
-
-        {:keys [output-tables dependencies]} (->> sql-transforms-by-db
+        transform-ids (into #{} (map :id) transforms)
+        {:keys [output-tables dependencies]} (->> transforms-by-db
                                                   (map (fn [[db-id db-transforms]]
                                                          (qp.store/with-metadata-provider db-id
                                                            {:output-tables (output-table-map db-transforms)
                                                             :dependencies (dependency-map db-transforms)})))
                                                   (apply merge-with merge))
 
-        sql-deps (update-vals dependencies #(into #{} (keep output-tables) %))]
-    (merge python-deps sql-deps)))
+        all-deps (update-vals dependencies #(into #{}
+                                                  (keep (fn [{:keys [table transform]}]
+                                                          (or (output-tables table)
+                                                              (transform-ids transform))))
+                                                  %))]
+    all-deps))
 
 (defn find-cycle
   "Finds a path containing a cycle in the directed graph `node->children`.
@@ -123,9 +133,12 @@
                                transforms)
         db-id (get-in to-check [:source :query :database])]
     (qp.store/with-metadata-provider db-id
-      (let [output-tables (output-table-map (filter #(= (get-in % [:source :query :database]) db-id)
-                                                    transforms))
-            node->children #(->> % transforms-by-id transform-deps (keep output-tables))
+      (let [db-transforms (filter #(= (get-in % [:source :query :database]) db-id) transforms)
+            output-tables (output-table-map db-transforms)
+            transform-ids (into #{} (map :id) db-transforms)
+            node->children #(->> % transforms-by-id transform-deps (keep (fn [{:keys [table transform]}]
+                                                                           (or (output-tables table)
+                                                                               (transform-ids transform)))))
             id->name (comp :name transforms-by-id)
             cycle (find-cycle node->children [transform-id])]
         (when cycle

enterprise/backend/src/metabase_enterprise/transforms/python_runner.clj

@@ -4,10 +4,12 @@
    [clojure.core.async :as a]
    [clojure.java.io :as io]
    [clojure.string :as str]
+   [medley.core :as m]
    [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
    [metabase-enterprise.transforms.settings :as transforms.settings]
    [metabase.driver :as driver]
    [metabase.query-processor.compile :as qp.compile]
+   [metabase.query-processor.pipeline :as qp.pipeline]
    ^{:clj-kondo/ignore [:discouraged-namespace]}
    [metabase.query-processor.store :as qp.store]
    [metabase.util.json :as json]
@@ -19,7 +21,6 @@
    (java.net URI)
    (java.nio.charset StandardCharsets)
    (java.time Duration)
-   (java.util.concurrent CancellationException)
    (software.amazon.awssdk.auth.credentials AwsBasicCredentials StaticCredentialsProvider)
    (software.amazon.awssdk.auth.credentials DefaultCredentialsProvider)
    (software.amazon.awssdk.core.sync RequestBody)
@@ -60,7 +61,8 @@
                   :type     :native
                   :native   native}]
       (qp.store/with-metadata-provider db-id
-        (driver/execute-reducible-query driver query {:canceled-chan cancel-chan} respond)))))
+        (binding [qp.pipeline/*canceled-chan* cancel-chan]
+          (driver/execute-reducible-query driver query {:canceled-chan cancel-chan} respond))))))
 
 (defn- root-type
   [base-type]
@@ -71,6 +73,8 @@
            :type/Boolean
            :type/DateTimeWithTZ
            :type/DateTime
+           :type/Dictionary
+           :type/JSON
            :type/Date
            :type/Text])))
 
@@ -100,23 +104,42 @@
                             :semantic_type  (some-> (effective-semantic-type-i-think col-meta) name)
                             :effective_type (some-> (or (:effective_type col-meta) (:database_type col-meta)) name)
                             ;; TODO get this passed through
-                            #_#_:field_id   (:id col-meta)})
+                            :field_id       (:id col-meta)})
                          cols-meta)
    :table_metadata {:table_id table-id}})
 
+(defn- maybe-fixup-v [col v]
+  ;; clickhouse returns bigdecimals for int64 values
+  (if (and (isa? (:base_type col) :type/Integer)
+           (or (instance? BigDecimal v)
+               (float? v)))
+    (bigint v)
+    v))
+
 (defn- write-table-data-to-file! [id temp-file cancel-chan]
-  (let [db-id         (t2/select-one-fn :db_id (t2/table-name :model/Table) :id id)
-        driver        (t2/select-one-fn :engine :model/Database db-id)
-        ;; TODO: limit
-        query         {:source-table id}
-        manifest-atom (atom nil)]
+  (let [db-id           (t2/select-one-fn :db_id (t2/table-name :model/Table) :id id)
+        driver          (t2/select-one-fn :engine :model/Database db-id)
+        all-fields-meta (t2/select [:model/Field :id :name :base_type :effective_type :semantic_type :database_type :database_position :nfc_path :parent_id]
+                                   :table_id id
+                                   :active true
+                                   {:order-by [[:database_position :asc]]})
+        fields-meta (filter #(and (nil? (:parent_id %)) (nil? (:nfc_path %))) all-fields-meta)
+        query {:source-table id}
+        manifest (generate-manifest id fields-meta)]
     (execute-mbql-query driver db-id query
                         (fn [{cols-meta :cols} reducible-rows]
-                          (reset! manifest-atom (generate-manifest id cols-meta))
                           (with-open [os (io/output-stream temp-file)]
-                            (write-to-stream! os (mapv :name cols-meta) reducible-rows)))
+                            (let [filtered-col-meta (m/index-by :name fields-meta)
+                                  filtered-cols (filter #(contains? filtered-col-meta (:name %)) cols-meta)
+                                  col-name->index (into {} (map-indexed (fn [i col] [(:name col) i]) cols-meta))
+                                  filtered-indices (mapv (fn [col] [(col-name->index (:name col)) col]) filtered-cols)
+                                  filtered-rows (eduction (map (fn [row] (mapv (fn [[idx col]]
+                                                                                 (let [v (nth row idx)]
+                                                                                   (maybe-fixup-v col v))) filtered-indices)))
+                                                          reducible-rows)]
+                              (write-to-stream! os (mapv :name filtered-cols) filtered-rows))))
                         cancel-chan)
-    @manifest-atom))
+    manifest))
 
 (defn- maybe-with-endpoint-s3-client [^S3ClientBuilder builder endpoint]
   (let [region (transforms.settings/python-storage-s-3-region)]
@@ -256,8 +279,10 @@
   (try
     (let [^DeleteObjectRequest request (build-delete-object-request bucket-name key)]
       (.deleteObject s3-client request))
-    (catch Exception _
-      ;; Ignore deletion errors - object might not exist or we might not have permissions
+    (catch Exception e
+      (log/debugf e "Error deleting s3 object %s" key)
+      ;; Ignore deletion errors - object might not exist, or we might not have permissions
+      ;; NOTE: we plan to put general retention on the bucket so that objects will eventually be deleted
       nil)))
 
 (defn- cleanup-s3-objects [^S3Client s3-client bucket-name s3-keys]
@@ -293,6 +318,9 @@
         bucket-name         (transforms.settings/python-storage-s-3-bucket)
         loc
         (fn [method relative-path]
+          ;; save a tracking root to a well known location
+          ;; later can ls these to collect those objects whose ttl has elapsed
+          ;; (OR setup a retention policy in s3?)
           (let [path (str work-dir-name "/" relative-path)]
             {:path   path
              :method method
@@ -301,16 +329,17 @@
     {:s3-client   (create-s3-client)                        ;; do not like mixing interactive things with descriptions, but its damn convenient to have it here for now
      :bucket-name bucket-name
      :objects
-     (-> (into
-          {:output          (loc :put "output.csv")
-           :output-manifest (loc :put "output-manifest.json")
-           :events          (loc :put "events.jsonl")}
-          (for [[table-name id] table-name->id]
-            {[:table id :manifest] (loc :get (str "-table-" (name table-name) "-" id ".manifest.json"))
-             [:table id :data]     (loc :get (str "-table-" (name table-name) "-" id ".jsonl"))})))}))
+     (into
+      {:output          (loc :put "output.csv")
+       :output-manifest (loc :put "output-manifest.json")
+       :events          (loc :put "events.jsonl")}
+      (for [[table-name id] table-name->id]
+        {[:table id :manifest] (loc :get (str "-table-" (name table-name) "-" id ".manifest.json"))
+         [:table id :data]     (loc :get (str "-table-" (name table-name) "-" id ".jsonl"))}))}))
 
 (defn open-s3-shared-storage!
-  "Returns a deref'able shared storage value, (.close) will cleanup any s3 objects named in storage (data files for tables and so on)."
+  "Returns a deref'able shared storage value, (.close) will optimistically delete any s3 objects named in storage (data files for tables, metadata files etc).
+  The intention is the bucket specifies a generic object retention policy to ensure objects are eventually deleted (e.g. because the process dies during writing and .close never gets called)"
   ^Closeable [table-name->id]
   (let [shared-storage (s3-shared-storage table-name->id)]
     (reify IDeref
@@ -318,10 +347,13 @@
       Closeable
       (close [_] (cleanup-s3-objects (:s3-client shared-storage) (:bucket-name shared-storage) (map :path (vals (:objects shared-storage))))))))
 
-(defn copy-tables-to-s3! [{:keys [run-id
-                                  shared-storage
-                                  table-name->id
-                                  cancel-chan]}]
+(defn copy-tables-to-s3!
+  "Writes table content to their corresponding objects named in shared-storage, see (open-shared-storage!).
+  Blocks until all tables are fully written and committed to shared storage."
+  [{:keys [run-id
+           shared-storage
+           table-name->id
+           cancel-chan]}]
   (doseq [id (vals table-name->id)
           :let [{:keys [s3-client bucket-name objects]} shared-storage
                 {data-path :path}     (get objects [:table id :data])
@@ -350,7 +382,10 @@
           (safe-delete temp-file)
           (safe-delete manifest-file))))))
 
-(defn execute-python-code-http-call! [{:keys [server-url code run-id table-name->id shared-storage]}]
+(defn execute-python-code-http-call!
+  "Calls the /execute endpoint of the python runner. Blocks until the run either succeeds or fails and returns
+  the response from the server."
+  [{:keys [server-url code run-id table-name->id shared-storage]}]
   (let [{:keys [objects]} shared-storage
         {:keys [output output-manifest events]} objects
 
@@ -358,6 +393,7 @@
         table-name->manifest-url (update-vals table-name->id (comp :url #(get objects [:table % :manifest])))
 
         payload                  {:code                code
+                                  :library             (t2/select-fn->fn :path :source :model/PythonLibrary)
                                   :timeout             30
                                   :request_id          run-id
                                   :output_url          (:url output)
@@ -366,12 +402,13 @@
                                   :table_mapping       table-name->url
                                   :manifest_mapping    table-name->manifest-url}
 
-        response                 (http/post (str server-url "/execute")
-                                            {:content-type     :json
-                                             :accept           :json
-                                             :body             (json/encode payload)
-                                             :throw-exceptions false
-                                             :as               :json})]
+        response                 (transforms.instrumentation/with-python-api-timing [run-id]
+                                   (http/post (str server-url "/execute")
+                                              {:content-type     :json
+                                               :accept           :json
+                                               :body             (json/encode payload)
+                                               :throw-exceptions false
+                                               :as               :json}))]
     ;; when a 500 is returned we observe a string in the body (despite the python returning json)
     ;; always try to parse the returned string as json before yielding (could tighten this up at some point)
     (update response :body (fn [string-if-error]
@@ -393,7 +430,7 @@
                       :async?       true}
                      identity identity))))
 
-;; temporary (just keep the files alive longer and stream them directly (i.e into tmp file for write)
+;; temporary, we should not need to realize data/events files into memory longer term
 (defn read-output-objects
   "Temporary function that strings/jsons stuff in S3 and returns it for compatibility."
   [{:keys [s3-client bucket-name objects]}]

enterprise/backend/src/metabase_enterprise/transforms/schedule.clj

@@ -61,12 +61,17 @@
     (log/info "Creating trigger for transform job" job-id "with schedule" schedule)
     (task/add-trigger! (build-trigger job-id schedule))))
 
+(defn existing-trigger
+  "Return the existing trigger for `job-id-or-trigger`, or nil, if the trigger doesn't exist."
+  [job-id-or-trigger]
+  (first (task/existing-triggers (job-key job-id-or-trigger)
+                                 (trigger-key job-id-or-trigger))))
+
 (defn- delete-trigger!
   "Delete the trigger for a transform job."
   ([job-id-or-trigger]
    (if (number? job-id-or-trigger)
-     (if-let [trigger (first (task/existing-triggers (job-key job-id-or-trigger)
-                                                     (trigger-key job-id-or-trigger)))]
+     (if-let [trigger (existing-trigger job-id-or-trigger)]
        (delete-trigger! trigger)
        (log/info "No trigger for this transform job exists"))
      (do

enterprise/backend/src/metabase_enterprise/transforms/settings.clj

@@ -6,7 +6,7 @@
 
 (set! *warn-on-reflection* true)
 
-(def ^:privatg is-not-prod? (or config/is-dev? config/is-test?))
+(def ^:private is-not-prod? (or config/is-dev? config/is-test?))
 
 (setting/defsetting transform-timeout
   (deferred-tru "The timeout for a transform job, in minutes.")

enterprise/backend/src/metabase_enterprise/transforms/util.clj

@@ -1,17 +1,19 @@
 (ns metabase-enterprise.transforms.util
   (:require
    [clojure.string :as str]
+   [java-time.api :as t]
    [metabase.driver :as driver]
    [metabase.lib.schema.common :as lib.schema.common]
    [metabase.query-processor.compile :as qp.compile]
    [metabase.sync.core :as sync]
+   [metabase.util :as u]
    [metabase.util.date-2 :as u.date]
    [metabase.util.log :as log]
    [metabase.util.malli :as mu]
    [metabase.util.malli.registry :as mr]
    [toucan2.core :as t2])
   (:import
-   (java.time Instant LocalDateTime OffsetDateTime ZonedDateTime ZoneId)
+   (java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)
    (java.util Date)))
 
 (set! *warn-on-reflection* true)
@@ -35,11 +37,12 @@
 
 (defn target-table-exists?
   "Test if the target table of a transform already exists."
-  [{:keys [source target] :as transform}]
-  (when (query-transform? transform)
-    (let [db-id (-> source :query :database)
-          {driver :engine :as database} (t2/select-one :model/Database db-id)]
-      (driver/table-exists? driver database target))))
+  [{:keys [source target] :as _transform}]
+  (let [db-id (or (-> source :query :database)
+                  ;; python transform target
+                  (-> target :database))
+        {driver :engine :as database} (t2/select-one :model/Database db-id)]
+    (driver/table-exists? driver database target)))
 
 (defn target-table
   "Load the `target` table of a transform from the database specified by `database-id`."
@@ -108,23 +111,35 @@
   [transform]
   (case (-> transform :target :type)
     "table"             :transforms/table))
-(defn local-timestamp
-  "Convert the timestamp t to a ZonedDateTime instance in the system timezone."
-  [t]
-  (condp instance? t
-    String         (recur (u.date/parse t))
-    Date           (recur (.toInstant ^Date t))
-    Instant        (ZonedDateTime/ofInstant t (ZoneId/systemDefault))
-    LocalDateTime  (ZonedDateTime/of t (ZoneId/systemDefault))
-    OffsetDateTime (.atZoneSameInstant ^OffsetDateTime t (ZoneId/systemDefault))
-    ZonedDateTime  (.withZoneSameInstant ^ZonedDateTime t (ZoneId/systemDefault))
-    (throw (ex-info (str "Cannot convert timestamp " t " of type " (type t) " to a ZonedDateTime")
-                    {:timestamp t}))))
 
-(defn local-timestamp-string
+(defn ->instant
+  "Convert a temporal value `t` to an Instant in the system timezone."
+  ^Instant [t]
+  (when t
+    (condp instance? t
+      Instant        t
+      Date           (.toInstant ^Date t)
+      OffsetDateTime (.toInstant ^OffsetDateTime t)
+      ZonedDateTime  (.toInstant ^ZonedDateTime t)
+      LocalDateTime  (recur (.atZone ^LocalDateTime t (t/zone-id)))
+      String         (recur (u.date/parse t))
+      LocalTime      (recur (.atDate ^LocalTime t (t/local-date)))
+      OffsetTime     (recur (.atDate ^OffsetTime t (t/local-date)))
+      LocalDate      (recur (.atStartOfDay ^LocalDate t))
+      (throw (ex-info (str "Cannot convert temporal " t " of type " (type t) " to an Instant")
+                      {:temporal t})))))
+
+(defn utc-timestamp-string
   "Convert the timestamp t to a string encoding the it in the system timezone."
   [t]
-  (-> t local-timestamp str))
+  (-> t ->instant str))
+
+(defn localize-run-timestamps
+  "Convert the timestamps of a `run` to ISO strings in UTC."
+  [run]
+  (-> run
+      (u/update-some :start_time utc-timestamp-string)
+      (u/update-some :end_time   utc-timestamp-string)))
 
 (mr/def ::column-definition
   [:map
@@ -158,8 +173,27 @@
    database-id :- pos-int?
    table-schema :- ::table-definition]
   (let [{:keys [columns] table-name :name} table-schema
-        column-definitions (into {} (map (fn [{:keys [name type]}] [name (driver/type->database-type driver type)]))
+        column-definitions (into {} (map (fn [{:keys [name type database-type]}]
+                                           (let [database-type (when-not (and (= driver :bigquery-cloud-sdk)
+                                                                              (#{"ARRAY" "RECORD"} database-type))
+                                                                 database-type)
+                                                 db-type (if database-type
+                                                           [[:raw database-type]]
+                                                           (driver/type->database-type driver type))]
+                                             [name db-type])))
                                  columns)
         primary-key-opts (select-keys table-schema [:primary-key])]
     (log/infof "Creating table %s with %d columns" table-name (count columns))
     (driver/create-table! driver database-id table-name column-definitions primary-key-opts)))
+
+(defn rename-table!
+  "kename a table in the database."
+  [driver database-id old-table-name new-table-name]
+  (log/infof "Renaming table %s to %s" old-table-name new-table-name)
+  (driver/rename-table! driver database-id old-table-name new-table-name))
+
+(defn drop-table!
+  "Drop a table in the database."
+  [driver database-id table-name]
+  (log/infof "Dropping table %s" table-name)
+  (driver/drop-table! driver database-id table-name))

enterprise/backend/test/metabase_enterprise/python_transform/api_test.clj

@@ -0,0 +1,89 @@
+(ns metabase-enterprise.python-transform.api-test
+  "Tests for /api/ee/python-transform endpoints."
+  (:require
+   [clojure.test :refer :all]
+   [metabase-enterprise.python-transform.models.python-library :as python-library]
+   [metabase.test :as mt]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(deftest get-library-path-test
+  (testing "GET /api/ee/python-transform/library/:path"
+    (mt/with-premium-features #{:transforms}
+      (testing "requires superuser permissions"
+        (is (= "You don't have permissions to do that."
+               (mt/user-http-request :rasta :get 403 "ee/python-transform/library/common"))))
+
+      (testing "returns nil when no library exists"
+        (t2/delete! :model/PythonLibrary)
+        (is (nil? (mt/user-http-request :crowberto :get 204 "ee/python-transform/library/common"))))
+
+      (testing "returns existing library"
+        (t2/delete! :model/PythonLibrary)
+        (python-library/update-python-library-source! "common" "def my_function():\n    return 42")
+        (is (=? {:source "def my_function():\n    return 42"
+                 :path "common"
+                 :created_at some?
+                 :updated_at some?}
+                (mt/user-http-request :crowberto :get 200 "ee/python-transform/library/common"))))
+
+      (testing "rejects invalid paths"
+        (is (= "Invalid library path. Only 'common' is currently supported."
+               (:message (mt/user-http-request :crowberto :get 400 "ee/python-transform/library/invalid-path"))))))))
+
+(deftest put-library-path-test
+  (testing "PUT /api/ee/python-transform/library/:path"
+    (mt/with-premium-features #{:transforms}
+      (testing "requires superuser permissions"
+        (is (= "You don't have permissions to do that."
+               (mt/user-http-request :rasta :put 403 "ee/python-transform/library/common"
+                                     {:source "def test(): pass"}))))
+
+      (testing "validates request body"
+        (testing "requires source field"
+          (is (=? {:errors
+                   {:source "string"}}
+                  (mt/user-http-request :crowberto :put 400 "ee/python-transform/library/common"
+                                        {}))))
+
+        (testing "allows empty string"
+          (t2/delete! :model/PythonLibrary)
+          (is (=? {:source ""
+                   :path "common"
+                   :id integer?
+                   :created_at some?
+                   :updated_at some?}
+                  (mt/user-http-request :crowberto :put 200 "ee/python-transform/library/common"
+                                        {:source ""})))))
+
+      (testing "creates new library when none exists"
+        (t2/delete! :model/PythonLibrary)
+        (is (=? {:source "def new_function():\n    return 100"
+                 :path "common"
+                 :id integer?
+                 :created_at some?
+                 :updated_at some?}
+                (mt/user-http-request :crowberto :put 200 "ee/python-transform/library/common"
+                                      {:source "def new_function():\n    return 100"})))
+        (is (= "def new_function():\n    return 100"
+               (t2/select-one-fn :source :model/PythonLibrary))))
+
+      (testing "updates existing library"
+        (t2/delete! :model/PythonLibrary)
+        (python-library/update-python-library-source! "common" "def old_function():\n    return 1")
+        (is (=? {:source "def updated_function():\n    return 2"
+                 :path "common"
+                 :id integer?
+                 :created_at some?
+                 :updated_at some?}
+                (mt/user-http-request :crowberto :put 200 "ee/python-transform/library/common"
+                                      {:source "def updated_function():\n    return 2"})))
+        (is (= "def updated_function():\n    return 2"
+               (t2/select-one-fn :source :model/PythonLibrary)))
+        (is (= 1 (t2/count :model/PythonLibrary)) "Should not create duplicate records"))
+
+      (testing "rejects invalid paths"
+        (is (= "Invalid library path. Only 'common' is currently supported."
+               (:message (mt/user-http-request :crowberto :put 400 "ee/python-transform/library/invalid-path"
+                                               {:source "def test(): pass"}))))))))

enterprise/backend/test/metabase_enterprise/python_transform/models/python_library_test.clj

@@ -0,0 +1,56 @@
+(ns metabase-enterprise.python-transform.models.python-library-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase-enterprise.python-transform.models.python-library :as python-library]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(deftest update-python-library-source-test
+  (testing "update-python-library-source!"
+    (testing "creates new record when none exists"
+      (t2/delete! :model/PythonLibrary)
+      (is (=? {:source "def new_func(): return 1"
+               :path "common"
+               :id integer?
+               :created_at some?
+               :updated_at some?}
+              (python-library/update-python-library-source! "common" "def new_func(): return 1")))
+      (is (= 1 (t2/count :model/PythonLibrary)))
+      (is (= "def new_func(): return 1"
+             (t2/select-one-fn :source :model/PythonLibrary))))
+
+    (testing "updates existing record"
+      (is (= 1 (t2/count :model/PythonLibrary)))
+      (is (=? {:source "def updated_func(): return 2"
+               :path "common"
+               :id integer?
+               :created_at some?
+               :updated_at some?}
+              (python-library/update-python-library-source! "common" "def updated_func(): return 2")))
+      (is (= 1 (t2/count :model/PythonLibrary)) "Should not create duplicate")
+      (is (= "def updated_func(): return 2"
+             (t2/select-one-fn :source :model/PythonLibrary))))
+
+    (testing "rejects invalid paths"
+      (is (thrown-with-msg? clojure.lang.ExceptionInfo
+                            #"Invalid library path"
+                            (python-library/update-python-library-source! "invalid-path" "def test(): pass")))
+      (is (= 1 (t2/count :model/PythonLibrary)) "Should not create library with invalid path"))))
+
+(deftest get-python-library-by-path-test
+  (testing "get-python-library-by-path"
+    (testing "returns library when path is valid"
+      (t2/delete! :model/PythonLibrary)
+      (python-library/update-python-library-source! "common" "def test(): pass")
+      (is (=? {:source "def test(): pass"
+               :path "common"
+               :id integer?
+               :created_at some?
+               :updated_at some?}
+              (python-library/get-python-library-by-path "common"))))
+
+    (testing "rejects invalid paths"
+      (is (thrown-with-msg? clojure.lang.ExceptionInfo
+                            #"Invalid library path"
+                            (python-library/get-python-library-by-path "invalid-path"))))))

enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/row_level_restrictions_test.clj

@@ -186,13 +186,12 @@
                    :query {:source-query {:source-table $$checkins
                                           :fields [$id $date $user_id $venue_id]
                                           :filter [:and
-                                                                          ;; This still gets :default bucketing!
-                                                                          ;; auto-bucket-datetimes puts :day bucketing
-                                                                          ;; on both parts of this filter, since it's
-                                                                          ;; matching a YYYY-mm-dd string. Then
-                                                                          ;; optimize-temporal-filters sees that the
-                                                                          ;; :type/Date column already has :day
-                                                                          ;; granularity, and switches both to :default
+                                                   ;; This still gets :default bucketing! auto-bucket-datetimes
+                                                   ;; puts :day bucketing on both parts of this filter, since it's
+                                                   ;; matching a YYYY-mm-dd string. Then optimize-temporal-filters
+                                                   ;; sees that the
+                                                   ;; :type/Date column already has :day
+                                                   ;; granularity, and switches both to :default
                                                    [:> !default.date [:absolute-datetime
                                                                       #t "2014-01-01"
                                                                       :default]]

enterprise/backend/test/metabase_enterprise/semantic_search/api_test.clj

@@ -3,31 +3,20 @@
    [clojure.core.memoize :as memoize]
    [clojure.test :refer :all]
    [metabase-enterprise.semantic-search.api :as semantic.api]
-   [metabase-enterprise.semantic-search.embedding :as semantic.embedding]
    [metabase-enterprise.semantic-search.env :as semantic.env]
    [metabase-enterprise.semantic-search.index :as semantic.index]
    [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
    [metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]
    [metabase-enterprise.semantic-search.test-util :as semantic.tu]
    [metabase.search.ingestion :as search.ingestion]
-   [metabase.test :as mt]
-   [metabase.test.fixtures :as fixtures]))
+   [metabase.test :as mt]))
 
-(use-fixtures :once (compose-fixtures
-                     (fixtures/initialize :db)
-                     #'semantic.tu/once-fixture))
-
-(defmacro with-mock-index-metadata!
-  [& body]
-  `(with-redefs [semantic.env/get-pgvector-datasource! (constantly semantic.tu/db)
-                 semantic.embedding/get-configured-model (constantly semantic.tu/mock-embedding-model)
-                 semantic.env/get-index-metadata (constantly semantic.tu/mock-index-metadata)]
-     ~@body))
+(use-fixtures :once #'semantic.tu/once-fixture)
 
 (deftest status-endpoint-test
   (testing "GET /api/ee/semantic-search/status"
     (mt/with-premium-features #{:semantic-search}
-      (with-mock-index-metadata!
+      (semantic.tu/with-test-db! {:mode :mock-initialized}
         (with-open [index-ref (semantic.tu/open-temp-index!)]
           (with-redefs [semantic.index-metadata/get-active-index-state (fn [_ _]
                                                                          {:index @index-ref
@@ -71,17 +60,18 @@
 (deftest re-init-endpoint-test
   (testing "POST /api/search/re-init with semantic search support"
     (mt/with-premium-features #{:semantic-search}
-      (semantic.tu/with-index!
+      (semantic.tu/with-test-db! {:mode :mock-indexed}
         (let [original-index      semantic.tu/mock-index
               original-table-name (:table-name original-index)
               new-index           (with-redefs [semantic.index/model-table-suffix (constantly 345)]
                                     (#'semantic.pgvector-api/fresh-index semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model :force-reset? true))
-              new-table-name      (:table-name new-index)]
+              new-table-name      (:table-name new-index)
+              pgvector (semantic.env/get-pgvector-datasource!)]
 
           (is (semantic.tu/table-exists-in-db? original-table-name))
           (is (not (semantic.tu/table-exists-in-db? new-table-name)))
 
-          (let [best-index (semantic.index-metadata/find-compatible-index! semantic.tu/db semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model)]
+          (let [best-index (semantic.index-metadata/find-compatible-index! pgvector semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model)]
             (is (=? original-index (:index best-index)))
             (is (:active best-index)))
 
@@ -96,7 +86,7 @@
 
             (is (zero? (semantic.tu/index-count new-index))))
 
-          (let [best-index (semantic.index-metadata/find-compatible-index! semantic.tu/db semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model)]
+          (let [best-index (semantic.index-metadata/find-compatible-index! pgvector semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model)]
             (is (=? new-index (:index best-index)))
             (is (:active best-index)))
 

enterprise/backend/test/metabase_enterprise/semantic_search/core_test.clj

@@ -2,6 +2,7 @@
   (:require
    [clojure.test :refer :all]
    [metabase-enterprise.semantic-search.core :as semantic.core]
+   [metabase-enterprise.semantic-search.env :as semantic.env]
    [metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]
    [metabase-enterprise.semantic-search.test-util :as semantic.tu]
    [metabase.app-db.core :as mdb]
@@ -10,19 +11,17 @@
    [metabase.search.in-place.legacy :as in-place.legacy]
    [metabase.search.in-place.scoring :as in-place.scoring]
    [metabase.search.settings :as search.settings]
-   [metabase.test :as mt]
-   [metabase.test.fixtures :as fixtures]))
+   [metabase.test :as mt]))
 
-(use-fixtures :once (compose-fixtures
-                     (fixtures/initialize :db)
-                     #'semantic.tu/once-fixture))
+(use-fixtures :once #'semantic.tu/once-fixture)
 
 (deftest fallback-engine-available-with-semantic-test
   (mt/with-premium-features #{:semantic-search}
     (mt/with-temporary-setting-values [search.settings/search-engine "semantic"]
       (with-open [_ (semantic.tu/open-temp-index!)]
-        (semantic.tu/cleanup-index-metadata! semantic.tu/db semantic.tu/mock-index-metadata)
-        (semantic.tu/with-index!
+        (semantic.tu/cleanup-index-metadata! (semantic.env/get-pgvector-datasource!)
+                                             semantic.tu/mock-index-metadata)
+        (semantic.tu/with-test-db! {:mode :mock-indexed}
           (is (search.engine/supported-engine? (if (= :mysql (mdb/db-type))
                                                  ;; appdb is not supported on :mysql, should fallback to in-place
                                                  :search.engine/in-place
@@ -56,7 +55,9 @@
                           in-place.legacy/results
                           (fn [_]
                             (reset! legacy-called? true)
-                            [{:id 2 :name "legacy-result" :model "card" :collection_id 1 :score 0.7}])]
+                            (reify clojure.lang.IReduceInit
+                              (reduce [_this rf init]
+                                (reduce rf init [{:id 2 :name "legacy-result" :model "card" :collection_id 1 :score 0.7}]))))]
               (testing "API defaults to semantic engine when configured"
                 (let [response (mt/user-http-request :crowberto :get 200 "search" :q "test")]
                   (is @semantic-called? "Semantic search should be called by default")
@@ -97,11 +98,15 @@
                                       {:result (dissoc result :score)
                                        :score  (:score result)})
                 search.engine/results (fn [ctx]
-                                        (case (:search-engine ctx)
-                                          :search.engine/semantic (semantic.core/results ctx)
-                                          (if (fn? fallback-results)
-                                            (fallback-results ctx)
-                                            fallback-results)))]
+                                        (let [fallback-results* #(if (fn? fallback-results)
+                                                                   (fallback-results ctx)
+                                                                   fallback-results)]
+                                          (case (:search-engine ctx)
+                                            :search.engine/semantic (semantic.core/results ctx)
+                                            :search.engine/in-place (reify clojure.lang.IReduceInit
+                                                                      (reduce [_this rf init]
+                                                                        (reduce rf init (fallback-results*))))
+                                            (fallback-results*))))]
     (thunk)))
 
 (defn- make-card-result

enterprise/backend/test/metabase_enterprise/semantic_search/dlq_test.clj

@@ -3,6 +3,7 @@
    [clojure.test :refer :all]
    [honey.sql :as sql]
    [metabase-enterprise.semantic-search.dlq :as semantic.dlq]
+   [metabase-enterprise.semantic-search.env :as semantic.env]
    [metabase-enterprise.semantic-search.gate :as semantic.gate]
    [metabase-enterprise.semantic-search.index :as semantic.index]
    [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
@@ -46,7 +47,7 @@
 
 (deftest dlq-table-creation-test
   (testing "DLQ table creation and existence checks"
-    (let [pgvector       semantic.tu/db
+    (let [pgvector       (semantic.env/get-pgvector-datasource!)
           index-metadata (semantic.tu/unique-index-metadata)
           index-id       42]
       (with-open [_ (semantic.tu/open-metadata! pgvector index-metadata)]
@@ -58,7 +59,7 @@
 
 (deftest dlq-entry-management-test
   (testing "DLQ entry add, query, and delete operations"
-    (let [pgvector       semantic.tu/db
+    (let [pgvector       (semantic.env/get-pgvector-datasource!)
           index-metadata (semantic.tu/unique-index-metadata)
           index-id       42]
       (with-open [_ (semantic.tu/open-metadata! pgvector index-metadata)
@@ -97,7 +98,7 @@
 
 (deftest dlq-entry-upsert-test
   (testing "DLQ entry upsert behavior on conflict"
-    (let [pgvector       semantic.tu/db
+    (let [pgvector       (semantic.env/get-pgvector-datasource!)
           index-metadata (semantic.tu/unique-index-metadata)
           index-id       42]
       (with-open [_ (semantic.tu/open-metadata! pgvector index-metadata)
@@ -129,7 +130,7 @@
             (is (= t2 (:error_gated_at (first results))))))))))
 
 (deftest dlq-poll-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         model          semantic.tu/mock-embedding-model
         index          (semantic.index-metadata/qualify-index (semantic.index/default-index model) index-metadata)
@@ -194,7 +195,7 @@
           (is (= 3 (count (semantic.dlq/poll pgvector index-metadata @index-id-ref 10)))))))))
 
 (deftest dlq-retry-loop-test
-  (let [pgvector         semantic.tu/db
+  (let [pgvector         (semantic.env/get-pgvector-datasource!)
         index-metadata   (semantic.tu/unique-index-metadata)
         model            semantic.tu/mock-embedding-model
         index            (semantic.index-metadata/qualify-index (semantic.index/default-index model) index-metadata)
@@ -296,7 +297,7 @@
                 (is (pos? (:failure-count result)))))))))))
 
 (deftest try-batch-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         model          semantic.tu/mock-embedding-model
         index          (semantic.index-metadata/qualify-index (semantic.index/default-index model) index-metadata)

enterprise/backend/test/metabase_enterprise/semantic_search/gate_test.clj

@@ -4,6 +4,7 @@
    [clojure.test :refer :all]
    [clojure.walk :as walk]
    [honey.sql :as sql]
+   [metabase-enterprise.semantic-search.env :as semantic.env]
    [metabase-enterprise.semantic-search.gate :as semantic.gate]
    [metabase-enterprise.semantic-search.index :as semantic.index]
    [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
@@ -89,7 +90,7 @@
   (:ts (jdbc/execute-one! pgvector ["select clock_timestamp() ts"])))
 
 (deftest gate-documents!-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         t0             (ts "2025-01-01T00:00:00Z")
         t1             (ts "2025-01-01T00:01:00Z")
@@ -168,7 +169,7 @@
                (comparable-gate-row (dissoc (get-gate-row! pgvector index-metadata (:id (version c3 t3))) :gated_at))))))))
 
 (deftest poll-test
-  (let [pgvector        semantic.tu/db
+  (let [pgvector        (semantic.env/get-pgvector-datasource!)
         index-metadata  (semantic.tu/unique-index-metadata)
         epoch-watermark {:last-poll Instant/EPOCH :last-seen Instant/EPOCH}
         c1              {:model "card" :id "123" :searchable_text "a"}
@@ -275,7 +276,7 @@
              (:last-seen watermark))))))
 
 (deftest flush-watermark!-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         model1         {:provider "foo" :model-name "m1" :vector-dimensions 42}
         model2         {:provider "foo" :model-name "m2" :vector-dimensions 42}
@@ -330,7 +331,7 @@
 
 (deftest gate-documents-metrics-test
   (mt/with-prometheus-system! [_ system]
-    (let [pgvector       semantic.tu/db
+    (let [pgvector       (semantic.env/get-pgvector-datasource!)
           index-metadata (semantic.tu/unique-index-metadata)
           t1             (ts "2025-01-01T00:01:00Z")
           c1             {:model "card" :id "123" :searchable_text "Dog Training Guide"}

enterprise/backend/test/metabase_enterprise/semantic_search/index_metadata_test.clj

@@ -1,6 +1,7 @@
 (ns metabase-enterprise.semantic-search.index-metadata-test
   (:require
    [clojure.test :refer :all]
+   [metabase-enterprise.semantic-search.env :as semantic.env]
    [metabase-enterprise.semantic-search.index :as semantic.index]
    [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
    [metabase-enterprise.semantic-search.test-util :as semantic.tu]
@@ -23,7 +24,7 @@
                                             semantic.index/hnsw-index-name)))))))
 
 (deftest create-tables-if-not-exists!-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         ;; could use open-tables here but I chose to be explicit
         sut            #(semantic.tu/closeable
@@ -46,7 +47,7 @@
    (fn [_] (semantic.tu/cleanup-index-metadata! pgvector index-metadata))))
 
 (deftest drop-tables-if-exists!-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         sut            semantic.index-metadata/drop-tables-if-exists!]
     (with-open [_ (open-tables! pgvector index-metadata)]
@@ -59,7 +60,7 @@
         (is (= now-expected (semantic.tu/get-table-names pgvector)))))))
 
 (deftest ensure-control-row-exists!-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         sut            semantic.index-metadata/ensure-control-row-exists!]
     (with-open [_ (open-tables! pgvector index-metadata)]
@@ -74,7 +75,7 @@
           (is (= control-snap (semantic.tu/get-control-rows pgvector index-metadata))))))))
 
 (deftest activate-index!-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         un-index       (semantic.index/default-index semantic.tu/mock-embedding-model)
         index1         (semantic.index-metadata/qualify-index (assoc un-index :table-name "i1") index-metadata)
@@ -98,7 +99,7 @@
             (is (=? [{:active_id index-id2}] (semantic.tu/get-control-rows pgvector index-metadata)))))))))
 
 (deftest get-active-index-state-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         un-index       (semantic.index/default-index semantic.tu/mock-embedding-model)
         index          (semantic.index-metadata/qualify-index un-index index-metadata)
@@ -134,7 +135,7 @@
   (run! #(add-index! pgvector index-metadata %) inactive))
 
 (deftest find-compatible-index!-test
-  (let [pgvector         semantic.tu/db
+  (let [pgvector         (semantic.env/get-pgvector-datasource!)
         embedding-model1 semantic.tu/mock-embedding-model
         embedding-model2 (assoc semantic.tu/mock-embedding-model
                                 :model-name "mock2")
@@ -190,7 +191,7 @@
                   (is (nil? (sut' model))))))))))))
 
 (deftest create-new-index-spec-test
-  (let [pgvector         semantic.tu/db
+  (let [pgvector         (semantic.env/get-pgvector-datasource!)
         index-metadata   (semantic.tu/unique-index-metadata)
         embedding-model  semantic.tu/mock-embedding-model
         sut              semantic.index-metadata/create-new-index-spec]
@@ -202,7 +203,7 @@
                 result))))))
 
 (deftest record-new-index-table!-test
-  (let [pgvector        semantic.tu/db
+  (let [pgvector        (semantic.env/get-pgvector-datasource!)
         index-metadata  (semantic.tu/unique-index-metadata)
         embedding-model semantic.tu/mock-embedding-model
         index           (default-index embedding-model index-metadata)

enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj

@@ -3,6 +3,7 @@
    [clojure.string :as str]
    [clojure.test :refer :all]
    [metabase-enterprise.semantic-search.embedding :as semantic.embedding]
+   [metabase-enterprise.semantic-search.env :as semantic.env]
    [metabase-enterprise.semantic-search.index :as semantic.index]
    [metabase-enterprise.semantic-search.test-util :as semantic.tu]
    [metabase.analytics.core :as analytics]
@@ -18,14 +19,14 @@
   (mt/with-premium-features #{:semantic-search}
     (with-open [index-ref (semantic.tu/open-temp-index!)]
       ;; open-temp-index-table! creates the temp table, so drop it in order to test create!.
-      (semantic.index/drop-index-table! semantic.tu/db semantic.tu/mock-index)
+      (semantic.index/drop-index-table! (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index)
       (testing "index table is not present before create!"
         (is (not (semantic.tu/table-exists-in-db? (:table-name @index-ref))))
         (is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref))))
         (is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/fts-index-name @index-ref))))
         (is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/fts-native-index-name @index-ref)))))
       (testing "index table is present after create!"
-        (semantic.index/create-index-table-if-not-exists! semantic.tu/db semantic.tu/mock-index {:force-reset? false})
+        (semantic.index/create-index-table-if-not-exists! (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index {:force-reset? false})
         (is (semantic.tu/table-exists-in-db? (:table-name @index-ref)))
         (is (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref)))
         (is (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/fts-index-name @index-ref)))
@@ -37,7 +38,7 @@
       (testing "index table is present before drop!"
         (is (semantic.tu/table-exists-in-db? (:table-name @index-ref))))
       (testing "index table is not present after drop!"
-        (semantic.index/drop-index-table! semantic.tu/db semantic.tu/mock-index)
+        (semantic.index/drop-index-table! (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index)
         (is (not (semantic.tu/table-exists-in-db? (:table-name @index-ref))))))))
 
 (deftest upsert-index!-test
@@ -343,7 +344,7 @@
             (testing "Permission filtering metrics"
               (reset! analytics-calls [])
               (mt/with-test-user :crowberto
-                (semantic.index/query-index semantic.tu/db semantic.tu/mock-index
+                (semantic.index/query-index (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index
                                             {:search-string "dog training"}))
 
               (let [permission-calls (filter #(= :metabase-search/semantic-permission-filter-ms (first %)) @analytics-calls)]
@@ -355,7 +356,7 @@
             (testing "Semantic search timing metrics"
               (reset! analytics-calls [])
               (mt/with-test-user :crowberto
-                (semantic.index/query-index semantic.tu/db semantic.tu/mock-index
+                (semantic.index/query-index (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index
                                             {:search-string "elephant migration"
                                              :filter-items-in-personal-collection "only"}))
 

enterprise/backend/test/metabase_enterprise/semantic_search/indexer_test.clj

@@ -3,6 +3,7 @@
    [clojure.test :refer :all]
    [honey.sql :as sql]
    [metabase-enterprise.semantic-search.dlq :as semantic.dlq]
+   [metabase-enterprise.semantic-search.env :as semantic.env]
    [metabase-enterprise.semantic-search.gate :as semantic.gate]
    [metabase-enterprise.semantic-search.index :as semantic.index]
    [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
@@ -33,7 +34,7 @@
   (Timestamp/from (Instant/parse s)))
 
 (deftest indexing-step-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         model          semantic.tu/mock-embedding-model
         index          (semantic.index-metadata/qualify-index (semantic.index/default-index model) index-metadata)
@@ -214,7 +215,7 @@
            (log/fatal "Indexing loop thread not exiting during test!")))))))
 
 (deftest indexing-loop-thread-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         index          semantic.tu/mock-index
         metadata-row   {:indexer_last_poll Instant/EPOCH
@@ -283,7 +284,7 @@
             (is (.join thread (Duration/ofSeconds 1)))))))))
 
 (deftest indexing-loop-exit-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         model          semantic.tu/mock-embedding-model
         index          (semantic.index-metadata/qualify-index (semantic.index/default-index model) index-metadata)
@@ -371,7 +372,7 @@
    (fn [_] (semantic.dlq/drop-dlq-table-if-exists! pgvector index-metadata index-id))))
 
 (deftest quartz-job-run!-test
-  (let [pgvector        semantic.tu/db
+  (let [pgvector        (semantic.env/get-pgvector-datasource!)
         index-metadata  (semantic.tu/unique-index-metadata)
         open-job-thread (fn [& args]
                           (let [caught-ex (volatile! nil)]
@@ -458,7 +459,7 @@
 
 (deftest dlq-step-test
   (mt/with-prometheus-system! [_ system]
-    (let [pgvector       semantic.tu/db
+    (let [pgvector       (semantic.env/get-pgvector-datasource!)
           index-metadata (semantic.tu/unique-index-metadata)
           model          semantic.tu/mock-embedding-model
           index          (semantic.index-metadata/qualify-index (semantic.index/default-index model) index-metadata)
@@ -541,7 +542,7 @@
 
 (deftest indexer-stall-and-recovery-test
   (mt/with-prometheus-system! [_ system]
-    (let [pgvector             semantic.tu/db
+    (let [pgvector             (semantic.env/get-pgvector-datasource!)
           index-metadata       (semantic.tu/unique-index-metadata)
           model                semantic.tu/mock-embedding-model
           index                (semantic.index-metadata/qualify-index (semantic.index/default-index model) index-metadata)
@@ -660,7 +661,7 @@
                         (mt/metric-value system :metabase-search/semantic-indexer-poll-to-poll-interval-ms)))))))))))
 
 (deftest dlq-integration-with-indexer-loop-test
-  (let [pgvector         semantic.tu/db
+  (let [pgvector         (semantic.env/get-pgvector-datasource!)
         index-metadata   (semantic.tu/unique-index-metadata)
         model            semantic.tu/mock-embedding-model
         index            (semantic.index-metadata/qualify-index (semantic.index/default-index model) index-metadata)
@@ -776,7 +777,7 @@
 
 (deftest indexer-loop-metric-test
   (mt/with-prometheus-system! [_ system]
-    (let [pgvector semantic.tu/db
+    (let [pgvector (semantic.env/get-pgvector-datasource!)
           index-metadata (semantic.tu/unique-index-metadata)
           metadata-row {:id                42
                         :indexer_last_poll (ts "2025-01-01T12:00:00Z")

enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj

@@ -9,6 +9,7 @@
    [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
    [metabase-enterprise.semantic-search.db.migration :as semantic.db.migration]
    [metabase-enterprise.semantic-search.db.migration.impl :as semantic.db.migration.impl]
+   [metabase-enterprise.semantic-search.env :as semantic.env]
    [metabase-enterprise.semantic-search.index :as semantic.index]
    [metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]
    [metabase-enterprise.semantic-search.test-util :as semantic.tu]
@@ -21,11 +22,10 @@
 (set! *warn-on-reflection* true)
 
 (use-fixtures :once #'semantic.tu/once-fixture)
-(use-fixtures :each #'semantic.tu/ensure-no-migration-table-fixture)
 
 (deftest migration-table-versions-test
   (mt/with-premium-features #{:semantic-search}
-    (semantic.tu/with-test-db! semantic.tu/default-test-db
+    (semantic.tu/with-test-db-defaults!
       (letfn [(migrate-and-get-db-version
                 [attempted-version]
                 (with-redefs [semantic.db.migration.impl/schema-version attempted-version
@@ -56,14 +56,14 @@
 
 (deftest migration-lock-coordination-test
   (mt/with-premium-features #{:semantic-search}
-    (semantic.tu/with-test-db! semantic.tu/default-test-db
+    (semantic.tu/with-test-db-defaults!
       (testing "Migration of simultaneous init attempt is blocked"
         (let [original-write-fn @#'semantic.db.migration/write-successful-migration!
               original-migrate-fn @#'semantic.db.connection/do-with-migrate-tx
+              original-maybe-migrate @#'semantic.db.migration/maybe-migrate!
               results (atom {:executed-migrations 0
                              :log []})]
-          (with-redefs-fn {;; TODO: why in ci init index won't succeed -- http on embedding
-                           #'semantic.pgvector-api/index-documents! (constantly nil)
+          (with-redefs-fn {#'semantic.pgvector-api/index-documents! (constantly nil)
                            #'semantic.db.connection/do-with-migrate-tx
                            (fn [& args]
                              (let [tid (.getId (Thread/currentThread))]
@@ -72,6 +72,12 @@
                                (apply original-migrate-fn args)
                                (swap! results update :log conj [tid :ended (t/local-date-time)]))
                              nil)
+                           ;; add delay into migration function to ensure that thread performing migration
+                           ;; finishes earlier
+                           #'semantic.db.migration/maybe-migrate!
+                           (fn [& args]
+                             (Thread/sleep 200)
+                             (apply original-maybe-migrate args))
                            #'semantic.db.migration/write-successful-migration!
                            (fn [& args]
                              (Thread/sleep 2000)
@@ -114,12 +120,12 @@
 (deftest expected-db-schema-after-migration-test
   (try
     (mt/with-premium-features #{:semantic-search}
-      (semantic.tu/with-test-db! semantic.tu/default-test-db
+      (semantic.tu/with-test-db-defaults!
         (with-redefs [semantic.pgvector-api/index-documents! (constantly nil)]
           (semantic.core/init! (semantic.tu/mock-documents) nil)
           (testing "migration table has expected columns"
             (is (map-contains-keys?
-                 (jdbc/execute-one! (semantic.db.datasource/ensure-initialized-data-source!)
+                 (jdbc/execute-one! (semantic.env/get-pgvector-datasource!)
                                     (sql/format {:select [:*]
                                                  :from [:migration]}))
                  (qualify :migration [:migrated_at
@@ -127,7 +133,7 @@
                                       :version]))))
           (testing "control table has expected columns"
             (is (map-contains-keys?
-                 (jdbc/execute-one! (semantic.db.datasource/ensure-initialized-data-source!)
+                 (jdbc/execute-one! (semantic.env/get-pgvector-datasource!)
                                     (sql/format {:select [:*]
                                                  :from [:index_control]}))
                  (qualify :index_control [:active_id
@@ -136,7 +142,7 @@
                                           :version]))))
           (testing "metadata table has expected columns"
             (is  (map-contains-keys?
-                  (jdbc/execute-one! semantic.tu/db
+                  (jdbc/execute-one! (semantic.env/get-pgvector-datasource!)
                                      (sql/format {:select [:*]
                                                   :from [:index_metadata]}))
                   (qualify :index_metadata [:id
@@ -183,7 +189,7 @@
                         "text_search_with_native_query_vector"
                         "verified"
                         "view_count"}
-                      (->>  (jdbc/execute! semantic.tu/db
+                      (->>  (jdbc/execute! (semantic.env/get-pgvector-datasource!)
                                            (sql/format {:select [:column_name]
                                                         :from [:information_schema.columns]
                                                         :where [[:= :table_name [:inline index-table]]]}))
@@ -191,7 +197,7 @@
                             set)))))
           (testing "index table has expected columns"
             (is (= ["document" "document_hash" "gated_at" "id" "model" "model_id" "updated_at"]
-                   (->> (jdbc/execute! semantic.tu/db
+                   (->> (jdbc/execute! (semantic.env/get-pgvector-datasource!)
                                        (sql/format {:select [:column_name]
                                                     :from [:information_schema.columns]
                                                     :where [[:= :table_name [:inline "index_gate"]]]}))
@@ -212,7 +218,7 @@
 
 (deftest dynamic-schema-migration-test
   (mt/with-premium-features #{:semantic-search}
-    (semantic.tu/with-test-db! semantic.tu/default-test-db
+    (semantic.tu/with-test-db-defaults!
       (with-redefs [semantic.pgvector-api/index-documents! (constantly nil)]
         (semantic.core/init! (semantic.tu/mock-documents) nil)
              ;; add column to index table

enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj

@@ -3,6 +3,7 @@
    [clojure.test :refer :all]
    [honey.sql :as sql]
    [metabase-enterprise.semantic-search.dlq :as semantic.dlq]
+   [metabase-enterprise.semantic-search.env :as semantic.env]
    [metabase-enterprise.semantic-search.index :as semantic.index]
    [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
    [metabase-enterprise.semantic-search.indexer :as semantic.indexer]
@@ -21,12 +22,11 @@
 (set! *warn-on-reflection* true)
 
 (use-fixtures :once #'semantic.tu/once-fixture)
-(use-fixtures :each #'semantic.tu/ensure-no-migration-table-fixture)
 
 ;; NOTE: isolation tests are absent, in prod there is only one index-metadata
 
 (deftest init-semantic-search!-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         model1         semantic.tu/mock-embedding-model
         model2         (assoc semantic.tu/mock-embedding-model :model-name "embed-harder")
@@ -69,7 +69,7 @@
                       (semantic.index-metadata/find-compatible-index! pgvector index-metadata model2))))))))))
 
 (deftest init-recreates-missing-index-table-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         model1         semantic.tu/mock-embedding-model
         sut*           semantic.pgvector-api/init-semantic-search!
@@ -98,7 +98,7 @@
     (is (thrown? Exception (apply sut args)))))
 
 (deftest index-documents!-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         model1         semantic.tu/mock-embedding-model
         model2         (assoc semantic.tu/mock-embedding-model :model-name "embedagain")
@@ -125,7 +125,7 @@
                        @calls))))))))))
 
 (deftest delete-documents!-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         dash           "dashboard"
         card           "card"
@@ -166,7 +166,7 @@
 (deftest query-test
   (semantic.tu/with-indexable-documents!
     (mt/with-dynamic-fn-redefs [semantic.index/model-table-suffix semantic.tu/mock-table-suffix]
-      (let [pgvector       semantic.tu/db
+      (let [pgvector       (semantic.env/get-pgvector-datasource!)
             index-metadata (semantic.tu/unique-index-metadata)
             model1         semantic.tu/mock-embedding-model
             model2         (assoc semantic.tu/mock-embedding-model :model-name "judge-embedd")
@@ -218,7 +218,7 @@
 
 (deftest e2e-index-a-sample-db-with-gate-test
   (let [docs            (mt/dataset test-data (vec (search.ingestion/searchable-documents)))
-        pgvector        semantic.tu/db
+        pgvector        (semantic.env/get-pgvector-datasource!)
         index-metadata  (semantic.tu/unique-index-metadata)
         embedding-model semantic.tu/mock-embedding-model
         open-job-thread (fn [& args]

enterprise/backend/test/metabase_enterprise/semantic_search/query_test.clj

@@ -1,7 +1,6 @@
 (ns metabase-enterprise.semantic-search.query-test
   (:require
    [clojure.test :refer :all]
-   [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
    [metabase-enterprise.semantic-search.index :as semantic.index]
    [metabase-enterprise.semantic-search.test-util :as semantic.tu]
    [metabase.permissions.models.data-permissions :as data-perms]
@@ -12,22 +11,21 @@
 
 (use-fixtures :once #'semantic.tu/once-fixture)
 
-(deftest database-initialised-test
-  (is (some? @semantic.db.datasource/data-source))
-  (is (= {:test 1} (semantic.db.datasource/test-connection!))))
-
 (deftest basic-query-test
   (testing "Simple queries with no filters"
     (mt/with-premium-features #{:semantic-search}
       (mt/as-admin
-        (semantic.tu/with-index!
-          (testing "Dog-related query finds dog content"
-            (let [results (semantic.tu/query-index {:search-string "puppy"})]
-              (is (= "Dog Training Guide" (-> results first :name)))))
+        (semantic.tu/with-test-db! {:mode :mock-indexed}
+          (semantic.tu/with-only-semantic-weights
+            (testing "Dog-related query finds dog content"
+              (let [results (-> (semantic.tu/query-index {:search-string "puppy"})
+                                semantic.tu/filter-for-mock-embeddings)]
+                (is (= "Dog Training Guide" (-> results first :name)))))
 
-          (testing "Bird-related query finds bird content"
-            (let [results (semantic.tu/query-index {:search-string "avian"})]
-              (is (= "Bird Watching Tips" (-> results first :name))))))))))
+            (testing "Bird-related query finds bird content"
+              (let [results (-> (semantic.tu/query-index {:search-string "avian"})
+                                semantic.tu/filter-for-mock-embeddings)]
+                (is (= "Bird Watching Tips" (-> results first :name)))))))))))
 
 (defn- index-of-name
   "Return the index of the item with :name `name` in `coll`"
@@ -41,75 +39,76 @@
   (testing "Tests for :search-native-query"
     (mt/with-premium-features #{:semantic-search}
       (mt/as-admin
-        (semantic.tu/with-index!
-          (let [not-top-10? #(or (nil? %) (< 10 %))]
-            (doseq [{:keys [name desc search-string search-native-query? expected-index?]}
-                    [{:name "Dog Training Guide"
-                      :desc "contains"
-                      :search-string "AVG(tricks)"
-                      :search-native-query? false
-                      :expected-index? not-top-10?}
-                     {:name "Dog Training Guide"
-                      :desc "contains"
-                      :search-string "AVG(tricks)"
-                      :search-native-query? true
-                      :expected-index? zero?}
-                     {:name "Dog Training Guide"
-                      :desc "contains"
-                      :search-string "GROUP BY breed"
-                      :search-native-query? false
-                      :expected-index? not-top-10?}
-                     {:name "Dog Training Guide"
-                      :desc "contains"
-                      :search-string "GROUP BY breed"
-                      :search-native-query? true
-                      :expected-index? zero?}
-                     {:name "Dog Training Guide"
-                      :desc "does not contain"
-                      :search-string "SUM(tricks)"
-                      :search-native-query? false
-                      :expected-index? not-top-10?}
-                     {:name "Dog Training Guide"
-                      :desc "does not contain"
-                      :search-string "SUM(tricks)"
-                      :search-native-query? true
-                      :expected-index? not-top-10?}
-                     {:name "Bird Watching Tips"
-                      :desc "does not contain (no native query)"
-                      :search-string "AVG(tricks)"
-                      :search-native-query? true
-                      :expected-index? not-top-10?}
-                     {:name "Dog Training Guide"
-                      :desc "semantic search"
-                      :search-string "puppy"
-                      :search-native-query? true
-                      :expected-index? zero?}
-                     {:name "Bird Watching Tips"
-                      :desc "semantic search"
-                      :search-string "avian"
-                      :search-native-query? true
-                      :expected-index? zero?}
-                     {:name "Dog Training Guide"
-                      :desc "non-native keyword query"
-                      :search-string "Training"
-                      :search-native-query? true
-                      :expected-index? zero?}
-                     {:name "Bird Watching Tips"
-                      :desc "non-native keyword query"
-                      :search-string "Watching"
-                      :search-native-query? true
-                      :expected-index? zero?}]]
-              (testing (format "\n%s %s %s %s" name desc search-string search-native-query?)
-                (is (-> (semantic.tu/query-index {:search-string search-string
-                                                  :search-native-query search-native-query?})
-                        (index-of-name name)
-                        expected-index?))))))))))
+        (semantic.tu/with-test-db! {:mode :mock-indexed}
+          (semantic.tu/with-only-semantic-weights
+            (let [not-top-10? #(or (nil? %) (< 10 %))]
+              (doseq [{:keys [name desc search-string search-native-query? expected-index?]}
+                      [{:name "Dog Training Guide"
+                        :desc "contains"
+                        :search-string "AVG(tricks)"
+                        :search-native-query? false
+                        :expected-index? not-top-10?}
+                       {:name "Dog Training Guide"
+                        :desc "contains"
+                        :search-string "AVG(tricks)"
+                        :search-native-query? true
+                        :expected-index? zero?}
+                       {:name "Dog Training Guide"
+                        :desc "contains"
+                        :search-string "GROUP BY breed"
+                        :search-native-query? false
+                        :expected-index? not-top-10?}
+                       {:name "Dog Training Guide"
+                        :desc "contains"
+                        :search-string "GROUP BY breed"
+                        :search-native-query? true
+                        :expected-index? zero?}
+                       {:name "Dog Training Guide"
+                        :desc "does not contain"
+                        :search-string "SUM(tricks)"
+                        :search-native-query? false
+                        :expected-index? not-top-10?}
+                       {:name "Dog Training Guide"
+                        :desc "does not contain"
+                        :search-string "SUM(tricks)"
+                        :search-native-query? true
+                        :expected-index? not-top-10?}
+                       {:name "Bird Watching Tips"
+                        :desc "does not contain (no native query)"
+                        :search-string "AVG(tricks)"
+                        :search-native-query? true
+                        :expected-index? not-top-10?}
+                       {:name "Dog Training Guide"
+                        :desc "semantic search"
+                        :search-string "puppy"
+                        :search-native-query? true
+                        :expected-index? zero?}
+                       {:name "Bird Watching Tips"
+                        :desc "semantic search"
+                        :search-string "avian"
+                        :search-native-query? true
+                        :expected-index? zero?}
+                       {:name "Dog Training Guide"
+                        :desc "non-native keyword query"
+                        :search-string "Training"
+                        :search-native-query? true
+                        :expected-index? zero?}
+                       {:name "Bird Watching Tips"
+                        :desc "non-native keyword query"
+                        :search-string "Watching"
+                        :search-native-query? true
+                        :expected-index? zero?}]]
+                (testing (format "\n%s %s %s %s" name desc search-string search-native-query?)
+                  (is (-> (semantic.tu/query-index {:search-string search-string
+                                                    :search-native-query search-native-query?})
+                          (index-of-name name)
+                          expected-index?)))))))))))
 
 (deftest model-filtering-test
   (testing "Filter results by model type"
     (mt/with-premium-features #{:semantic-search}
       (mt/as-admin
-        (semantic.tu/with-index!
+        (semantic.tu/with-test-db! {:mode :mock-indexed}
           (testing "Filter for cards only"
             (let [card-results (semantic.tu/query-index {:search-string "avian" :models ["card"]})
                   filtered-results (semantic.tu/filter-for-mock-embeddings card-results)]
@@ -132,7 +131,7 @@
   (testing "Filter results by archived status"
     (mt/with-premium-features #{:semantic-search}
       (mt/as-admin
-        (semantic.tu/with-index!
+        (semantic.tu/with-test-db! {:mode :mock-indexed}
           (testing "Include only non-archived items"
             (let [active-results (semantic.tu/query-index {:search-string "feline" :archived? false})]
               (is (every? #(not (:archived %)) active-results))))
@@ -152,7 +151,7 @@
   (testing "Filter results by creator"
     (mt/with-premium-features #{:semantic-search}
       (mt/as-admin
-        (semantic.tu/with-index!
+        (semantic.tu/with-test-db! {:mode :mock-indexed}
           (testing "Filter by single creator"
             (let [crowberto-results (semantic.tu/query-index {:search-string "aquatic" :created-by [(mt/user->id :crowberto)]})
                   filtered-results (semantic.tu/filter-for-mock-embeddings crowberto-results)]
@@ -169,7 +168,7 @@
   (testing "Filter results by verified status"
     (mt/with-premium-features #{:semantic-search}
       (mt/as-admin
-        (semantic.tu/with-index!
+        (semantic.tu/with-test-db! {:mode :mock-indexed}
           (testing "Include only verified items"
             (let [verified-results (semantic.tu/query-index {:search-string "puppy" :verified true})
                   filtered-results (semantic.tu/filter-for-mock-embeddings verified-results)]
@@ -185,7 +184,7 @@
   (testing "Complex filters with multiple criteria"
     (mt/with-premium-features #{:semantic-search}
       (mt/as-admin
-        (semantic.tu/with-index!
+        (semantic.tu/with-test-db! {:mode :mock-indexed}
           (testing "Non-archived cards by specific creator"
             (let [results (semantic.tu/query-index {:search-string "equine"
                                                     :models ["card"]
@@ -219,7 +218,7 @@
 
 (deftest permissions-test
   (mt/with-premium-features #{:semantic-search}
-    (semantic.tu/with-index!
+    (semantic.tu/with-test-db! {:mode :mock-indexed}
       (let [monsters-table (t2/select-one-pk :model/Table :name "Monsters Table")
             q (fn [model s] (map :name (semantic.tu/query-index {:search-string s, :models [model]})))
             all-users (perms-group/all-users)
@@ -255,11 +254,14 @@
     (mt/with-premium-features #{:semantic-search}
       (mt/as-admin
         (binding [semantic.index/*batch-size* 1]
-          (semantic.tu/with-index!
-            (testing "Horse-related query finds horse content"
-              (let [results (semantic.tu/query-index {:search-string "marine mammal"})]
-                (is (= "Whale Communication" (-> results first :name)))))
+          (semantic.tu/with-test-db! {:mode :mock-indexed}
+            (semantic.tu/with-only-semantic-weights
+              (testing "Horse-related query finds horse content"
+                (let [results (-> (semantic.tu/query-index {:search-string "marine mammal"})
+                                  semantic.tu/filter-for-mock-embeddings)]
+                  (is (= "Whale Communication" (-> results first :name)))))
 
-            (testing "Tiger-related query finds tiger content"
-              (let [results (semantic.tu/query-index {:search-string "endangered species"})]
-                (is (= "Tiger Conservation" (-> results first :name)))))))))))
+              (testing "Tiger-related query finds tiger content"
+                (let [results (-> (semantic.tu/query-index {:search-string "endangered species"})
+                                  semantic.tu/filter-for-mock-embeddings)]
+                  (is (= "Tiger Conservation" (-> results first :name))))))))))))

enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.clj

@@ -5,6 +5,7 @@
    [honey.sql :as sql]
    [java-time.api :as t]
    [metabase-enterprise.semantic-search.core :as semantic.core]
+   [metabase-enterprise.semantic-search.env :as semantic.env]
    [metabase-enterprise.semantic-search.repair :as semantic.repair]
    [metabase-enterprise.semantic-search.test-util :as semantic.tu]
    [metabase-enterprise.semantic-search.util :as semantic.util]
@@ -47,13 +48,21 @@
   [gate-contents id]
   (some #(when (= (:id %) id) %) gate-contents))
 
+(defn- clear-gate-table!
+  "Clear all entries from the gate table to ensure clean test state"
+  [pgvector gate-table-name]
+  (jdbc/execute! pgvector
+                 (-> {:delete-from [(keyword gate-table-name)]}
+                     (sql/format :quoted true))))
+
 (deftest repair-index-integration-test
   (testing "repair-index! properly handles document additions and deletions via gate table"
     (mt/with-premium-features #{:semantic-search}
-      (semantic.tu/with-index!
-        (let [pgvector       semantic.tu/db
-              index-metadata semantic.tu/mock-index-metadata
+      (semantic.tu/with-test-db! {:mode :mock-indexed}
+        (let [pgvector       (semantic.env/get-pgvector-datasource!)
+              index-metadata (semantic.env/get-index-metadata)
               gate-table     (:gate-table-name index-metadata)
+              _              (clear-gate-table! pgvector gate-table)
               initial-docs   [(create-test-document "card" 1 "Dog Training Guide")
                               (create-test-document "card" 2 "Cat Behavior Study")
                               (create-test-document "dashboard" 3 "Animal Stats")]]
@@ -83,21 +92,21 @@
 (deftest repair-table-cleanup-test
   (testing "The repair table gets cleaned up properly at the end of a repair-index! job"
     (mt/with-premium-features #{:semantic-search}
-      (semantic.tu/with-index!
-        (let [pgvector       semantic.tu/db
+      (semantic.tu/with-test-db! {:mode :mock-indexed}
+        (let [pgvector       (semantic.env/get-pgvector-datasource!)
               index-metadata semantic.tu/mock-index-metadata
               gate-table     (:gate-table-name index-metadata)
-              initial-docs   [(create-test-document "card" 1 "Dog Training Guide")]]
+              initial-docs   [(create-test-document "card" 6 "Dog Training Guide")]]
           (semantic.core/update-index! initial-docs)
           (semantic.tu/index-all!)
 
           (testing "repair table is cleaned up after successful repair"
             (let [test-repair-table-name "repair_table_cleanup_test"]
               (with-redefs [semantic.repair/repair-table-name (constantly test-repair-table-name)]
-                (semantic.core/repair-index! [(create-test-document "card" 2 "New Test Card")])
+                (semantic.core/repair-index! [(create-test-document "card" 7 "New Test Card")])
 
                 (let [gate-contents (gate-table-contents pgvector gate-table)]
-                  (is (tombstone? (gate-entry-by-id gate-contents "card_1")))
-                  (is (some? (gate-entry-by-id gate-contents "card_2")))
+                  (is (tombstone? (gate-entry-by-id gate-contents "card_6")))
+                  (is (some? (gate-entry-by-id gate-contents "card_7")))
                   (is (not (semantic.util/table-exists? pgvector test-repair-table-name))
                       "Repair table should be cleaned up after repair-index! completes"))))))))))

enterprise/backend/test/metabase_enterprise/semantic_search/scoring_test.clj

@@ -6,7 +6,6 @@
    [metabase-enterprise.semantic-search.scoring :as semantic.scoring]
    [metabase-enterprise.semantic-search.test-util :as semantic.tu]
    [metabase.app-db.core :as mdb]
-   [metabase.search.appdb.scoring-test :refer [with-weights]]
    [metabase.search.config :as search.config]
    [metabase.test :as mt])
   (:import
@@ -34,7 +33,7 @@
   "Populate the index with the given documents."
   {:style/indent :defn}
   [documents & body]
-  `(with-open [_# (semantic.tu/open-temp-index!)]
+  `(semantic.tu/with-test-db! {:mode :mock-initialized}
      (semantic.tu/upsert-index! (map add-doc-defaults ~documents))
      ~@body))
 
@@ -56,13 +55,17 @@
 (defn search-results
   "Like search-results* but with a sanity check that search without weights returns a different result."
   [ranker-key search-string & {:as raw-ctx}]
-  (let [result   (with-weights {ranker-key 1} (search-results* search-string raw-ctx))
-        inverted (with-weights {ranker-key -1} (search-results* search-string raw-ctx))]
+  (let [result   (semantic.tu/with-weights {ranker-key  1} (search-results* search-string raw-ctx))
+        inverted (semantic.tu/with-weights {ranker-key -1} (search-results* search-string raw-ctx))]
     ;; note that this may not be a strict reversal, due to ties.
     (is (not= inverted result)
         "sanity check: search-no-weights should be different")
     result))
 
+;;
+;; index-based scorers
+;;
+
 (deftest rrf-test
   (mt/with-premium-features #{:semantic-search}
     (with-index-contents!
@@ -143,7 +146,7 @@
         (is (= [["dataset" 1 "card ancient"]
                 ["metric"  3 "card old"]
                 ["card"    2 "card recent"]]
-               (with-weights {:model 1.0 :model/dataset 1.0}
+               (semantic.tu/with-weights {:model 1.0 :model/dataset 1.0}
                  (search-results* "card"))))))))
 
 (deftest recency-test
@@ -181,9 +184,9 @@
          {:model "dashboard" :id 2 :name "view dashboard" :view_count 0}
          {:model "dataset"   :id 3 :name "view dataset"   :view_count 0}]
         ;; fix some test flakes where dataset 3 exists and has some sort of recent views
-        (with-weights (assoc (search.config/weights :default)
-                             :user-recency 0
-                             :rrf 0)
+        (semantic.tu/with-weights (assoc (search.config/weights :default)
+                                         :user-recency 0
+                                         :rrf 0)
           (is (=? [{:model "dashboard", :id 2, :name "view dashboard"}
                    {:model "card",      :id 1, :name "view card"}
                    {:model "dataset",   :id 3, :name "view dataset"}]
@@ -202,8 +205,8 @@
 (defn indifferent?
   "Check that the results and their order do not depend on the given ranker."
   [ranker-key search-string & {:as raw-ctx}]
-  (= (with-weights {ranker-key 1} (search-results* search-string raw-ctx))
-     (with-weights {ranker-key -1} (search-results* search-string raw-ctx))))
+  (= (semantic.tu/with-weights {ranker-key  1} (search-results* search-string raw-ctx))
+     (semantic.tu/with-weights {ranker-key -1} (search-results* search-string raw-ctx))))
 
 (deftest dashboard-count-test-2
   (mt/with-premium-features #{:semantic-search}
@@ -213,6 +216,44 @@
          {:model "card" :id 2 :name "card" :dashboardcard_count 201}]
         (is (indifferent? :dashboard "card"))))))
 
+;;;
+;;; "premium" index-based scorers
+;;;
+
+(deftest official-collection-test
+  (with-index-contents!
+    [{:model "collection" :id 1 :name "collection normal" :official_collection false}
+     {:model "collection" :id 2 :name "collection official" :official_collection true}]
+    (testing "official collections has higher rank"
+      (mt/with-premium-features #{:semantic-search :official-collections}
+        (is (= [["collection" 2 "collection official"]
+                ["collection" 1 "collection normal"]]
+               (search-results :official-collection "collection")))))
+    (testing "only if feature is enabled"
+      (mt/with-premium-features #{:semantic-search}
+        (is (= [["collection" 1 "collection normal"]
+                ["collection" 2 "collection official"]]
+               (search-results* "collection")))))))
+
+(deftest verified-test
+  (with-index-contents!
+    [{:model "card" :id 1 :name "card normal" :verified false}
+     {:model "card" :id 2 :name "card verified" :verified true}]
+    (testing "verified items have higher rank"
+      (mt/with-premium-features #{:semantic-search :content-verification}
+        (is (= [["card" 2 "card verified"]
+                ["card" 1 "card normal"]]
+               (search-results :verified "card")))))
+    (testing "only if feature is enabled"
+      (mt/with-premium-features #{:semantic-search}
+        (is (= [["card" 1 "card normal"]
+                ["card" 2 "card verified"]]
+               (search-results* "card")))))))
+
+;;
+;; appdb-based scorers
+;;
+
 (deftest bookmark-test
   (mt/with-premium-features #{:semantic-search}
     ;; TODO (BOT-360) enable for :mysql

enterprise/backend/test/metabase_enterprise/semantic_search/task/index_cleanup_test.clj

@@ -69,7 +69,7 @@
 
 (deftest stale-index-cleanup-test
   (mt/with-premium-features #{:semantic-search}
-    (let [pgvector semantic.tu/db
+    (let [pgvector (semantic.env/get-pgvector-datasource!)
           index-metadata (semantic.tu/unique-index-metadata)
           retention-hours 24
           old-time (t/minus (t/offset-date-time) (t/hours (inc retention-hours)))
@@ -133,7 +133,7 @@
 
 (deftest tombstone-cleanup-test
   (mt/with-premium-features #{:semantic-search}
-    (let [pgvector semantic.tu/db
+    (let [pgvector (semantic.env/get-pgvector-datasource!)
           index-metadata (semantic.tu/unique-index-metadata)
           retention-hours (semantic.settings/tombstone-retention-hours)
           old-time (t/minus (t/offset-date-time) (t/hours (inc retention-hours)))
@@ -228,7 +228,7 @@
 
 (deftest repair-table-cleanup-test
   (mt/with-premium-features #{:semantic-search}
-    (let [pgvector semantic.tu/db
+    (let [pgvector (semantic.env/get-pgvector-datasource!)
           retention-hours 2
           old-time (t/minus (t/instant) (t/hours (inc retention-hours))) ; 3 hours ago
           recent-time (t/minus (t/instant) (t/hours 1))]                 ; 1 hour ago

enterprise/backend/test/metabase_enterprise/semantic_search/task/metric_collector_test.clj

@@ -67,11 +67,10 @@
 (deftest metric-collector-test
   (mt/with-premium-features #{:semantic-search}
     (mt/with-prometheus-system! [_ system]
-      (let [pgvector       semantic.tu/db
+      (let [pgvector       (semantic.env/get-pgvector-datasource!)
             index-metadata (semantic.tu/unique-index-metadata)
             model semantic.tu/mock-embedding-model]
-        (with-redefs [semantic.env/get-pgvector-datasource! (fn [] pgvector)
-                      semantic.env/get-index-metadata (fn [] index-metadata)
+        (with-redefs [semantic.env/get-index-metadata (fn [] index-metadata)
                       semantic.env/get-configured-embedding-model (fn [] model)]
           (testing "Missing tables are handled gracefully"
             (let [result (try

enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj

@@ -15,13 +15,13 @@
    [metabase-enterprise.semantic-search.indexer :as semantic.indexer]
    [metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]
    [metabase-enterprise.semantic-search.util :as semantic.util]
+   [metabase.search.config :as search.config]
    [metabase.search.ingestion :as search.ingestion]
    [metabase.test :as mt]
    [metabase.util :as u]
    [metabase.util.json :as json]
    [metabase.util.log :as log]
    [next.jdbc :as jdbc]
-   [next.jdbc.protocols :as jdbc.protocols]
    [next.jdbc.result-set :as jdbc.rs])
   (:import
    (clojure.lang IDeref)
@@ -31,7 +31,11 @@
 
 (set! *warn-on-reflection* true)
 
-;; If I won't find any use for following muted code in follow-up tasks I'll delete it -- lbrdnk
+;; Purpose of this fixure is to block running tests if db-url is not set. That's true for enterprise app-db tests in CI.
+(defn once-fixture
+  [f]
+  (when semantic.db.datasource/db-url
+    (f)))
 
 (def default-test-db "my_test_db")
 
@@ -60,54 +64,116 @@
   [db-name & body]
   `(do-with-temp-datasource! ~db-name (fn [] ~@body)))
 
+(defmulti do-with-setup-test-db!
+  "Setup pgvector database for tests."
+  {:arglists '([mode thunk])}
+  (fn [mode _thunk] mode))
+
+(defmethod do-with-setup-test-db! :blank
+  [_mode thunk]
+  (thunk))
+
+(declare mock-embedding-model
+         mock-index-metadata
+         mock-table-suffix)
+
+(defmethod do-with-setup-test-db! :mock-initialized
+  [_mode thunk]
+  (with-redefs [semantic.embedding/get-configured-model        (fn [] mock-embedding-model)
+                semantic.index-metadata/default-index-metadata mock-index-metadata
+                semantic.index/model-table-suffix              mock-table-suffix]
+    (let [pgvector (semantic.env/get-pgvector-datasource!)
+          index-metadata (semantic.env/get-index-metadata)
+          embedding-model (semantic.env/get-configured-embedding-model)]
+      (semantic.pgvector-api/init-semantic-search! pgvector index-metadata embedding-model)
+      (thunk))))
+
+;; TODO: declare with macro (the do- less version) throws weird errors -- investigate!
+(declare do-with-indexable-documents!)
+
+(defmethod do-with-setup-test-db! :mock-gated
+  [mode thunk]
+  ((get-method do-with-setup-test-db! :mock-initialized)
+   mode
+   (fn []
+     (do-with-indexable-documents!
+      (fn []
+        (let [pgvector (semantic.env/get-pgvector-datasource!)
+              index-metadata (semantic.env/get-index-metadata)]
+          (semantic.pgvector-api/gate-updates! pgvector
+                                               index-metadata
+                                               (search.ingestion/searchable-documents))
+          (thunk)))))))
+
+(declare index-all!)
+
+(defmethod do-with-setup-test-db! :mock-indexed
+  [mode thunk]
+  ((get-method do-with-setup-test-db! :mock-gated)
+   mode
+   (fn []
+     (index-all!)
+     (thunk))))
+
+;; Reminder: this can be adjusted so (1) each database is unique and (2) redefs are thread local (latter is not simple
+;; but possible I believe), so we can take advantage of parallel tests.
 (defn do-with-test-db!
   "Impl [[with-test-db]]"
-  [db-name thunk]
+  [{:keys [dbname mode cleanup]
+    :or {dbname default-test-db
+         mode :blank
+         cleanup :before}
+    :as _opts}
+   thunk]
   (with-temp-datasource! "postgres"
     (try
-      (jdbc/execute! (semantic.db.datasource/ensure-initialized-data-source!)
-                     [(str "DROP DATABASE IF EXISTS " db-name " (FORCE)")])
-      (log/fatal "creating database")
-      (jdbc/execute! (semantic.db.datasource/ensure-initialized-data-source!)
-                     [(str "CREATE DATABASE " db-name)])
-      (log/fatal "created database")
+      (when (#{:before :both} cleanup)
+        (jdbc/execute! (semantic.env/get-pgvector-datasource!)
+                       [(str "DROP DATABASE IF EXISTS " dbname " (FORCE)")]))
+      (log/debugf "Creating database %s" dbname)
+      (jdbc/execute! (semantic.env/get-pgvector-datasource!)
+                     [(str "CREATE DATABASE " dbname)])
+      (log/debugf "Created test pgvector database %s" dbname)
       (catch java.sql.SQLException e
-        (log/fatal "creation failed")
+        (log/debugf "Creation of test pgvector database %s failed" dbname)
         (throw e))))
-  (with-temp-datasource! db-name
-    (thunk)))
 
+  (with-temp-datasource! dbname
+    (do-with-setup-test-db! mode thunk))
+
+  (when (#{:after :both} cleanup)
+    (with-temp-datasource! "postgres"
+      (try
+        (jdbc/execute! (semantic.env/get-pgvector-datasource!)
+                       [(str "DROP DATABASE IF EXISTS " dbname " (FORCE)")])
+        (catch java.sql.SQLException e
+          (log/debugf "Test pgvector database teardown %s failed" dbname)
+          (throw e))))))
+
+;; TODO: When we are parallelizing tests, we'd have to make
+;;       - `with-test-db!` use thread-safe version of with-redefs or similar,
+;;       - instead of `default-test-db` we could make unique database per test by means of eg. some counter.
 (defmacro with-test-db!
   "Drop, create database dbname on pgvector and redefine datasource accordingly. Not thread safe."
-  [db-name & body]
-  `(do-with-test-db! ~db-name (fn [] ~@body)))
+  [opts & body]
+  `(do-with-test-db! ~opts (fn [] ~@body)))
 
-(def ^:private init-delay
-  (delay
-    (when-not @semantic.db.datasource/data-source
-      (semantic.db.datasource/init-db!))))
+(defmacro with-test-db-defaults!
+  "Tiny wrapper to avoid at this point redundant {} arg of [[with-test-db!]]."
+  [& body]
+  `(with-test-db! {} ~@body))
 
-(defn once-fixture [f]
-  (when semantic.db.datasource/db-url
-    @init-delay
-    (f)))
+(defmacro with-weights
+  "Execute `body` overriding search weights with `weight-map`."
+  [weight-map & body]
+  `(mt/with-dynamic-fn-redefs [search.config/weights (constantly ~weight-map)]
+     ~@body))
 
-(declare db)
-
-#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
-(defn ensure-no-migration-table-fixture [f]
-  (semantic.db.migration/drop-migration-table! db)
-  (f)
-  (semantic.db.migration/drop-migration-table! db))
-
-(def db
-  "Proxies the semantic.db.datasource/data-source, avoids the deref and prettifies a little"
-  ;; proxy because semantic.db.datasource/data-source is not initialised until the fixture runs
-  (reify jdbc.protocols/Sourceable
-    (get-datasource [_] (jdbc.protocols/get-datasource @semantic.db.datasource/data-source))))
-
-(comment
-  (jdbc/execute! db ["select 1"]))
+(defmacro with-only-semantic-weights
+  "Execute `body` with only the semantic search hybrid scorer weights active."
+  [& body]
+  `(with-weights {:rrf 1}
+     ~@body))
 
 (def mock-embeddings
   "Static mapping from strings to (made-up) 4-dimensional embedding vectors for testing. Each pair of strings represents a
@@ -183,14 +249,16 @@
 (defmethod semantic.embedding/get-embeddings-batch "mock" [_ texts] (get-mock-embeddings-batch texts))
 (defmethod semantic.embedding/pull-model           "mock" [_])
 
+#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
 (defn query-index [search-context]
-  (:results (semantic.index/query-index db mock-index search-context)))
+  (:results (semantic.index/query-index (semantic.env/get-pgvector-datasource!) mock-index search-context)))
 
+;; TODO: this should go!!!
 (defn upsert-index! [documents & {:keys [index] :or {index mock-index} :as opts}]
-  (semantic.index/upsert-index! db index documents opts))
+  (semantic.index/upsert-index! (semantic.env/get-pgvector-datasource!) index documents opts))
 
 (defn delete-from-index! [model ids]
-  (semantic.index/delete-from-index! db mock-index model ids))
+  (semantic.index/delete-from-index! (semantic.env/get-pgvector-datasource!) mock-index model ids))
 
 (defn dog-training-native-query []
   (mt/native-query {:query "SELECT AVG(tricks) FROM dogs WHERE age > 7 GROUP BY breed"}))
@@ -235,74 +303,116 @@
 
 (defn open-temp-index! ^Closeable [& {:keys [index] :or {index mock-index}}]
   (closeable
-   (do (semantic.index/create-index-table-if-not-exists! db index {:force-reset? true})
+   (do (semantic.index/create-index-table-if-not-exists! (semantic.env/get-pgvector-datasource!) index {:force-reset? true})
        index)
    (fn cleanup-temp-index-table! [{:keys [table-name] :as index}]
      (try
-       (semantic.index/drop-index-table! db index)
+       (semantic.index/drop-index-table! (semantic.env/get-pgvector-datasource!) index)
        (catch Exception e
          (log/error e "Warning: failed to clean up test table" table-name))))))
 
-(declare cleanup-index-metadata!)
+(defn do-with-indexable-documents!
+  "Wrap the thunk into with-temp, creating entities used throughout semantic search test.
 
-(defn open-temp-index-and-metadata! ^Closeable []
-  (closeable
-   (do (cleanup-index-metadata! db mock-index-metadata)
-       (semantic.pgvector-api/init-semantic-search! db mock-index-metadata mock-embedding-model)
-       mock-index-metadata)
-   (fn cleanup-temp-index-and-metadata! [index-metadata]
-     (try
-       (cleanup-index-metadata! db index-metadata)
-       (catch Exception e
-         (log/error e "Warning: failed to clean up index and metadata" index-metadata))))))
+  N.B. *disable-updates* is bound to avoid processing of those entities by logic in [[search.ingestion]] ns.
+  The processing is triggered by means :hook/search-index which :model/Card (and others) derive.
+
+  As of 2025-09-10, processing triggered by insertion, combined with manual gating of documents that callers
+  of this fn do, would result in duplicate processing and deletion of those entitities from index
+  due to [[search.ingestion/bulk-ingest!]].
+
+  For details see the https://metaboat.slack.com/archives/C07SJT1P0ET/p1757452434713309?thread_ts=1757410361.879029&cid=C07SJT1P0ET"
+  [thunk]
+  ;; NB: *disable-updates*
+  (binding [search.ingestion/*disable-updates* true]
+    (mt/dataset
+      test-data
+      (mt/with-temp [:model/Collection
+                     {col1 :id}
+                     {:name "Wildlife Collection" :archived false}
+
+                     :model/Collection
+                     {col2 :id}
+                     {:name "Archived Animals" :archived true}
+
+                     :model/Collection
+                     {col3 :id}
+                     {:name "Cryptozoology", :archived false}
+
+                     :model/Card
+                     {card1 :id}
+                     {:name "Dog Training Guide" :collection_id col1 :creator_id (mt/user->id :crowberto)
+                      :archived false :query_type "native" :dataset_query (dog-training-native-query)}
+
+                     :model/Card
+                     _
+                     {:name "Bird Watching Tips" :collection_id col1 :creator_id (mt/user->id :rasta) :archived false}
+
+                     :model/Card
+                     _
+                     {:name "Cat Behavior Study" :collection_id col2 :creator_id (mt/user->id :crowberto) :archived true}
+
+                     :model/Card
+                     _
+                     {:name "Horse Racing Analysis" :collection_id col1 :creator_id (mt/user->id :rasta) :archived false}
+
+                     :model/Card
+                     _
+                     {:name "Fish Tank Setup" :collection_id col2 :creator_id (mt/user->id :crowberto) :archived true}
+
+                     :model/Card
+                     _
+                     {:name "Bigfoot Sightings" :collection_id col3 :creator_id (mt/user->id :crowberto) :archived false}
+
+                     :model/ModerationReview
+                     _
+                     {:moderated_item_type "card"
+                      :moderated_item_id card1
+                      :moderator_id (mt/user->id :crowberto)
+                      :status "verified"
+                      :most_recent true}
+
+                     :model/Dashboard
+                     _
+                     {:name "Elephant Migration" :collection_id col1 :creator_id (mt/user->id :rasta) :archived false}
+
+                     :model/Dashboard
+                     _
+                     {:name "Lion Pride Dynamics" :collection_id col1 :creator_id (mt/user->id :crowberto) :archived false}
+
+                     :model/Dashboard
+                     _
+                     {:name "Penguin Colony Study" :collection_id col2 :creator_id (mt/user->id :rasta) :archived true}
+
+                     :model/Dashboard
+                     _
+                     {:name "Whale Communication" :collection_id col1 :creator_id (mt/user->id :crowberto) :archived false}
+
+                     :model/Dashboard
+                     _
+                     {:name "Tiger Conservation" :collection_id col2 :creator_id (mt/user->id :rasta) :archived true}
+
+                     :model/Dashboard
+                     _
+                     {:name "Loch Ness Stuff" :collection_id col3 :creator_id (mt/user->id :crowberto), :archived false}
+
+                     :model/Database
+                     {db-id :id}
+                     {:name "Animal Database"}
+
+                     :model/Table
+                     _
+                     {:name "Species Table", :db_id db-id}
+
+                     :model/Table
+                     _
+                     {:name "Monsters Table", :db_id db-id, :active true}]
+        (thunk)))))
 
 (defmacro with-indexable-documents!
-  "Add a collection of test documents to that can be indexed to the appdb."
+  "Wrapper for [[do-with-indexable-documents!]]."
   [& body]
-  `(mt/dataset ~(symbol "test-data")
-     (mt/with-temp [:model/Collection       {col1# :id}  {:name "Wildlife Collection" :archived false}
-
-                    :model/Collection       {col2# :id}  {:name "Archived Animals" :archived true}
-
-                    :model/Collection       {col3# :id}  {:name "Cryptozoology", :archived false}
-
-                    :model/Card             {card1# :id} {:name "Dog Training Guide" :collection_id col1# :creator_id (mt/user->id :crowberto) :archived false
-                                                          :query_type "native" :dataset_query (dog-training-native-query)}
-
-                    :model/Card             {}           {:name "Bird Watching Tips" :collection_id col1# :creator_id (mt/user->id :rasta) :archived false}
-
-                    :model/Card             {}           {:name "Cat Behavior Study" :collection_id col2# :creator_id (mt/user->id :crowberto) :archived true}
-
-                    :model/Card             {}           {:name "Horse Racing Analysis" :collection_id col1# :creator_id (mt/user->id :rasta) :archived false}
-
-                    :model/Card             {}           {:name "Fish Tank Setup" :collection_id col2# :creator_id (mt/user->id :crowberto) :archived true}
-
-                    :model/Card             {}           {:name "Bigfoot Sightings" :collection_id col3# :creator_id (mt/user->id :crowberto), :archived false}
-
-                    :model/ModerationReview {}           {:moderated_item_type "card"
-                                                          :moderated_item_id card1#
-                                                          :moderator_id (mt/user->id :crowberto)
-                                                          :status "verified"
-                                                          :most_recent true}
-
-                    :model/Dashboard        {}           {:name "Elephant Migration" :collection_id col1# :creator_id (mt/user->id :rasta) :archived false}
-
-                    :model/Dashboard        {}           {:name "Lion Pride Dynamics" :collection_id col1# :creator_id (mt/user->id :crowberto) :archived false}
-
-                    :model/Dashboard        {}           {:name "Penguin Colony Study" :collection_id col2# :creator_id (mt/user->id :rasta) :archived true}
-
-                    :model/Dashboard        {}           {:name "Whale Communication" :collection_id col1# :creator_id (mt/user->id :crowberto) :archived false}
-
-                    :model/Dashboard        {}           {:name "Tiger Conservation" :collection_id col2# :creator_id (mt/user->id :rasta) :archived true}
-
-                    :model/Dashboard        {}           {:name "Loch Ness Stuff" :collection_id col3# :creator_id (mt/user->id :crowberto), :archived false}
-
-                    :model/Database         {db-id# :id} {:name "Animal Database"}
-
-                    :model/Table            {}           {:name "Species Table", :db_id db-id#}
-
-                    :model/Table            {}           {:name "Monsters Table", :db_id db-id#, :active true}]
-       ~@body)))
+  `(do-with-indexable-documents! (fn [] ~@body)))
 
 (defn index-all!
   "Run indexer synchonously until we've exhausted polling all documents"
@@ -310,44 +420,25 @@
   (let [metadata-row   {:indexer_last_poll Instant/EPOCH
                         :indexer_last_seen Instant/EPOCH}
         indexing-state (semantic.indexer/init-indexing-state metadata-row)
-        step (fn [] (semantic.indexer/indexing-step db mock-index-metadata mock-index indexing-state))]
+        pgvector (semantic.env/get-pgvector-datasource!)
+        step (fn [] (semantic.indexer/indexing-step pgvector mock-index-metadata mock-index indexing-state))]
     (while (do (step) (pos? (:last-novel-count @indexing-state))))))
 
-(defmacro blocking-index!
-  "Execute body ensuring [[index-all!]] is invoked at the end"
-  [& body]
-  `(let [ret# (do ~@body)]
-     (index-all!)
-     ret#))
-
-(defmacro with-index!
-  "Ensure a clean, small index for testing populated with a few collections, cards, and dashboards."
-  [& body]
-  `(with-indexable-documents!
-     (with-redefs [semantic.embedding/get-configured-model        (fn [] mock-embedding-model)
-                   semantic.index-metadata/default-index-metadata mock-index-metadata
-                   semantic.index/model-table-suffix              mock-table-suffix]
-       (with-open [_# (open-temp-index-and-metadata!)]
-         (binding [search.ingestion/*force-sync* true]
-           (blocking-index!
-            (semantic.pgvector-api/gate-updates! (semantic.env/get-pgvector-datasource!)
-                                                 mock-index-metadata
-                                                 (search.ingestion/searchable-documents)))
-           ~@body)))))
-
+#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
 (defn table-exists-in-db?
   "Check if a table actually exists in the database"
   [table-name]
   (when table-name
     (try
-      (semantic.util/table-exists? db (name table-name))
+      (semantic.util/table-exists? (semantic.env/get-pgvector-datasource!) (name table-name))
       (catch Exception _ false))))
 
+#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
 (defn table-has-index?
   [table-name index-name]
   (when table-name
     (try
-      (let [result (jdbc/execute! db
+      (let [result (jdbc/execute! (semantic.env/get-pgvector-datasource!)
                                   ["SELECT EXISTS (SELECT 1 FROM pg_indexes WHERE tablename = ? AND indexname = ?)"
                                    (name table-name)
                                    (name index-name)])]
@@ -422,7 +513,7 @@
 (defn index-count
   "Count the number of documents in the index."
   [index]
-  (let [result (jdbc/execute-one! db
+  (let [result (jdbc/execute-one! (semantic.env/get-pgvector-datasource!)
                                   (-> (sql.helpers/select [:%count.* :count])
                                       (sql.helpers/from (keyword (:table-name index)))
                                       semantic.index/sql-format-quoted)
@@ -434,17 +525,18 @@
   "Query the full index table and return all documents with decoded embeddings.
   Not used in tests, but useful for debugging."
   []
-  (->> (jdbc/execute! db
+  (->> (jdbc/execute! (semantic.env/get-pgvector-datasource!)
                       (-> (sql.helpers/select :model :model_id :content :creator_id :embedding)
                           (sql.helpers/from (keyword (:table-name mock-index)))
                           semantic.index/sql-format-quoted)
                       {:builder-fn jdbc.rs/as-unqualified-lower-maps})
        (mapv decode-embedding)))
 
+#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
 (defn query-embeddings
   "Query the `mock-index` table and return the decoded `:embedding`s for the given `model`"
   [{:keys [model model_id]}]
-  (->> (jdbc/execute! db
+  (->> (jdbc/execute! (semantic.env/get-pgvector-datasource!)
                       (-> (sql.helpers/select :model :model_id :content :creator_id :embedding)
                           (sql.helpers/from (keyword (:table-name mock-index)))
                           (sql.helpers/where :and
@@ -454,10 +546,11 @@
                       {:builder-fn jdbc.rs/as-unqualified-lower-maps})
        (mapv decode-embedding)))
 
+#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
 (defn query-tsvectors
   "Query the `mock-index` table and return the unwrapped tsvector columns for the given `model`"
   [{:keys [model model_id]}]
-  (->> (jdbc/execute! db
+  (->> (jdbc/execute! (semantic.env/get-pgvector-datasource!)
                       (-> (sql.helpers/select :model :model_id :content :creator_id
                                               :text_search_vector :text_search_with_native_query_vector)
                           (sql.helpers/from (keyword (:table-name mock-index)))
@@ -480,10 +573,11 @@
            (query-embeddings {:model "dashboard"
                               :model_id "456"})))))
 
+#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
 (defn check-index-has-no-mock-docs []
   (let [{:keys [table-name]}     mock-index
         table-exists-sql         "select exists(select * from information_schema.tables where table_name = ?) table_exists"
-        [{:keys [table_exists]}] (jdbc/execute! db [table-exists-sql table-name])]
+        [{:keys [table_exists]}] (jdbc/execute! (semantic.env/get-pgvector-datasource!) [table-exists-sql table-name])]
     (when table_exists
       (check-index-has-no-mock-card)
       (check-index-has-no-mock-dashboard))))

enterprise/backend/test/metabase_enterprise/semantic_search/without_init_test.clj

@@ -17,14 +17,15 @@
 (defn- without-init! [test-func]
   (mt/with-premium-features #{:semantic-search}
     (try
-      (semantic.tu/cleanup-index-metadata! semantic.tu/db semantic.tu/mock-index-metadata)
+      (semantic.tu/cleanup-index-metadata! (semantic.env/get-pgvector-datasource!)
+                                           semantic.tu/mock-index-metadata)
       (with-redefs [semantic.db.datasource/data-source          (atom nil)
                     semantic.env/get-index-metadata             (constantly semantic.tu/mock-index-metadata)
                     semantic.env/get-configured-embedding-model (constantly semantic.tu/mock-embedding-model)
                     semantic.index/model-table-suffix           semantic.tu/mock-table-suffix]
         (test-func))
       (finally
-        (semantic.tu/cleanup-index-metadata! semantic.tu/db semantic.tu/mock-index-metadata)))))
+        (semantic.tu/cleanup-index-metadata! (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index-metadata)))))
 
 (deftest update-index!-without-init!-test
   (without-init!

enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj

@@ -963,6 +963,64 @@
                    :content "11 = 11"}
                   (t2/select-one :model/NativeQuerySnippet :entity_id (:entity_id snippet)))))))))
 
+(deftest snippet-template-tags-import-test
+  (testing "Template tags import preserves nil, empty, and populated values"
+
+    (testing "Missing template_tags field -> {} when selected"
+      (mt/with-empty-h2-app-db!
+        (let [snippet-data {:serdes/meta [{:model "NativeQuerySnippet"
+                                           :id    "test-entity-1"
+                                           :label "test_snippet_1"}]
+                            :name        "no-tags-field"
+                            :content     "WHERE id = {{id}}"
+                            :creator_id  "test@example.com"
+                            :entity_id   "test-entity-1"}
+              ingestion    (ingestion-in-memory [snippet-data])]
+          (serdes.load/load-metabase! ingestion)
+          (let [template-tags (t2/select-one-fn :template_tags :model/NativeQuerySnippet :entity_id "test-entity-1")]
+            ;; Toucan hooks compute the template-tags:
+            (is (=? {"id" {:type         :text
+                           :display-name "ID"
+                           :name         "id"}}
+                    template-tags))))))
+
+    (testing "Empty map template_tags -> preserved as empty map"
+      (mt/with-empty-h2-app-db!
+        (let [snippet-data {:serdes/meta   [{:model "NativeQuerySnippet"
+                                             :id    "test-entity-2"
+                                             :label "test_snippet_2"}]
+                            :name          "empty-tags"
+                            :content       "SELECT 1"
+                            :template_tags {}
+                            :creator_id    "test@example.com"
+                            :entity_id     "test-entity-2"}
+              ingestion    (ingestion-in-memory [snippet-data])]
+          (serdes.load/load-metabase! ingestion)
+          (let [template-tags (t2/select-one-fn :template_tags :model/NativeQuerySnippet :entity_id "test-entity-2")]
+            (is (= {} template-tags))))))
+
+    (testing "Snippet template tags get preserved rather than recalculated"
+      (mt/with-empty-h2-app-db!
+        (let [snippet-data {:serdes/meta   [{:model "NativeQuerySnippet"
+                                             :id    "test-entity-3"
+                                             :label "test_snippet_3"}]
+                            :name          "with-tags"
+                            :content       "WHERE id = {{snippet: id}}"
+                            :template_tags {"id" {:type         :snippet
+                                                  :name         "snippet: id"
+                                                  :snippet-name "id"
+                                                  ;; Definitely not calculated that way:
+                                                  :display-name "Snippet: WOOP"}}
+                            :creator_id    "test@example.com"
+                            :entity_id     "test-entity-3"}
+              ingestion    (ingestion-in-memory [snippet-data])]
+          (serdes.load/load-metabase! ingestion)
+          (let [template-tags (t2/select-one-fn :template_tags :model/NativeQuerySnippet :entity_id "test-entity-3")]
+            (is (=? {"snippet: id" {:type         :snippet
+                                    :name         "snippet: id"
+                                    :display-name "Snippet: WOOP"}}
+                    template-tags))))))))
+
 (deftest load-action-test
   (let [serialized (atom nil)
         eid (u/generate-nano-id)]

enterprise/backend/test/metabase_enterprise/transforms/api/transform_job_test.clj

@@ -3,9 +3,13 @@
    [clojure.test :refer :all]
    [metabase-enterprise.transforms.models.transform-job]
    [metabase-enterprise.transforms.models.transform-tag]
+   [metabase-enterprise.transforms.schedule :as transforms.schedule]
+   [metabase-enterprise.transforms.test-util :refer [parse-instant utc-timestamp]]
    [metabase.test :as mt]
    [toucan2.core :as t2]))
 
+(set! *warn-on-reflection* true)
+
 (deftest create-job-test
   (testing "POST /api/ee/transform-job"
     (mt/with-premium-features #{:transforms}
@@ -77,16 +81,58 @@
 (deftest list-jobs-test
   (testing "GET /api/ee/transform-job"
     (mt/with-premium-features #{:transforms}
-      (mt/with-temp [:model/TransformJob job1 {:name "Job 1" :schedule "0 0 0 * * ?"}
-                     :model/TransformJob job2 {:name "Job 2" :schedule "0 0 */4 * * ?"}]
-        (let [response (mt/user-http-request :crowberto :get 200 "ee/transform-job")
-              job-ids  (set (map :id response))]
-          (is (contains? job-ids (:id job1)))
-          (is (contains? job-ids (:id job2)))
-          (is (every? #(or
-                        (and (not= (:id %) (:id job1))
-                             (not= (:id %) (:id job2)))
-                        (nil? (:last_run %))) response)))))))
+      (let [at-5-second-schedule "5 * * * * ?"]
+        (mt/with-temp [:model/TransformTag {t1-id :id} {:name "test-tag"}
+                       :model/TransformTag {t2-id :id} {:name "second test-tag"}
+                       :model/TransformJob {j1-id :id} {:name "Job 1" :schedule at-5-second-schedule}
+                       :model/TransformJob {j2-id :id} {:name "Job 2" :schedule "0 0 0 * * ?"}
+                       :model/TransformJob {j3-id :id} {:name "Job 3" :schedule "0 0 0 * * ?"}
+                       :model/TransformJobTransformTag _ {:job_id j1-id :tag_id t1-id :position 0}
+                       :model/TransformJobTransformTag _ {:job_id j2-id :tag_id t1-id :position 1}
+                       :model/TransformJobTransformTag _ {:job_id j3-id :tag_id t2-id :position 0}
+                       :model/TransformJobRun _ {:job_id j1-id :status "timeout" :run_method "cron"
+                                                 :start_time (parse-instant "2025-08-25T10:12:11")
+                                                 :end_time (parse-instant "2025-08-26T10:52:17")}
+                       :model/TransformJobRun _ {:job_id j2-id :status "started" :run_method "manual"
+                                                 :start_time (parse-instant "2025-08-26T10:12:11")
+                                                 :end_time nil
+                                                 :is_active true}]
+          (let [our-job-ids #{j1-id j2-id j3-id}
+                returned-jobs (fn [response] (into [] (filter (comp our-job-ids :id)) response))
+                returned-job-ids (fn [response] (into #{} (keep (comp our-job-ids :id)) response))]
+            (testing "listing without filtering"
+              (let [response (mt/user-http-request :crowberto :get 200 "ee/transform-job")]
+                (is (= our-job-ids (returned-job-ids response)))))
+            (testing "filtering by transform_tag_ids"
+              (let [response (mt/user-http-request :crowberto :get 200 "ee/transform-job" :transform_tag_ids [t2-id])]
+                (is (=? [{:schedule "0 0 0 * * ?"
+                          :tag_ids [t2-id]
+                          :name "Job 3"
+                          :id j3-id}]
+                        (returned-jobs response)))))
+            (testing "filtering by last_run_start_time"
+              (let [response (mt/user-http-request :crowberto :get 200 "ee/transform-job" :last_run_start_time "2025-08-26~")]
+                (is (= #{j2-id} (returned-job-ids response)))))
+            (testing "filtering by last_run_end_time without scheduled job"
+              (let [response (mt/user-http-request :crowberto :get 200 "ee/transform-job" :next_run_start_time "2025-08-26~")]
+                (is (= #{} (returned-job-ids response)))))
+            (testing "filtering by last_run_end_time with scheduled job"
+              (mt/with-temp-scheduler!
+                (transforms.schedule/initialize-job! {:id j1-id, :schedule at-5-second-schedule})
+                (try
+                  (let [response (mt/user-http-request :crowberto :get 200 "ee/transform-job" :next_run_start_time "2025-08-27~")]
+                    (is (=? [{:id j1-id
+                              :last_run {:job_id j1-id
+                                         :run_method "cron"
+                                         :start_time (utc-timestamp "2025-08-25T10:12:11")
+                                         :end_time (utc-timestamp "2025-08-26T10:52:17")}
+                              :name "Job 1"
+                              :next_run {:start_time #"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ"}
+                              :schedule at-5-second-schedule
+                              :tag_ids [t1-id]}]
+                            (returned-jobs response))))
+                  (finally
+                    (transforms.schedule/delete-job! j1-id)))))))))))
 
 (deftest update-job-test
   (testing "PUT /api/ee/transform-job/:id"

enterprise/backend/test/metabase_enterprise/transforms/api_test.clj

@@ -8,7 +8,7 @@
    [metabase-enterprise.transforms.models.transform-transform-tag]
    [metabase-enterprise.transforms.query-test-util :as query-test-util]
    [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
-   [metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]
+   [metabase-enterprise.transforms.test-util :as transforms.tu :refer [parse-instant with-transform-cleanup! utc-timestamp]]
    [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.driver :as driver]
    [metabase.legacy-mbql.normalize :as mbql.normalize]
@@ -17,11 +17,8 @@
    [metabase.lib.metadata :as lib.metadata]
    [metabase.query-processor :as qp]
    [metabase.test :as mt]
-   [metabase.test.util :as tu]
    [metabase.util :as u]
-   [toucan2.core :as t2])
-  (:import
-   (java.time LocalDateTime ZonedDateTime ZoneId)))
+   [toucan2.core :as t2]))
 
 (set! *warn-on-reflection* true)
 
@@ -363,8 +360,8 @@
                                         :target  (assoc target :database (mt/id))}
                     {transform-id :id} (mt/user-http-request :crowberto :post 200 "ee/transform"
                                                              original)]
-                (do (test-run transform-id)
-                    (wait-for-table table-name 5000))
+                (test-run transform-id)
+                (wait-for-table table-name 5000)
                 (is (true? (driver/table-exists? driver/*driver* (mt/db) target)))
                 (is (= [["Alice" 25] ["Bob" 30]]
                        (transforms.tu/table-rows table-name)))))))))))
@@ -432,20 +429,6 @@
   (let [response (apply mt/user-http-request :crowberto :get 200 "ee/transform/run" filters)]
     (filter our-pred (:data response))))
 
-(defn- parse-timestamp
-  ^ZonedDateTime [timestamp-string]
-  (-> timestamp-string
-      LocalDateTime/parse
-      (.atZone (ZoneId/systemDefault))))
-
-(defn- parse-instant
-  [timestamp-string]
-  (-> timestamp-string parse-timestamp .toInstant))
-
-(defn- zoned-timestamp
-  [timestamp-string]
-  (-> timestamp-string parse-timestamp str))
-
 (deftest get-runs-filter-by-multiple-statuses-test
   (testing "GET /api/ee/transform/run - filter by multiple statuses"
     (mt/with-premium-features #{:transforms}
@@ -457,9 +440,9 @@
                      :model/TransformRun {r1-id  :id} {:transform_id t0-id :status "succeeded" :run_method "manual"
                                                        :start_time (parse-instant "2025-08-26T10:12:11")
                                                        :end_time (parse-instant "2025-08-27T10:52:17")}
-                     :model/TransformRun {_r2-id :id} {:transform_id t1-id :status "succeeded" :run_method "cron"
-                                                       :start_time (parse-instant "2025-08-22T10:12:11")
-                                                       :end_time (parse-instant "2025-08-22T10:12:17")}
+                     :model/TransformRun {r2-id :id} {:transform_id t1-id :status "succeeded" :run_method "cron"
+                                                      :start_time (parse-instant "2025-08-22T10:12:11")
+                                                      :end_time (parse-instant "2025-08-22T10:12:17")}
                      :model/TransformRun {r3-id :id} {:transform_id t1-id :status "succeeded" :run_method "manual"
                                                       :start_time (parse-instant "2025-08-22T23:57:34")
                                                       :end_time (parse-instant "2025-08-23T00:17:41")}
@@ -474,15 +457,15 @@
                                                        :end_time nil :is_active true}]
         (let [our-run-pred (comp #{t0-id t1-id} :transform_id)
               t0-runs [{:id r1-id
-                        :start_time (zoned-timestamp "2025-08-26T10:12:11")
-                        :end_time (zoned-timestamp "2025-08-27T10:52:17")
+                        :start_time (utc-timestamp "2025-08-26T10:12:11")
+                        :end_time (utc-timestamp "2025-08-27T10:52:17")
                         :run_method "manual"
                         :status "succeeded"
                         :transform {:id t0-id}
                         :transform_id t0-id}
                        {:id r0-id
-                        :start_time (zoned-timestamp "2025-08-25T10:12:11")
-                        :end_time (zoned-timestamp "2025-08-26T10:52:17")
+                        :start_time (utc-timestamp "2025-08-25T10:12:11")
+                        :end_time (utc-timestamp "2025-08-26T10:52:17")
                         :run_method "cron"
                         :status "timeout"
                         :transform {:id t0-id}
@@ -494,21 +477,33 @@
               (is (every? #(contains? statuses (:status %)) our-runs))))
           (testing "Filter by 'start_time'"
             (is (=? [{:id r1-id
-                      :start_time (zoned-timestamp "2025-08-26T10:12:11")
-                      :end_time (zoned-timestamp "2025-08-27T10:52:17")
+                      :start_time (utc-timestamp "2025-08-26T10:12:11")
+                      :end_time (utc-timestamp "2025-08-27T10:52:17")
                       :run_method "manual"
                       :status "succeeded"
                       :transform {:id t0-id}
                       :transform_id t0-id}]
                     (transform-runs our-run-pred :start_time "2025-08-26~")))
             (let [our-runs (transform-runs our-run-pred :start_time "~2025-08-25")]
-              (is (= 6 (count our-runs)))
-              (is (every? (comp neg-int? #(compare % "2025-08-26") :start_time) our-runs)))
+              (is (= 6 (count our-runs))))
             (let [our-runs (transform-runs our-run-pred :start_time "2025-08-22~2025-08-23")]
-              (is (= 2 (count our-runs)))
-              (is (every? (every-pred (comp nat-int? #(compare % "2025-08-22") :start_time)
-                                      (comp neg-int? #(compare % "2025-08-24") :start_time))
-                          our-runs))))
+              (is (=? [{:transform {:id t1-id}
+                        :run_method "manual"
+                        :is_active nil
+                        :start_time (utc-timestamp "2025-08-22T23:57:34")
+                        :end_time (utc-timestamp "2025-08-23T00:17:41")
+                        :transform_id t1-id
+                        :status "succeeded"
+                        :id r3-id}
+                       {:transform {:id t1-id}
+                        :run_method "cron"
+                        :is_active nil
+                        :start_time (utc-timestamp "2025-08-22T10:12:11")
+                        :end_time (utc-timestamp "2025-08-22T10:12:17")
+                        :transform_id t1-id
+                        :status "succeeded"
+                        :id r2-id}]
+                      our-runs))))
           (testing "Filter by 'end_time'"
             (is (=? t0-runs
                     (transform-runs our-run-pred :end_time "2025-08-26~")))
@@ -526,8 +521,8 @@
             (is (=? [{:id r3-id
                       :status "succeeded"
                       :run_method "manual"
-                      :start_time (zoned-timestamp "2025-08-22T23:57:34")
-                      :end_time (zoned-timestamp "2025-08-23T00:17:41")
+                      :start_time (utc-timestamp "2025-08-22T23:57:34")
+                      :end_time (utc-timestamp "2025-08-23T00:17:41")
                       :transform {:id t1-id}
                       :transform_id t1-id}]
                     (transform-runs our-run-pred :run_methods ["manual"] :start_time "~2025-08-25" :end_time "~2025-08-23")))))))))

enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj

@@ -5,15 +5,18 @@
    [metabase-enterprise.transforms.execute :as transforms.execute]
    [metabase-enterprise.transforms.query-test-util :as query-test-util]
    [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
-   [metabase-enterprise.transforms.test-util :refer [with-transform-cleanup!]]
+   [metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]
    [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.driver :as driver]
+   [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
    [metabase.lib.core :as lib]
    [metabase.lib.metadata :as lib.metadata]
    [metabase.query-processor :as qp]
+   [metabase.sync.core :as sync]
    [metabase.test :as mt]
    [metabase.test.data.sql :as sql.tx]
-   [toucan2.core :as t2]))
+   [toucan2.core :as t2]
+   [toucan2.util :as u]))
 
 (set! *warn-on-reflection* true)
 
@@ -146,3 +149,193 @@
           (let [table-exists? (driver/table-exists? driver db-id {:schema schema-name :name table-name})]
             (is (some? table-exists?) "Table should exist in the database schema")
             (driver/drop-table! driver db-id (:name table-schema))))))))
+
+(deftest transform-schema-created-if-needed-test
+  (mt/test-drivers (mt/normal-driver-select {:+features [:transforms/table :schemas]})
+    (mt/dataset transforms-dataset/transforms-test
+      (with-transform-cleanup! [target-table {:type   "table"
+                                              :schema (str "transform_schema_" (mt/random-name))
+                                              :name   "widget_products"}]
+        (let [mp (mt/metadata-provider)
+              transforms-products (lib.metadata/table mp (mt/id :transforms_products))
+              products-category (lib.metadata/field mp (mt/id :transforms_products :category))
+              products-id (lib.metadata/field mp (mt/id :transforms_products :id))
+              query (-> (lib/query mp transforms-products)
+                        (lib/filter (lib/= products-category "Widget"))
+                        (lib/order-by products-id :asc))]
+          (mt/with-temp [:model/Transform transform {:name   "transform"
+                                                     :source {:type  :query
+                                                              :query query}
+                                                     :target target-table}]
+            (transforms.execute/run-mbql-transform! transform {:run-method :manual})
+            (let [table-result      (wait-for-table (:name target-table) 10000)
+                  query-result (->> (lib/query mp table-result)
+                                    (qp/process-query)
+                                    (mt/formatted-rows [int str str 2.0 str])
+                                    (sort-by first <))]
+              (is (= [[1 "Widget A" "Widget" 19.99 "2024-01-01T10:00:00Z"]
+                      [7 "Widget B" "Widget" 24.99 "2024-01-07T10:00:00Z"]
+                      [9 "Widget C" "Widget" 14.99 "2024-01-09T10:00:00Z"]
+                      [10 "Widget D" "Widget" 34.99 "2024-01-10T10:00:00Z"]
+                      [15 "Widget E" "Widget" 44.99 "2024-01-15T10:00:00Z"]]
+                     query-result)))))))))
+
+;; TODO(rileythomp, 2025-08-28): Make this test driver agnostic
+(deftest no-create-schema-permissions-test
+  (mt/test-driver :postgres
+    (mt/dataset transforms-dataset/transforms-test
+      (let [details (:details (mt/db))
+            password (:password details)
+            no-schema-user (u/lower-case-en (mt/random-name))
+            spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))]
+        (try
+          (driver/execute-raw-queries! driver/*driver* spec
+                                       [[(format "CREATE ROLE %s WITH LOGIN PASSWORD '%s';" no-schema-user password)]
+                                        [(format "REVOKE CREATE ON DATABASE \"%s\" FROM %s;" (:db details) no-schema-user)]
+                                        [(format "GRANT USAGE ON SCHEMA public TO %s;" no-schema-user)]
+                                        [(format "GRANT SELECT ON ALL TABLES IN SCHEMA public TO %s;" no-schema-user)]
+                                        [(format "GRANT CREATE ON SCHEMA public TO %s;" no-schema-user)]])
+          (mt/with-temp [:model/Database db {:engine :postgres
+                                             :details (assoc details
+                                                             :user no-schema-user)}]
+            (mt/with-db db
+              (sync/sync-database! db {:scan :schema})
+              (let [mp (mt/metadata-provider)
+                    transforms-products (lib.metadata/table mp (mt/id :transforms_products))
+                    products-category (lib.metadata/field mp (mt/id :transforms_products :category))
+                    products-id (lib.metadata/field mp (mt/id :transforms_products :id))
+                    query (-> (lib/query mp transforms-products)
+                              (lib/filter (lib/= products-category "Widget"))
+                              (lib/order-by products-id :asc))]
+                (testing "user without create schema permissions should be able to create tables in existing schema"
+                  (with-transform-cleanup! [target-table {:type   "table"
+                                                          :schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))
+                                                          :name   "widget_products"}]
+                    (mt/with-temp [:model/Transform transform {:name   "transform"
+                                                               :source {:type  :query
+                                                                        :query query}
+                                                               :target target-table}]
+                      (transforms.execute/run-mbql-transform! transform {:run-method :manual})
+                      (let [table-result      (wait-for-table (:name target-table) 10000)
+                            query-result (->> (lib/query mp table-result)
+                                              (qp/process-query)
+                                              (mt/formatted-rows [int str str 2.0 str])
+                                              (sort-by first <))]
+                        (is (= [[1 "Widget A" "Widget" 19.99 "2024-01-01T10:00:00Z"]
+                                [7 "Widget B" "Widget" 24.99 "2024-01-07T10:00:00Z"]
+                                [9 "Widget C" "Widget" 14.99 "2024-01-09T10:00:00Z"]
+                                [10 "Widget D" "Widget" 34.99 "2024-01-10T10:00:00Z"]
+                                [15 "Widget E" "Widget" 44.99 "2024-01-15T10:00:00Z"]]
+                               query-result))))))
+                (testing "user without create schema permissions should not be able to create a new schema"
+                  (with-transform-cleanup! [target-table {:type   "table"
+                                                          :schema (str "transform_schema_" (mt/random-name))
+                                                          :name   "widget_products"}]
+                    (mt/with-temp [:model/Transform transform {:name   "transform"
+                                                               :source {:type  :query
+                                                                        :query query}
+                                                               :target target-table}]
+
+                      (is (thrown-with-msg?
+                           clojure.lang.ExceptionInfo
+                           #"ERROR: permission denied for database transforms-test"
+                           (transforms.execute/run-mbql-transform! transform {:run-method :manual})))))))))
+          (finally
+            (driver/execute-raw-queries! driver/*driver* spec
+                                         [[(format "DROP OWNED BY %s;" no-schema-user)]
+                                          [(format "DROP ROLE IF EXISTS %s;" no-schema-user)]])))))))
+
+(deftest atomic-python-transform-swap-test
+  (testing "Python transform execution with atomic table swap"
+    (mt/test-drivers #{:h2 :postgres}
+      (mt/with-premium-features #{:transforms}
+        (mt/dataset transforms-dataset/transforms-test
+          (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
+            (with-transform-cleanup! [{table-name :name :as target} {:type   "table"
+                                                                     :schema schema
+                                                                     :name   "atomic_python_swap"}]
+              (let [initial-transform {:name   "Python Transform Initial"
+                                       :source {:type  "python"
+                                                :source-tables {}
+                                                :body  (str "import pandas as pd\n"
+                                                            "\n"
+                                                            "def transform():\n"
+                                                            "    return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}
+                                       :target (assoc target :database (mt/id))}]
+                (mt/with-temp [:model/Transform transform initial-transform]
+                  (transforms.execute/execute-python-transform! transform {:run-method :manual})
+                  (wait-for-table table-name 10000)
+
+                  (let [initial-rows (transforms.tu/table-rows table-name)]
+                    (is (= [["Alice" 25] ["Bob" 30]] initial-rows) "Initial data should be Alice and Bob")
+
+                    (t2/update! :model/Transform (:id transform)
+                                {:source {:type "python"
+                                          :source-tables {}
+                                          :body (str "import pandas as pd\n"
+                                                     "\n"
+                                                     "def transform():\n"
+                                                     "    return pd.DataFrame({'name': ['Charlie', 'Diana', 'Eve'], 'age': [35, 40, 45]})")}}))
+
+                  ;; Test atomic swap by intercepting drop operation
+                  (let [drop-latch (java.util.concurrent.CountDownLatch. 1)
+                        original-drop-table! transforms.util/drop-table!]
+                    (with-redefs [transforms.util/drop-table! (fn [driver db-id table-name]
+                                                                ;; Wait for latch before proceeding with drop
+                                                                (.await drop-latch)
+                                                                (original-drop-table! driver db-id table-name))]
+                      ;; Start transform in background
+                      (let [transform-future (future
+                                               (transforms.execute/execute-python-transform!
+                                                (t2/select-one :model/Transform (:id transform))
+                                                {:run-method :manual}))]
+                        ;; Give transform time to create temp table and reach the drop operation
+                        (Thread/sleep 1000)
+                        ;; Verify old table is still accessible during transform
+                        (is (= [["Alice" 25] ["Bob" 30]]
+                               (transforms.tu/table-rows table-name))
+                            "Original data should still be accessible during transform")
+                        ;; Release the latch to allow drop and complete transform
+                        (.countDown drop-latch)
+                        ;; Wait for transform to complete
+                        @transform-future
+                        ;; Verify final state
+                        (is (= [["Charlie" 35] ["Diana" 40] ["Eve" 45]]
+                               (transforms.tu/table-rows table-name))
+                            "Table should now contain Charlie, Diana, and Eve")))))))))))))
+
+(deftest python-transform-temp-table-cleanup-test
+  (testing "Python transform cleans up temp tables on success"
+    (mt/test-drivers #{:h2 :postgres}
+      (mt/with-premium-features #{:transforms}
+        (mt/dataset transforms-dataset/transforms-test
+          (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
+            (with-transform-cleanup! [{table-name :name :as target} {:type   "table"
+                                                                     :schema schema
+                                                                     :name   "python_cleanup_test"}]
+              (let [transform-def {:name   "Python Transform Cleanup"
+                                   :source {:type  "python"
+                                            :source-tables {}
+                                            :body  (str "import pandas as pd\n"
+                                                        "\n"
+                                                        "def transform():\n"
+                                                        "    return pd.DataFrame({'col1': [1, 2, 3], 'col2': ['a', 'b', 'c']})")}
+                                   :target (assoc target :database (mt/id))}]
+                (mt/with-temp [:model/Transform transform transform-def]
+                  ;; Run initial transform
+                  (transforms.execute/execute-python-transform! transform {:run-method :manual})
+                  (wait-for-table table-name 10000)
+
+                  ;; Run again to trigger temp table creation and swap
+                  (transforms.execute/execute-python-transform! transform {:run-method :manual})
+
+                  ;; Check that no temp tables remain
+                  (let [db-id (mt/id)
+                        tables (t2/select :model/Table :db_id db-id :active true)
+                        temp-table-pattern (re-pattern (str ".*" table-name "_temp_.*"))]
+                    (is (not-any? #(re-matches temp-table-pattern (:name %)) tables)
+                        "No temp tables should remain after successful Python transform")
+
+                    ;; Verify the final data is correct
+                    (is (= [[1 "a"] [2 "b"] [3 "c"]] (transforms.tu/table-rows table-name))
+                        "Table should contain the expected data after swap")))))))))))

enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj

@@ -1,25 +1,46 @@
 (ns ^:mb/driver-tests metabase-enterprise.transforms.ordering-test
   (:require
    [clojure.test :refer :all]
+   [metabase-enterprise.transforms.execute :as transforms.execute]
    [metabase-enterprise.transforms.ordering :as ordering]
+   [metabase.driver :as driver]
+   [metabase.driver.sql :as driver.sql]
    [metabase.query-processor.compile :as qp.compile]
    [metabase.test :as mt]
+   [metabase.test.data.sql :as sql.tx]
    [toucan2.core :as t2]))
 
-(defn- make-transform [{:keys [query name]}]
-  {:source {:type "query",
-            :query query}
-   :name "transform1"
-   :target {:schema "public"
-            :name (or name "orders_2")
-            :type "table"}})
+(defn- make-transform [query & [name schema]]
+  (let [name (or name (mt/random-name))
+        default-schema (some-> driver/*driver* driver.sql/default-schema)
+        schema (or schema default-schema "public")]
+    {:source {:type :query
+              :query query}
+     :name (str "transform_" name)
+     :target {:schema schema
+              :name name
+              :type :table}}))
+
+(defn- make-python-transform [source-tables & [name schema target-db]]
+  (let [name (or name (mt/random-name))
+        default-schema (some-> driver/*driver* driver.sql/default-schema)
+        schema (or schema default-schema "public")
+        target-db (or target-db (mt/id))]
+    {:source {:type "python"
+              :source-tables source-tables
+              :body "# Python code here\nresult = df"}
+     :name (str "python_transform_" name)
+     :target {:database target-db
+              :schema schema
+              :name name
+              :type "table"}}))
 
 (deftest basic-ordering-test
   (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
     (mt/with-temp [:model/Transform {t1 :id} (make-transform
-                                              {:query {:database (mt/id),
-                                                       :type "query",
-                                                       :query {:source-table (mt/id :orders)}}})]
+                                              {:database (mt/id),
+                                               :type "query",
+                                               :query {:source-table (mt/id :orders)}})]
       (is (= {t1 #{}}
              (ordering/transform-ordering (t2/select :model/Transform :id t1)))))))
 
@@ -30,14 +51,15 @@
                    :model/Field _ {:table_id table
                                    :name "foo"}
                    :model/Transform {parent :id} (make-transform
-                                                  {:query {:database (mt/id),
-                                                           :type "query",
-                                                           :query {:source-table (mt/id :orders)}}})
+                                                  {:database (mt/id),
+                                                   :type "query",
+                                                   :query {:source-table (mt/id :orders)}}
+                                                  "orders_2")
                    :model/Transform {child :id} (make-transform
-                                                 {:query {:database (mt/id)
-                                                          :type "query"
-                                                          :query {:source-table table}}
-                                                  :name "orders_3"})]
+                                                 {:database (mt/id)
+                                                  :type "query"
+                                                  :query {:source-table table}}
+                                                 "orders_3")]
       (is (= {parent #{}
               child #{parent}}
              (ordering/transform-ordering (t2/select :model/Transform :id [:in [parent child]])))))))
@@ -46,36 +68,134 @@
   (mt/with-metadata-provider (mt/id)
     (#'ordering/transform-deps transform)))
 
+(deftest not-run-transform-dependency-ordering-test
+  (mt/test-driver :postgres
+    (testing "dependencies are correctly identified when no transforms have been run yet"
+      (mt/with-temp [:model/Transform {t1 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type :native,
+                                                 :native {:query "SELECT * FROM orders LIMIT 100"}}
+                                                "orders_transform")
+                     :model/Transform {t2 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type :native,
+                                                 :native {:query "SELECT * FROM products LIMIT 100"}}
+                                                "products_transform")
+                     :model/Transform {t3 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type :native,
+                                                 :native {:query "SELECT * FROM orders_transform ot
+                                                                  LEFT JOIN products p
+                                                                  ON ot.product_id = p.id
+                                                                  LIMIT 100"}}
+                                                "orders_transform_products")
+                     :model/Transform {t4 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type :native,
+                                                 :native {:query "SELECT * FROM orders_transform ot
+                                                                  LEFT JOIN products_transform pt
+                                                                  ON ot.product_id = pt.id
+                                                                  LIMIT 100"}}
+                                                "orders_transform_products_transform")]
+        (is (= #{{:transform t1} {:transform t2}}
+               (transform-deps-for-db (t2/select-one :model/Transform  t4))))
+        (is (= {t1 #{}
+                t2 #{}
+                t3 #{t1}
+                t4 #{t1 t2}}
+               (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3 t4]]))))))
+    (testing "dependencies are correctly identified when some transform have been run and some haven't"
+      (mt/with-temp [:model/Transform {t1 :id :as transform1} (make-transform
+                                                               {:database (mt/id),
+                                                                :type :native,
+                                                                :native {:query "SELECT * FROM checkins LIMIT 100"}}
+                                                               "checkins_transform")
+                     :model/Transform {t2 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type :native,
+                                                 :native {:query "SELECT * FROM venues LIMIT 100"}}
+                                                "venues_transform")
+                     :model/Transform {t3 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type :native,
+                                                 :native {:query "SELECT * FROM venues_transform vt
+                                                                  LEFT JOIN checkins_transform ct
+                                                                  ON vt.id = ct.venue_id
+                                                                  LIMIT 100"}}
+                                                "venues_transform_2")]
+        (try
+          (transforms.execute/run-mbql-transform! transform1 {:run-method :manual})
+          (let [table1 (t2/select-one-pk :model/Table :name "checkins_transform")]
+            (is (= #{{:table table1} {:transform t2}}
+                   (transform-deps-for-db (t2/select-one :model/Transform  t3)))))
+          (is (= {t1 #{}
+                  t2 #{}
+                  t3 #{t1 t2}}
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]]))))
+          (finally
+            (t2/delete! :model/Table :name "checkins_transform")))))))
+
+(deftest bigquery-native-transform-dependency-ordering-test
+  (mt/test-driver :bigquery-cloud-sdk
+    (let [[dataset] (sql.tx/qualified-name-components :bigquery-cloud-sdk nil)]
+      (testing "dependencies are correctly identified when some transform have been run and some haven't"
+        (mt/with-temp [:model/Transform {t1 :id} (make-transform
+                                                  {:database (mt/id),
+                                                   :type :native,
+                                                   :native {:query (format "SELECT * FROM `%s.checkins` LIMIT 100" dataset)}}
+                                                  "checkins_transform" dataset)
+                       :model/Table {table1 :id} {:schema dataset
+                                                  :name "checkins_transform"}
+                       :model/Transform {t2 :id} (make-transform
+                                                  {:database (mt/id),
+                                                   :type :native,
+                                                   :native {:query (format "SELECT * FROM `%s.venues` LIMIT 100" dataset)}}
+                                                  "venues_transform" dataset)
+                       :model/Transform {t3 :id} (make-transform
+                                                  {:database (mt/id),
+                                                   :type :native,
+                                                   :native {:query (format "SELECT * FROM `%s.venues_transform` vt
+                                                                            LEFT JOIN `%s.checkins_transform` ct
+                                                                            ON vt.id = ct.venue_id
+                                                                            LIMIT 100" dataset dataset)}}
+                                                  "venues_transform_2" dataset)]
+          (is (= #{{:table table1} {:transform t2}}
+                 (transform-deps-for-db (t2/select-one :model/Transform  t3))))
+          (is (= {t1 #{}
+                  t2 #{}
+                  t3 #{t1 t2}}
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]])))))))))
+
 (deftest basic-dependencies-test
   (mt/with-temp [:model/Transform {t1 :id} (make-transform
-                                            {:query {:database (mt/id),
-                                                     :type "query",
-                                                     :query {:source-table (mt/id :orders)}}})]
-    (is (= #{(mt/id :orders)}
+                                            {:database (mt/id),
+                                             :type "query",
+                                             :query {:source-table (mt/id :orders)}})]
+    (is (= #{{:table (mt/id :orders)}}
            (transform-deps-for-db (t2/select-one :model/Transform :id t1))))))
 
 (deftest joined-dependencies-test
   (mt/with-temp [:model/Transform {t1 :id} (make-transform
-                                            {:query {:database (mt/id),
-                                                     :type "query",
-                                                     :query {:source-table (mt/id :orders),
-                                                             :joins
-                                                             [{:fields "all",
-                                                               :strategy "left-join",
-                                                               :alias "Products",
-                                                               :condition
-                                                               [:=
-                                                                [:field
-                                                                 (mt/id :orders :product_id)
-                                                                 {:base-type "type/Integer"}]
-                                                                [:field
-                                                                 (mt/id :products :id)
-                                                                 {:base-type "type/Integer",
-                                                                  :join-alias "Products"}]],
-                                                               :source-table (mt/id :products)}]},
-                                                     :parameters []}})]
-    (is (= #{(mt/id :orders)
-             (mt/id :products)}
+                                            {:database (mt/id),
+                                             :type "query",
+                                             :query {:source-table (mt/id :orders),
+                                                     :joins
+                                                     [{:fields "all",
+                                                       :strategy "left-join",
+                                                       :alias "Products",
+                                                       :condition
+                                                       [:=
+                                                        [:field
+                                                         (mt/id :orders :product_id)
+                                                         {:base-type "type/Integer"}]
+                                                        [:field
+                                                         (mt/id :products :id)
+                                                         {:base-type "type/Integer",
+                                                          :join-alias "Products"}]],
+                                                       :source-table (mt/id :products)}]},
+                                             :parameters []})]
+    (is (= #{{:table (mt/id :orders)}
+             {:table (mt/id :products)}}
            (transform-deps-for-db (t2/select-one :model/Transform :id t1))))))
 
 (deftest card-dependencies-test
@@ -99,27 +219,27 @@
                                                                      :source-table (mt/id :products)}]},
                                                            :parameters []}}
                    :model/Transform {t1 :id} (make-transform
-                                              {:query {:database (mt/id),
-                                                       :type "query",
-                                                       :query
-                                                       {:aggregation [["count"]],
-                                                        :breakout [["field" "Products__category" {:base-type "type/Text"}]],
-                                                        :source-table (str "card__" card)},
-                                                       :parameters []}})]
-      (is (= #{(mt/id :orders)
-               (mt/id :products)}
+                                              {:database (mt/id),
+                                               :type "query",
+                                               :query
+                                               {:aggregation [["count"]],
+                                                :breakout [["field" "Products__category" {:base-type "type/Text"}]],
+                                                :source-table (str "card__" card)},
+                                               :parameters []})]
+      (is (= #{{:table (mt/id :orders)}
+               {:table (mt/id :products)}}
              (transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))
 
 (deftest native-dependencies-test
   (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
     (mt/with-temp [:model/Transform {t1 :id} (make-transform
-                                              {:query {:database (mt/id),
-                                                       :type :native
-                                                       :native (qp.compile/compile
-                                                                {:database (mt/id),
-                                                                 :type "query",
-                                                                 :query {:source-table (mt/id :orders)}})}})]
-      (is (= #{(mt/id :orders)}
+                                              {:database (mt/id),
+                                               :type :native
+                                               :native (qp.compile/compile
+                                                        {:database (mt/id),
+                                                         :type "query",
+                                                         :query {:source-table (mt/id :orders)}})})]
+      (is (= #{{:table (mt/id :orders)}}
              (transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))
 
 (deftest native-card-dependencies-test
@@ -145,21 +265,20 @@
                                                                      :source-table (mt/id :products)}]},
                                                            :parameters []}}
                    :model/Transform {t1 :id} (make-transform
-                                              {:query {:database (mt/id),
-                                                       :type :native,
-                                                       :native {:query
-                                                                (mt/native-query-with-card-template-tag
-                                                                 (:engine (mt/db))
-                                                                 (str "#" card)),
-
-                                                                :template-tags
-                                                                {(keyword (str "#" card))
-                                                                 {:type "card"
-                                                                  :name "card"
-                                                                  :display-name "card"
-                                                                  :card-id card}}}}})]
-      (is (= #{(mt/id :orders)
-               (mt/id :products)}
+                                              {:database (mt/id),
+                                               :type :native,
+                                               :native {:query
+                                                        (mt/native-query-with-card-template-tag
+                                                         (:engine (mt/db))
+                                                         (str "#" card)),
+                                                        :template-tags
+                                                        {(keyword (str "#" card))
+                                                         {:type "card"
+                                                          :name "card"
+                                                          :display-name "card"
+                                                          :card-id card}}}})]
+      (is (= #{{:table (mt/id :orders)}
+               {:table (mt/id :products)}}
              (transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))
 
 (defn- rotations
@@ -181,3 +300,173 @@
         cycles (rotations [3 4 6 7])]
     (is (contains? cycles
                    (ordering/find-cycle test-graph)))))
+
+(deftest get-transform-cycle-test
+  (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+    (testing "cycle is detected in transform referencing itself"
+      (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                :name "table_1"}
+                     :model/Field _ {:table_id table1
+                                     :name "foo"}
+                     :model/Transform {t1 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type "query",
+                                                 :query {:source-table table1}}
+                                                "table_1")]
+        (is (= {:cycle-str "transform_table_1"
+                :cycle [t1]}
+               (ordering/get-transform-cycle (t2/select-one :model/Transform :id t1))))))
+    (testing "cycle is caught in 2 transforms referencing each other"
+      (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                :name "table_1"}
+                     :model/Field _ {:table_id table1
+                                     :name "foo"}
+                     :model/Table {table2 :id} {:schema "public"
+                                                :name "table_2"}
+                     :model/Field _ {:table_id table2
+                                     :name "foo"}
+                     :model/Transform {t1 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type "query",
+                                                 :query {:source-table table1}}
+                                                "table_2")
+                     :model/Transform {t2 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type "query",
+                                                 :query {:source-table table2}}
+                                                "table_1")]
+        (is (= {:cycle-str "transform_table_2 -> transform_table_1",
+                :cycle [t1 t2]}
+               (ordering/get-transform-cycle (t2/select-one :model/Transform :id t1))))))
+    (testing "cycle is detected in 3 transforms referencing each other"
+      (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                :name "table_1"}
+                     :model/Field _ {:table_id table1
+                                     :name "foo"}
+                     :model/Table {table2 :id} {:schema "public"
+                                                :name "table_2"}
+                     :model/Field _ {:table_id table2
+                                     :name "foo"}
+                     :model/Table {table3 :id} {:schema "public"
+                                                :name "table_3"}
+                     :model/Field _ {:table_id table3
+                                     :name "foo"}
+                     :model/Transform {t1 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type "query",
+                                                 :query {:source-table table1}}
+                                                "table_2")
+                     :model/Transform {t2 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type "query",
+                                                 :query {:source-table table2}}
+                                                "table_3")
+                     :model/Transform {t3 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type "query",
+                                                 :query {:source-table table3}}
+                                                "table_1")]
+        (is (= {:cycle-str "transform_table_2 -> transform_table_1 -> transform_table_3",
+                :cycle [t1 t3 t2]}
+               (ordering/get-transform-cycle (t2/select-one :model/Transform :id t1)))))))
+
+  (deftest python-transform-basic-dependencies-test
+    (testing "Python transforms with source-tables dependencies are extracted correctly"
+      (mt/with-temp [:model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)
+                                                                       "products" (mt/id :products)})]
+        (is (= #{{:table (mt/id :orders)}
+                 {:table (mt/id :products)}}
+               (transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))
+
+  (deftest python-transform-ordering-test
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+      (testing "Python transforms are ordered correctly based on source-tables dependencies"
+        (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                  :name "output_1"}
+                       :model/Field _ {:table_id table1
+                                       :name "foo"}
+                       :model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)} "output_1")
+                       :model/Transform {t2 :id} (make-python-transform {"output_1" table1} "output_2")]
+          (is (= {t1 #{}
+                  t2 #{t1}}
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2]]))))))))
+
+  (deftest python-transform-multiple-dependencies-test
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+      (testing "Python transforms with multiple table dependencies"
+        (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                  :name "output_1"}
+                       :model/Field _ {:table_id table1
+                                       :name "foo"}
+                       :model/Table {table2 :id} {:schema "public"
+                                                  :name "output_2"}
+                       :model/Field _ {:table_id table2
+                                       :name "bar"}
+                       :model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)} "output_1")
+                       :model/Transform {t2 :id} (make-python-transform {"products" (mt/id :products)} "output_2")
+                       :model/Transform {t3 :id} (make-python-transform {"output_1" table1
+                                                                         "output_2" table2} "final_output")]
+          (is (= {t1 #{}
+                  t2 #{}
+                  t3 #{t1 t2}}
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]]))))))))
+
+  (deftest mixed-transform-ordering-test
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+      (testing "Python and query transforms are ordered together correctly"
+        (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                  :name "sql_output"}
+                       :model/Field _ {:table_id table1
+                                       :name "foo"}
+                       :model/Table {table2 :id} {:schema "public"
+                                                  :name "python_output"}
+                       :model/Field _ {:table_id table2
+                                       :name "bar"}
+                       ;; SQL transform that depends on orders table
+                       :model/Transform {t1 :id} (make-transform
+                                                  {:database (mt/id),
+                                                   :type "query",
+                                                   :query {:source-table (mt/id :orders)}}
+                                                  "sql_output")
+                       ;; Python transform that depends on the SQL transform's output
+                       :model/Transform {t2 :id} (make-python-transform {"sql_output" table1} "python_output")
+                       ;; Another SQL transform that depends on the Python transform's output
+                       :model/Transform {t3 :id} (make-transform
+                                                  {:database (mt/id)
+                                                   :type "query"
+                                                   :query {:source-table table2}}
+                                                  "final_output")]
+          (is (= {t1 #{}
+                  t2 #{t1}
+                  t3 #{t2}}
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]])))))))))
+
+#_(deftest mixed-database-transform-ordering-test
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+      (testing "Python and query transforms are ordered together correctly, across multiple databases"
+      ;; orders (db-1) -> sql_output (db-1) -> python_output (db-2) -> final_output (db-2)
+        (mt/with-temp [:model/Database db-2    {}
+                       :model/Table    table-1 {:db_id (mt/id),    :schema "public", :name "sql_output"}
+                       :model/Table    table-2 {:db_id (:id db-2), :schema "public", :name "python_output"}
+                       :model/Field    _       {:table_id (:id table-1), :name "foo"}
+                       :model/Field    _       {:table_id (:id table-2), :name "bar"}
+
+                     ;; SQL transform that depends on orders table
+                       :model/Transform {t1 :id} (make-transform
+                                                  {:database (mt/id)
+                                                   :type     "query"
+                                                   :query    {:source-table (mt/id :orders)}}
+                                                  (:name table-1))
+                     ;; Python transform that depends on the SQL transform's output
+                       :model/Transform {t2 :id} (make-python-transform {"sql_output" (:id table-1)} (:name table-2))
+                     ;; Another SQL transform that depends on the Python transform's output
+                       :model/Transform {t3 :id} (make-transform
+                                                  {:database (:id db-2)
+                                                   :type     "query"
+                                                   :query    {:source-table (:id table-2)}}
+                                                  "final_output")]
+          (is (= {t1 #{}
+                  t2 #{t1}
+                  t3 #{t2}}
+
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]]))))))))

enterprise/backend/test/metabase_enterprise/transforms/python_runner_test.clj

@@ -8,15 +8,16 @@
    [java-time.api :as t]
    [metabase-enterprise.transforms.python-runner :as python-runner]
    [metabase-enterprise.transforms.schedule :as transforms.schedule]
+   [metabase-enterprise.transforms.settings :as transforms.settings]
    [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
    [metabase-enterprise.transforms.test-util :as transforms.tu]
    [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.driver :as driver]
-   [metabase-enterprise.transforms.settings :as transforms.settings]
    [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
    [metabase.sync.core :as sync]
    [metabase.task.core :as task]
    [metabase.test :as mt]
+   [metabase.test.data.interface :as tx]
    [metabase.test.data.sql :as sql.tx]
    [metabase.util :as u]
    [metabase.util.json :as json]
@@ -65,12 +66,13 @@
                                                                         :run-id         test-id
                                                                         :table-name->id table-name->id
                                                                         :shared-storage @shared-storage-ref})
-          {:keys [output events]} (python-runner/read-output-objects @shared-storage-ref)]
+          {:keys [output output-manifest events]} (python-runner/read-output-objects @shared-storage-ref)]
       ;; not sure about munging this all together but its what tests expect for now
       (merge (:body response)
-             {:output output
-              :stdout (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
-              :stderr (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))}))))
+             {:output          output
+              :output-manifest output-manifest
+              :stdout          (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
+              :stderr          (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))}))))
 
 (deftest ^:parallel transform-function-basic-test
   (testing "executes transform function and returns CSV output"
@@ -80,7 +82,7 @@
                               "    return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")
           result         (execute {:code transform-code})]
       (is (=? {:output "name,age\nAlice,25\nBob,30\n"
-               :stdout "Successfully saved 2 rows to CSV\nSuccessfully saved output manifest with 2 fields"
+               :stdout "Successfully saved 2 rows to S3\nSuccessfully saved output manifest with 2 fields"
                :stderr ""}
               result)))))
 
@@ -109,7 +111,7 @@
                                                  "    raise ValueError('Something went wrong')")})
           expected-template (str "ERROR: Transform function failed: Something went wrong\n"
                                  "Traceback (most recent call last):\n"
-                                 "  File \"/app/transform_runner.py\", line ___LINE___, in main\n"
+                                 "  File \"/app/external/transform_runner.py\", line ___LINE___, in main\n"
                                  "    result = script.transform()\n"
                                  "             ^^^^^^^^^^^^^^^^^^\n"
                                  "  File \"___PATH___/script.py\", line 2, in transform\n"
@@ -131,7 +133,7 @@
                               "    return pd.DataFrame(data)")
           result         (execute {:code transform-code})]
       (is (=? {:output "x,y,z\n1,10,a\n2,20,b\n3,30,c\n"
-               :stdout "Successfully saved 3 rows to CSV\nSuccessfully saved output manifest with 3 fields"
+               :stdout "Successfully saved 3 rows to S3\nSuccessfully saved output manifest with 3 fields"
                :stderr ""}
               result)))))
 
@@ -145,7 +147,7 @@
                                 "    return pd.DataFrame(data)")
             result         (execute {:code transform-code})]
         (is (=? {:output "name,score\nCharlie,85\nDana,92\n"
-                 :stdout "Successfully saved 2 rows to CSV\nSuccessfully saved output manifest with 2 fields"
+                 :stdout "Successfully saved 2 rows to S3\nSuccessfully saved output manifest with 2 fields"
                  :stderr ""}
                 result))))))
 
@@ -168,53 +170,52 @@
                                        :tables {"students" (mt/id :students)}})]
 
           (is (=? {:output   "id,name,score\n1,Alice,85\n2,Bob,92\n3,Charlie,88\n4,Dana,90\n"
-                   :metadata #(= (json/decode+kw %)
-                                 {:version         "0.1.0"
-                                  :fields          [{:base_type      "Integer",
-                                                     :dtype          "int64",
-                                                     :database_type  "int4",
-                                                     :effective_type "int4",
-                                                     :name           "id",
-                                                     :semantic_type  nil,
-                                                     :root_type      "Integer"}
-                                                    {:base_type      "Text",
-                                                     :dtype          "object",
-                                                     :database_type  "varchar",
-                                                     :effective_type "varchar",
-                                                     :name           "name",
-                                                     :semantic_type  nil,
-                                                     :root_type      "Text"}
-                                                    {:base_type      "Integer",
-                                                     :dtype          "int64",
-                                                     :database_type  "int4",
-                                                     :effective_type "int4",
-                                                     :name           "score",
-                                                     :semantic_type  nil,
-                                                     :root_type      "Integer"}],
-                                  :source_metadata {:fields         [{:base_type      "Integer",
-                                                                      :dtype          "int64",
-                                                                      :database_type  "int4",
-                                                                      :effective_type "int4",
-                                                                      :name           "id",
-                                                                      :semantic_type  nil,
-                                                                      :root_type      "Integer"}
-                                                                     {:base_type      "Text",
-                                                                      :dtype          "object",
-                                                                      :database_type  "varchar",
-                                                                      :effective_type "varchar",
-                                                                      :name           "name",
-                                                                      :semantic_type  nil,
-                                                                      :root_type      "Text"}
-                                                                     {:base_type      "Integer",
-                                                                      :dtype          "int64",
-                                                                      :database_type  "int4",
-                                                                      :effective_type "int4",
-                                                                      :name           "score",
-                                                                      :semantic_type  nil,
-                                                                      :root_type      "Integer"}],
-                                                    :table_metadata {:table_id (mt/id :students)},
-                                                    :version        "0.1.0"}})
-                   :stdout   (str "Successfully saved 4 rows to CSV\n"
+                   :output-manifest {:fields [{:base_type "Integer",
+                                               :database_type "int4",
+                                               :dtype "int64",
+                                               :effective_type "Integer",
+                                               :name "id",
+                                               :root_type "Integer",
+                                               :semantic_type "PK"}
+                                              {:base_type "Text",
+                                               :database_type "varchar",
+                                               :dtype "object",
+                                               :effective_type "Text",
+                                               :name "name",
+                                               :root_type "Text",
+                                               :semantic_type "Name"}
+                                              {:base_type "Integer",
+                                               :database_type "int4",
+                                               :dtype "int64",
+                                               :effective_type "Integer",
+                                               :name "score",
+                                               :root_type "Integer",
+                                               :semantic_type "Score"}],
+                                     :source_metadata {:fields [{:base_type "Integer",
+                                                                 :database_type "int4",
+                                                                 :dtype "int64",
+                                                                 :effective_type "Integer",
+                                                                 :name "id",
+                                                                 :root_type "Integer",
+                                                                 :semantic_type "PK"}
+                                                                {:base_type "Text",
+                                                                 :database_type "varchar",
+                                                                 :dtype "object",
+                                                                 :effective_type "Text",
+                                                                 :name "name",
+                                                                 :root_type "Text",
+                                                                 :semantic_type "Name"}
+                                                                {:base_type "Integer",
+                                                                 :database_type "int4",
+                                                                 :dtype "int64",
+                                                                 :effective_type "Integer",
+                                                                 :name "score",
+                                                                 :root_type "Integer",
+                                                                 :semantic_type "Score"}],
+                                                       :table_metadata {:table_id (mt/malli=? int?)},
+                                                       :version "0.1.0"},
+                                     :version "0.1.0"}
+                   :stdout   (str "Successfully saved 4 rows to S3\n"
                                   "Successfully saved output manifest with 3 fields")
                    :stderr   ""}
                   result)))))))
@@ -244,11 +245,10 @@
                                        :tables {"students" (mt/id :students)}})]
 
           (is (=? {:output   "student_count,average_score\n4,88.75\n"
-                   :metadata #(= (json/decode+kw %)
-                                 {:version        "0.1.0",
-                                  :fields         [{:name "student_count", :dtype "int64"}
-                                                   {:name "average_score", :dtype "float64"}]})
-                   :stdout   (str "Successfully saved 1 rows to CSV\n"
+                   :output-manifest {:version "0.1.0",
+                                     :fields  [{:name "student_count", :dtype "int64"}
+                                               {:name "average_score", :dtype "float64"}]}
+                   :stdout   (str "Successfully saved 1 rows to S3\n"
                                   "Successfully saved output manifest with 2 fields")
                    :stderr   ""}
                   result)))))))
@@ -299,7 +299,7 @@
             [row1 row2 row3] rows
             header-to-index (zipmap headers (range))
             get-col (fn [row col-name] (nth row (header-to-index col-name)))
-            metadata (some-> (:metadata result) json/decode+kw)]
+            metadata (:output-manifest result)]
 
         (is (= (set ["id" "name" "description" "count" "price" "is_active" "created_date" "updated_at" "scheduled_for"])
                (set headers)))
@@ -360,12 +360,78 @@
                                     :timeout 30000})))
                 (is (true? (t2/exists? :model/Table :name (:name target))))))))))))
 
+(deftest transform-function-with-library-test
+  (testing "transform function can use libraries"
+    (mt/test-drivers #{:postgres}
+      (mt/with-temp [:model/PythonLibrary _ {:path   "circle"
+                                             :source "import math\n\ndef calculate_circle_area(radius):\n    return math.pi * radius ** 2"}
+                     :model/PythonLibrary _ {:path   "utils"
+                                             :source "def format_currency(amount):\n    return f\"${amount:,.2f}\""}]
+        (let [transform-code (str "import pandas as pd\n"
+                                  "from circle import calculate_circle_area\n"
+                                  "from utils import format_currency\n"
+                                  "\n"
+                                  "def transform():\n"
+                                  "    data = [\n"
+                                  "        {'radius': 5, 'area': calculate_circle_area(5), 'price': format_currency(78.54)},\n"
+                                  "        {'radius': 10, 'area': calculate_circle_area(10), 'price': format_currency(314.16)}\n"
+                                  "    ]\n"
+                                  "    return pd.DataFrame(data)")
+              result         (execute {:code transform-code})]
+          (is (=? {:output #(and (str/includes? % "radius,area,price")
+                                 (str/includes? % "5,78.5")
+                                 (str/includes? % "$78.54")
+                                 (str/includes? % "10,314.1")
+                                 (str/includes? % "$314.16"))
+                   :stdout "Successfully saved 2 rows to S3\nSuccessfully saved output manifest with 3 fields"
+                   :stderr ""}
+                  result)))))))
+
+(deftest transform-function-without-libraries-test
+  (testing "transform function works when no libraries exist"
+    (mt/test-drivers #{:postgres}
+      (with-redefs [t2/select-fn->fn (fn [k v model]
+                                       (when (and (= k :path)
+                                                  (= v :source)
+                                                  (= model :model/PythonLibrary))
+                                         {}))]
+        (let [transform-code (str "import pandas as pd\n"
+                                  "\n"
+                                  "def transform():\n"
+                                  "    return pd.DataFrame({'status': ['ok']})")
+              result         (execute {:code transform-code})]
+          (is (=? {:output "status\nok\n"
+                   :stdout "Successfully saved 1 rows to CSV\nSuccessfully saved output manifest with 1 fields"
+                   :stderr ""}
+                  result)))))))
+
+(deftest transform-function-library-import-error-test
+  (testing "transform function handles missing library gracefully"
+    (mt/test-drivers #{:postgres}
+      (with-redefs [t2/select-fn->fn (fn [k v model]
+                                       (when (and (= k :path)
+                                                  (= v :source)
+                                                  (= model :model/PythonLibrary))
+                                         {"utils" "def helper():\n    return 42"}))]
+        (let [transform-code (str "import pandas as pd\n"
+                                  "from common import some_function  # This library doesn't exist\n"
+                                  "\n"
+                                  "def transform():\n"
+                                  "    return pd.DataFrame({'value': [some_function()]})")
+              result         (execute {:code transform-code})]
+          (is (=? {:error     "Execution failed"
+                   :exit-code 1
+                   :stderr    #(str/includes? % "No module named 'common'")}
+                  result)))))))
+
 (deftest transform-type-roundtrip-test
-  (mt/test-drivers #{:postgres :h2 :mysql :bigquery-cloud-sdk :redshift :snowflake :sqlserver}
+  (mt/test-drivers #{:postgres :h2 :mysql :bigquery-cloud-sdk :redshift :snowflake :sqlserver :clickhouse}
     (mt/with-empty-db
       (let [driver       driver/*driver*
             db-id        (mt/id)
-            table-name   (mt/random-name)
+            table-name   (if (= driver :redshift)
+                           (tx/db-qualified-table-name (get-in (mt/db) [:settings :database-source-dataset-name]) (mt/random-name))
+                           (mt/random-name))
             schema-name  (sql.tx/session-schema driver)
             qualified-table-name (if schema-name
                                    (keyword schema-name table-name)
@@ -392,14 +458,16 @@
                                           (mapv :name (:columns table-schema))
                                           {:type :rows :data row-values})
 
-            _ (sync/sync-database! (mt/db))
+            _ (sync/sync-database! (mt/db) {:scan :schema})
+
             transform-code (str "import pandas as pd\n"
                                 "\n"
                                 "def transform(" table-name "):\n"
                                 "    return " table-name)
             result (execute {:code transform-code
                              :tables {table-name (mt/id qualified-table-name)}})
-            metadata (some-> (:metadata result) json/decode+kw)]
+
+            metadata (:output-manifest result)]
 
         (testing "All expected columns are present"
           (is (= #{"id" "price" "active" "created_tz" "created_at" "created_date" "description"}
@@ -409,7 +477,7 @@
           (is (= {"id"           :type/Integer
                   "price"        :type/Float
                   "active"       :type/Boolean
-                  "created_tz"   (if (= driver/*driver* :mysql) :type/DateTime :type/DateTimeWithTZ)
+                  "created_tz"   :type/DateTimeWithTZ
                   "created_at"   :type/DateTime
                   "created_date" :type/Date
                   "description"  :type/Text}
@@ -417,4 +485,4 @@
                    [name (transforms.util/dtype->base-type dtype)]))))
 
         ;; cleanup
-        (driver/drop-table! driver db-id (:name table-schema))))))
+        (driver/drop-table! driver db-id qualified-table-name)))))

enterprise/backend/test/metabase_enterprise/transforms/test_util.clj

@@ -5,7 +5,11 @@
    [metabase.driver :as driver]
    [metabase.test :as mt]
    [metabase.util :as u]
-   [toucan2.core :as t2]))
+   [toucan2.core :as t2])
+  (:import
+   (java.time Instant LocalDateTime ZonedDateTime ZoneId)))
+
+(set! *warn-on-reflection* true)
 
 (defn drop-target!
   "Drop transform target `target` and clean up its metadata.
@@ -53,3 +57,20 @@
   (mt/rows (mt/process-query {:database (mt/id)
                               :query    {:source-table (t2/select-one-pk :model/Table :name table-name)}
                               :type     :query})))
+
+(defn parse-timestamp
+  "Parse a local datetime and convert it to a ZonedDateTime in the default timezone."
+  ^ZonedDateTime [timestamp-string]
+  (-> timestamp-string
+      LocalDateTime/parse
+      (.atZone (ZoneId/systemDefault))))
+
+(defn parse-instant
+  "Parse a local datetime and convert it to an Instant in the default timezone."
+  ^Instant [timestamp-string]
+  (-> timestamp-string parse-timestamp .toInstant))
+
+(defn utc-timestamp
+  "Parse a local datetime and convert it to a string encoding a ZonedDateTime in the default timezone."
+  ^String [timestamp-string]
+  (-> timestamp-string parse-instant str))

enterprise/frontend/src/.eslintrc.js

@@ -58,6 +58,10 @@ module.exports = {
             ...baseRestrictredConfig,
             paths: [
               ...baseRestrictredConfig.paths,
+              {
+                name: "zod",
+                message: "Please import from `zod/mini` instead.",
+              },
               {
                 name: "metabase/lib/redux",
                 importNames: ["useStore", "useDispatch"],

enterprise/frontend/src/embedding-sdk-bundle/components/public/CollectionBrowser/CollectionBrowser.schema.ts

@@ -0,0 +1,30 @@
+import {
+  any,
+  function as functionSchema,
+  optional,
+  strictObject,
+} from "zod/mini";
+import type { infer as zInfer } from "zod/v4/core/core";
+
+import type { ValidateInferredSchema } from "embedding-sdk-bundle/types/schema";
+
+import type { CollectionBrowserProps } from "./CollectionBrowser";
+
+const rawPropsSchema = strictObject({
+  EmptyContentComponent: optional(any()),
+  className: optional(any()),
+  collectionId: optional(any()),
+  onClick: optional(any()),
+  pageSize: optional(any()),
+  style: optional(any()),
+  visibleColumns: optional(any()),
+  visibleEntityTypes: optional(any()),
+});
+const propsSchema: ValidateInferredSchema<
+  CollectionBrowserProps,
+  zInfer<typeof rawPropsSchema>
+> = rawPropsSchema;
+
+export const collectionBrowserPropsSchema = functionSchema({
+  input: [propsSchema],
+});

enterprise/frontend/src/embedding-sdk-bundle/components/public/CollectionBrowser/CollectionBrowser.tsx

@@ -22,6 +22,8 @@ import CollectionBreadcrumbs from "metabase/nav/containers/CollectionBreadcrumbs
 import { Stack } from "metabase/ui";
 import type { CollectionId, CollectionItemModel } from "metabase-types/api";
 
+import { collectionBrowserPropsSchema } from "./CollectionBrowser.schema";
+
 const USER_FACING_ENTITY_NAMES = [
   "collection",
   "dashboard",
@@ -200,6 +202,7 @@ const CollectionBrowserWrapper = ({
   return <CollectionBrowserInner collectionId={collectionId} {...restProps} />;
 };
 
-export const CollectionBrowser = withPublicComponentWrapper(
-  CollectionBrowserWrapper,
+export const CollectionBrowser = Object.assign(
+  withPublicComponentWrapper(CollectionBrowserWrapper),
+  { schema: collectionBrowserPropsSchema },
 );

enterprise/frontend/src/embedding-sdk-bundle/components/public/CreateDashboardModal/CreateDashboardModal.schema.ts

@@ -0,0 +1,27 @@
+import {
+  any,
+  function as functionSchema,
+  nonoptional,
+  optional,
+  strictObject,
+} from "zod/mini";
+import type { infer as zInfer } from "zod/v4/core/core";
+
+import type { ValidateInferredSchema } from "embedding-sdk-bundle/types/schema";
+
+import type { CreateDashboardModalProps } from "./CreateDashboardModal";
+
+const rawPropsSchema = strictObject({
+  initialCollectionId: optional(any()),
+  isOpen: optional(any()),
+  onClose: optional(any()),
+  onCreate: nonoptional(any()),
+});
+const propsSchema: ValidateInferredSchema<
+  CreateDashboardModalProps,
+  zInfer<typeof rawPropsSchema>
+> = rawPropsSchema;
+
+export const createDashboardModalSchema = functionSchema({
+  input: [propsSchema],
+});

enterprise/frontend/src/embedding-sdk-bundle/components/public/CreateDashboardModal/CreateDashboardModal.tsx

@@ -8,6 +8,8 @@ import { useCollectionQuery, useLocale } from "metabase/common/hooks";
 import { CreateDashboardModal as CreateDashboardModalCore } from "metabase/dashboard/containers/CreateDashboardModal";
 import { useSelector } from "metabase/lib/redux";
 
+import { createDashboardModalSchema } from "./CreateDashboardModal.schema";
+
 /**
  * @expand
  * @category CreateDashboardModal
@@ -68,6 +70,7 @@ const CreateDashboardModalInner = ({
   );
 };
 
-export const CreateDashboardModal = withPublicComponentWrapper(
-  CreateDashboardModalInner,
+export const CreateDashboardModal = Object.assign(
+  withPublicComponentWrapper(CreateDashboardModalInner),
+  { schema: createDashboardModalSchema },
 );

enterprise/frontend/src/embedding-sdk-bundle/components/public/InteractiveQuestion/InteractiveQuestion.schema.ts

@@ -0,0 +1,56 @@
+import {
+  any,
+  function as functionSchema,
+  nonoptional,
+  optional,
+  strictObject,
+  type infer as zInfer,
+} from "zod/mini";
+
+import type { ValidateInferredSchema } from "embedding-sdk-bundle/types/schema";
+
+import type { InteractiveQuestionProps } from "./InteractiveQuestion";
+
+const rawPropsSchema = strictObject({
+  children: optional(any()),
+  className: optional(any()),
+  componentPlugins: optional(
+    strictObject({
+      mapQuestionClickActions: optional(any()),
+      dashboard: optional(any()),
+    }),
+  ),
+  deserializedCard: optional(any()),
+  entityTypes: optional(any()),
+  height: optional(any()),
+  initialSqlParameters: optional(any()),
+  isSaveEnabled: optional(any()),
+  onBeforeSave: optional(any()),
+  onNavigateBack: optional(any()),
+  onRun: optional(any()),
+  onSave: optional(any()),
+  options: optional(any()),
+  plugins: optional(
+    strictObject({
+      mapQuestionClickActions: optional(any()),
+      dashboard: optional(any()),
+    }),
+  ),
+  questionId: nonoptional(any()),
+  style: optional(any()),
+  targetCollection: optional(any()),
+  targetDashboardId: optional(any()),
+  title: optional(any()),
+  width: optional(any()),
+  withChartTypeSelector: optional(any()),
+  withDownloads: optional(any()),
+  withResetButton: optional(any()),
+});
+const propsSchema: ValidateInferredSchema<
+  InteractiveQuestionProps,
+  zInfer<typeof rawPropsSchema>
+> = rawPropsSchema;
+
+export const interactiveQuestionSchema = functionSchema({
+  input: [propsSchema],
+});

enterprise/frontend/src/embedding-sdk-bundle/components/public/InteractiveQuestion/InteractiveQuestion.tsx

@@ -26,6 +26,8 @@ import {
   type SdkQuestionProps,
 } from "embedding-sdk-bundle/components/public/SdkQuestion/SdkQuestion";
 
+import { interactiveQuestionSchema } from "./InteractiveQuestion.schema";
+
 /**
  * @interface
  * @expand
@@ -100,4 +102,5 @@ const subComponents: InteractiveQuestionComponents = {
 export const InteractiveQuestion = Object.assign(
   _InteractiveQuestion,
   subComponents,
+  { schema: interactiveQuestionSchema },
 );

enterprise/frontend/src/embedding-sdk-bundle/components/public/MetabotQuestion/MetabotChatEmbedding.module.css

@@ -18,7 +18,7 @@
   position: relative;
   max-width: 41.5rem;
   width: 100%;
-  box-shadow: 0 2px 13.8px -1px #1871bf52;
+  border: 1px solid var(--mb-color-brand);
   border-radius: 22px;
 }
 

enterprise/frontend/src/embedding-sdk-bundle/components/public/MetabotQuestion/MetabotChatEmbedding.tsx

@@ -1,5 +1,5 @@
 import cx from "classnames";
-import { useCallback, useEffect, useState } from "react";
+import { useEffect, useState } from "react";
 import { t } from "ttag";
 
 import { useSdkDispatch } from "embedding-sdk-bundle/store";
@@ -12,7 +12,6 @@ import {
   Tooltip,
   UnstyledButton,
 } from "metabase/ui";
-import { MetabotIcon } from "metabase-enterprise/metabot/components/MetabotIcon";
 import {
   useMetabotAgent,
   useMetabotChatHandlers,
@@ -30,10 +29,15 @@ export const MetabotChatEmbedding = () => {
   const metabot = useMetabotAgent();
   const { handleSubmitInput, handleResetInput } = useMetabotChatHandlers();
 
-  const resetInput = useCallback(() => {
+  const resetInput = () => {
     handleResetInput();
     setInputExpanded(false);
-  }, [handleResetInput]);
+  };
+
+  const startNewConversation = () => {
+    resetInput();
+    metabot.resetConversation();
+  };
 
   const [inputExpanded, setInputExpanded] = useState(false);
   const handleMaybeExpandInput = () => {
@@ -66,6 +70,9 @@ export const MetabotChatEmbedding = () => {
     dispatch(resetConversationId());
   }, [dispatch]);
 
+  const isOngoingConversation = metabot.messages.length > 0;
+  const isPromptDefined = metabot.prompt.length > 0;
+
   return (
     <Box className={Styles.container} data-testid="metabot-chat">
       <Flex
@@ -74,20 +81,20 @@ export const MetabotChatEmbedding = () => {
           metabot.isDoingScience && Styles.innerContainerLoading,
           inputExpanded && Styles.innerContainerExpanded,
         )}
-        gap="sm"
       >
         <Flex
-          w="33px"
-          h="24px"
+          w="16px"
+          h="16px"
           style={{
             flexShrink: 0,
           }}
           justify="center"
+          align="center"
         >
           {metabot.isDoingScience ? (
             <Loader size="sm" />
           ) : (
-            <MetabotIcon isLoading={false} />
+            <Icon name="ai" c="var(--mb-color-brand)" size="1rem" />
           )}
         </Flex>
         <Textarea
@@ -123,7 +130,7 @@ export const MetabotChatEmbedding = () => {
             }
           }}
         />
-        {metabot.isDoingScience ? (
+        {metabot.isDoingScience && (
           <UnstyledButton
             h="1rem"
             data-testid="metabot-cancel-request"
@@ -133,17 +140,40 @@ export const MetabotChatEmbedding = () => {
               <Icon name="stop" c="var(--mb-color-text-primary)" size="1rem" />
             </Tooltip>
           </UnstyledButton>
-        ) : (
-          <UnstyledButton
-            h="1rem"
-            onClick={resetInput}
-            data-testid="metabot-close-chat"
-            style={{
-              visibility: metabot.prompt.length > 0 ? "visible" : "hidden",
-            }}
-          >
-            <Icon name="close" c="var(--mb-color-text-primary)" size="1rem" />
-          </UnstyledButton>
+        )}
+
+        {!metabot.isDoingScience && (
+          <>
+            {isOngoingConversation && !isPromptDefined && (
+              <UnstyledButton
+                h="1rem"
+                onClick={startNewConversation}
+                data-testid="metabot-new-conversation"
+              >
+                <Tooltip label={t`Start new chat`}>
+                  <Icon
+                    name="edit_document_outlined"
+                    c="var(--mb-color-text-primary)"
+                    size="1rem"
+                  />
+                </Tooltip>
+              </UnstyledButton>
+            )}
+
+            {isPromptDefined && (
+              <UnstyledButton
+                h="1rem"
+                onClick={resetInput}
+                data-testid="metabot-close-chat"
+              >
+                <Icon
+                  name="close"
+                  c="var(--mb-color-text-primary)"
+                  size="1rem"
+                />
+              </UnstyledButton>
+            )}
+          </>
         )}
       </Flex>
     </Box>

enterprise/frontend/src/embedding-sdk-bundle/components/public/MetabotQuestion/MetabotQuestion.schema.ts

@@ -0,0 +1,5 @@
+import { function as functionSchema } from "zod/mini";
+
+export const metabotQuestionSchema = functionSchema({
+  input: [],
+});

enterprise/frontend/src/embedding-sdk-bundle/components/public/MetabotQuestion/MetabotQuestion.tsx

@@ -19,6 +19,7 @@ import {
 import { useMetabotReactions } from "metabase-enterprise/metabot/hooks/use-metabot-reactions";
 
 import { MetabotChatEmbedding } from "./MetabotChatEmbedding";
+import { metabotQuestionSchema } from "./MetabotQuestion.schema";
 import { QuestionDetails } from "./QuestionDetails";
 import { QuestionTitle } from "./QuestionTitle";
 
@@ -109,6 +110,7 @@ const MetabotQuestionWrapped = () => {
   );
 };
 
-export const MetabotQuestion = withPublicComponentWrapper(
-  MetabotQuestionWrapped,
+export const MetabotQuestion = Object.assign(
+  withPublicComponentWrapper(MetabotQuestionWrapped),
+  { schema: metabotQuestionSchema },
 );