WIP runner

* Make with-index! use new, clean, separate database per test

* Add with-test-db! setup modes

* Comment

* Adjust with-test-db! and related

* Drop with-index! in favor of modal with-test-db! in first test

* semantic.tu/db removal

* Drop use of semantic.tu/db in favor of get-pgvector-datasource!

* Drop semantic.tu/db  from tu

* Remove once-fixture

* Remove ensure-no-migration-table-fixture

* Comment

* Comment

* Cleanup

* Make the rest of the query-test ns use with-test-db!

* Remove redundant test

* Remove remaining usage of with-index!

* Drop remnants of with-index! and make linter happy

* Cleanup

* Comment

* Remove with-mock-index-metadata! in favor of with-temp-db!

* Adjust with-index-contents!

* Fixture to avoid running app-db tests

* Fixture

* Require

* whitespace

.clj-kondo/config.edn

@@ -1,7 +1,7 @@
 ;; -*- comment-column: 100; -*-
 {:config-paths ["macros" "src" "config"]
  :linters
- { ;;
+ {;;
   ;; enabled optional linters (these are disabled by default)
   ;;
 
@@ -186,7 +186,7 @@
   :metabase/disallow-hardcoded-driver-names-in-tests
   ;; don't include `:h2` since that will probably lead to a mountain of tests where we are specifically testing
   ;; middleware or compilation behavior with just H2 since it's the default.
-  {:drivers #{ ;; core drivers
+  {:drivers #{;; core drivers
               #_:h2
               :postgres
               :mysql
@@ -418,6 +418,7 @@
   metabase-enterprise.transforms.api-test/with-transform-tags                           clojure.core/let
   metabase-enterprise.transforms.api-test/with-worker-runs                              clojure.core/let
   metabase-enterprise.transforms.test-util/with-transform-cleanup!                      clojure.core/let
+  metabase-enterprise.transforms.execute/with-transform-lifecycle                       clojure.core/let
   metabase.actions.test-util/with-actions                                               clojure.core/let
   metabase.api.common/let-404                                                           clojure.core/let
   metabase.app-db.custom-migrations.util/with-temp-scheduler!                           clojure.core/fn
@@ -734,7 +735,9 @@
   ;; `metabase-enterprise.sandbox.query-processor*-test`
   ;;
   {:pattern "^metabase(?:((?:-enterprise\\.sandbox)?\\.query-processor)|(?:\\.driver\\.sql(?!server)(?!ite))).*-test$"
-   :name    general-qp-and-driver-test-namespaces}]
+   :name    general-qp-and-driver-test-namespaces}
+  {:pattern "^metabase\\.python-runner\\..*"
+   :name    python-runner-namespaces}]
 
  :config-in-ns
  {test-namespaces

.clj-kondo/config/modules/config.edn

@@ -574,7 +574,6 @@
    :api  #{metabase.legacy-mbql.normalize
            metabase.legacy-mbql.predicates
            metabase.legacy-mbql.schema
-           metabase.legacy-mbql.schema.helpers
            metabase.legacy-mbql.util}
    :uses #{lib
            models
@@ -591,8 +590,7 @@
            metabase.lib.field
            metabase.lib.field.resolution
            metabase.lib.filter
-           metabase.lib.filter.desugar
-           metabase.lib.filter.desugar.jvm
+           metabase.lib.filter.desugar.jvm ; only so [[metabase.legacy-mbql.jvm-util]] can use defined regexes
            metabase.lib.init
            metabase.lib.join.util
            metabase.lib.metadata
@@ -887,6 +885,19 @@
            enterprise/advanced-permissions
            enterprise/sandbox}}
 
+  python-runner
+  {:team "Workflows"
+   :api  #{metabase.python-runner.api
+           metabase.python-runner.core
+           metabase.python-runner.init}
+   :uses #{api
+           driver
+           models
+           query-processor
+           request
+           util
+           enterprise/transforms}}
+
   queries
   {:team "Querying"
    :api  #{metabase.queries.api
@@ -1948,8 +1959,11 @@
   {:team "Querying"
    :api  #{metabase-enterprise.transforms.api
            metabase-enterprise.transforms.core
-           metabase-enterprise.transforms.init}
-   :uses #{api
+           metabase-enterprise.transforms.init
+           ;; TODO: Remove this from API once we remove the python-runner callback to metabase
+           metabase-enterprise.transforms.settings}
+   :uses #{analytics
+           api
            app-db
            config
            driver
@@ -1965,6 +1979,17 @@
            sync
            task
            task-history
+           ;; TODO: Remove this once we're using a better file format than CSV
+           upload
+           util}}
+
+  enterprise/python-transform
+  {:team "Transform"
+   :api  #{metabase-enterprise.python-transform.api
+           metabase-enterprise.python-transform.core
+           metabase-enterprise.python-transform.init}
+   :uses #{api
+           models
            util}}
 
   enterprise/upload-management

.clj-kondo/src/hooks/clojure/core/require.clj

@@ -52,7 +52,13 @@
 
 (defn- lint-require* [node current-ns config]
   (let [[_require & args]   (:children node)
-        required-namespaces (keep unwrap-require args)]
+        required-namespaces (->> (keep unwrap-require args)
+                                 ;; `require` runs through `cc/load-libs` which allows
+                                 ;; #{:as :reload :reload-all :require :use :verbose :refer :as-alias}
+                                 ;; as args. If you use a REPL tool that applies clj-kondo linting (e.g., Clojure MCP)
+                                 ;; you may wish to `:reload` or `:reload-all` a namespace. So we shouldn't assume all
+                                 ;; args are keywords:
+                                 (remove keyword?))]
     (lint* node current-ns config required-namespaces)))
 
 (defn- lint-requiring-resolve* [node current-ns config]

.github/actions/await-port/action.yml

@@ -11,6 +11,18 @@ runs:
     - name: Wait for port ${{ inputs.port }} to be ready
       shell: bash
       run: |
+        echo "Waiting for port ${{ inputs.port }} to be ready..."
+        timeout=300  # 5 minutes timeout
+        elapsed=0
         while ! nc -z localhost ${{ inputs.port }}; do
-          sleep 0.1;
+          if [ $elapsed -ge $timeout ]; then
+            echo "❌ Timeout waiting for port ${{ inputs.port }} after ${timeout} seconds"
+            exit 1
+          fi
+          sleep 1
+          elapsed=$((elapsed + 1))
+          if [ $((elapsed % 10)) -eq 0 ]; then
+            echo "  Still waiting for port ${{ inputs.port }}... ($elapsed seconds)"
+          fi
         done
+        echo "✅ Port ${{ inputs.port }} is ready!"

.github/file-paths.yaml

@@ -41,10 +41,17 @@ frontend_sources: &frontend_sources
   - "enterprise/frontend/!(test)**/**/{!(*.spec),}.{js,jsx,ts,tsx,css}"
   - "frontend/!(test)**/**/{!(*.spec),}.{js,jsx,ts,tsx,css}"
 
+frontend_embedding_sources:
+  - "enterprise/frontend/src/{embedding,embedding-*}/**/{!(*.spec),}.{js,jsx,ts,tsx,css}"
+  - "enterprise/frontend/src/metabase-enterprise/{embedding,embedding-*}**/{!(*.spec),}.{js,jsx,ts,tsx,css}"
+
 frontend_embedding_sdk_sources:
   - *frontend_configs
   - "enterprise/frontend/!(test)**/**/{!(*.spec),}.{js,jsx,ts,tsx,css}"
 
+frontend_embedding_sdk_package_sources:
+  - "enterprise/frontend/src/embedding-sdk-package/**/!(*.spec).{js,jsx,ts,tsx,css}"
+
 frontend_specs: &frontend_specs
   - *shared_specs
   - "frontend/test/**"

.github/scripts/conventional-commit-pr-title-reminder/utils.js

@@ -1,24 +1,66 @@
-exports.updateComment = updateComment;
+const TYPES = new Set([
+  "feat",
+  "fix",
+  "perf",
+  "docs",
+  "style",
+  "refactor",
+  "test",
+  "build",
+  "ci",
+]);
+
+/**
+ * @param title {string}
+ */
+function isConventionalTitle(title) {
+  const match = /^(?<type>\w+)\((?<rawScopes>[^)]+)\):/.exec(title);
+
+  if (!match) {
+    return false;
+  }
+
+  const [, type, rawScopes] = match;
+  const scopes = rawScopes
+    .split(",")
+    .map((scope) => scope.trim())
+    .filter(Boolean);
+
+  return TYPES.has(type) && scopes.length > 0;
+}
 
 /** @param {import('@types/github-script').AsyncFunctionArguments} AsyncFunctionArguments */
-async function updateComment({ github, context }) {
+async function updateCommentForEmbedding({ github, context }) {
   const githubUsername = context.payload.sender.login;
-  const comment = `${CONVENTIONAL_COMMIT_REMINDER_COMMENT_IDENTIFIER}
-@${githubUsername} You have modified embedding SDK code. Please make sure the PR title follows [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) style.
-Here are all the supported types that will be shown in the changelog:
+
+  const comment = `${CONVENTIONAL_COMMENT_IDENTIFIER}
+@${githubUsername} You have modified embedding code. Please make sure the PR title follows [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) style.
+Here are all the supported types:
 - \`feat\`
 - \`fix\`
 - \`perf\`
+- \`docs\`
+- \`style\`
+- \`refactor\`
+- \`test\`
+- \`build\`
+- \`ci\`
 
-These types \`docs\`, \`style\`, \`refactor\`, \`test\`, \`build\`, and \`ci\` are also encouraged for non-changelog related tasks.
-
-Please also make sure to include \`sdk\` scope, otherwise, the changelog will not include this task.
+Please also make sure to include a scope.
 
 For example, these are valid PR titles:
-- \`feat(sdk): Add interactive dashboard component\`
-- \`feat(sdk): Support theming pie chart\`
-- \`fix(sdk): Fix static dashboard crash\``;
+- \`feat(sdk-bundle): Add interactive dashboard component\`
+- \`fix(sdk-package): Fixed MetabaseProviderPropsStore logic\`
+- \`fix(sdk-package, embed-js): Fix API\``;
 
+  return updateComment(comment, { github, context });
+}
+
+/**
+ * @param comment {string}
+ * @param {import('@types/github-script').AsyncFunctionArguments} AsyncFunctionArguments
+ */
+async function updateComment(comment, { github, context }) {
   const owner = context.repo.owner;
   const repo = context.repo.repo;
   const pullRequestNumber = context.issue.number;
@@ -49,20 +91,19 @@ For example, these are valid PR titles:
   }
 }
 
-const CONVENTIONAL_COMMIT_REMINDER_COMMENT_IDENTIFIER =
-  "<!---conventional commit reminder comment ID-->";
+const CONVENTIONAL_COMMENT_IDENTIFIER =
+  "<!---conventional comment identifier-->";
+
 /**
- * @typedef Comment
- * @property {string} body
- *
- * @param {Comment[]} comments
+ * @param comments {{body: string}[]}
  */
 function getExistingConventionalCommitReminderComment(comments) {
   for (const comment of comments) {
-    if (
-      comment.body.startsWith(CONVENTIONAL_COMMIT_REMINDER_COMMENT_IDENTIFIER)
-    ) {
+    if (comment.body.startsWith(CONVENTIONAL_COMMENT_IDENTIFIER)) {
       return comment;
     }
   }
 }
+
+exports.isConventionalTitle = isConventionalTitle;
+exports.updateCommentForEmbedding = updateCommentForEmbedding;

.github/scripts/conventional-commit-pr-title-reminder/utils.unit.spec.js

@@ -0,0 +1,45 @@
+const { isConventionalTitle } = require("./utils");
+
+describe("isConventionalTitle", () => {
+  it("returns true for a simple conventional commit", () => {
+    expect(isConventionalTitle("feat: add new feature")).toBe(false);
+  });
+
+  it("returns true for a commit with a scope", () => {
+    expect(isConventionalTitle("fix(api): correct API bug")).toBe(true);
+  });
+
+  it("returns true for a commit with multiple scopes, comma separated", () => {
+    expect(isConventionalTitle("feat(build,deps): update dependencies")).toBe(
+      true,
+    );
+  });
+
+  it("returns true for a commit with multiple scopes, comma separated with space", () => {
+    expect(isConventionalTitle("feat(build, deps): update dependencies")).toBe(
+      true,
+    );
+  });
+
+  it("returns false for a commit with round brackets in a message", () => {
+    expect(isConventionalTitle("feat: (something) update dependencies")).toBe(
+      false,
+    );
+  });
+
+  it("returns false for missing colon", () => {
+    expect(isConventionalTitle("feat add feature")).toBe(false);
+  });
+
+  it("returns false for invalid type", () => {
+    expect(isConventionalTitle("invalidType: some message")).toBe(false);
+  });
+
+  it("returns false for empty message", () => {
+    expect(isConventionalTitle("feat:")).toBe(false);
+  });
+
+  it("returns false for empty input", () => {
+    expect(isConventionalTitle("")).toBe(false);
+  });
+});

.github/scripts/get-issue-data.js

@@ -0,0 +1,84 @@
+const githubSlackMap = require("../../release/github-slack-map.json");
+
+async function getIssueData({ context, github, manualIssueNumber }) {
+  let issueNumber;
+  let issueData;
+  let prReviews = [];
+  const triggerType = context.eventName;
+
+  if (context.eventName === "issues") {
+    issueNumber = context.payload.issue.number;
+    issueData = context.payload.issue;
+  } else if (context.eventName === "workflow_dispatch") {
+    issueNumber = parseInt(manualIssueNumber);
+    const { data } = await github.rest.issues.get({
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      issue_number: issueNumber,
+    });
+    issueData = data;
+  } else if (context.eventName === "pull_request_review") {
+    const prNumber = context.payload.pull_request.number;
+    const { data: prData } = await github.rest.pulls.get({
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      pull_number: prNumber,
+    });
+
+    // Extract issue number from branch name (e.g., "claude-fix/issue-12345-some-title")
+    const branchMatch = prData.head.ref.match(/claude-fix\/issue-(\d+)/);
+    if (!branchMatch) {
+      throw new Error(
+        `Could not extract issue number from branch name: ${prData.head.ref}`,
+      );
+    }
+
+    issueNumber = parseInt(branchMatch[1]);
+
+    const { data } = await github.rest.issues.get({
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      issue_number: issueNumber,
+    });
+    issueData = data;
+
+    const { data: reviews } = await github.rest.pulls.listReviews({
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      pull_number: prNumber,
+    });
+
+    prReviews = reviews.filter(
+      (review) =>
+        review.state === "CHANGES_REQUESTED" &&
+        isMetabaseMember(review.user.login),
+    );
+  }
+
+  const { data: comments } = await github.rest.issues.listComments({
+    owner: context.repo.owner,
+    repo: context.repo.repo,
+    issue_number: issueNumber,
+  });
+
+  const employeeComments = comments.filter((comment) =>
+    isMetabaseMember(comment.user.login),
+  );
+
+  return {
+    issueNumber,
+    issueData,
+    employeeComments,
+    prReviews,
+    triggerType,
+    isIssueAuthorEmployee: isMetabaseMember(issueData.user.login),
+  };
+}
+
+function isMetabaseMember(username) {
+  return username in githubSlackMap;
+}
+
+module.exports = {
+  getIssueData,
+};

.github/workflows/backend.yml

@@ -195,6 +195,7 @@ jobs:
       - name: Run cljfmt
         run: ./bin/mage cljfmt-all --force-check
 
+
   be-tests-result:
     needs:
       - be-linter-eastwood

.github/workflows/claude-code-bug-fix.yml

@@ -4,97 +4,22 @@ on:
   issues:
     types:
       - labeled
+  pull_request_review:
+    types:
+      - submitted
   workflow_dispatch:
     inputs:
       issue_number:
         description: "Issue number to fix"
         required: true
         type: string
+      override_employee_check:
+        description: "Override employee check for issue creator"
+        required: false
+        default: false
+        type: boolean
 
 jobs:
-  check-employee-and-issue:
-    runs-on: ubuntu-latest
-    timeout-minutes: 2
-    permissions:
-      contents: read
-      issues: read
-    if: |
-      (github.event_name == 'issues' && github.event.label.name == 'Claude Bug Fix' && github.event.issue.state == 'open') ||
-      (github.event_name == 'workflow_dispatch')
-    outputs:
-      is_employee: ${{ steps.check_employee.outputs.is_employee }}
-      issue_number: ${{ steps.get_issue.outputs.issue_number }}
-      issue_data: ${{ steps.get_issue.outputs.issue_data }}
-    steps:
-      - name: Generate GitHub App Token
-        uses: actions/create-github-app-token@v1
-        id: app-token
-        with:
-          app-id: ${{ secrets.METABASE_BOT_APP_ID }}
-          private-key: ${{ secrets.METABASE_BOT_APP_PRIVATE_KEY }}
-
-      - name: Get issue data
-        id: get_issue
-        uses: actions/github-script@v7
-        with:
-          script: |
-            let issueNumber, issueData;
-
-            if (context.eventName === 'issues') {
-              // Original issue label event
-              issueNumber = context.payload.issue.number;
-              issueData = context.payload.issue;
-            } else if (context.eventName === 'workflow_dispatch') {
-              // Manual trigger with issue number input
-              issueNumber = parseInt('${{ github.event.inputs.issue_number }}');
-              const { data } = await github.rest.issues.get({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                issue_number: issueNumber
-              });
-              issueData = data;
-            }
-
-            core.setOutput('issue_number', issueNumber);
-            core.setOutput('issue_data', JSON.stringify(issueData));
-
-            console.log(`Issue #${issueNumber}: ${issueData.title}`);
-            console.log(`Issue author: ${issueData.user.login}`);
-
-      - name: Check if issue creator is a Metabase employee
-        id: check_employee
-        uses: actions/github-script@v7
-        env:
-          ISSUE_DATA: ${{ steps.get_issue.outputs.issue_data }}
-        with:
-          github-token: ${{ steps.app-token.outputs.token }}
-          script: |
-            const issueData = JSON.parse(process.env.ISSUE_DATA);
-            const issueAuthor = issueData.user.login;
-
-            try {
-              // Check if user is a member of the metabase organization (including private members)
-              const { data: membership } = await github.rest.orgs.getMembershipForUser({
-                org: 'metabase',
-                username: issueAuthor
-              });
-
-              if (membership.state === 'active') {
-                console.log(`User ${issueAuthor} is an active member of the Metabase organization. Proceeding with bug fix automation.`);
-                core.setOutput('is_employee', 'true');
-              } else {
-                console.log(`User ${issueAuthor} is not an active member of the Metabase organization. Skipping bug fix automation.`);
-                core.setOutput('is_employee', 'false');
-              }
-            } catch (error) {
-              if (error.status === 404) {
-                console.log(`User ${issueAuthor} is not a member of the Metabase organization. Skipping bug fix automation.`);
-              } else {
-                console.error('Error checking organization membership:', error);
-              }
-              core.setOutput('is_employee', 'false');
-            }
-
   auto-fix-bug:
     runs-on: ubuntu-latest
     timeout-minutes: 30
@@ -103,8 +28,10 @@ jobs:
       issues: write
       pull-requests: write
       id-token: write
-    needs: check-employee-and-issue
-    if: needs.check-employee-and-issue.outputs.is_employee == 'true'
+    if: |
+      (github.event_name == 'issues' && github.event.label.name == 'Claude Bug Fix' && github.event.issue.state == 'open') ||
+      (github.event_name == 'workflow_dispatch') ||
+      (github.event_name == 'pull_request_review' && github.event.review.state == 'changes_requested' && startsWith(github.event.pull_request.head.ref, 'claude-fix/'))
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -112,67 +39,95 @@ jobs:
           fetch-depth: 0
           token: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Get issue data and check employee status
+        id: get_issue
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const { getIssueData } = require('${{ github.workspace }}/.github/scripts/get-issue-data.js');
+
+            try {
+              const result = await getIssueData({
+                context,
+                github,
+                manualIssueNumber: '${{ github.event.inputs.issue_number }}'
+              });
+
+              const { issueNumber, issueData, employeeComments, prReviews, triggerType, isIssueAuthorEmployee } = result;
+
+              // Check employee status and fail if not authorized
+              if (!result.isIssueAuthorEmployee && '${{ github.event.inputs.override_employee_check }}' !== 'true') {
+                  core.setFailed(`User ${result.issueData.user.login} is not a Metabase employee. Skipping bug fix automation.`);
+              }
+
+              core.setOutput('issue_number', issueNumber);
+              core.setOutput('issue_data', JSON.stringify(issueData));
+              core.setOutput('issue_comments', JSON.stringify(employeeComments));
+              core.setOutput('pr_reviews', JSON.stringify(prReviews));
+              core.setOutput('trigger_type', triggerType);
+            } catch (error) {
+              core.setFailed(error.message);
+            }
+
       - name: Set up git configuration
         run: |
           git config --global user.name "Claude Code[bot]"
           git config --global user.email "noreply@anthropic.com"
 
-      - name: Prepare dynamic issue info
-        id: prepare_issue_info
+      - name: Prepare issue context and bug fix prompt
+        id: prepare_prompt
         uses: actions/github-script@v7
         env:
-          ISSUE_DATA: ${{ needs.check-employee-and-issue.outputs.issue_data }}
+          ISSUE_DATA: ${{ steps.get_issue.outputs.issue_data }}
+          ISSUE_COMMENTS: ${{ steps.get_issue.outputs.issue_comments }}
+          PR_REVIEWS: ${{ steps.get_issue.outputs.pr_reviews }}
+          TRIGGER_TYPE: ${{ steps.get_issue.outputs.trigger_type }}
         with:
           script: |
             const issueData = JSON.parse(process.env.ISSUE_DATA);
-            const issueNumber = '${{ needs.check-employee-and-issue.outputs.issue_number }}';
+            const issueComments = JSON.parse(process.env.ISSUE_COMMENTS);
+            const prReviews = JSON.parse(process.env.PR_REVIEWS);
+            const triggerType = process.env.TRIGGER_TYPE;
+            const issueNumber = '${{ steps.get_issue.outputs.issue_number }}';
 
-            // Convert issue title to kebab case
+            // Convert issue title to kebab case and create branch name
             const issueTitle = issueData.title || 'untitled';
             const kebabTitle = issueTitle
               .toLowerCase()
               .replace(/[^a-z0-9]+/g, '-')
               .replace(/^-+|-+$/g, '')
               .replace(/-+/g, '-')
-              .substring(0, 50); // Limit length
+              .substring(0, 50);
+            const branchName = `claude-fix/issue-${issueNumber}-${kebabTitle}`;
 
-            // Create base branch name
-            const baseBranchName = `claude-fix/issue-${issueNumber}-${kebabTitle}`;
-            core.exportVariable('BASE_BRANCH_NAME', baseBranchName);
+            // Format data for bash environment
+            const formattedComments = issueComments.length > 0
+              ? issueComments.map(comment =>
+                  `**Comment by @${comment.user.login} on ${comment.created_at}:**\n${comment.body}`
+                ).join('\n\n---\n\n')
+              : '';
+            const formattedReviews = prReviews.length > 0
+              ? prReviews.map(review =>
+                  `**Review by @${review.user.login} on ${review.submitted_at} (Changes Requested):**\n${review.body || 'No review comment provided'}`
+                ).join('\n\n---\n\n')
+              : '';
 
-            // Export issue info for the prompt
+            core.exportVariable('BRANCH_NAME', branchName);
             core.exportVariable('ISSUE_NUMBER', issueNumber);
             core.exportVariable('ISSUE_TITLE', issueData.title);
             core.exportVariable('ISSUE_AUTHOR', issueData.user.login);
             core.exportVariable('ISSUE_CREATED', issueData.created_at);
             core.exportVariable('ISSUE_BODY', issueData.body || 'No description provided');
+            core.exportVariable('ISSUE_COMMENTS_FORMATTED', formattedComments);
+            core.exportVariable('PR_REVIEWS_FORMATTED', formattedReviews);
+            core.exportVariable('TRIGGER_TYPE', triggerType);
 
-      - name: Determine branch strategy
+      - name: Generate bug fix prompt
         run: |
-          # Use the base branch name (no counters)
-          BRANCH_NAME="${{ env.BASE_BRANCH_NAME }}"
-
-          # Check if this exact branch already exists
-          if git branch -r --list "origin/${BRANCH_NAME}" | grep -q .; then
-            echo "Branch already exists: $BRANCH_NAME"
-            echo "Will push to existing branch and skip PR creation"
-            echo "BRANCH_EXISTS=true" >> $GITHUB_ENV
-          else
-            echo "Creating new branch: $BRANCH_NAME"
-            echo "Will create new PR"
-            echo "BRANCH_EXISTS=false" >> $GITHUB_ENV
-          fi
-
-          # Export the branch name
-          echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
-
-      - name: Prepare bug fix prompt
-        id: prepare_prompt
-        run: |
-          # Prepare the prompt for Claude
           cat << 'EOF' >> $GITHUB_ENV
           BUG_FIX_PROMPT<<PROMPT_END
-          You are helping to automatically fix a bug reported in GitHub issue #${{ env.ISSUE_NUMBER }}.
+          You are helping to fix a bug reported in GitHub issue #${{ env.ISSUE_NUMBER }}.
 
           ## Bug Report Details
           **Title:** ${{ env.ISSUE_TITLE }}
@@ -182,8 +137,27 @@ jobs:
           **Description:**
           ${{ env.ISSUE_BODY }}
 
+          **Comments:**
+          ${{ env.ISSUE_COMMENTS_FORMATTED }}
+          EOF
+
+          # Add review feedback section only if this is a review trigger
+          if [ "${{ env.TRIGGER_TYPE }}" = "pull_request_review" ]; then
+            cat << 'EOF' >> $GITHUB_ENV
+
+          ## Previous Fix Attempt - Review Feedback (MUST ADDRESS)
+          Your previous automated fix attempt received the following feedback that needs to be addressed:
+
+          ${{ env.PR_REVIEWS_FORMATTED }}
+          EOF
+          fi
+
+          # Add the rest of the prompt (always the same)
+          cat << 'EOF' >> $GITHUB_ENV
+
           ## Instructions
-          1. **Analyze the bug report** - Understand what the issue is describing
+          1. **Analyze the issue description and comments** - Understand what the issue is describing
+             a. Pay particular attention to reproduction steps and if there's an agreed upon fix.
           2. **Search the codebase** - Use Grep and Glob tools to find relevant files
           3. **Implement a fix** - Follow existing code patterns and conventions
           4. **Test the fix** - Run appropriate tests using the commands in CLAUDE.md:
@@ -205,15 +179,26 @@ jobs:
           - Only modify actual production code in the main repository directories
           - The scratchpad will be cleaned up automatically - nothing from it will be committed
 
-          ## If You Cannot Fix The Bug
-          If you determine that you cannot create a definitive fix, create a file called `FAILED_FIX_ANALYSIS.md` with:
-          1. **Why No Fix** - Specific reason why you couldn't implement a fix
-          2. **Tool Limitations** - If you needed additional tools/access that weren't available
-
           Start by understanding the bug and then systematically work toward a solution.
           PROMPT_END
           EOF
 
+      - name: Determine branch strategy
+        run: |
+          BRANCH_NAME="${{ env.BRANCH_NAME }}"
+          if git branch -r --list "origin/${BRANCH_NAME}" | grep -q .; then
+            echo "Branch already exists: $BRANCH_NAME"
+            echo "Will push to existing branch and skip PR creation"
+            echo "BRANCH_EXISTS=true" >> $GITHUB_ENV
+          else
+            echo "Creating new branch: $BRANCH_NAME"
+            echo "Will create new PR"
+            echo "BRANCH_EXISTS=false" >> $GITHUB_ENV
+          fi
+
+          # Export the branch name
+          echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
+
       - name: Create scratchpad directory for Claude
         run: |
           # Create a scratchpad directory for Claude to use for temporary files
@@ -249,20 +234,6 @@ jobs:
       - name: Check for changes and analysis
         id: check_changes
         run: |
-          # Check if Claude created an analysis file first and store its content
-          if [ -f "FAILED_FIX_ANALYSIS.md" ]; then
-            echo "has_analysis=true" >> $GITHUB_OUTPUT
-            echo "Analysis file found"
-            # Store analysis content in a GitHub output (base64 encoded to handle multiline)
-            ANALYSIS_CONTENT=$(base64 -w 0 FAILED_FIX_ANALYSIS.md)
-            echo "analysis_content=$ANALYSIS_CONTENT" >> $GITHUB_OUTPUT
-            # Remove analysis file so it doesn't count as a code change
-            rm -f FAILED_FIX_ANALYSIS.md
-          else
-            echo "has_analysis=false" >> $GITHUB_OUTPUT
-            echo "No analysis file found"
-          fi
-
           # Now check for actual code changes (excluding analysis file)
           if [ -n "$(git status --porcelain)" ]; then
             echo "has_changes=true" >> $GITHUB_OUTPUT
@@ -278,25 +249,16 @@ jobs:
         id: prepare_reviewers
         uses: actions/github-script@v7
         env:
-          ISSUE_DATA: ${{ needs.check-employee-and-issue.outputs.issue_data }}
+          ISSUE_DATA: ${{ steps.get_issue.outputs.issue_data }}
         with:
           script: |
-            // Get existing team labels from the issue
             const issueData = JSON.parse(process.env.ISSUE_DATA);
             const issueLabels = issueData.labels.map(label => label.name);
             const teamLabels = issueLabels.filter(label => label.startsWith('.Team/'));
-
-            console.log('Issue labels:', issueLabels);
-            console.log('Team labels found:', teamLabels);
-
-            // Add issue creator as reviewer
             const issueCreator = issueData.user.login;
-
             core.setOutput('team_labels', teamLabels.join(','));
             core.setOutput('reviewers', issueCreator);
 
-            console.log('Reviewer:', issueCreator);
-
       - name: Create Pull Request
         if: steps.check_changes.outputs.has_changes == 'true'
         uses: peter-evans/create-pull-request@v6
@@ -363,25 +325,3 @@ jobs:
         with:
           script: |
             console.log('🤖 No code changes were made by Claude.');
-
-            // Check if Claude provided a detailed analysis
-            if ('${{ steps.check_changes.outputs.has_analysis }}' === 'true') {
-              try {
-                // Decode the base64-encoded analysis content
-                const analysisBase64 = '${{ steps.check_changes.outputs.analysis_content }}';
-                const analysisContent = Buffer.from(analysisBase64, 'base64').toString('utf8');
-                console.log('📋 Claude provided detailed analysis:');
-                console.log('='.repeat(50));
-                console.log(analysisContent);
-                console.log('='.repeat(50));
-              } catch (error) {
-                console.log('❌ Could not decode analysis content:', error);
-              }
-            } else {
-              console.log('ℹ️  No detailed analysis file was created by Claude.');
-              console.log('This could be because:');
-              console.log('- The issue requires more context or clarification');
-              console.log('- The fix is complex and requires human judgment');
-              console.log('- The issue is not a straightforward code bug');
-              console.log('- Claude couldn\'t locate the relevant code to modify');
-            }

.github/workflows/conventional-commit-pr-title-reminder.yml

@@ -4,39 +4,53 @@ on:
   pull_request:
     branches:
       - master
-    paths:
-      - enterprise/frontend/src/embedding-sdk-bundle/**
 
 jobs:
+  files-changed:
+    name: Check which files changed
+    runs-on: ubuntu-22.04
+    timeout-minutes: 3
+    outputs:
+      frontend_embedding_sources: ${{ steps.changes.outputs.frontend_embedding_sources }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Test which files changed
+        uses: dorny/paths-filter@v3.0.0
+        id: changes
+        with:
+          token: ${{ github.token }}
+          filters: .github/file-paths.yaml
+
   pr-title-reminder:
     runs-on: ubuntu-22.04
     timeout-minutes: 5
     permissions:
       pull-requests: write
+    needs: [ files-changed ]
+    if: |
+      !cancelled() &&
+      needs.files-changed.outputs.frontend_embedding_sources == 'true'
     steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+
       - name: Check if the PR title is already following Conventional Commits style
         uses: actions/github-script@v7
         id: is-conventional-commit
         with:
           result-encoding: string
           script: | # js
+            const { isConventionalTitle } = require(".github/scripts/conventional-commit-pr-title-reminder/utils.js")
             const pullRequestTitle = context.payload.pull_request.title
-            const conventionalCommitRegex = /^(?<type>\w+)\((?<scope>\w+)\):/
-            const match = pullRequestTitle.match(conventionalCommitRegex)
-            if (match) {
-              const { type, scope } = match.groups
-              return ['feat', 'fix', 'perf', 'docs', 'style', 'refactor', 'test', 'build', 'ci'].includes(type) && scope === 'sdk'
-            }
 
-      - name: Checkout repository so `gh` CLI works
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.head_ref }}
+            return isConventionalTitle(pullRequestTitle)
 
       - name: Remind PR authors to use conventional commit in the title
         if: ${{ steps.is-conventional-commit.outputs.result != 'true' }}
         uses: actions/github-script@v7
         with:
-          script: |
-            const { updateComment } = require(".github/scripts/conventional-commit-pr-title-reminder/utils.js")
-            await updateComment({github, context})
+          script: | #js
+            const { updateCommentForEmbedding } = require(".github/scripts/conventional-commit-pr-title-reminder/utils.js")
+
+            await updateCommentForEmbedding({github, context})

.github/workflows/drivers.yml

@@ -338,6 +338,12 @@ jobs:
     name: H2
     steps:
     - uses: actions/checkout@v4
+      with:
+        submodules: true
+        token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+    - name: Start python-runner
+      working-directory: python-runner
+      run: make run
     - name: Test H2 driver
       uses: ./.github/actions/test-driver
       with:
@@ -442,6 +448,12 @@ jobs:
     name: "${{ matrix.version.name }} ${{ matrix.job.name }}"
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
+          token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      - name: Start python-runner
+        working-directory: python-runner
+        run: make run
       - name: Test ${{ matrix.version.name }}
         uses: ./.github/actions/test-driver
         with:
@@ -745,6 +757,12 @@ jobs:
           POSTGRES_HOST_AUTH_METHOD: trust
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
+          token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      - name: Start python-runner
+        working-directory: python-runner
+        run: make run
       - name: Test ${{ matrix.version.name }} (${{ matrix.job.name }})
         uses: ./.github/actions/test-driver
         with:

.github/workflows/e2e-component-tests-embedding-sdk-backward-compatibility.yml

@@ -140,7 +140,10 @@ jobs:
         run: node e2e/runner/run_cypress_ci.js snapshot --browser ${{ steps.cypress-prep.outputs.chrome-path }}
 
       - name: Run component tests for Embedding SDK
-        run: node e2e/runner/run_cypress_ci.js component --browser ${{ steps.cypress-prep.outputs.chrome-path }}
+        run: |
+          node e2e/runner/run_cypress_ci.js component \
+            --browser ${{ steps.cypress-prep.outputs.chrome-path }} \
+            --env grepTags="-@skip-backward-compatibility",grepOmitFiltered=true
 
       - name: Upload Test Results
         uses: ./.github/actions/upload-test-results

.github/workflows/embedding-sdk-labels-reminder.yml

@@ -0,0 +1,49 @@
+name: Embedding SDK Labels Reminder
+
+on:
+  pull_request:
+    types: [auto_merge_enabled, review_requested, labeled, unlabeled, synchronize]
+    branches:
+      - "master"
+  merge_group:
+
+jobs:
+  files-changed:
+    name: Check which files changed
+    runs-on: ubuntu-22.04
+    timeout-minutes: 3
+    outputs:
+      frontend_embedding_sdk_package_sources: ${{ steps.changes.outputs.frontend_embedding_sdk_package_sources }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Test which files changed
+        uses: dorny/paths-filter@v3.0.0
+        id: changes
+        with:
+          token: ${{ github.token }}
+          filters: .github/file-paths.yaml
+
+  embedding-sdk-package-no-release-notes-reminder:
+    needs: [ files-changed ]
+    if: |
+      !cancelled() &&
+      (needs.files-changed.outputs.frontend_embedding_sdk_package_sources == 'true')
+    name: Embedding SDK package release-notes/no-release-notes label reminder
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v4
+      - uses: jesusvasquez333/verify-pr-label-action@v1.4.0
+        id: verify-release-notes-labels
+        with:
+          github-token: '${{ secrets.GITHUB_TOKEN }}'
+          valid-labels: 'release-notes, no-release-notes'
+          disable-reviews: true
+      - name: Comment on PR if label should be added
+        if: ${{ always() }}
+        uses: ./.github/actions/notify-pull-request
+        with:
+          remove-previous-comment: true
+          add-new-comment: ${{ steps.verify-release-notes-labels.outcome == 'failure' }}
+          message: "⚠️ Changes were made to the <b>Embedding SDK Package</b>. Review this PR and apply one of the following labels.<br><br>- <code>release-notes</code> — select if this PR must be included in the main Metabase changelog release notes.<br>- <code>no-release-notes</code> — select if this PR should not be included in the main Metabase changelog release notes."
+          include-log: true

.github/workflows/perf-test-labeled.yml

@@ -9,9 +9,13 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  uberjar:
+    if: ${{ github.event.label.name == 'Run-perf' }}
+    uses: ./.github/workflows/uberjar.yml
+    secrets: inherit
+
   run-perf-test:
+    needs: [uberjar]
     if: ${{ github.event.label.name == 'Run-perf' }}
     uses: ./.github/workflows/perf-test.yml
-    with:
-      wait_for_uberjar: true
     secrets: inherit

.github/workflows/perf-test.yml

@@ -3,11 +3,6 @@ run-name: Run perf test for ${{ github.ref_name }} by @${{ github.actor }}
 
 on:
   workflow_call:
-    inputs:
-      wait_for_uberjar:
-        description: "Wait for uberjar build"
-        required: false
-        type: boolean
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -31,63 +26,6 @@ jobs:
         uses: aws-actions/amazon-ecr-login@v2
         with:
           registries: "${{ secrets.PR_ENV_AWS_ACCOUNT_ID }}"
-      - name: Wait for uberjar
-        id: wait_for_uberjar
-        if: ${{ inputs.wait_for_uberjar == true }}
-        run: |
-          ## Get workflow run id for uberjar build
-          curl -Ls --output e2e-tests.json \
-          -H "Accept: application/vnd.github+json" \
-          -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
-          -H "X-GitHub-Api-Version: 2022-11-28" \
-          https://api.github.com/repos/${{ github.repository }}/actions/workflows/run-tests.yml/runs?head_sha=${{ github.event.pull_request.head.sha || github.sha }}
-          ID=$(jq -r '.workflow_runs[0].id' e2e-tests.json)
-          echo "Run ID: ${ID}"
-
-          ## Get workflow run id for uberjar build
-          while [ -z "$JOB_ID" ]; do
-            NEXT_URL="https://api.github.com/repos/${{ github.repository }}/actions/runs/${ID}/jobs?filter=latest"
-
-            # Search for build job
-            echo "Searching for build job..."
-            while [ -z "$JOB_ID" ] && [ -n "$NEXT_URL" ]; do
-              # Get one page of jobs
-              RESPONSE=$(curl -s -i \
-                -H "Accept: application/vnd.github+json" \
-                -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
-                -H "X-GitHub-Api-Version: 2022-11-28" \
-                "$NEXT_URL")
-
-              # Extract headers and body
-              HEADERS=$(echo "$RESPONSE" | sed -n '1,/^\r$/p')
-              BODY=$(echo "$RESPONSE" | sed '1,/^\r$/d')
-              LINK_HEADER=$(echo "$HEADERS" | grep -i '^link:' | sed 's/^link: //')
-
-              # Check for next page in Link header
-              if echo "$LINK_HEADER" | grep -q 'rel="next"'; then
-                NEXT_URL=$(echo "$LINK_HEADER" | sed -E 's/.*<([^>]+)>; rel="next".*/\1/')
-              else
-                NEXT_URL=""
-              fi
-
-              # Check for build job
-              JOB_ID=$(echo "$BODY" | jq -r '.jobs[] | select(.name | contains("build (ee)")) | .id')
-            done
-          done
-          echo "Job ID: ${JOB_ID}"
-
-          ## Wait for uberjar build to complete
-          while [ true ]; do
-            curl -Ls --output uberjar.json \
-            -H "Accept: application/vnd.github+json" \
-            -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
-            -H "X-GitHub-Api-Version: 2022-11-28" \
-            https://api.github.com/repos/${{ github.repository }}/actions/jobs/${JOB_ID}
-            jq -r '.steps[] | select(.name == "Prepare uberjar artifact") | .status' uberjar.json | grep -q "completed" && break
-            echo "Waiting for uberjar build..."
-            sleep 10
-          done
-          echo "run_id=$(jq -r '.workflow_runs[0].id' e2e-tests.json)" >> $GITHUB_OUTPUT
       - name: Retrieve uberjar artifact for ee
         uses: ./.github/actions/fetch-artifact
         with:

.github/workflows/release-embedding-sdk.yml

@@ -285,10 +285,6 @@ jobs:
         run: |
           ./bin/embedding-sdk/release_utils.bash update_package_json_template ${{ needs.determine-sdk-version.outputs.sdk_version }}
 
-      - name: Update changelog
-        run: |
-          yarn embedding-sdk:generate-changelog -o changelog-diff
-
       - name: Build SDK package
         run: yarn run build-embedding-sdk-package
 
@@ -301,12 +297,6 @@ jobs:
           name: metabase-sdk
           path: ./resources/embedding-sdk
 
-      - name: Upload changelog diff
-        uses: actions/upload-artifact@v4
-        with:
-          name: sdk-changelog-diff
-          path: ./changelog-diff
-
   publish-npm:
     needs: [build-sdk, determine-sdk-version]
     runs-on: ubuntu-22.04
@@ -335,19 +325,7 @@ jobs:
         run: |
           bash ./bin/embedding-sdk/release_utils.bash update_package_json_template ${{ env.sdk_version }}
 
-      - name: Retrieve SDK changelog diff
-        if: inputs.branch != 'master'
-        uses: actions/download-artifact@v4
-        with:
-          name: sdk-changelog-diff
-
-      - name: Update changelog
-        if: inputs.branch != 'master'
-        run: |
-          cat changelog-diff enterprise/frontend/src/embedding-sdk-bundle/CHANGELOG.md > new-changelog
-          mv new-changelog enterprise/frontend/src/embedding-sdk-bundle/CHANGELOG.md
-
-      - name: Create a PR updating readme + published version, and changelog (using GitHub Metabase Automation account)
+      - name: Create a PR updating readme + published version (using GitHub Metabase Automation account)
         run: |
           git checkout -b update-sdk-version-${{ env.sdk_version }}
 

.gitignore

@@ -163,3 +163,4 @@ CLAUDE.local.md
 
 /logs
 /.clojure-mcp
+__pycache__/

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "python-runner"]
+	path = python-runner
+	url = https://github.com/metabase/python-runner-container.git
+	branch = prototype

bin/embedding-sdk/generate-sdk-package-files.js

@@ -89,6 +89,6 @@ generateSdkPackage();
 copyFileToOutput("LICENSE.txt");
 copyFileToOutput("frontend/src/embedding-sdk-package/README.md", "README.md");
 copyFileToOutput(
-  "frontend/src/embedding-sdk-bundle/CHANGELOG.md",
+  "frontend/src/embedding-sdk-package/CHANGELOG.md",
   "CHANGELOG.md",
 );

deps.edn

@@ -36,7 +36,7 @@
   com.clearspring.analytics/stream          {:mvn/version "2.9.8"               ; Various sketching algorithms
                                              :exclusions  [it.unimi.dsi/fastutil
                                                            org.slf4j/slf4j-api]}
-  com.clojure-goes-fast/jvm-alloc-rate-meter{:mvn/version "0.1.4"}              ; Monitor heap allocation rate.
+  com.clojure-goes-fast/jvm-alloc-rate-meter {:mvn/version "0.1.4"}              ; Monitor heap allocation rate.
   com.clojure-goes-fast/jvm-hiccup-meter    {:mvn/version "0.1.1"}              ; Monitor GC pauses and other system-induced pauses.
   com.draines/postal                        {:mvn/version "2.0.5"               ; SMTP library
                                              :exclusions [com.sun.mail/jakarta.mail]}
@@ -45,7 +45,7 @@
   com.github.albfernandez/juniversalchardet {:mvn/version "2.5.0"}
   com.github.blagerweij/liquibase-sessionlock {:mvn/version "1.6.9"}            ; session lock for liquibase migrations
   com.github.jknack/handlebars              ^:antq/exclude                      ; 4.4.0 requires java 17+
-                                            {:mvn/version "4.3.1"}              ; templating engine
+  {:mvn/version "4.3.1"}              ; templating engine
   com.github.oliyh/martian-clj-http         {:mvn/version "0.1.31"}             ; openapi client
   com.github.pangloss/transducers           {:git/sha "c91d39046c8c64dc31b84ebb27154a8084fcda4c" ; like medley for transducers
                                              :git/url "https://github.com/pangloss/transducers"}
@@ -55,14 +55,14 @@
   com.github.vertical-blank/sql-formatter   {:mvn/version "2.0.5"}              ; Java SQL formatting library https://github.com/vertical-blank/sql-formatter
   com.google.guava/guava                    {:mvn/version "33.4.8-jre"}         ; dep for BigQuery, Spark, and GA. Require here rather than letting different dep versions stomp on each other — see comments on #9697
   com.h2database/h2                         ^:antq/exclude                      ; https://metaboat.slack.com/archives/CKZEMT1MJ/p1727191010259979
-                                            {:mvn/version "2.1.214"}            ; embedded SQL database
+  {:mvn/version "2.1.214"}            ; embedded SQL database
   com.ibm.icu/icu4j                         {:mvn/version "77.1"}               ; Used to transliterate Unicode characters into Latin characters
   com.knuddels/jtokkit                      {:mvn/version "1.1.0"}              ; Java tokenizer library for OpenAI models, used for token counting in semantic search
   com.mchange/c3p0                          {:mvn/version "0.10.1"}             ; DB connection pool
   com.snowplowanalytics/snowplow-java-tracker ^:antq/exclude                    ; 2+ depend on apache httpclient 5.x, but saml and clj-http depend on 4.x
-                                            {:mvn/version "1.0.1"}              ; Snowplow analytics
+  {:mvn/version "1.0.1"}              ; Snowplow analytics
   com.sun.mail/jakarta.mail                 ^:antq/exclude                      ; 2+ changes imports from javax.mail to jakarta.mail
-                                            {:mvn/version "1.6.8"}              ; SMTP lib, required by postal
+  {:mvn/version "1.6.8"}              ; SMTP lib, required by postal
   com.taoensso/nippy                        {:mvn/version "3.6.0"}              ; Fast serialization (i.e., GZIP) library for Clojure
   com.vladsch.flexmark/flexmark             {:mvn/version "0.64.8"}             ; Markdown parsing
   com.vladsch.flexmark/flexmark-ext-autolink
@@ -92,7 +92,7 @@
   ;; TODO: replace back to normal artifact
   io.github.metabase/macaw                  {:mvn/version "0.2.27"}             ; Parse native SQL queries
   io.github.resilience4j/resilience4j-retry ^:antq/exclude                      ; 2.x do not support Java 8
-                                            {:mvn/version "1.7.1"}              ; Support for retrying operations
+  {:mvn/version "1.7.1"}              ; Support for retrying operations
   io.prometheus/simpleclient_hotspot        {:mvn/version "0.16.0"}             ; prometheus jvm collector
   junegunn/grouper                          {:mvn/version "0.1.1"}              ; Batch processing helper
   kixi/stats                                {:mvn/version "0.5.7"               ; Various statistic measures implemented as transducers
@@ -198,6 +198,8 @@
                                                            org.eclipse.jetty.ee9/jetty-ee9-servlet
                                                            org.eclipse.jetty.ee9.websocket/jetty-ee9-websocket-jetty-server]}
   slingshot/slingshot                       {:mvn/version "0.12.2"}             ; enhanced throw/catch, used by other deps
+  software.amazon.awssdk/s3                 {:mvn/version "2.33.0"}
+  software.amazon.awssdk/sts                {:mvn/version "2.33.0"}           ; AWS SDK v2 for S3
   stencil/stencil                           {:mvn/version "0.5.0"}              ; Mustache templates for Clojure
   user-agent/user-agent                     {:mvn/version "0.1.1"}              ; User-Agent string parser, for Login History page & elsewhere
   weavejester/dependency                    {:mvn/version "1.0.0"}              ; Dependency graphs and topological sorting
@@ -836,8 +838,13 @@
   :test/lib
   {:exec-args {:only ["test/metabase/lib"
                       "test/metabase/lib_be"]}}
+  ;; test the python transforms
+  ;;    clj -X:dev:ee:ee-dev:driver:test:test/python-transform
+  :test/python-transform
+  {:exec-args {:only [metabase-enterprise.transforms.api-test
+                      "enterprise/backend/test/metabase_enterprise/transforms"]}}
 
-  ;; test the Query Processor
+;; test the Query Processor
   ;;
   ;;    clj -X:dev:ee:ee-dev:test:test/qp
   :test/qp

docs/ai/metabot.md

@@ -68,7 +68,9 @@ You can also ask Metabot to tell you about specific tables in your database.
 
 ## How Metabot uses the query builder
 
-When you ask Metabot to create a chart from natural language, it first looks for existing questions that might answer your request. If it finds a relevant question, it'll point you to that instead of creating something new. Otherwise, Metabot will use the [query builder](../questions/query-builder/editor.md) to create a new chart for you. Keep in mind that Metabot can only handle basic query builder operations. It's limited to single-level aggregation and grouping, so if you need more complex analysis, you can take over and refine the query yourself, or switch to the native editor.
+When you ask Metabot to create a chart from natural language, it first looks for existing questions that might answer your request. If it finds a relevant question, it'll point you to that instead of creating something new. Otherwise, Metabot will use the [query builder](../questions/query-builder/editor.md) to create a new chart for you. 
+
+Keep in mind that Metabot is still learning the query builder. Metabot can only handle basic query builder operations, and it lacks access to the library of [custom expressions](../questions/query-builder/expressions-list.md). Metabot is also limited to single-level aggregation and grouping, so if you need more complex analysis, you can take over and refine the query yourself, or switch to the SQL editor.
 
 ## Metabot in the native editor
 

docs/api.html

@@ -15864,7 +15864,35 @@ redirect_from:
       "get" : {
         "summary" : "GET /api/ee/transform-job/",
         "description" : "Get all transform jobs.",
-        "parameters" : [ ],
+        "parameters" : [ {
+          "in" : "query",
+          "name" : "last_run_start_time",
+          "required" : false,
+          "schema" : {
+            "type" : "string",
+            "minLength" : 1
+          }
+        }, {
+          "in" : "query",
+          "name" : "next_run_start_time",
+          "required" : false,
+          "schema" : {
+            "type" : "string",
+            "minLength" : 1
+          }
+        }, {
+          "in" : "query",
+          "name" : "transform_tag_ids",
+          "required" : false,
+          "schema" : {
+            "type" : "array",
+            "items" : {
+              "description" : "value must be an integer greater or equal to than zero.",
+              "type" : "integer",
+              "minimum" : 0
+            }
+          }
+        } ],
         "responses" : {
           "2XX" : {
             "description" : "Successful response"

docs/configuring-metabase/config-template.md

@@ -117,6 +117,7 @@ config:
     db-connection-timeout-ms: 10000
     db-query-timeout-minutes: 20
     default-maps-enabled: true
+    disable-cors-on-localhost: false
     download-row-limit: null
     email-from-address: notifications@metabase.com
     email-from-address-override: notifications@metabase.com
@@ -134,7 +135,7 @@ config:
     email-smtp-username: null
     email-smtp-username-override: null
     embedding-app-origins-interactive: null
-    embedding-app-origins-sdk: ""
+    embedding-app-origins-sdk: ''
     embedding-homepage: hidden
     embedding-secret-key: null
     enable-embedding-interactive: false

docs/configuring-metabase/environment-variables.md

@@ -393,6 +393,15 @@ Timeout in minutes for databases query execution, both Metabase application data
 
 Whether or not the default GeoJSON maps are enabled.
 
+### `MB_DISABLE_CORS_ON_LOCALHOST`
+
+- Type: boolean
+- Default: `false`
+- [Exported as](../installation-and-operation/serialization.md): `disable-cors-on-localhost`.
+- [Configuration file name](./config-file.md): `disable-cors-on-localhost`
+
+Prevents the server from sending CORS headers for requests originating from localhost.
+
 ### `MB_DOWNLOAD_ROW_LIMIT`
 
 - Type: positive-integer
@@ -562,7 +571,7 @@ Allow these space delimited origins to embed Metabase interactive.
 > Only available on Metabase [Pro](https://www.metabase.com/product/pro) and [Enterprise](https://www.metabase.com/product/enterprise) plans.
 
 - Type: string
-- Default: `""`
+- Default: ``
 - [Configuration file name](./config-file.md): `embedding-app-origins-sdk`
 
 Allow Metabase SDK access to these space delimited origins.

docs/developers-guide/driver-changelog.md

@@ -15,10 +15,16 @@ title: Driver interface changelog
 - Added metabase.driver/compile-transform, metabase.driver/compile-drop-table, metabase.driver/execute-raw-queries!,
   metabase.driver/run-transform!, metabase.driver/drop-transform-target!, metabase.driver/native-query-deps,
   metabase.driver/connection-spec, metabase.driver/table-exists?, metabase.driver.sql/normalize-name,
-  metabase.driver.sql/default-schema, and metabase.driver.sql/find-table to implement sql transforms.
+  and metabase.driver.sql/default-schema to implement sql transforms.
 
 - Added the driver multi-methods `metabase.driver/schema-exists?` and `metabase.driver/create-schema-if-needed!` which should be implemented by drivers that support `:schemas` and `:transforms/table`.
 
+- Added `metabase.driver/type->database-type` multimethod that returns the database type for a given Metabase type (from the type hierarchy) as a HoneySQL spec. Unlike `upload-type->database-type` which is specific to upload types, this method handles general Metabase base types.
+
+- Added `metabase.driver/rename-table!` multimethod to rename a table from one name to another. Both table names may be qualified by schema (e.g., `schema.table`). This is currently used for transform operations and uploads.
+
+- Added `metabase.driver/insert-from-source!` multimethod that abstracts data insertion from various sources into existing tables. This multimethod dispatches on both the driver and the data source type (`:rows` or `:csv-file`). It allows drivers to optimize based on the data source type and returns the number of rows inserted. Default implementations are provided for both `:rows` and `:csv-file` source types.
+
 - `metabase.driver/humanize-connection-error-message` now takes a list of all messages in the exception chain,
   instead of just the top-level message as a string.
 

docs/embedding/sdk/next-js.md

@@ -14,11 +14,11 @@ See a [sample Next.js app that uses the SDK](https://github.com/metabase/metabas
 
 ## SDK components with Server Side Rendering (SSR) or React Server Components
 
-As of Embedding Analytics SDK v57, SDK components automatically skip server-side rendering (SSR) and render only on the client.
+As of Embedded Analytics SDK v57, SDK components automatically skip server-side rendering (SSR) and render only on the client.
 
 ### Compatibility layer for Server Side Rendering (SSR) (DEPRECATED)
 
-As of Embedding analytics SDK 57, the compatibility layer for server-side rendering (SSR) is deprecated and no longer required. If you use the compatibility layer, change your imports from `@metabase/embedding-sdk-react/next` to `@metabase/embedding-sdk-react`.
+As of Embedded Analytics SDK 57, the compatibility layer for server-side rendering (SSR) is deprecated and no longer required. If you use the compatibility layer, change your imports from `@metabase/embedding-sdk-react/next` to `@metabase/embedding-sdk-react`.
 
 ## Handling authentication
 

docs/embedding/sdk/upgrade.md

@@ -10,8 +10,8 @@ Here's a basic overview of the steps you'll want to take when upgrading your SDK
 ## 1. Read the release post and changelog for Metabase and the Embedded analytics SDK
 
 - [Release posts](https://www.metabase.com/releases) give a good overview of what's in each release, and call out breaking changes (which are rare).
-- [Metabase changelogs](https://www.metabase.com/changelog) list Metabase changes.
-- [Embedded analytics SDK changelogs](https://www.metabase.com/changelog/56) list changes specific to the SDK Package.
+- [Metabase + Embedding analytics SDK changelogs](https://www.metabase.com/changelog) list Metabase and Embedding analytics SDK changes.
+- [Embedded analytics SDK package changelog](https://github.com/metabase/metabase/blob/release-x.56.x/enterprise/frontend/src/embedding-sdk-bundle/CHANGELOG.md) list changes specific to the SDK lightweight `@metabase/embedding-sdk-react` package.
 
 Check for any relevant changes, especially deprecations or breaking changes that require you to update your application's code. If there are deprecation changes, we'll have docs that'll walk you through what changes you'll need to make and why.
 

e2e/runner/cypress-runner-run-tests.js

@@ -127,7 +127,10 @@ const configs = {
       openMode: args["--open"] || process.env.OPEN_UI === "true",
     };
 
-    return sdkComponentConfig;
+    const userArgs = await parseArguments(args);
+
+    const finalConfig = Object.assign({}, sdkComponentConfig, userArgs);
+    return finalConfig;
   },
 };
 

e2e/test-component/scenarios/embedding-sdk/metabot-question.cy.spec.tsx

@@ -115,6 +115,32 @@ describe("scenarios > embedding-sdk > metabot-question", () => {
     });
   });
 
+  it("should start a new conversation when clicking a `start new conversation` button", () => {
+    setup(metabotResponse);
+
+    mountSdkContent(<MetabotQuestion />);
+
+    getSdkRoot().within(() => {
+      cy.findByTestId("metabot-chat-input").type("Show orders {enter}");
+
+      H.mockMetabotResponse({
+        statusCode: 200,
+        body: metabotRetryResponse,
+      });
+
+      cy.findAllByTestId("metabot-chat-message").should("have.length", 2);
+
+      cy.findByTestId("metabot-new-conversation").click();
+
+      cy.findAllByTestId("metabot-chat-message").should("not.exist");
+      cy.findByTestId("metabot-chat-input").should("have.value", "");
+
+      cy.findByTestId("metabot-chat-input").type("Show orders {enter}");
+
+      cy.findAllByTestId("metabot-chat-message").should("have.length", 2);
+    });
+  });
+
   it("should set correct metabot_id both for the a new message and when retrying", () => {
     setup(metabotResponse);
 

e2e/test-component/scenarios/embedding-sdk/sdk-bundle.cy.spec.tsx

@@ -26,14 +26,13 @@ const sdkBundleCleanup = () => {
 
 describe(
   "scenarios > embedding-sdk > sdk-bundle",
-  // These test in some cases load a new SDK Bundle that in combination with the Component Testing is memory-consuming
-  { numTestsKeptInMemory: 1 },
+  {
+    tags: ["@skip-backward-compatibility"],
+    // These test in some cases load a new SDK Bundle that in combination with the Component Testing is memory-consuming
+    numTestsKeptInMemory: 1,
+  },
   () => {
     beforeEach(() => {
-      cy.intercept("POST", "/api/dashboard/*/dashcard/*/card/*/query").as(
-        "dashcardQuery",
-      );
-
       signInAsAdminAndEnableEmbeddingSdk();
 
       cy.signOut();
@@ -44,10 +43,6 @@ describe(
     [{ strictMode: false }, { strictMode: true }].forEach(({ strictMode }) => {
       describe(`Common cases ${strictMode ? "with" : "without"} strict mode`, () => {
         it("should display an SDK question", () => {
-          cy.window().then((win) => {
-            cy.spy(win.console, "warn").as("consoleWarn");
-          });
-
           mountSdkContent(
             <InteractiveQuestion questionId={ORDERS_QUESTION_ID} />,
             { strictMode },
@@ -199,6 +194,13 @@ describe(
         it("should show a custom loader when the SDK bundle is loading", () => {
           sdkBundleCleanup();
 
+          cy.intercept("GET", "/api/card/*", (request) => {
+            // Delay request for 500ms to avoid flakiness
+            request.continue(
+              () => new Promise((resolve) => setTimeout(resolve, 500)),
+            );
+          });
+
           mountSdkContent(
             <InteractiveQuestion questionId={ORDERS_QUESTION_ID} />,
             {
@@ -285,7 +287,7 @@ describe(
     describe("Components", () => {
       it("should display an SDK question with custom layout components", () => {
         cy.window().then((win) => {
-          cy.spy(win.console, "warn").as("consoleWarn");
+          cy.spy(win.console, "error").as("consoleError");
         });
 
         mountSdkContent(
@@ -303,15 +305,13 @@ describe(
 
           cy.findByTestId("visualization-root").should("be.visible");
         });
+
+        cy.get("@consoleError").should("not.be.called");
       });
 
       it("should show an error on a component level if SDK components are not wrapped within the MetabaseProvider", () => {
         sdkBundleCleanup();
 
-        cy.window().then((win) => {
-          cy.spy(win.console, "warn").as("consoleWarn");
-        });
-
         cy.mount(<InteractiveQuestion questionId={ORDERS_QUESTION_ID} />);
 
         getSdkRoot().within(() => {
@@ -320,6 +320,41 @@ describe(
           ).should("exist");
         });
       });
+
+      it("should show a console error if SDK Package component uses a prop that is not yet available in SDK bundle", () => {
+        sdkBundleCleanup();
+
+        cy.window().then((win) => {
+          cy.spy(win.console, "error").as("consoleError");
+        });
+
+        mountSdkContent(
+          <InteractiveQuestion
+            questionId={ORDERS_QUESTION_ID}
+            {...{ foo: "bar" }}
+          />,
+        );
+
+        cy.get("@consoleError").should(
+          "be.calledWithMatch",
+          "this property is not recognized by the component",
+        );
+      });
+
+      it("should show a console error if SDK Package component does not use a prop that is still expected by SDK bundle", () => {
+        sdkBundleCleanup();
+
+        cy.window().then((win) => {
+          cy.spy(win.console, "error").as("consoleError");
+        });
+
+        mountSdkContent(<InteractiveQuestion />);
+
+        cy.get("@consoleError").should(
+          "be.calledWithMatch",
+          "this property is required by the component",
+        );
+      });
     });
 
     describe("Error handling", { retries: 3 }, () => {
@@ -344,7 +379,7 @@ describe(
 
           cy.findByTestId("sdk-error-container").should(
             "contain.text",
-            "Error loading the Embedding Analytics SDK",
+            "Error loading the Embedded Analytics SDK",
           );
         });
 
@@ -363,7 +398,7 @@ describe(
 
           cy.findByTestId("sdk-error-container").should(
             "contain.text",
-            "Custom error: Error loading the Embedding Analytics SDK",
+            "Custom error: Error loading the Embedded Analytics SDK",
           );
         });
       });

e2e/test/scenarios/admin-2/people.cy.spec.js

@@ -767,7 +767,7 @@ describe("scenarios > admin > people > group managers", () => {
       cy.url().should("match", /\/$/);
     });
 
-    it("can manage members from the people page", () => {
+    it("can manage members from the people page", { tags: "@flaky" }, () => {
       // Open membership select for a user
       // eslint-disable-next-line no-unscoped-text-selectors -- deprecated usage
       cy.findByText(noCollectionUserName)

e2e/test/scenarios/admin/i18n/content-translation/dashboards.cy.spec.ts

@@ -7,6 +7,10 @@ import type {
   DashboardDetails,
   StructuredQuestionDetails,
 } from "e2e/support/helpers";
+import {
+  ORDERS_COUNT_BY_CREATED_AT_AND_PRODUCT_CATEGORY,
+  PRODUCTS_COUNT_BY_CATEGORY_PIE,
+} from "e2e/support/test-visualizer-data";
 import type { DictionaryArray } from "metabase-types/api";
 const { ORDERS, ORDERS_ID } = SAMPLE_DATABASE;
 
@@ -23,6 +27,169 @@ import { uploadTranslationDictionaryViaAPI } from "./helpers/e2e-content-transla
 const { H } = cy;
 
 describe("scenarios > content translation > static embedding > dashboards", () => {
+  describe("values translation", () => {
+    beforeEach(() => {
+      H.restore();
+      cy.signInAsAdmin();
+      H.activateToken("bleeding-edge");
+
+      uploadTranslationDictionaryViaAPI([
+        { locale: "fr", msgid: "Gadget", msgstr: "Le gadget" },
+        { locale: "fr", msgid: "Doohickey", msgstr: "Le doohickey" },
+        { locale: "fr", msgid: "Gizmo", msgstr: "Le gizmo" },
+        { locale: "fr", msgid: "Widget", msgstr: "Le widget" },
+      ]);
+
+      cy.intercept("POST", "/api/card/*/query").as("cardQuery");
+      cy.intercept("GET", "/api/embed/dashboard/*").as("dashboard");
+
+      cy.signInAsAdmin();
+
+      H.createQuestion(ORDERS_COUNT_BY_CREATED_AT_AND_PRODUCT_CATEGORY, {
+        idAlias: "productsCountByCreatedAtQuestionId",
+        wrapId: true,
+      });
+
+      H.createQuestion(
+        {
+          ...PRODUCTS_COUNT_BY_CATEGORY_PIE,
+          visualization_settings: {
+            "pie.rows": [
+              ["Widget", "#69C8C8"],
+              ["Gadget", "#C7EAEA"],
+              ["Gizmo", "#98D9D9"],
+              ["Doohickey", "#F3F3F4"],
+            ].map(([key, color]) => ({
+              key,
+              name: key,
+              originalName: key,
+              color,
+              defaultColor: false,
+              enabled: true,
+              hidden: false,
+              isOther: false,
+            })),
+          },
+        },
+        {
+          idAlias: "productsCountByCategoryPieQuestionId",
+          wrapId: true,
+        },
+      );
+    });
+
+    it("should assign the proper colors to a pie", () => {
+      H.createDashboard({
+        name: "the_dashboard",
+      }).then(({ body: { id: dashboardId } }) => {
+        H.visitDashboard(dashboardId);
+        H.editDashboard();
+        H.openQuestionsSidebar();
+
+        H.sidebar().findByText(PRODUCTS_COUNT_BY_CATEGORY_PIE.name).click();
+        H.saveDashboard();
+
+        H.openStaticEmbeddingModal({
+          acceptTerms: false,
+        });
+        H.publishChanges("dashboard", () => {});
+
+        H.visitEmbeddedPage(
+          {
+            resource: { dashboard: dashboardId as number },
+            params: {},
+          },
+          {
+            additionalHashOptions: {
+              locale: "fr",
+            },
+          },
+        );
+
+        cy.wait("@dashboard");
+        cy.wait("@cardQuery");
+
+        H.getDashboardCard(0).within(() => {
+          cy.findAllByText("Le gadget").should("exist");
+          cy.findAllByText("Le doohickey").should("exist");
+          cy.findAllByText("Le gizmo").should("exist");
+          cy.findAllByText("Le widget").should("exist");
+
+          // Verify colors
+          cy.findByTestId("chart-legend").within(() => {
+            cy.get("button [color]").then(($elements) => {
+              const actualColors = Array.from($elements).map((el) =>
+                el.getAttribute("color"),
+              );
+              expect(actualColors).to.deep.equal([
+                "#69C8C8",
+                "#C7EAEA",
+                "#98D9D9",
+                "#F3F3F4",
+              ]);
+            });
+          });
+        });
+      });
+    });
+
+    it("should translate static embedding dashboard values on visualizer cards (metabase#62373)", () => {
+      H.visitDashboard(ORDERS_DASHBOARD_ID);
+
+      H.editDashboard();
+      H.removeDashboardCard(0);
+      H.openQuestionsSidebar();
+
+      // Add the regular question
+      H.sidebar()
+        .findByText(ORDERS_COUNT_BY_CREATED_AT_AND_PRODUCT_CATEGORY.name)
+        .click();
+
+      // Add the visualizer question
+      H.clickVisualizeAnotherWay(
+        ORDERS_COUNT_BY_CREATED_AT_AND_PRODUCT_CATEGORY.name,
+      );
+      H.selectVisualization("bar");
+      H.saveDashcardVisualizerModal({ mode: "create" });
+      H.saveDashboard();
+
+      H.openStaticEmbeddingModal({
+        acceptTerms: false,
+      });
+      H.publishChanges("dashboard", () => {});
+
+      H.visitEmbeddedPage(
+        {
+          resource: { dashboard: ORDERS_DASHBOARD_ID as number },
+          params: {},
+        },
+        {
+          additionalHashOptions: {
+            locale: "fr",
+          },
+        },
+      );
+
+      cy.wait("@dashboard");
+      cy.wait("@cardQuery");
+      cy.wait("@cardQuery");
+
+      H.getDashboardCard(0).within(() => {
+        cy.findAllByText("Le gadget").should("exist");
+        cy.findAllByText("Le doohickey").should("exist");
+        cy.findAllByText("Le gizmo").should("exist");
+        cy.findAllByText("Le widget").should("exist");
+      });
+
+      H.getDashboardCard(1).within(() => {
+        cy.findAllByText("Le gadget").should("exist");
+        cy.findAllByText("Le doohickey").should("exist");
+        cy.findAllByText("Le gizmo").should("exist");
+        cy.findAllByText("Le widget").should("exist");
+      });
+    });
+  });
+
   describe("measure names", () => {
     before(() => {
       cy.intercept("POST", "/api/ee/content-translation/upload-dictionary").as(

e2e/test/scenarios/admin/transforms.cy.spec.ts

@@ -701,37 +701,43 @@ H.describeWithSnowplowEE("scenarios > admin > transforms", () => {
       assertTableDoesNotExistError();
     });
 
-    it("should be able to delete the target and restore the same target back", () => {
-      cy.log("create and run a transform");
-      createMbqlTransform({ visitTransform: true });
-      runTransformAndWaitForSuccess();
+    it(
+      "should be able to delete the target and restore the same target back",
+      { tags: "@flaky" },
+      () => {
+        cy.log("create and run a transform");
+        createMbqlTransform({ visitTransform: true });
+        runTransformAndWaitForSuccess();
 
-      cy.log("delete the old target without creating the new one");
-      getTransformPage().button("Change target").click();
-      H.modal().within(() => {
-        cy.findByLabelText("Table name").clear().type(TARGET_TABLE_2);
-        cy.findByLabelText("Delete transform_table").click();
-        cy.button("Change target and delete the old one").click();
-        cy.wait("@deleteTransformTable");
-        cy.wait("@updateTransform");
-      });
+        cy.log("delete the old target without creating the new one");
+        getTransformPage().button("Change target").click();
+        H.modal().within(() => {
+          cy.findByLabelText("Table name").clear().type(TARGET_TABLE_2);
+          cy.findByLabelText("Delete transform_table").click();
+          cy.button("Change target and delete the old one").click();
+          cy.wait("@deleteTransformTable");
+          cy.wait("@updateTransform");
+        });
 
-      cy.log("change the target back to the original one");
-      getTransformPage().button("Change target").click();
-      H.modal().within(() => {
-        cy.findByLabelText("Table name").clear().type(TARGET_TABLE);
-        cy.button("Change target").click();
-        cy.wait("@updateTransform");
-      });
+        cy.log("change the target back to the original one");
+        getTransformPage().button("Change target").click();
+        H.modal().within(() => {
+          cy.findByLabelText("Table name").clear().type(TARGET_TABLE);
+          cy.button("Change target").click();
+          cy.wait("@updateTransform");
+        });
 
-      cy.log("run the transform to re-create the original target");
-      runTransformAndWaitForSuccess();
+        cy.log("run the transform to re-create the original target");
+        runTransformAndWaitForSuccess();
 
-      cy.log("verify the target is available");
-      getTableLink().click();
-      H.queryBuilderHeader().findByText("Transform Table").should("be.visible");
-      H.assertQueryBuilderRowCount(3);
-    });
+        cy.log("verify the target is available");
+        getTableLink().click();
+        H.queryBuilderHeader()
+          .findByText("Transform Table")
+          .should("be.visible");
+        H.assertQueryBuilderRowCount(3);
+      },
+    );
 
     it("should not allow to overwrite an existing table when changing the target", () => {
       createMbqlTransform({ visitTransform: true });
@@ -1321,7 +1327,7 @@ describe("scenarios > admin > transforms > jobs", () => {
   });
 
   describe("tags", () => {
-    it("should be able to add and remove tags", () => {
+    it("should be able to add and remove tags", { tags: "@flaky" }, () => {
       H.createTransformJob({ name: "New job" }, { visitTransformJob: true });
       getTagsInput().click();
 
@@ -2239,8 +2245,8 @@ function assertOptionSelected(name: string) {
 
 function assertOptionNotSelected(name: string) {
   cy.findByRole("option", { name }).should(
-    "not.have.attr",
+    "have.attr",
     "aria-selected",
-    "true",
+    "false",
   );
 }

e2e/test/scenarios/collections/uploads.cy.spec.js

@@ -15,80 +15,84 @@ H.describeWithSnowplow(
       cy.intercept("POST", "/api/table/*/replace-csv").as("replaceCSV");
     });
 
-    it("Can upload a CSV file to an empty postgres schema", () => {
-      const testFile = VALID_CSV_FILES[0];
-      const EMPTY_SCHEMA_NAME = "empty_uploads";
+    it(
+      "Can upload a CSV file to an empty postgres schema",
+      { tags: "@flaky" },
+      () => {
+        const testFile = VALID_CSV_FILES[0];
+        const EMPTY_SCHEMA_NAME = "empty_uploads";
 
-      cy.intercept("PUT", "/api/setting/*").as("saveSettings");
-      cy.intercept("GET", "/api/database").as("databaseList");
+        cy.intercept("PUT", "/api/setting/*").as("saveSettings");
+        cy.intercept("GET", "/api/database").as("databaseList");
 
-      H.restore("postgres-writable");
-      cy.signInAsAdmin();
+        H.restore("postgres-writable");
+        cy.signInAsAdmin();
 
-      H.queryWritableDB(
-        "DROP SCHEMA IF EXISTS empty_uploads CASCADE;",
-        "postgres",
-      );
-      H.queryWritableDB(
-        "CREATE SCHEMA IF NOT EXISTS empty_uploads;",
-        "postgres",
-      );
-
-      cy.request("POST", "/api/collection", {
-        name: "Uploads Collection",
-        parent_id: null,
-      }).then(({ body: { id: collectionId } }) => {
-        cy.wrap(collectionId).as("collectionId");
-      });
-      cy.visit("/admin/settings/uploads");
-
-      cy.findByLabelText("Upload Settings Form")
-        .findByPlaceholderText("Select a database")
-        .click();
-      H.popover().findByText("Writable Postgres12").click();
-      cy.findByLabelText("Upload Settings Form")
-        .findByPlaceholderText("Select a schema")
-        .click();
-
-      H.popover().findByText(EMPTY_SCHEMA_NAME).click();
-
-      cy.findByLabelText("Upload Settings Form")
-        .button("Enable uploads")
-        .click();
-
-      cy.wait(["@saveSettings", "@databaseList"]);
-
-      uploadFileToCollection(testFile);
-
-      const tableQuery = `SELECT * FROM information_schema.tables WHERE table_name LIKE '%${testFile.tableName}_%' ORDER BY table_name DESC LIMIT 1;`;
-
-      H.queryWritableDB(tableQuery, "postgres").then((result) => {
-        expect(result.rows.length).to.equal(1);
-        const tableName = result.rows[0].table_name;
         H.queryWritableDB(
-          `SELECT count(*) FROM ${EMPTY_SCHEMA_NAME}.${tableName};`,
+          "DROP SCHEMA IF EXISTS empty_uploads CASCADE;",
           "postgres",
-        ).then((result) => {
-          expect(Number(result.rows[0].count)).to.equal(testFile.rowCount);
+        );
+        H.queryWritableDB(
+          "CREATE SCHEMA IF NOT EXISTS empty_uploads;",
+          "postgres",
+        );
+
+        cy.request("POST", "/api/collection", {
+          name: "Uploads Collection",
+          parent_id: null,
+        }).then(({ body: { id: collectionId } }) => {
+          cy.wrap(collectionId).as("collectionId");
         });
-      });
+        cy.visit("/admin/settings/uploads");
 
-      cy.log(
-        "Ensure that table is visible in admin without refreshing (metabase#38041)",
-      );
+        cy.findByLabelText("Upload Settings Form")
+          .findByPlaceholderText("Select a database")
+          .click();
+        H.popover().findByText("Writable Postgres12").click();
+        cy.findByLabelText("Upload Settings Form")
+          .findByPlaceholderText("Select a schema")
+          .click();
 
-      cy.findByTestId("app-bar")
-        .findByRole("button", { name: "Settings" })
-        .click();
-      H.popover().findByText("Admin settings").click();
+        H.popover().findByText(EMPTY_SCHEMA_NAME).click();
 
-      cy.findByRole("link", { name: "Table Metadata" }).click();
+        cy.findByLabelText("Upload Settings Form")
+          .button("Enable uploads")
+          .click();
 
-      H.DataModel.TablePicker.getDatabase("Writable Postgres12").click();
-      H.DataModel.TablePicker.getSchema(EMPTY_SCHEMA_NAME).click();
-      H.DataModel.TablePicker.getTables().should("have.length", 1);
-      H.DataModel.TablePicker.getTable("Dog Breeds").should("be.visible");
-    });
+        cy.wait(["@saveSettings", "@databaseList"]);
+
+        uploadFileToCollection(testFile);
+
+        const tableQuery = `SELECT * FROM information_schema.tables WHERE table_name LIKE '%${testFile.tableName}_%' ORDER BY table_name DESC LIMIT 1;`;
+
+        H.queryWritableDB(tableQuery, "postgres").then((result) => {
+          expect(result.rows.length).to.equal(1);
+          const tableName = result.rows[0].table_name;
+          H.queryWritableDB(
+            `SELECT count(*) FROM ${EMPTY_SCHEMA_NAME}.${tableName};`,
+            "postgres",
+          ).then((result) => {
+            expect(Number(result.rows[0].count)).to.equal(testFile.rowCount);
+          });
+        });
+
+        cy.log(
+          "Ensure that table is visible in admin without refreshing (metabase#38041)",
+        );
+
+        cy.findByTestId("app-bar")
+          .findByRole("button", { name: "Settings" })
+          .click();
+        H.popover().findByText("Admin settings").click();
+
+        cy.findByRole("link", { name: "Table Metadata" }).click();
+
+        H.DataModel.TablePicker.getDatabase("Writable Postgres12").click();
+        H.DataModel.TablePicker.getSchema(EMPTY_SCHEMA_NAME).click();
+        H.DataModel.TablePicker.getTables().should("have.length", 1);
+        H.DataModel.TablePicker.getTable("Dog Breeds").should("be.visible");
+      },
+    );
 
     ["postgres", "mysql"].forEach((dialect) => {
       describe(`CSV Uploading (${dialect})`, () => {

e2e/test/scenarios/custom-column/custom-column-reproductions.cy.spec.js

@@ -2495,3 +2495,67 @@ describe("issue 62987", () => {
     });
   });
 });
+
+describe("issue 63180", () => {
+  beforeEach(() => {
+    H.restore();
+    cy.signInAsNormalUser();
+    H.createQuestion(
+      {
+        query: {
+          "source-table": ORDERS_ID,
+          expressions: {
+            Foo: ["+", 1, 2],
+          },
+        },
+      },
+      { visitQuestion: true },
+    );
+
+    H.openNotebook();
+  });
+
+  it("should not be possible to close the custom expression editor when creating a new expression from a combine or extract shortcut (metabase#63180)", () => {
+    function testCombineColumns() {
+      H.getNotebookStep("expression").icon("add").click();
+      H.expressionEditorWidget().within(() => {
+        cy.findByText("Combine columns").click();
+        cy.button("Done").scrollIntoView().click();
+      });
+
+      cy.log("clicking outside the editor should not close it");
+      H.getNotebookStep("data").click();
+      H.expressionEditorWidget().should("be.visible");
+      H.modal().should("not.exist");
+
+      cy.log("clearing the expression should allow clicking outside to work");
+      H.CustomExpressionEditor.clear();
+      H.getNotebookStep("data").click();
+      H.expressionEditorWidget().should("not.exist");
+      H.modal().should("not.exist");
+    }
+
+    function testExtractColumns() {
+      H.getNotebookStep("expression").icon("add").click();
+      H.expressionEditorWidget().within(() => {
+        cy.findByText("Extract columns").click();
+        cy.findByText("Email").click();
+        cy.findByText("Domain").click();
+      });
+
+      cy.log("clicking outside the editor should not close it");
+      H.getNotebookStep("data").click();
+      H.expressionEditorWidget().should("be.visible");
+      H.modal().should("not.exist");
+
+      cy.log("clearing the expression should allow clicking outside to work");
+      H.CustomExpressionEditor.clear();
+      H.getNotebookStep("data").click();
+      H.expressionEditorWidget().should("not.exist");
+      H.modal().should("not.exist");
+    }
+
+    testCombineColumns();
+    testExtractColumns();
+  });
+});

e2e/test/scenarios/dashboard/dashboard-reproductions.cy.spec.js

@@ -2180,3 +2180,28 @@ describe("issue 53370", () => {
     //
   });
 });
+
+describe("issue 63176", () => {
+  beforeEach(() => {
+    H.restore();
+    cy.signInAsNormalUser();
+  });
+
+  it("should not be possible to save a dashboard with an empty name and the correct error should be displayed (metabase#63176)", () => {
+    cy.visit("/");
+    H.newButton().click();
+    H.popover().findByText("Dashboard").click();
+    H.modal().within(() => {
+      cy.findByPlaceholderText("What is the name of your dashboard?").type(" ");
+      cy.button("Create").click();
+
+      cy.findByText("value must be a non-blank string.").should("be.visible");
+      cy.findByPlaceholderText("What is the name of your dashboard?").should(
+        "have.attr",
+        "aria-invalid",
+        "true",
+      );
+      cy.button("Failed").should("be.visible");
+    });
+  });
+});

e2e/test/scenarios/embedding/embedding-reproductions.cy.spec.js

@@ -957,20 +957,24 @@ describe("issue 40660", () => {
     });
   });
 
-  it("static dashboard content shouldn't overflow its container (metabase#40660)", () => {
-    H.openStaticEmbeddingModal({
-      activeTab: "parameters",
-      previewMode: "preview",
-    });
+  it(
+    "static dashboard content shouldn't overflow its container (metabase#40660)",
+    { tags: "@flaky" },
+    () => {
+      H.openStaticEmbeddingModal({
+        activeTab: "parameters",
+        previewMode: "preview",
+      });
 
-    H.getIframeBody().within(() => {
-      cy.findByTestId("embed-frame").scrollTo("bottom");
+      H.getIframeBody().within(() => {
+        cy.findByTestId("embed-frame").scrollTo("bottom");
 
-      cy.findByRole("link", { name: "Powered by Metabase" }).should(
-        "be.visible",
-      );
-    });
-  });
+        cy.findByRole("link", { name: "Powered by Metabase" }).should(
+          "be.visible",
+        );
+      });
+    },
+  );
 });
 
 // Skipped since it does not make sense when CSP is disabled

e2e/test/scenarios/models/models-list-view.cy.spec.js

@@ -123,6 +123,17 @@ describe("scenarios > models list view", () => {
         cy.findByText("2.07").should("not.exist");
       });
 
+      cy.log("Verify that empty column preview displays placeholder value");
+      cy.get("@rightColumns").within(() => {
+        cy.get("input").type("DISC");
+      });
+      H.popover().within(() => {
+        cy.findByText("DISCOUNT").click();
+      });
+      cy.get("@listPreview").within(() => {
+        cy.findByText("$123.46").should("be.visible");
+      });
+
       cy.log("Update list item icon");
       cy.get("@listPreview").within(() => {
         cy.findByRole("img").should("have.attr", "aria-label", "document icon");

e2e/test/scenarios/models/reproductions.cy.spec.ts

@@ -143,31 +143,35 @@ describe("issue 35711", () => {
     cy.signInAsAdmin();
   });
 
-  it("can edit metadata of a model with a custom column (metabase#35711)", () => {
-    H.createQuestion(
-      {
-        type: "model",
-        query: {
-          "source-table": ORDERS_ID,
-          expressions: {
-            "Custom column": ["-", DISCOUNT_FIELD_REF, 1],
+  it(
+    "can edit metadata of a model with a custom column (metabase#35711)",
+    { tags: "@flaky" },
+    () => {
+      H.createQuestion(
+        {
+          type: "model",
+          query: {
+            "source-table": ORDERS_ID,
+            expressions: {
+              "Custom column": ["-", DISCOUNT_FIELD_REF, 1],
+            },
+            limit: 5, // optimization
           },
-          limit: 5, // optimization
         },
-      },
-      { visitQuestion: true },
-    );
+        { visitQuestion: true },
+      );
 
-    H.openQuestionActions();
-    H.popover().findByText("Edit metadata").click();
-    H.waitForLoaderToBeRemoved();
+      H.openQuestionActions();
+      H.popover().findByText("Edit metadata").click();
+      H.waitForLoaderToBeRemoved();
 
-    reorderTaxAndTotalColumns();
-    assertNoError();
+      reorderTaxAndTotalColumns();
+      assertNoError();
 
-    cy.findByTestId("editor-tabs-query-name").click();
-    assertNoError();
-  });
+      cy.findByTestId("editor-tabs-query-name").click();
+      assertNoError();
+    },
+  );
 });
 
 describe("issues 25884 and 34349", () => {
@@ -977,20 +981,26 @@ describe("issue 39993", () => {
     cy.intercept("PUT", "/api/card/*").as("updateModel");
   });
 
-  it("should preserve viz settings for models with custom expressions (metabase#39993)", () => {
-    H.createQuestion(modelDetails).then(({ body: card }) =>
-      H.visitModel(card.id),
-    );
-    H.openQuestionActions();
-    H.popover().findByText("Edit metadata").click();
-    H.waitForLoaderToBeRemoved();
-    cy.log("drag & drop the custom column 100 px to the left");
-    H.moveDnDKitElement(H.tableHeaderColumn(columnName), { horizontal: -100 });
-    cy.button("Save changes").click();
-    cy.wait("@updateModel");
-    cy.findAllByTestId("header-cell").eq(0).should("have.text", "Exp");
-    cy.findAllByTestId("header-cell").eq(1).should("have.text", "ID");
-  });
+  it(
+    "should preserve viz settings for models with custom expressions (metabase#39993)",
+    { tags: "@flaky" },
+    () => {
+      H.createQuestion(modelDetails).then(({ body: card }) =>
+        H.visitModel(card.id),
+      );
+      H.openQuestionActions();
+      H.popover().findByText("Edit metadata").click();
+      H.waitForLoaderToBeRemoved();
+      cy.log("drag & drop the custom column 100 px to the left");
+      H.moveDnDKitElement(H.tableHeaderColumn(columnName), {
+        horizontal: -100,
+      });
+      cy.button("Save changes").click();
+      cy.wait("@updateModel");
+      cy.findAllByTestId("header-cell").eq(0).should("have.text", "Exp");
+      cy.findAllByTestId("header-cell").eq(1).should("have.text", "ID");
+    },
+  );
 });
 
 describe("issue 34574", () => {
@@ -1154,36 +1164,40 @@ describe("issue 36161", () => {
     cy.intercept("POST", "/api/dataset").as("dataset");
   });
 
-  it("should allow to override metadata for custom columns (metabase#36161)", () => {
-    H.visitModel(ORDERS_MODEL_ID);
-    cy.wait("@dataset");
+  it(
+    "should allow to override metadata for custom columns (metabase#36161)",
+    { tags: "@flaky" },
+    () => {
+      H.visitModel(ORDERS_MODEL_ID);
+      cy.wait("@dataset");
 
-    H.openQuestionActions("Edit query definition");
-    H.getNotebookStep("data").button("Pick columns").click();
-    H.popover().findByText("Select all").click();
-    H.getNotebookStep("data").button("Custom column").click();
-    H.enterCustomColumnDetails({ formula: "[ID]", name: "ID2" });
-    H.popover().button("Done").click();
-    H.getNotebookStep("expression").icon("add").click();
-    H.enterCustomColumnDetails({ formula: "[ID]", name: "ID3" });
-    H.popover().button("Done").click();
-    H.runButtonOverlay().click();
-    cy.wait("@dataset");
-    cy.findByTestId("editor-tabs-columns-name").click();
-    H.openColumnOptions("ID2");
-    H.renameColumn("ID2", "ID2 custom");
-    H.openColumnOptions("ID3");
-    H.renameColumn("ID3", "ID3 custom");
-    H.saveMetadataChanges();
+      H.openQuestionActions("Edit query definition");
+      H.getNotebookStep("data").button("Pick columns").click();
+      H.popover().findByText("Select all").click();
+      H.getNotebookStep("data").button("Custom column").click();
+      H.enterCustomColumnDetails({ formula: "[ID]", name: "ID2" });
+      H.popover().button("Done").click();
+      H.getNotebookStep("expression").icon("add").click();
+      H.enterCustomColumnDetails({ formula: "[ID]", name: "ID3" });
+      H.popover().button("Done").click();
+      H.runButtonOverlay().click();
+      cy.wait("@dataset");
+      cy.findByTestId("editor-tabs-columns-name").click();
+      H.openColumnOptions("ID2");
+      H.renameColumn("ID2", "ID2 custom");
+      H.openColumnOptions("ID3");
+      H.renameColumn("ID3", "ID3 custom");
+      H.saveMetadataChanges();
 
-    H.openNotebook();
-    H.getNotebookStep("data").button("Filter").click();
-    H.popover().within(() => {
-      cy.findByText("ID").should("be.visible");
-      cy.findByText("ID2 custom").should("be.visible");
-      cy.findByText("ID3 custom").should("be.visible");
-    });
-  });
+      H.openNotebook();
+      H.getNotebookStep("data").button("Filter").click();
+      H.popover().within(() => {
+        cy.findByText("ID").should("be.visible");
+        cy.findByText("ID2 custom").should("be.visible");
+        cy.findByText("ID3 custom").should("be.visible");
+      });
+    },
+  );
 });
 
 describe("issue 34514", () => {

e2e/test/scenarios/onboarding/setup/setup-embedding.cy.spec.ts

@@ -75,195 +75,201 @@ H.describeWithSnowplowEE("scenarios > setup embedding (EMB-477)", () => {
       .should("be.visible");
   });
 
-  it("[cloud-hosted] should allow users to go through the embedding setup and onboarding flow", () => {
-    // mock `is-hosted`, we can't set the token before the user is created, we'll set it right after that
-    H.activateToken("pro-cloud");
+  it(
+    "[cloud-hosted] should allow users to go through the embedding setup and onboarding flow",
+    { tags: "@flaky" },
+    () => {
+      // mock `is-hosted`, we can't set the token before the user is created, we'll set it right after that
+      H.activateToken("pro-cloud");
 
-    mockCloudHosted();
+      mockCloudHosted();
 
-    cy.visit(
-      "/setup/embedding?first_name=Firstname&last_name=Lastname&email=testy@metabase.test&site_name=Epic Team",
-    );
-
-    cy.log("0: Welcome step");
-    assertEmbeddingOnboardingPageLoaded();
-
-    H.expectUnstructuredSnowplowEvent({
-      event: "embedding_setup_step_seen",
-      event_detail: "welcome",
-    });
-
-    step().within(() => {
-      cy.findByText(
-        /Let's get you up and running with a starting setup for embedded analytics/,
-      ).should("be.visible");
-      cy.button("Start").should("be.visible").click();
-    });
-
-    cy.log("1: User creation step");
-    sidebar().within(() => {
-      cy.findByRole("listitem", { current: "step" }).should(
-        "have.text",
-        "Set up your account",
-      );
-    });
-    step().within(() => {
-      cy.findByRole("heading", { name: "What should we call you?" }).should(
-        "be.visible",
+      cy.visit(
+        "/setup/embedding?first_name=Firstname&last_name=Lastname&email=testy@metabase.test&site_name=Epic Team",
       );
 
+      cy.log("0: Welcome step");
+      assertEmbeddingOnboardingPageLoaded();
+
       H.expectUnstructuredSnowplowEvent({
         event: "embedding_setup_step_seen",
-        event_detail: "user-creation",
+        event_detail: "welcome",
       });
 
-      cy.findByLabelText("First name").should("have.value", "Firstname");
-      cy.findByLabelText("Last name").should("have.value", "Lastname");
-      cy.findByLabelText("Email").should("have.value", "testy@metabase.test");
-      cy.findByLabelText("Company or team name").should(
-        "have.value",
-        "Epic Team",
-      );
+      step().within(() => {
+        cy.findByText(
+          /Let's get you up and running with a starting setup for embedded analytics/,
+        ).should("be.visible");
+        cy.button("Start").should("be.visible").click();
+      });
 
-      fillOutUserForm();
+      cy.log("1: User creation step");
+      sidebar().within(() => {
+        cy.findByRole("listitem", { current: "step" }).should(
+          "have.text",
+          "Set up your account",
+        );
+      });
+      step().within(() => {
+        cy.findByRole("heading", { name: "What should we call you?" }).should(
+          "be.visible",
+        );
 
-      cy.intercept("POST", "/api/setup").as("setup");
-      cy.button("Next").should("be.enabled").click();
-    });
-
-    cy.log(
-      "Now we have a user we simulate being on cloud by setting a Metatabase Token",
-    );
-    cy.wait("@setup");
-
-    // set a real token so we have real token-features for the rest of the flow
-    H.activateToken("pro-cloud");
-
-    cy.log("2: Data connection step");
-    sidebar().within(() => {
-      cy.findByRole("listitem", { current: "step" }).should(
-        "have.text",
-        "Connect to your data",
-      );
-    });
-    step().within(() => {
-      cy.findByRole("heading", { name: "Connect to your data" }).should(
-        "be.visible",
-      );
-
-      cy.findByRole("option", { name: "PostgreSQL" }).click();
-      fillOutDatabaseForm();
-
-      cy.log("simulate database that takes some time to fully sync");
-      const NO_OF_MIN_RETRY = 4;
-      let currentRetry = 0;
-      cy.intercept("GET", "api/database", (req) => {
-        req.on("response", (res) => {
-          currentRetry++;
-          const [postgresDb] = res.body.data;
-          if (
-            postgresDb.initial_sync_status !== "complete" ||
-            currentRetry < NO_OF_MIN_RETRY
-          ) {
-            // Simulate syncing not complete until we reach a certain number of retries
-            postgresDb.initial_sync_status = "incomplete";
-          }
+        H.expectUnstructuredSnowplowEvent({
+          event: "embedding_setup_step_seen",
+          event_detail: "user-creation",
         });
+
+        cy.findByLabelText("First name").should("have.value", "Firstname");
+        cy.findByLabelText("Last name").should("have.value", "Lastname");
+        cy.findByLabelText("Email").should("have.value", "testy@metabase.test");
+        cy.findByLabelText("Company or team name").should(
+          "have.value",
+          "Epic Team",
+        );
+
+        fillOutUserForm();
+
+        cy.intercept("POST", "/api/setup").as("setup");
+        cy.button("Next").should("be.enabled").click();
       });
-      cy.button("Connect database").should("be.enabled").click();
-    });
 
-    cy.log("3: Content generation step");
-    sidebar().within(() => {
-      cy.findByRole("listitem", { current: "step" }).should(
-        "have.text",
-        "Generate starter content",
+      cy.log(
+        "Now we have a user we simulate being on cloud by setting a Metatabase Token",
       );
-    });
+      cy.wait("@setup");
 
-    const SECOND = 1000;
-    // Waiting for the database sync to complete
-    step()
-      .findByRole("heading", {
-        name: "Select Tables to Embed",
-        timeout: 10 * SECOND,
-      })
-      .should("be.visible");
+      // set a real token so we have real token-features for the rest of the flow
+      H.activateToken("pro-cloud");
 
-    expectNoDatabaseStatus();
-
-    step().within(() => {
-      cy.findByRole("checkbox", { name: "feedback" })
-        .should("be.enabled")
-        .check();
-      cy.findByRole("checkbox", { name: "orders" })
-        .should("be.enabled")
-        .check();
-      cy.findByRole("checkbox", { name: "products" })
-        .should("be.enabled")
-        .check();
-
-      cy.button("Continue").should("be.enabled").click();
-    });
-
-    cy.log("4: Processing step");
-
-    step().within(() => {
-      cy.findByRole("heading", { name: "Setting Up Your Analytics" }).should(
-        "be.visible",
-      );
-      /**
-       * Since this loader will go away automatically it's crucial to wait for it to go away
-       * before start asserting the next steps.
-       */
-      cy.findByText("Creating dashboards...").should("not.exist");
-    });
-
-    cy.log("5: Embed in your app step");
-    sidebar().within(() => {
-      cy.findByRole("listitem", { current: "step" }).should(
-        "have.text",
-        "Add to your app",
-      );
-    });
-
-    step().within(() => {
-      cy.findByRole("heading", { name: "Add to your app" }).should(
-        "be.visible",
-      );
-      cy.findByRole("tab", { name: 'A look at "Feedback"' }).click();
-      H.getIframeBody().within(() => {
-        cy.findByText('A look at "Feedback"').should("be.visible");
+      cy.log("2: Data connection step");
+      sidebar().within(() => {
+        cy.findByRole("listitem", { current: "step" }).should(
+          "have.text",
+          "Connect to your data",
+        );
       });
-      cy.findByRole("code").contains(
-        new RegExp(
-          `<iframe src="${Cypress.config("baseUrl")}/dashboard/\\d+" width="800px" height="500px" />`,
-        ),
+      step().within(() => {
+        cy.findByRole("heading", { name: "Connect to your data" }).should(
+          "be.visible",
+        );
+
+        cy.findByRole("option", { name: "PostgreSQL" }).click();
+        fillOutDatabaseForm();
+
+        cy.log("simulate database that takes some time to fully sync");
+        const NO_OF_MIN_RETRY = 4;
+        let currentRetry = 0;
+        cy.intercept("GET", "api/database", (req) => {
+          req.on("response", (res) => {
+            currentRetry++;
+            const [postgresDb] = res.body.data;
+            if (
+              postgresDb.initial_sync_status !== "complete" ||
+              currentRetry < NO_OF_MIN_RETRY
+            ) {
+              // Simulate syncing not complete until we reach a certain number of retries
+              postgresDb.initial_sync_status = "incomplete";
+            }
+          });
+        });
+        cy.button("Connect database").should("be.enabled").click();
+      });
+
+      cy.log("3: Content generation step");
+      sidebar().within(() => {
+        cy.findByRole("listitem", { current: "step" }).should(
+          "have.text",
+          "Generate starter content",
+        );
+      });
+
+      const SECOND = 1000;
+      // Waiting for the database sync to complete
+      step()
+        .findByRole("heading", {
+          name: "Select Tables to Embed",
+          timeout: 10 * SECOND,
+        })
+        .should("be.visible");
+
+      expectNoDatabaseStatus();
+
+      step().within(() => {
+        cy.findByRole("checkbox", { name: "feedback" })
+          .should("be.enabled")
+          .check();
+        cy.findByRole("checkbox", { name: "orders" })
+          .should("be.enabled")
+          .check();
+        cy.findByRole("checkbox", { name: "products" })
+          .should("be.enabled")
+          .check();
+
+        cy.button("Continue").should("be.enabled").click();
+      });
+
+      cy.log("4: Processing step");
+
+      step().within(() => {
+        cy.findByRole("heading", { name: "Setting Up Your Analytics" }).should(
+          "be.visible",
+        );
+        /**
+         * Since this loader will go away automatically it's crucial to wait for it to go away
+         * before start asserting the next steps.
+         */
+        cy.findByText("Creating dashboards...").should("not.exist");
+      });
+
+      cy.log("5: Embed in your app step");
+      sidebar().within(() => {
+        cy.findByRole("listitem", { current: "step" }).should(
+          "have.text",
+          "Add to your app",
+        );
+      });
+
+      step().within(() => {
+        cy.findByRole("heading", { name: "Add to your app" }).should(
+          "be.visible",
+        );
+        cy.findByRole("tab", { name: 'A look at "Feedback"' }).click();
+        H.getIframeBody().within(() => {
+          cy.findByText('A look at "Feedback"').should("be.visible");
+        });
+        cy.findByRole("code").contains(
+          new RegExp(
+            `<iframe src="${Cypress.config("baseUrl")}/dashboard/\\d+" width="800px" height="500px" />`,
+          ),
+        );
+
+        cy.button("I see Metabase").click();
+
+        cy.findByRole("heading", { name: "You're on your way!" }).should(
+          "be.visible",
+        );
+        cy.findByRole("link", { name: "Take me to Metabase" }).click();
+      });
+
+      cy.log("5: Metabase");
+      cy.button("New").should("be.visible");
+
+      cy.log(
+        "Only one 'Automatically Generated Dashboards' collection should be created",
       );
-
-      cy.button("I see Metabase").click();
-
-      cy.findByRole("heading", { name: "You're on your way!" }).should(
-        "be.visible",
-      );
-      cy.findByRole("link", { name: "Take me to Metabase" }).click();
-    });
-
-    cy.log("5: Metabase");
-    cy.button("New").should("be.visible");
-
-    cy.log(
-      "Only one 'Automatically Generated Dashboards' collection should be created",
-    );
-    sidebar().within(() => {
-      cy.findByText("Automatically Generated Dashboards").should("be.visible");
-      cy.findByText("Automatically Generated Dashboards").should(
-        "have.length",
-        1,
-      );
-    });
-    expectNoDatabaseStatus();
-  });
+      sidebar().within(() => {
+        cy.findByText("Automatically Generated Dashboards").should(
+          "be.visible",
+        );
+        cy.findByText("Automatically Generated Dashboards").should(
+          "have.length",
+          1,
+        );
+      });
+      expectNoDatabaseStatus();
+    },
+  );
 });
 
 /**

e2e/test/scenarios/permissions/admin-permissions.cy.spec.js

@@ -310,7 +310,7 @@ describe("scenarios > admin > permissions", { tags: "@OSS" }, () => {
     });
 
     context("group focused view", () => {
-      it("shows filterable list of groups", () => {
+      it("shows filterable list of groups", { tags: "@flaky" }, () => {
         cy.visit("/admin/permissions");
 
         // no groups selected initially and it shows an empty state

e2e/test/scenarios/permissions/sandboxing/sandboxing-via-ui.cy.spec.ts

@@ -78,12 +78,16 @@ describe(
       H.restore("sandboxing-snapshot" as any);
     });
 
-    it("shows all data before sandboxing policy is applied", () => {
-      signInAs(gizmoViewer);
-      assertNoResultsOrValuesAreSandboxed(dashboard, sandboxableQuestions);
-      signInAs(widgetViewer);
-      assertNoResultsOrValuesAreSandboxed(dashboard, sandboxableQuestions);
-    });
+    it(
+      "shows all data before sandboxing policy is applied",
+      { tags: "@flaky" },
+      () => {
+        signInAs(gizmoViewer);
+        assertNoResultsOrValuesAreSandboxed(dashboard, sandboxableQuestions);
+        signInAs(widgetViewer);
+        assertNoResultsOrValuesAreSandboxed(dashboard, sandboxableQuestions);
+      },
+    );
 
     describe("we can apply a sandbox policy", () => {
       beforeEach(() => {

e2e/test/scenarios/question-reproductions/reproductions-3.cy.spec.js

@@ -1955,7 +1955,7 @@ describe("issue 45063", { tags: "@flaky" }, () => {
       });
     });
 
-    it("should work with models", () => {
+    it("should work with models", { tags: "@flaky" }, () => {
       createGuiModel({ sourceTableId: ORDERS_ID });
       verifyRemappedFilter({
         visitCard: () => cy.get("@modelId").then(H.visitModel),

e2e/test/scenarios/table-editing/table-editing.cy.spec.ts

@@ -304,15 +304,20 @@ describe("scenarios > table-editing", () => {
           cy.findAllByRole("button").last().click();
         });
 
-        cy.wait("@updateTableData").then(({ response }) => {
+        cy.wait("@updateTableData").then(({ response, request }) => {
           const targetDate = dayjs(
             new Date(2024, 4, day, hour, minute, 0),
           ).format("YYYY-MM-DDTHH:mm:ss");
-          const savedDate = response?.body.outputs[0].row.CREATED_AT;
 
-          expect(response?.body.outputs[0].op).to.equal("updated");
-          // Omit timezone offset
-          expect(savedDate.startsWith(targetDate)).to.be.true;
+          const requestDate = request.body.params.CREATED_AT;
+          const responseDate = response?.body.outputs[0].row.CREATED_AT;
+
+          // Check if request date matches the response date.
+          // In theory FE should preserve the initial input date timezone offset (might be based on the CI environment)
+          expect(requestDate).to.equal(responseDate);
+
+          // Check if date param matches the input date (without timezone offset at offset 19)
+          expect(requestDate.slice(0, 19)).to.equal(targetDate);
         });
 
         H.undoToast().findByText("Successfully updated").should("be.visible");
@@ -447,6 +452,10 @@ describe("scenarios > table-editing", () => {
       cy.findAllByTestId("row-select-checkbox").eq(15).click();
       cy.findAllByTestId("row-select-checkbox").eq(16).click();
 
+      cy.log("should not show edit icon when rows are selected");
+      cy.findByTestId("row-edit-icon").should("not.exist");
+
+      cy.log("should bulk delete rows");
       cy.findByTestId("toast-card").findByText("Delete").click();
 
       H.modal().within(() => {
@@ -462,6 +471,9 @@ describe("scenarios > table-editing", () => {
       cy.findByTestId("toast-card").should("not.exist");
 
       H.undoToast().findByText("Successfully deleted").should("be.visible");
+
+      cy.log("should show edit icon when no rows are selected");
+      cy.findAllByTestId("row-edit-icon").should("exist");
     });
   });
 });

enterprise/backend/src/metabase_enterprise/api/routes.clj

@@ -26,6 +26,7 @@
    [metabase-enterprise.metabot-v3.api]
    [metabase-enterprise.metabot-v3.tools.api]
    [metabase-enterprise.permission-debug.api]
+   [metabase-enterprise.python-transform.api]
    [metabase-enterprise.sandbox.api.routes]
    [metabase-enterprise.scim.routes]
    [metabase-enterprise.semantic-search.api]
@@ -107,6 +108,7 @@
    "/metabot-tools"                metabase-enterprise.metabot-v3.tools.api/routes
    "/metabot-v3"                   (premium-handler metabase-enterprise.metabot-v3.api/routes :metabot-v3)
    "/permission_debug"             (premium-handler metabase-enterprise.permission-debug.api/routes :advanced-permissions)
+   "/python-transform"             (premium-handler metabase-enterprise.python-transform.api/routes :transforms)
    "/scim"                         (premium-handler metabase-enterprise.scim.routes/routes :scim)
    "/semantic-search"              (premium-handler metabase-enterprise.semantic-search.api/routes :semantic-search)
    "/serialization"                (premium-handler metabase-enterprise.serialization.api/routes :serialization)

enterprise/backend/src/metabase_enterprise/python_transform/api.clj

@@ -0,0 +1,29 @@
+(ns metabase-enterprise.python-transform.api
+  (:require
+   [metabase-enterprise.python-transform.models.python-library :as python-library]
+   [metabase.api.common :as api]
+   [metabase.api.macros :as api.macros]
+   [metabase.api.routes.common :refer [+auth]]
+   [metabase.api.util.handlers :as handlers]))
+
+(api.macros/defendpoint :get "/library/:path"
+  "Get the Python library for user modules."
+  [{:keys [path]} :- [:map [:path :string]]
+   _query-params]
+  (api/check-superuser)
+  (some-> (python-library/get-python-library-by-path path)
+          (select-keys [:source :path :created_at :updated_at])))
+
+(api.macros/defendpoint :put "/library/:path"
+  "Update the Python library source code for user modules."
+  [{:keys [path]} :- [:map [:path :string]]
+   _query-params
+   body :- [:map {:closed true}
+            [:source :string]]]
+  (api/check-superuser)
+  (python-library/update-python-library-source! path (:source body)))
+
+(def ^{:arglists '([request respond raise])} routes
+  "`/api/ee/python-transform` routes."
+  (handlers/routes
+   (api.macros/ns-handler *ns* +auth)))

enterprise/backend/src/metabase_enterprise/python_transform/core.clj

@@ -0,0 +1,5 @@
+(ns metabase-enterprise.python-transform.core
+  (:require
+   [metabase-enterprise.python-transform.models.python-library]))
+
+(comment metabase-enterprise.python-transform.models.python-library/keep-me)

enterprise/backend/src/metabase_enterprise/python_transform/models/python_library.clj

@@ -0,0 +1,41 @@
+(ns metabase-enterprise.python-transform.models.python-library
+  (:require
+   [metabase.app-db.core :as app-db]
+   [metabase.util.i18n :refer [tru]]
+   [methodical.core :as methodical]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(methodical/defmethod t2/table-name :model/PythonLibrary [_model] :python_library)
+
+(doseq [trait [:metabase/model :hook/timestamped?]]
+  (derive :model/PythonLibrary trait))
+
+(def ^:private allowed-paths
+  "Set of allowed library paths. Currently only 'common' is supported."
+  #{"common"})
+
+(defn- validate-path!
+  "Validates that the given path is allowed. Throws an exception if not."
+  [path]
+  (when-not (contains? allowed-paths path)
+    (throw (ex-info (tru "Invalid library path. Only ''common'' is currently supported.")
+                    {:status-code 400
+                     :path path
+                     :allowed-paths allowed-paths}))))
+
+(defn get-python-library-by-path
+  "Get the Python library by path."
+  [path]
+  (validate-path! path)
+  (t2/select-one :model/PythonLibrary :path path))
+
+(defn update-python-library-source!
+  "Update the Python library source code. Creates a new record if none exists. Returns the updated library."
+  [path source]
+  (validate-path! path)
+  (let [id (app-db/update-or-insert! :model/PythonLibrary
+                                     {:path path}
+                                     (constantly {:path path :source source}))]
+    (t2/select-one :model/PythonLibrary id)))

enterprise/backend/src/metabase_enterprise/semantic_search/core.clj

@@ -12,7 +12,8 @@
    [metabase.analytics.core :as analytics]
    [metabase.premium-features.core :refer [defenterprise]]
    [metabase.search.engine :as search.engine]
-   [metabase.util.log :as log]))
+   [metabase.util.log :as log]
+   [toucan2.realize :as t2.realize]))
 
 (defn- fallback-engine
   "Find the highest priority search engine available for fallback."
@@ -58,7 +59,12 @@
           (analytics/inc! :metabase-search/semantic-fallback-triggered {:fallback-engine fallback})
           (analytics/observe! :metabase-search/semantic-results-before-fallback final-count)
           (let [fallback-results (try
-                                   (search.engine/results (assoc search-ctx :search-engine fallback))
+                                   (cond->> (search.engine/results (assoc search-ctx :search-engine fallback))
+                                     ;; The in-place engine returns a reducible (but not seqable) result that needs to
+                                     ;; be realized before we concat and dedup with the semantic engine results.
+                                     (= :search.engine/in-place fallback)
+                                     (into [] (comp (map t2.realize/realize)
+                                                    (take (- (semantic.settings/semantic-search-results-limit) final-count)))))
                                    (catch Throwable t
                                      (log/warn t "Semantic search fallback errored, ignoring")
                                      []))

enterprise/backend/src/metabase_enterprise/semantic_search/index.clj

@@ -127,7 +127,7 @@
    :creator_id          creator_id
    :database_id         database_id
    :last_editor_id      last_editor_id
-   :name                (:name doc)
+   :name                (or (:name doc) "")
    :content             searchable_text
    :display_type        display_type
    :archived            (some-> archived to-boolean)
@@ -145,18 +145,18 @@
    :text_search_vector  (if (:name doc)
                           [:||
                            (search/weighted-tsvector "A" (:name doc))
-                           (search/weighted-tsvector "B" (:searchable_text doc ""))]
-                          (search/weighted-tsvector "A" (:searchable_text doc "")))
+                           (search/weighted-tsvector "B" (or (:searchable_text doc) ""))]
+                          (search/weighted-tsvector "A" (or (:searchable_text doc) "")))
    :text_search_with_native_query_vector
    (if (:name doc)
      [:||
       (search/weighted-tsvector "A" (:name doc))
       (search/weighted-tsvector "B"
-                                (str/join " " (remove str/blank? [(:searchable_text doc "")
-                                                                  (:native_query doc "")])))]
+                                (str/join " " (remove str/blank? [(or (:searchable_text doc) "")
+                                                                  (or (:native_query doc) "")])))]
      (search/weighted-tsvector "A"
-                               (str/join " " (remove str/blank? [(:searchable_text doc "")
-                                                                 (:native_query doc "")]))))})
+                               (str/join " " (remove str/blank? [(or (:searchable_text doc) "")
+                                                                 (or (:native_query doc) "")]))))})
 
 (defn index-size
   "Fetches the number of documents in the index table."

enterprise/backend/src/metabase_enterprise/semantic_search/repair.clj

@@ -13,7 +13,7 @@
    [medley.core :as m]
    [metabase.util :as u]
    [metabase.util.log :as log]
-   [nano-id.core :as nano-id]
+   [metabase.util.string :as u.str]
    [next.jdbc :as jdbc]
    [next.jdbc.result-set :as jdbc.rs]))
 
@@ -93,7 +93,7 @@
   Format: repair_<millis-since-epoch>_<short-id>"
   []
   (let [millis-since-epoch (t/to-millis-from-epoch (t/instant))
-        short-id           (u/lower-case-en (subs (nano-id/nano-id) 0 6))]
+        short-id           (u/lower-case-en (u.str/random-string 6))]
     (format "repair_%d_%s" millis-since-epoch short-id)))
 
 (defn with-repair-table!

enterprise/backend/src/metabase_enterprise/transforms/api.clj

@@ -1,7 +1,10 @@
 (ns metabase-enterprise.transforms.api
   (:require
+   [clojure.core.async :as a]
+   [clojure.string :as str]
    [metabase-enterprise.transforms.api.transform-job]
    [metabase-enterprise.transforms.api.transform-tag]
+   [metabase-enterprise.transforms.canceling :as transforms.canceling]
    [metabase-enterprise.transforms.execute :as transforms.execute]
    [metabase-enterprise.transforms.models.transform :as transform.model]
    [metabase-enterprise.transforms.models.transform-run :as transform-run]
@@ -29,12 +32,21 @@
 (set! *warn-on-reflection* true)
 
 (mr/def ::transform-source
-  [:map
-   [:type [:= "query"]]
-   [:query [:map [:database :int]]]])
+  [:multi {:dispatch (comp keyword :type)}
+   [:query
+    [:map
+     [:type [:= "query"]]
+     [:query [:map [:database :int]]]]]
+   [:python
+    [:map {:closed true}
+     [:source-database {:optional true} :int]
+     [:source-tables   [:map-of :string :int]]
+     [:type [:= "python"]]
+     [:body :string]]]])
 
 (mr/def ::transform-target
   [:map
+   [:database {:optional true} :int]
    [:type [:enum "table"]]
    [:schema {:optional true} [:or ms/NonBlankString :nil]]
    [:name :string]])
@@ -57,19 +69,22 @@
 
 (defn- source-database-id
   [transform]
-  (-> transform :source :query :database))
+  (if (transforms.util/python-transform? transform)
+    (-> transform :source :source-database)
+    (-> transform :source :query :database)))
 
 (defn- check-database-feature
   [transform]
-  (let [database (api/check-400 (t2/select-one :model/Database (source-database-id transform))
-                                (deferred-tru "The source database cannot be found."))
-        feature (transforms.util/required-database-feature transform)]
-    (api/check-400 (not (:is_sample database))
-                   (deferred-tru "Cannot run transforms on the sample database."))
-    (api/check-400 (not (:is_audit database))
-                   (deferred-tru "Cannot run transforms on audit databases."))
-    (api/check-400 (driver.u/supports? (:engine database) feature database)
-                   (deferred-tru "The database does not support the requested transform target type."))))
+  (when (transforms.util/query-transform? transform)
+    (let [database (api/check-400 (t2/select-one :model/Database (source-database-id transform))
+                                  (deferred-tru "The source database cannot be found."))
+          feature (transforms.util/required-database-feature transform)]
+      (api/check-400 (not (:is_sample database))
+                     (deferred-tru "Cannot run transforms on the sample database."))
+      (api/check-400 (not (:is_audit database))
+                     (deferred-tru "Cannot run transforms on audit databases."))
+      (api/check-400 (driver.u/supports? (:engine database) feature database)
+                     (deferred-tru "The database does not support the requested transform target type.")))))
 
 (api.macros/defendpoint :get "/"
   "Get a list of transforms."
@@ -172,9 +187,10 @@
           new (merge old body)
           target-fields #(-> % :target (select-keys [:schema :name]))]
       (check-database-feature new)
-      (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]
-        (throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)
-                        {:status-code 400})))
+      (when (transforms.util/query-transform? old)
+        (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]
+          (throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)
+                          {:status-code 400}))))
       (api/check (not (and (not= (target-fields old) (target-fields new))
                            (transforms.util/target-table-exists? new)))
                  403
@@ -207,11 +223,14 @@
 (api.macros/defendpoint :post "/:id/cancel"
   "Cancel the current run for a given transform."
   [{:keys [id]} :- [:map
-                    [:id :string]]]
+                    [:id ms/PositiveInt]]]
   (log/info "canceling transform " id)
   (api/check-superuser)
-  (let [run (api/check-404 (transform-run/running-run-for-run-id id))]
-    (transform-run-cancelation/mark-cancel-started-run! (:id run)))
+  (let [transform (api/check-404 (t2/select-one :model/Transform id))
+        run (api/check-404 (transform-run/running-run-for-transform-id id))]
+    (transform-run-cancelation/mark-cancel-started-run! (:id run))
+    (when (transforms.util/python-transform? transform)
+      (transforms.canceling/cancel-run! (:id run))))
   nil)
 
 (api.macros/defendpoint :post "/:id/run"
@@ -222,9 +241,16 @@
   (api/check-superuser)
   (let [transform (api/check-404 (t2/select-one :model/Transform id))
         start-promise (promise)]
-    (u.jvm/in-virtual-thread*
-     (transforms.execute/run-mbql-transform! transform {:start-promise start-promise
-                                                        :run-method :manual}))
+    (if (transforms.util/python-transform? transform)
+      (u.jvm/in-virtual-thread*
+       (transforms.execute/execute-python-transform! transform {:start-promise start-promise
+                                                                :message-log   (atom {:pre-python  []
+                                                                                      :python      nil
+                                                                                      :post-python []})
+                                                                :run-method :manual}))
+      (u.jvm/in-virtual-thread*
+       (transforms.execute/run-mbql-transform! transform {:start-promise start-promise
+                                                          :run-method :manual})))
     (when (instance? Throwable @start-promise)
       (throw @start-promise))
     (let [result @start-promise
@@ -234,6 +260,53 @@
                               :run_id run-id})
           (assoc :status 202)))))
 
+;; hack
+(def ^:private python-test-run-id Integer/MAX_VALUE)
+
+(api.macros/defendpoint :post "/test-python"
+  "Test Python code execution without creating a transform."
+  [_route-params
+   _query-params
+   body :- [:map
+            [:code :string]
+            [:tables [:map-of :string :int]]]]
+  (log/info "test python code execution")
+  (api/check-superuser)
+  (let [run-id python-test-run-id
+        cancel-chan (a/promise-chan)]
+    (transforms.canceling/chan-start-run! run-id cancel-chan)
+    (try
+      (let [{:keys [response output events]} (transforms.execute/test-python-transform! (:code body) (:tables body) run-id cancel-chan)
+            {:keys [body status]} response]
+        (if (= status 200)
+          (do
+            (log/info "Python test execution succeeded")
+            (-> (response/response {:message (deferred-tru "Python code executed successfully")
+                                    :result  {:body (assoc body :output output)}})
+                (assoc :status 200)))
+          (do
+            (log/error "Error executing Python test code")
+            (-> (response/response {:message   (deferred-tru "Python code execution failed")
+                                    :error     body
+                                    :stdout    (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
+                                    :stderr    (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))
+                                    :exit_code (:exit_code (:exit_code body))})
+                (assoc :status status)))))
+      (finally
+        (transforms.canceling/chan-end-run! run-id)))))
+
+(api.macros/defendpoint :post "/test-python/cancel"
+  "Cancel the current test-python execution."
+  [_route-params
+   _query-params]
+  (log/info "canceling test python execution")
+  (api/check-superuser)
+  (if (transforms.canceling/chan-signal-cancel! python-test-run-id)
+    (-> (response/response {:message (deferred-tru "Python test canceled")})
+        (assoc :status 200))
+    (-> (response/response {:message (deferred-tru "No running Python test to cancel")})
+        (assoc :status 404))))
+
 (def ^{:arglists '([request respond raise])} routes
   "`/api/ee/transform` routes."
   (handlers/routes

enterprise/backend/src/metabase_enterprise/transforms/canceling.clj

@@ -49,7 +49,9 @@
    (chan-signal-cancel! run-id)
    (transform-run/timeout-run! run-id)))
 
-(defn- cancel-run! [run-id]
+(defn cancel-run!
+  "Cancel run"
+  [run-id]
   (when (chan-signal-cancel! run-id)
     (transform-run/cancel-run! run-id)))
 

enterprise/backend/src/metabase_enterprise/transforms/execute.clj

@@ -1,17 +1,26 @@
 (ns metabase-enterprise.transforms.execute
   (:require
    [clojure.core.async :as a]
+   [clojure.java.io :as io]
+   [clojure.string :as str]
    [metabase-enterprise.transforms.canceling :as canceling]
+   [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
    [metabase-enterprise.transforms.models.transform-run :as transform-run]
+   [metabase-enterprise.transforms.python-runner :as python-runner]
    [metabase-enterprise.transforms.settings :as transforms.settings]
    [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.driver :as driver]
    [metabase.driver.util :as driver.u]
    [metabase.lib.schema.common :as schema.common]
    [metabase.query-processor.pipeline :as qp.pipeline]
+   [metabase.util :as u]
+   [metabase.util.jvm :as u.jvm]
    [metabase.util.log :as log]
    [metabase.util.malli.registry :as mr]
-   [toucan2.core :as t2]))
+   [toucan2.core :as t2])
+  (:import
+   (java.io Closeable File)
+   (java.time Duration)))
 
 (set! *warn-on-reflection* true)
 
@@ -37,24 +46,38 @@
    (log/info "Syncing target" (pr-str target) "for transform")
    (transforms.util/activate-table-and-mark-computed! database target)))
 
-(defn run-transform!
+(defn run-cancelable-transform!
   "Run a compiled transform"
-  [run-id driver {:keys [db-id conn-spec output-schema] :as transform-details} opts]
-  ;; local run is responsible for status
+  [run-id driver {:keys [db conn-spec output-schema]} run-transform!]
+  ;; local run is responsible for status, using canceling lifecycle
   (try
-    (when-not (driver/schema-exists? driver db-id output-schema)
-      (driver/create-schema-if-needed! driver conn-spec output-schema))
+    (when (driver.u/supports? driver :schemas db)
+      (when-not (driver/schema-exists? driver (:id db) output-schema)
+        (driver/create-schema-if-needed! driver conn-spec output-schema)))
     (canceling/chan-start-timeout-vthread! run-id (transforms.settings/transform-timeout))
-    (binding [qp.pipeline/*canceled-chan* (a/promise-chan)]
-      (canceling/chan-start-run! run-id qp.pipeline/*canceled-chan*)
-      (driver/run-transform! driver transform-details opts))
-    (transform-run/succeed-started-run! run-id)
+    (let [cancel-chan (a/promise-chan)
+          ret (binding [qp.pipeline/*canceled-chan* cancel-chan]
+                (canceling/chan-start-run! run-id cancel-chan)
+                (run-transform! cancel-chan))]
+      (transform-run/succeed-started-run! run-id)
+      ret)
     (catch Throwable t
       (transform-run/fail-started-run! run-id {:message (.getMessage t)})
       (throw t))
     (finally
       (canceling/chan-end-run! run-id))))
 
+(defn- try-start-unless-already-running [id run-method]
+  (try
+    (transform-run/start-run! id {:run_method run-method})
+    (catch java.sql.SQLException e
+      (if (= (.getSQLState e) "23505")
+        (throw (ex-info "Transform is already running"
+                        {:error :already-running
+                         :transform-id id}
+                        e))
+        (throw e)))))
+
 (defn run-mbql-transform!
   "Run `transform` and sync its target table.
 
@@ -67,7 +90,7 @@
      (let [db (get-in source [:query :database])
            {driver :engine :as database} (t2/select-one :model/Database db)
            feature (transforms.util/required-database-feature transform)
-           transform-details {:db-id db
+           transform-details {:db db
                               :transform-type (keyword (:type target))
                               :conn-spec (driver/connection-spec driver database)
                               :query (transforms.util/compile-source source)
@@ -78,20 +101,13 @@
          (throw (ex-info "The database does not support the requested transform target type."
                          {:driver driver, :database database, :feature feature})))
        ;; mark the execution as started and notify any observers
-       (let [{run-id :id} (try
-                            (transform-run/start-run! id {:run_method run-method})
-                            (catch java.sql.SQLException e
-                              (if (= (.getSQLState e) "23505")
-                                (throw (ex-info "Transform is already running"
-                                                {:error :already-running
-                                                 :transform-id id}
-                                                e))
-                                (throw e))))]
+       (let [{run-id :id} (try-start-unless-already-running id run-method)]
          (when start-promise
            (deliver start-promise [:started run-id]))
          (log/info "Executing transform" id "with target" (pr-str target))
-         (run-transform! run-id driver transform-details opts)
-         (sync-target! target database run-id)))
+         (run-cancelable-transform! run-id driver transform-details (fn [_cancel-chan] (driver/run-transform! driver transform-details opts)))
+         (transforms.instrumentation/with-stage-timing [run-id :sync :table-sync]
+           (sync-target! target database run-id))))
      (catch Throwable t
        (log/error t "Error executing transform")
        (when start-promise
@@ -99,3 +115,228 @@
          ;; but we assume nobody would catch the exception anyway
          (deliver start-promise t))
        (throw t)))))
+
+(defn- log! [message-log s]
+  (when message-log
+    (swap! message-log (fn [m] (update m (if (:python m) :post-python :pre-python) conj s))))
+  nil)
+
+(defn- replace-python-logs! [message-log events]
+  (when message-log (swap! message-log assoc :python events))
+  nil)
+
+(defn- save-log-as-message! [run-id message-log]
+  (when message-log
+    (let [{:keys [pre-python python post-python]} @message-log]
+      (t2/update! :model/TransformRun
+                  :id run-id
+                  {:message (str/join "\n" (concat pre-python
+                                                   (for [{:keys [stream message]} python]
+                                                     (if (= "stdout" stream)
+                                                       (str "\033[32m" message "\033[0m")
+                                                       (str "\033[31m" message "\033[0m")))
+                                                   post-python))}))))
+
+(defn- log-loop! [run-id message-log]
+  (let [poll #(python-runner/get-logs run-id)]
+    (try
+      (loop []
+        (if (.isInterrupted (Thread/currentThread))
+          (log/debug "Message update loop interrupted")
+          (do (Thread/sleep 1000)
+              (let [{:keys [status body]} (poll)]
+                (cond
+                  (<= 200 status 299)
+                  (let [{:keys [execution_id events]} body]
+                    (if-not (= run-id execution_id)
+                      (do (log/debugf "Run id did not match expected: %s actual: %s" run-id execution_id)
+                          (recur))
+                      (do
+                        (replace-python-logs! message-log events)
+                        (save-log-as-message! run-id message-log)
+                        (recur))))
+                  (= 404 status)
+                  (do
+                    (log/debugf "No logs yet (or run finished), run-id: %s" run-id)
+                    (recur))
+                  :else
+                  (do
+                    (log/warnf "Unexpected status polling for logs %s %s, run-id: %s" status body run-id)
+                    (log/debug "Exiting due to poll error")))))))
+      (catch InterruptedException _)
+      (catch Throwable e
+        (log/errorf e "An exception was caught during msg update loop, run-id: %s" run-id)))))
+
+(defn- open-log-future! ^Closeable [run-id message-log]
+  (let [cleanup (fn [fut]
+                  (future-cancel fut)
+                  (if (= ::timeout (try (deref fut 10000 ::timeout) (catch Throwable _)))
+                    (log/fatalf "Log polling task did not respond to interrupt, run-id: %s" run-id)
+                    (log/debugf "Log polling task done, run-id: %s")))
+        fut     (u.jvm/in-virtual-thread*
+                 (log-loop! run-id message-log))]
+    (reify Closeable
+      (close [_] (cleanup fut)))))
+
+(defn- debug-info-str [{:keys [exit-code events]}]
+  (str/join "\n"
+            ;; todo this was temporary before the log stuff, needs a rethink at some point
+            ["stdout"
+             "======"
+             (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
+             "stderr"
+             "======"
+             (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))
+             "======"
+             (format "exit code %d" exit-code)]))
+
+(defn- create-table-and-insert-data!
+  "Create a table from metadata and insert data from source."
+  [driver db-id table-name metadata data-source]
+  (let [table-schema {:name (if (keyword? table-name) table-name (keyword table-name))
+                      :columns (mapv (fn [{:keys [name dtype base_type database_type]}]
+                                       {:name name
+                                        :type (or (some->> base_type (keyword "type"))
+                                                  (transforms.util/dtype->base-type dtype))
+                                        :database-type database_type
+                                        :nullable? true})
+                                     (:fields metadata))}
+        data-source (assoc data-source :table-schema table-schema)]
+    (transforms.util/create-table-from-schema! driver db-id table-schema)
+    (driver/insert-from-source! driver db-id table-name (mapv :name (:columns table-schema)) data-source)))
+
+(defn- transfer-file-to-db [driver db {:keys [target] :as transform} metadata temp-file]
+  (let [table-name (transforms.util/qualified-table-name driver target)
+        table-exists? (transforms.util/target-table-exists? transform)
+        data-source {:type :csv-file
+                     :file temp-file}]
+    (if table-exists?
+      ;; Table exists - use temp table + atomic swap pattern
+      (let [temp-table-name (keyword (str (u/qualified-name table-name) "_temp_" (System/currentTimeMillis)))]
+        (log/info "Existing table deletected, Create then swap")
+        (try
+          (create-table-and-insert-data! driver (:id db) temp-table-name metadata data-source)
+          ;; TODO: These operations should be within a transaction for atomicity
+          (transforms.util/drop-table! driver (:id db) table-name)
+          (transforms.util/rename-table! driver (:id db) temp-table-name table-name)
+          (catch Exception e
+            (log/error e "Failed to transfer data to table")
+            (try
+              (transforms.util/drop-table! driver (:id db) temp-table-name)
+              (catch Exception _))
+            (throw e))))
+      ;; Table doesn't exist - create directly with target name
+      (do
+        (log/info "New table")
+        (create-table-and-insert-data! driver (:id db) table-name metadata data-source)))))
+
+(defn test-python-transform!
+  "Execute a transform in test mode (does not write result into a table)."
+  [code tables->id run-id cancel-chan]
+  (with-open [shared-storage-ref (python-runner/open-s3-shared-storage! tables->id)]
+    (let [server-url              (transforms.settings/python-execution-server-url)
+          _                       (python-runner/copy-tables-to-s3! {:run-id         run-id
+                                                                     :shared-storage @shared-storage-ref
+                                                                     :table-name->id tables->id
+                                                                     :cancel-chan    cancel-chan})
+          _                       (python-runner/open-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan
+          response
+          (python-runner/execute-python-code-http-call!
+           {:server-url     server-url
+            :code           code
+            :run-id         run-id
+            :table-name->id tables->id
+            :shared-storage @shared-storage-ref})
+          {:keys [output events]} (python-runner/read-output-objects @shared-storage-ref)]
+      {:response response
+       :events   events
+       :output   output})))
+
+(defn- run-python-transform! [{:keys [source] :as transform} db run-id cancel-chan message-log]
+  (with-open [log-thread-ref     (open-log-future! run-id message-log)
+              shared-storage-ref (python-runner/open-s3-shared-storage! (:source-tables source))]
+    (let [driver     (:engine db)
+          server-url (transforms.settings/python-execution-server-url)
+          _          (python-runner/copy-tables-to-s3! {:run-id                  run-id
+                                                        :shared-storage @shared-storage-ref
+                                                        :table-name->id (:source-tables source)
+                                                        :cancel-chan    cancel-chan})
+          _          (python-runner/open-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan
+          {:keys [status body] :as response}
+          (python-runner/execute-python-code-http-call!
+           {:server-url     server-url
+            :code           (:body source)
+            :run-id         run-id
+            :table-name->id (:source-tables source)
+            :shared-storage @shared-storage-ref})
+          {:keys [exit_code]} body
+          ;; TODO temporary to keep more code stable while refactoring
+          ;; no need to materialize these early (i.e output we can stream directly into a tmp file or db if small)
+          {:keys [output output-manifest events]} (python-runner/read-output-objects @shared-storage-ref)]
+      (.close log-thread-ref)           ; early close to force any writes to flush
+      (when (seq events)
+        (replace-python-logs! message-log events))
+      (if (not= 200 status)
+        (throw (ex-info (debug-info-str {:exit-code exit_code :events events}) ;; todo do better here
+                        {:status-code     400
+                         :api-status-code status
+                         :body            body
+                         :events          events
+                         :error           (:error body)}))
+        (try
+          (let [temp-file (File/createTempFile "transform-output-" ".csv")]
+            (when-not (seq (:fields output-manifest))
+              (throw (ex-info "No fields in metadata"
+                              {:metadata output-manifest
+                               :raw-body body
+                               :events   events})))
+            (try
+              (with-open [writer (io/writer temp-file)]
+                (.write writer ^String output))
+              (let [file-size (.length temp-file)]
+                (transforms.instrumentation/with-stage-timing [run-id :data-transfer :file-to-dwh]
+                  (transfer-file-to-db driver db transform output-manifest temp-file))
+                (transforms.instrumentation/record-data-transfer! run-id :file-to-dwh file-size nil))
+              (finally
+                (.delete temp-file))))
+          response
+          (catch Exception e
+            (log/error e "Failed to to create resulting table")
+            (throw (ex-info "Failed to create the resulting table" {:error (.getMessage e)}))))))))
+
+(defn execute-python-transform!
+  "Execute a Python transform by calling the python runner.
+
+  This is executing synchronously, but supports being kicked off in the background
+  by delivering the `start-promise` just before the start when the beginning of the execution has been booked
+  in the database."
+  [transform {:keys [run-method start-promise message-log]}]
+  (when (transforms.util/python-transform? transform)
+    (try
+      (let [message-log (or message-log (atom {:pre-python  []
+                                               :python      nil
+                                               :post-python []}))
+            {:keys [target] transform-id :id} transform
+            {driver :engine :as db} (t2/select-one :model/Database (:database target))
+            {run-id :id} (try-start-unless-already-running transform-id run-method)]
+        (some-> start-promise (deliver [:started run-id]))
+        (log! message-log "Executing Python transform")
+        (log/info "Executing Python transform" transform-id "with target" (pr-str target))
+        (let [start-ms (u/start-timer)
+              transform-details
+              {:db db
+               :transform-type (keyword (:type target))
+               :conn-spec (driver/connection-spec driver db)
+               :output-schema (:schema target)
+               :output-table (transforms.util/qualified-table-name driver target)}
+              result   (run-cancelable-transform! run-id driver transform-details (fn [cancel-chan] (run-python-transform! transform db run-id cancel-chan message-log)))]
+          (transforms.instrumentation/with-stage-timing [run-id :sync :table-sync]
+            (sync-target! target db run-id))
+          (log! message-log (format "Python execution finished in %s" (Duration/ofMillis (u/since-ms start-ms))))
+          (save-log-as-message! run-id message-log)
+          {:run_id run-id
+           :result result}))
+      (catch Throwable t
+        (log/error t "Error executing Python transform")
+        (log! message-log "Error executing python transform")
+        (throw t)))))

enterprise/backend/src/metabase_enterprise/transforms/instrumentation.clj

@@ -0,0 +1,181 @@
+(ns metabase-enterprise.transforms.instrumentation
+  (:require
+   [metabase.analytics.prometheus :as prometheus]
+   [metabase.util :as u]
+   [metabase.util.log :as log]
+   [metabase.util.malli :as mu]
+   [metabase.util.malli.registry :as mr]))
+
+(set! *warn-on-reflection* true)
+
+(mr/def ::stage-type
+  [:enum :data-transfer :computation :sync])
+
+(mr/def ::stage-label
+  [:enum
+   :dwh-to-file
+   :file-to-s3
+   :file-to-dwh
+
+   :python-execution
+   :mbql-query
+   :table-sync])
+
+(mu/defn record-stage-start!
+  "Record the start of a transform stage."
+  [job-run-id :- pos-int?
+   stage-type :- ::stage-type
+   stage-label :- ::stage-label]
+  (log/infof "Transform stage started: run-id=%d type=%s label=%s" job-run-id (name stage-type) (name stage-label))
+  (prometheus/inc! :metabase-transforms/stage-started
+                   {:job-run-id (str job-run-id)
+                    :stage-type (name stage-type)
+                    :stage-label (name stage-label)}))
+
+(mu/defn record-stage-completion!
+  "Record the successful completion of a transform stage."
+  [job-run-id :- pos-int?
+   stage-type :- ::stage-type
+   stage-label :- ::stage-label
+   duration-ms :- int?]
+  (log/infof "Transform stage completed: run-id=%d type=%s label=%s duration=%dms" job-run-id (name stage-type) (name stage-label) duration-ms)
+  (let [labels {:job-run-id (str job-run-id)
+                :stage-type (name stage-type)
+                :stage-label (name stage-label)}]
+    (prometheus/inc! :metabase-transforms/stage-completed labels)
+    (prometheus/observe! :metabase-transforms/stage-duration-ms labels duration-ms)))
+
+(mu/defn record-stage-failure!
+  "Record the failure of a transform stage."
+  [job-run-id :- pos-int?
+   stage-type :- ::stage-type
+   stage-label :- ::stage-label
+   duration-ms :- [:maybe int?]]
+  (log/warnf "Transform stage failed: run-id=%d type=%s label=%s duration=%s" job-run-id (name stage-type) (name stage-label) (if duration-ms (str duration-ms "ms") "unknown"))
+  (let [labels {:job-run-id (str job-run-id)
+                :stage-type (name stage-type)
+                :stage-label (name stage-label)}]
+    (prometheus/inc! :metabase-transforms/stage-failed labels)
+    (when duration-ms
+      (prometheus/observe! :metabase-transforms/stage-duration-ms labels duration-ms))))
+
+(defn with-timing
+  "Generic timing function that executes body while recording metrics.
+
+   Takes a map of optional functions:
+   - :start-fn - function to call when starting
+   - :success-fn - function to call on success (receives duration-ms as last arg)
+   - :error-fn - function to call on error (receives duration-ms as last arg)
+
+   Usage:
+   (with-timing {:start-fn #(record-stage-start! %1 %2 %3)
+                 :success-fn #(record-stage-completion! %1 %2 %3 %4)
+                 :error-fn #(record-stage-failure! %1 %2 %3 %4)}
+                [job-run-id stage-type stage-label]
+                #(do-work)) "
+  [callbacks args body-fn]
+  (let [start-time (u/start-timer)]
+    (when-let [start-fn (:start-fn callbacks)]
+      (apply start-fn args))
+    (try
+      (let [result (body-fn)]
+        (when-let [success-fn (:success-fn callbacks)]
+          (apply success-fn (conj (vec args) (long (u/since-ms start-time)))))
+        result)
+      (catch Throwable t
+        (when-let [error-fn (:error-fn callbacks)]
+          (apply error-fn (conj (vec args) (long (u/since-ms start-time)))))
+        (throw t)))))
+
+(defmacro with-stage-timing
+  "Execute body while timing a transform stage. Automatically records start, completion or failure.
+
+   Usage:
+   (with-stage-timing [job-run-id :data-transfer :dwh-to-file]
+     ;; stage implementation
+     result)"
+  [args & body]
+  `(with-timing {:start-fn record-stage-start!
+                 :success-fn record-stage-completion!
+                 :error-fn record-stage-failure!}
+     ~args
+     (^:once fn* [] ~@body)))
+
+(mu/defn record-data-transfer!
+  "Record metrics about data transfer (size and rows)."
+  [job-run-id :- pos-int?
+   stage-label :- ::stage-label
+   bytes :- [:maybe int?]
+   rows :- [:maybe int?]]
+  (log/infof "Data transfer recorded: run-id=%d stage=%s%s%s"
+             job-run-id (name stage-label)
+             (if bytes (str " bytes=" bytes) "")
+             (if rows (str " rows=" rows) ""))
+  (let [labels {:job-run-id (str job-run-id)
+                :stage-label (name stage-label)}]
+    (when bytes
+      (prometheus/observe! :metabase-transforms/data-transfer-bytes labels bytes))
+    (when rows
+      (prometheus/observe! :metabase-transforms/data-transfer-rows labels rows))))
+
+(defn record-job-start!
+  "Record the start of a transform job run."
+  [job-id run-method]
+  (log/infof "Transform job started: job-id=%d run-method=%s" job-id (name run-method))
+  (prometheus/inc! :metabase-transforms/job-runs-total
+                   {:job-id (str job-id)
+                    :run-method (name run-method)}))
+
+(defn record-job-completion!
+  "Record the successful completion of a transform job run."
+  [job-id run-method duration-ms]
+  (log/infof "Transform job completed: job-id=%d run-method=%s duration=%dms" job-id (name run-method) duration-ms)
+  (let [labels {:job-id (str job-id)
+                :run-method (name run-method)}]
+    (prometheus/inc! :metabase-transforms/job-runs-completed labels)
+    (prometheus/observe! :metabase-transforms/job-run-duration-ms labels duration-ms)))
+
+(defn record-job-failure!
+  "Record the failure of a transform job run."
+  [job-id run-method duration-ms]
+  (log/warnf "Transform job failed: job-id=%d run-method=%s duration=%s" job-id (name run-method) (if duration-ms (str duration-ms "ms") "unknown"))
+  (let [labels {:job-id (str job-id)
+                :run-method (name run-method)}]
+    (prometheus/inc! :metabase-transforms/job-runs-failed labels)
+    (when duration-ms
+      (prometheus/observe! :metabase-transforms/job-run-duration-ms labels duration-ms))))
+
+(defmacro with-job-timing
+  "Execute body while timing a transform job run. Automatically records start, completion or failure.
+
+   Usage:
+   (with-job-timing [job-id run-method]
+     ;; job implementation
+     result)"
+  [[job-id run-method] & body]
+  `(with-timing {:start-fn record-job-start!
+                 :success-fn record-job-completion!
+                 :error-fn record-job-failure!}
+     [~job-id ~run-method]
+     (^:once fn* [] ~@body)))
+
+(mu/defn record-python-api-call!
+  "Record metrics about Python API calls."
+  [job-run-id :- pos-int?
+   duration-ms :- int?
+   status :- [:enum :success :error :timeout]]
+  (log/infof "Python API call %s: run-id=%d duration=%dms" (name status) job-run-id duration-ms)
+  (let [labels {:job-run-id (str job-run-id)}]
+    (prometheus/inc! :metabase-transforms/python-api-calls-total
+                     (assoc labels :status (name status)))
+    (prometheus/observe! :metabase-transforms/python-api-call-duration-ms labels duration-ms)))
+
+(defmacro with-python-api-timing
+  "Execute body while timing a Python API call."
+  [[job-run-id] & body]
+  `(with-timing {:success-fn (fn [job-run-id# duration-ms#]
+                               (record-python-api-call! job-run-id# duration-ms# :success))
+                 :error-fn (fn [job-run-id# duration-ms#]
+                             (record-python-api-call! job-run-id# duration-ms# :error))}
+     [~job-run-id]
+     (^:once fn* [] ~@body)))

enterprise/backend/src/metabase_enterprise/transforms/jobs.clj

@@ -6,10 +6,12 @@
    [clojurewerkz.quartzite.triggers :as triggers]
    [flatland.ordered.set :as ordered-set]
    [metabase-enterprise.transforms.execute :as transforms.execute]
+   [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
    [metabase-enterprise.transforms.models.job-run :as transforms.job-run]
    [metabase-enterprise.transforms.models.transform-run :as transform-run]
    [metabase-enterprise.transforms.ordering :as transforms.ordering]
    [metabase-enterprise.transforms.settings :as transforms.settings]
+   [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.task.core :as task]
    [metabase.util.log :as log]
    [toucan2.core :as t2]))
@@ -64,14 +66,21 @@
         (map transforms-by-id complete)))))
 
 (defn- run-transform! [run-id run-method {transform-id :id :as transform}]
-  (when (transform-run/running-run-for-run-id transform-id)
+  (when
+   (transform-run/running-run-for-transform-id transform-id)
     (log/warn "Transform" (pr-str transform-id) "already running, waiting")
     (loop []
       (Thread/sleep 2000)
-      (when (transform-run/running-run-for-run-id transform-id)
+      (when (transform-run/running-run-for-transform-id transform-id)
         (recur))))
+
   (log/info "Executing job transform" (pr-str transform-id))
-  (transforms.execute/run-mbql-transform! transform {:run-method run-method})
+  (if (transforms.util/python-transform? transform)
+    (transforms.instrumentation/with-stage-timing [run-id :computation :python-execution]
+      (transforms.execute/execute-python-transform! transform {:run-method run-method}))
+    (transforms.instrumentation/with-stage-timing [run-id :computation :mbql-query]
+      (transforms.execute/run-mbql-transform! transform {:run-method run-method})))
+
   (transforms.job-run/add-run-activity! run-id))
 
 (defn run-transforms!
@@ -110,12 +119,13 @@
     (let [transforms (job-transform-ids job-id)]
       (log/info "Executing transform job" (pr-str job-id) "with transforms" (pr-str transforms))
       (let [{run-id :id} (transforms.job-run/start-run! job-id run-method)]
-        (try
-          (run-transforms! run-id transforms opts)
-          (transforms.job-run/succeed-started-run! run-id)
-          (catch Throwable t
-            (transforms.job-run/fail-started-run! run-id {:message (.getMessage t)})
-            (throw t)))))))
+        (transforms.instrumentation/with-job-timing [job-id run-method]
+          (try
+            (run-transforms! run-id transforms opts)
+            (transforms.job-run/succeed-started-run! run-id)
+            (catch Throwable t
+              (transforms.job-run/fail-started-run! run-id {:message (.getMessage t)})
+              (throw t))))))))
 
 (def ^:private job-key "metabase-enterprise.transforms.jobs.timeout-job")
 

enterprise/backend/src/metabase_enterprise/transforms/models/transform_run.clj

@@ -147,11 +147,11 @@
                         :message   "Canceled by user but could not guarantee run stopped."})
     (cancel/delete-old-canceling-runs!)))
 
-(defn running-run-for-run-id
+(defn running-run-for-transform-id
   "Return a single active transform run or nil."
-  [id]
+  [transform-id]
   (t2/select-one :model/TransformRun
-                 :id id
+                 :transform_id transform-id
                  :is_active true))
 
 (defn- timestamp-constraint

enterprise/backend/src/metabase_enterprise/transforms/models/transform_run_cancelation.clj

@@ -14,14 +14,10 @@
   ([run-id]
    (mark-cancel-started-run! run-id {}))
   ([run-id properties]
-   (t2/insert! :model/TransformRunCancelation
-               (assoc properties
-                      :run_id run-id)
-               {:where [:exists {:select [1]
-                                 :from   [:transform_run]
-                                 :where  [:and
-                                          [:= :transform_run.id run-id]
-                                          :transform_run.is_active]}]})))
+   (when (t2/exists? :transform_run :id run-id :is_active true)
+     (t2/insert! :model/TransformRunCancelation
+                 (assoc properties
+                        :run_id run-id)))))
 
 (defn reducible-canceled-local-runs
   "Return a reducible sequence of local canceled runs."

enterprise/backend/src/metabase_enterprise/transforms/ordering.clj

@@ -16,23 +16,29 @@
 (set! *warn-on-reflection* true)
 
 (defn- transform-deps [transform]
-  (let [query (-> (get-in transform [:source :query])
-                  transforms.util/massage-sql-query
-                  qp.preprocess/preprocess
-                  ;; legacy usage -- don't do things like this going forward
-                  #_{:clj-kondo/ignore [:discouraged-var]}
-                  lib/->legacy-MBQL)]
-    (case (:type query)
-      :native (driver/native-query-deps (-> (qp.store/metadata-provider)
-                                            lib.metadata/database
-                                            :engine)
-                                        (get-in query [:native :query]))
-      :query (into #{}
-                   (keep #(clojure.core.match/match %
-                            [:source-table source] (when (int? source)
-                                                     source)
-                            _ nil))
-                   (tree-seq coll? seq query)))))
+  (if (transforms.util/python-transform? transform)
+    ;; Python transforms have dependencies via source-tables mapping
+    (into #{}
+          (map #(hash-map :table %))
+          (vals (get-in transform [:source :source-tables])))
+    ;; Query transforms have dependencies via SQL query analysis
+    (let [query (-> (get-in transform [:source :query])
+                    transforms.util/massage-sql-query
+                    qp.preprocess/preprocess
+                    ;; legacy usage -- don't do things like this going forward
+                    #_{:clj-kondo/ignore [:discouraged-var]}
+                    lib/->legacy-MBQL)]
+      (case (:type query)
+        :native (driver/native-query-deps (-> (qp.store/metadata-provider)
+                                              lib.metadata/database
+                                              :engine)
+                                          (get-in query [:native :query]))
+        :query (into #{}
+                     (keep #(clojure.core.match/match %
+                              [:source-table source] (when (int? source)
+                                                       {:table source})
+                              _ nil))
+                     (tree-seq coll? seq query))))))
 
 (defn- dependency-map [transforms]
   (into {}
@@ -56,20 +62,32 @@
 
   The result is a map of transform id -> #{transform ids the transform depends on}. Dependencies are limited to just
   the transforms in the original list -- if a transform depends on some transform not in the list, the 'extra'
-  dependency is ignored."
+  dependency is ignored. Both query and Python transforms can have dependencies on tables produced by other transforms."
   [transforms]
-  (let [transforms-by-db (->> transforms
+
+  (let [;; Group all transforms by their database
+        transforms-by-db (->> transforms
                               (map (fn [transform]
-                                     {(get-in transform [:source :query :database]) [transform]}))
+                                     (let [db-id (if (transforms.util/python-transform? transform)
+                                                   (get-in transform [:target :database])
+                                                   (get-in transform [:source :query :database]))]
+                                       {db-id [transform]})))
                               (apply merge-with into))
 
+        transform-ids (into #{} (map :id) transforms)
         {:keys [output-tables dependencies]} (->> transforms-by-db
                                                   (map (fn [[db-id db-transforms]]
                                                          (qp.store/with-metadata-provider db-id
                                                            {:output-tables (output-table-map db-transforms)
                                                             :dependencies (dependency-map db-transforms)})))
-                                                  (apply merge-with merge))]
-    (update-vals dependencies #(into #{} (keep output-tables) %))))
+                                                  (apply merge-with merge))
+
+        all-deps (update-vals dependencies #(into #{}
+                                                  (keep (fn [{:keys [table transform]}]
+                                                          (or (output-tables table)
+                                                              (transform-ids transform))))
+                                                  %))]
+    all-deps))
 
 (defn find-cycle
   "Finds a path containing a cycle in the directed graph `node->children`.
@@ -115,9 +133,12 @@
                                transforms)
         db-id (get-in to-check [:source :query :database])]
     (qp.store/with-metadata-provider db-id
-      (let [output-tables (output-table-map (filter #(= (get-in % [:source :query :database]) db-id)
-                                                    transforms))
-            node->children #(->> % transforms-by-id transform-deps (keep output-tables))
+      (let [db-transforms (filter #(= (get-in % [:source :query :database]) db-id) transforms)
+            output-tables (output-table-map db-transforms)
+            transform-ids (into #{} (map :id) db-transforms)
+            node->children #(->> % transforms-by-id transform-deps (keep (fn [{:keys [table transform]}]
+                                                                           (or (output-tables table)
+                                                                               (transform-ids transform)))))
             id->name (comp :name transforms-by-id)
             cycle (find-cycle node->children [transform-id])]
         (when cycle

enterprise/backend/src/metabase_enterprise/transforms/python_runner.clj

@@ -0,0 +1,443 @@
+(ns metabase-enterprise.transforms.python-runner
+  (:require
+   [clj-http.client :as http]
+   [clojure.core.async :as a]
+   [clojure.java.io :as io]
+   [clojure.string :as str]
+   [medley.core :as m]
+   [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
+   [metabase-enterprise.transforms.settings :as transforms.settings]
+   [metabase.driver :as driver]
+   [metabase.query-processor.compile :as qp.compile]
+   [metabase.query-processor.pipeline :as qp.pipeline]
+   ^{:clj-kondo/ignore [:discouraged-namespace]}
+   [metabase.query-processor.store :as qp.store]
+   [metabase.util.json :as json]
+   [metabase.util.log :as log]
+   [toucan2.core :as t2])
+  (:import
+   (clojure.lang IDeref)
+   (java.io BufferedWriter Closeable File OutputStream OutputStreamWriter)
+   (java.net URI)
+   (java.nio.charset StandardCharsets)
+   (java.time Duration)
+   (software.amazon.awssdk.auth.credentials AwsBasicCredentials StaticCredentialsProvider)
+   (software.amazon.awssdk.auth.credentials DefaultCredentialsProvider)
+   (software.amazon.awssdk.core.sync RequestBody)
+   (software.amazon.awssdk.regions Region)
+   (software.amazon.awssdk.services.s3 S3Client S3ClientBuilder S3Configuration)
+   (software.amazon.awssdk.services.s3.model DeleteObjectRequest GetObjectRequest NoSuchKeyException PutObjectRequest)
+   (software.amazon.awssdk.services.s3.presigner S3Presigner S3Presigner$Builder)
+   (software.amazon.awssdk.services.s3.presigner.model GetObjectPresignRequest PresignedGetObjectRequest PresignedPutObjectRequest PutObjectPresignRequest)))
+
+(set! *warn-on-reflection* true)
+
+(defn- safe-delete
+  "Safely delete a file."
+  [^File file]
+  (try (.delete file) (catch Exception _)))
+
+(defn- write-to-stream! [^OutputStream os col-names reducible-rows]
+  (let [writer (-> os
+                   (OutputStreamWriter. StandardCharsets/UTF_8)
+                   (BufferedWriter.))]
+
+    (run! (fn [row]
+            (let [row-map (zipmap col-names row)]
+              (json/encode-to row-map writer {})
+              (.newLine writer)))
+          reducible-rows)
+
+    (doto writer
+      (.flush)
+      (.close))))
+
+(defn- execute-mbql-query
+  [driver db-id query respond cancel-chan]
+  (driver/with-driver driver
+    (let [native (qp.compile/compile {:type :query, :database db-id :query query})
+
+          query  {:database db-id
+                  :type     :native
+                  :native   native}]
+      (qp.store/with-metadata-provider db-id
+        (binding [qp.pipeline/*canceled-chan* cancel-chan]
+          (driver/execute-reducible-query driver query {:canceled-chan cancel-chan} respond))))))
+
+(defn- root-type
+  [base-type]
+  (when base-type
+    (some #(when (isa? base-type %) %)
+          [:type/Float
+           :type/Integer
+           :type/Boolean
+           :type/DateTimeWithTZ
+           :type/DateTime
+           :type/Dictionary
+           :type/JSON
+           :type/Date
+           :type/Text])))
+
+(defn- closest-ancestor [t pred]
+  (loop [remaining (conj clojure.lang.PersistentQueue/EMPTY [t])]
+    (when-let [t (first remaining)]
+      (if (pred t)
+        t
+        (recur (into (pop t) (parents t)))))))
+
+(defn- effective-semantic-type-i-think
+  "Kinda sketchy but maybe reasonable way to infer the effective-semantic-type"
+  [{:keys [base_type effective_type semantic_type]}]
+  (or semantic_type (closest-ancestor (or effective_type base_type) #(isa? % :Semantic/*))))
+
+(defn- generate-manifest
+  "Generate a metadata manifest for the table columns."
+  [table-id cols-meta]
+  {:version        "0.1.0"
+   :fields         (mapv (fn [col-meta]
+                           {:name           (:name col-meta)
+                            :base_type      (some-> (:base_type col-meta) name)
+                            :database_type  (some-> (:database_type col-meta) name)
+                            :root_type      (some-> (root-type (:base_type col-meta)) name)
+                            ;; replace nil values with values indicating how they behave in practice.
+                            ;; there may be better ways of doing this already, but i worry it's just implicit in QP
+                            :semantic_type  (some-> (effective-semantic-type-i-think col-meta) name)
+                            :effective_type (some-> (or (:effective_type col-meta) (:database_type col-meta)) name)
+                            ;; TODO get this passed through
+                            :field_id       (:id col-meta)})
+                         cols-meta)
+   :table_metadata {:table_id table-id}})
+
+(defn- maybe-fixup-v [col v]
+  ;; clickhouse returns bigdecimals for int64 values
+  (if (and (isa? (:base_type col) :type/Integer)
+           (or (instance? BigDecimal v)
+               (float? v)))
+    (bigint v)
+    v))
+
+(defn- write-table-data-to-file! [id temp-file cancel-chan]
+  (let [db-id           (t2/select-one-fn :db_id (t2/table-name :model/Table) :id id)
+        driver          (t2/select-one-fn :engine :model/Database db-id)
+        all-fields-meta (t2/select [:model/Field :id :name :base_type :effective_type :semantic_type :database_type :database_position :nfc_path :parent_id]
+                                   :table_id id
+                                   :active true
+                                   {:order-by [[:database_position :asc]]})
+        fields-meta (filter #(and (nil? (:parent_id %)) (nil? (:nfc_path %))) all-fields-meta)
+        query {:source-table id}
+        manifest (generate-manifest id fields-meta)]
+    (execute-mbql-query driver db-id query
+                        (fn [{cols-meta :cols} reducible-rows]
+                          (with-open [os (io/output-stream temp-file)]
+                            (let [filtered-col-meta (m/index-by :name fields-meta)
+                                  filtered-cols (filter #(contains? filtered-col-meta (:name %)) cols-meta)
+                                  col-name->index (into {} (map-indexed (fn [i col] [(:name col) i]) cols-meta))
+                                  filtered-indices (mapv (fn [col] [(col-name->index (:name col)) col]) filtered-cols)
+                                  filtered-rows (eduction (map (fn [row] (mapv (fn [[idx col]]
+                                                                                 (let [v (nth row idx)]
+                                                                                   (maybe-fixup-v col v))) filtered-indices)))
+                                                          reducible-rows)]
+                              (write-to-stream! os (mapv :name filtered-cols) filtered-rows))))
+                        cancel-chan)
+    manifest))
+
+(defn- maybe-with-endpoint-s3-client [^S3ClientBuilder builder endpoint]
+  (let [region (transforms.settings/python-storage-s-3-region)]
+    (case [(some? endpoint) (some? region)]
+      [true true]   (doto builder
+                      (.endpointOverride (URI/create endpoint))
+                      (.region (Region/of region)))
+      [true false]  (log/warn "Ignoring endpoint because region is not defined")
+      [false true]  (.region builder (Region/of region))
+      [false false] builder)))
+
+(defn- maybe-with-endpoint-s3-presigner [^S3Presigner$Builder builder endpoint]
+  (let [region (transforms.settings/python-storage-s-3-region)]
+    (case [(some? endpoint) (some? region)]
+      [true true]   (doto builder
+                      (.endpointOverride (URI/create endpoint))
+                      (.region (Region/of region)))
+      [true false]  (log/warn "Ignoring endpoint because region is not defined")
+      [false true]  (.region builder (Region/of region))
+      [false false] (.region builder (Region/US_EAST_1)))))
+
+(defn- create-s3-configuration
+  "Create S3Configuration with path-style access setting"
+  ^S3Configuration []
+  (-> (S3Configuration/builder)
+      (.pathStyleAccessEnabled (transforms.settings/python-storage-s-3-path-style-access))
+      (.build)))
+
+(defn- build-put-object-request
+  "Build a PutObjectRequest with bucket and key"
+  [^String bucket-name ^String key]
+  (-> (PutObjectRequest/builder)
+      (.bucket bucket-name)
+      (.key key)
+      (.build)))
+
+(defn- build-get-object-request
+  "Build a GetObjectRequest with bucket and key"
+  [^String bucket-name ^String key]
+  (-> (GetObjectRequest/builder)
+      (.bucket bucket-name)
+      (.key key)
+      (.build)))
+
+(defn- build-delete-object-request
+  "Build a DeleteObjectRequest with bucket and key"
+  [^String bucket-name ^String key]
+  (-> (DeleteObjectRequest/builder)
+      (.bucket bucket-name)
+      (.key key)
+      (.build)))
+
+(def ^:private ^Duration presigned-url-duration
+  "Default duration for presigned URLs"
+  (Duration/ofHours 1))
+
+(defn- maybe-with-credentials-s3-client [^S3ClientBuilder builder]
+  (let [access-key (transforms.settings/python-storage-s-3-access-key)
+        secret-key (transforms.settings/python-storage-s-3-secret-key)]
+    (if (or access-key secret-key)
+      (if-not (and access-key secret-key)
+        (do (log/warnf "Ignoring %s because %s is not defined"
+                       (if access-key "access-key" "secret-key")
+                       (if (not access-key) "access-key" "secret-key"))
+            (.credentialsProvider builder (DefaultCredentialsProvider/create)))
+        (.credentialsProvider builder
+                              (StaticCredentialsProvider/create
+                               (AwsBasicCredentials/create access-key secret-key))))
+      (.credentialsProvider builder (DefaultCredentialsProvider/create)))))
+
+(defn- maybe-with-credentials-s3-presigner [^S3Presigner$Builder builder]
+  (let [access-key (transforms.settings/python-storage-s-3-access-key)
+        secret-key (transforms.settings/python-storage-s-3-secret-key)]
+    (if (or access-key secret-key)
+      (if-not (and access-key secret-key)
+        (do (log/warnf "Ignoring %s because %s is not defined"
+                       (if access-key "access-key" "secret-key")
+                       (if (not access-key) "access-key" "secret-key"))
+            builder)
+        (.credentialsProvider builder
+                              (StaticCredentialsProvider/create
+                               (AwsBasicCredentials/create access-key secret-key))))
+      builder)))
+
+;; We just recreate the client every time, to keep things simple if config is changed.
+(defn- create-s3-client
+  "Create S3 client for host operations (uploads, reads)"
+  ^S3Client []
+  (let [^S3ClientBuilder builder (S3Client/builder)]
+    (doto builder
+      (maybe-with-endpoint-s3-client (transforms.settings/python-storage-s-3-endpoint))
+      maybe-with-credentials-s3-client
+      (.serviceConfiguration (create-s3-configuration)))
+    (.build builder)))
+
+(defn- create-s3-presigner-for-container
+  "Create S3 presigner for container operations (presigned URLs).
+  Uses container-endpoint if different from host endpoint."
+  ^S3Presigner []
+  (let [container-endpoint (transforms.settings/python-storage-s-3-container-endpoint)
+        builder (S3Presigner/builder)]
+    (doto builder
+      (maybe-with-endpoint-s3-presigner (or container-endpoint (transforms.settings/python-storage-s-3-endpoint)))
+      maybe-with-credentials-s3-presigner
+      (.serviceConfiguration (create-s3-configuration)))
+    (.build builder)))
+
+(defn- upload-file-to-s3
+  "Upload file using host client"
+  [^S3Client s3-client ^String bucket-name ^String key ^File file]
+  (let [^PutObjectRequest request (build-put-object-request bucket-name key)]
+    (.putObject s3-client request (RequestBody/fromFile file))))
+
+(defn- generate-presigned-get-url
+  "Generate GET URL using container presigner"
+  [^S3Presigner presigner ^String bucket-name ^String key]
+  (let [^GetObjectRequest get-request (build-get-object-request bucket-name key)
+        ^GetObjectPresignRequest presign-request (-> (GetObjectPresignRequest/builder)
+                                                     (.signatureDuration presigned-url-duration)
+                                                     (.getObjectRequest get-request)
+                                                     (.build))
+        presigned ^PresignedGetObjectRequest (.presignGetObject presigner presign-request)]
+    (.toString (.url presigned))))
+
+(defn- generate-presigned-put-url
+  "Generate PUT URL using container presigner"
+  [^S3Presigner presigner ^String bucket-name ^String key]
+  (let [^PutObjectRequest put-request (build-put-object-request bucket-name key)
+        ^PutObjectPresignRequest presign-request (-> (PutObjectPresignRequest/builder)
+                                                     (.signatureDuration presigned-url-duration)
+                                                     (.putObjectRequest put-request)
+                                                     (.build))
+        presigned ^PresignedPutObjectRequest (.presignPutObject presigner presign-request)]
+    (.toString (.url presigned))))
+
+(defn- delete-s3-object [^S3Client s3-client ^String bucket-name ^String key]
+  (try
+    (let [^DeleteObjectRequest request (build-delete-object-request bucket-name key)]
+      (.deleteObject s3-client request))
+    (catch Exception e
+      (log/debugf e "Error deleting s3 object %s" key)
+      ;; Ignore deletion errors - object might not exist, or we might not have permissions
+      ;; NOTE: we plan to put general retention on the bucket so that objects will eventually be deleted
+      nil)))
+
+(defn- cleanup-s3-objects [^S3Client s3-client bucket-name s3-keys]
+  (run! (partial delete-s3-object s3-client bucket-name) s3-keys))
+
+(defn- read-from-s3
+  ([s3-client bucket-name key] (read-from-s3 s3-client bucket-name key ::throw))
+  ([^S3Client s3-client ^String bucket-name ^String key not-found]
+   (try
+     (let [^GetObjectRequest request (build-get-object-request bucket-name key)
+           response                  (.getObject s3-client request)]
+       (slurp response))
+     (catch NoSuchKeyException e
+       (if (identical? ::throw not-found)
+         (throw e)
+         not-found)))))
+
+(defn get-logs
+  "Return the logs of the current running python process"
+  [run-id]
+  (let [server-url (transforms.settings/python-execution-server-url)]
+    (http/get (str server-url "/logs")
+              {:content-type     :json
+               :accept           :json
+               :throw-exceptions false
+               :as               :json
+               :query-params     {:request_id run-id}})))
+
+(defn- s3-shared-storage [table-name->id]
+  (let [prefix              (some-> (transforms.settings/python-storage-s-3-prefix) (str "/"))
+        work-dir-name       (str prefix "run-" (System/nanoTime) "-" (rand-int 10000))
+        container-presigner (create-s3-presigner-for-container)
+        bucket-name         (transforms.settings/python-storage-s-3-bucket)
+        loc
+        (fn [method relative-path]
+          ;; save a tracking root to a well known location
+          ;; later can ls these to collect those objects whose ttl has elapsed
+          ;; (OR setup a retention policy in s3?)
+          (let [path (str work-dir-name "/" relative-path)]
+            {:path   path
+             :method method
+             :url    (case method :put (generate-presigned-put-url container-presigner bucket-name path)
+                           :get (generate-presigned-get-url container-presigner bucket-name path))}))]
+    {:s3-client   (create-s3-client)                        ;; do not like mixing interactive things with descriptions, but its damn convenient to have it here for now
+     :bucket-name bucket-name
+     :objects
+     (into
+      {:output          (loc :put "output.csv")
+       :output-manifest (loc :put "output-manifest.json")
+       :events          (loc :put "events.jsonl")}
+      (for [[table-name id] table-name->id]
+        {[:table id :manifest] (loc :get (str "-table-" (name table-name) "-" id ".manifest.json"))
+         [:table id :data]     (loc :get (str "-table-" (name table-name) "-" id ".jsonl"))}))}))
+
+(defn open-s3-shared-storage!
+  "Returns a deref'able shared storage value, (.close) will optimistically delete any s3 objects named in storage (data files for tables, metadata files etc).
+  The intention is the bucket specifies a generic object retention policy to ensure objects are eventually deleted (e.g. because the process dies during writing and .close never gets called)"
+  ^Closeable [table-name->id]
+  (let [shared-storage (s3-shared-storage table-name->id)]
+    (reify IDeref
+      (deref [_] shared-storage)
+      Closeable
+      (close [_] (cleanup-s3-objects (:s3-client shared-storage) (:bucket-name shared-storage) (map :path (vals (:objects shared-storage))))))))
+
+(defn copy-tables-to-s3!
+  "Writes table content to their corresponding objects named in shared-storage, see (open-shared-storage!).
+  Blocks until all tables are fully written and committed to shared storage."
+  [{:keys [run-id
+           shared-storage
+           table-name->id
+           cancel-chan]}]
+  (doseq [id (vals table-name->id)
+          :let [{:keys [s3-client bucket-name objects]} shared-storage
+                {data-path :path}     (get objects [:table id :data])
+                {manifest-path :path} (get objects [:table id :manifest])]]
+    (let [temp-file       (File/createTempFile data-path "")
+          manifest-file   (File/createTempFile manifest-path "")]
+      (try
+        ;; Write table data to temporary file and get manifest
+        (let [manifest (transforms.instrumentation/with-stage-timing [run-id :data-transfer :dwh-to-file]
+                         (write-table-data-to-file! id temp-file cancel-chan))]
+          ;; Write manifest to file
+          (with-open [writer (io/writer manifest-file)]
+            (json/encode-to manifest writer {}))
+          (let [file-size (.length temp-file)
+                manifest-size (.length manifest-file)]
+            (transforms.instrumentation/record-data-transfer! run-id :dwh-to-file file-size nil)
+
+            ;; Upload both files to S3
+            (transforms.instrumentation/with-stage-timing [run-id :data-transfer :file-to-s3]
+              (upload-file-to-s3 s3-client bucket-name data-path temp-file)
+              (upload-file-to-s3 s3-client bucket-name manifest-path manifest-file))
+
+            (transforms.instrumentation/record-data-transfer! run-id :file-to-s3 (+ file-size manifest-size) nil)))
+        (finally
+          ;; Clean up temporary files
+          (safe-delete temp-file)
+          (safe-delete manifest-file))))))
+
+(defn execute-python-code-http-call!
+  "Calls the /execute endpoint of the python runner. Blocks until the run either succeeds or fails and returns
+  the response from the server."
+  [{:keys [server-url code run-id table-name->id shared-storage]}]
+  (let [{:keys [objects]} shared-storage
+        {:keys [output output-manifest events]} objects
+
+        table-name->url          (update-vals table-name->id (comp :url #(get objects [:table % :data])))
+        table-name->manifest-url (update-vals table-name->id (comp :url #(get objects [:table % :manifest])))
+
+        payload                  {:code                code
+                                  :library             (t2/select-fn->fn :path :source :model/PythonLibrary)
+                                  :timeout             30
+                                  :request_id          run-id
+                                  :output_url          (:url output)
+                                  :output_manifest_url (:url output-manifest)
+                                  :events_url          (:url events)
+                                  :table_mapping       table-name->url
+                                  :manifest_mapping    table-name->manifest-url}
+
+        response                 (transforms.instrumentation/with-python-api-timing [run-id]
+                                   (http/post (str server-url "/execute")
+                                              {:content-type     :json
+                                               :accept           :json
+                                               :body             (json/encode payload)
+                                               :throw-exceptions false
+                                               :as               :json}))]
+    ;; when a 500 is returned we observe a string in the body (despite the python returning json)
+    ;; always try to parse the returned string as json before yielding (could tighten this up at some point)
+    (update response :body (fn [string-if-error]
+                             (if (string? string-if-error)
+                               (try
+                                 (json/decode+kw string-if-error)
+                                 (catch Exception _
+                                   {:error string-if-error}))
+                               string-if-error)))))
+
+(defn open-cancellation-process!
+  "Starts a core.async process that optimistically sends a cancellation request to the python executor if cancel-chan receives a value.
+  Returns a channel that will receive either the async http call j.u.c.FutureTask in the case of cancellation, or nil when the cancel-chan is closed."
+  [server-url run-id cancel-chan]
+  (a/go (when (a/<! cancel-chan)
+          (http/post (str server-url "/cancel")
+                     {:content-type :json
+                      :body         (json/encode {:request_id run-id})
+                      :async?       true}
+                     identity identity))))
+
+;; temporary, we should not need to realize data/events files into memory longer term
+(defn read-output-objects
+  "Temporary function that strings/jsons stuff in S3 and returns it for compatibility."
+  [{:keys [s3-client bucket-name objects]}]
+  (let [{:keys [output output-manifest events]} objects
+        output-content          (read-from-s3 s3-client bucket-name (:path output) nil)
+        output-manifest-content (read-from-s3 s3-client bucket-name (:path output-manifest) "{}")
+        events-content          (read-from-s3 s3-client bucket-name (:path events))]
+    {:output output-content
+     :output-manifest (json/decode+kw output-manifest-content)
+     :events (mapv json/decode+kw (str/split-lines events-content))}))

enterprise/backend/src/metabase_enterprise/transforms/schedule.clj

@@ -101,6 +101,7 @@
         trigger (build-trigger job-id schedule)]
     (task/schedule-task! job trigger)))
 
+;; TODO: should this be part of transformjob hooks?
 (defn update-job!
   "Update the trigger for a transform job."
   [job-id schedule]

enterprise/backend/src/metabase_enterprise/transforms/settings.clj

@@ -1,17 +1,119 @@
 (ns metabase-enterprise.transforms.settings
   (:require
+   [metabase.config.core :as config]
    [metabase.settings.core :as setting]
    [metabase.util.i18n :refer [deferred-tru]]))
 
 (set! *warn-on-reflection* true)
 
+(def ^:private is-not-prod? (or config/is-dev? config/is-test?))
+
 (setting/defsetting transform-timeout
   (deferred-tru "The timeout for a transform job, in minutes.")
-  :type       :integer
+  :type :integer
   :visibility :internal
-  :default    (* 4 60)
-  :feature    :transforms
-  :doc        false
-  :export?    false
+  :default (* 4 60)
+  :feature :transforms
+  :doc false
+  :export? false
   :encryption :no
-  :audit      :getter)
+  :audit :getter)
+
+(setting/defsetting python-execution-server-url
+  (deferred-tru "URL for the Python execution server that runs transform functions.")
+  :type :string
+  :visibility :admin
+  :default (when is-not-prod? "http://localhost:5001")
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-endpoint
+  (deferred-tru "S3 endpoint URL for storing Python execution artifacts.")
+  :type :string
+  :visibility :admin
+  :default (when is-not-prod? "http://localhost:4566")
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-region
+  (deferred-tru "AWS region for S3 storage.")
+  :type :string
+  :visibility :admin
+  :default (when is-not-prod? "us-east-1")
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-bucket
+  (deferred-tru "S3 bucket name for storing Python execution artifacts.")
+  :type :string
+  :visibility :admin
+  :default (if is-not-prod? "metabase-python-runner" "metabase-cloud-python-runner-staging")
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-prefix
+  (deferred-tru "Prefix to use for S3 objects. Necessary in prod where we only have access to a particular prefix.")
+  :type :string
+  :visibility :admin
+  :feature :transforms
+  :default (if is-not-prod? "test-prefix" "hosting-4b9351bb22b84791")
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-access-key
+  (deferred-tru "AWS access key ID for S3 authentication.")
+  :type :string
+  :visibility :admin
+  :feature :transforms
+  :default (when is-not-prod? "test")
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-secret-key
+  (deferred-tru "AWS secret access key for S3 authentication.")
+  :type :string
+  :visibility :admin
+  :feature :transforms
+  :default (when is-not-prod? "test")
+  :doc false
+  :export? false
+  :encryption :when-encryption-key-set
+  :audit :never)
+
+(setting/defsetting python-storage-s-3-container-endpoint
+  (deferred-tru "Alternative S3 endpoint accessible from containers. Leave empty if same as main endpoint.")
+  :type :string
+  :visibility :admin
+  :default (when is-not-prod? "http://localstack:4566")
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-path-style-access
+  (deferred-tru "Use path-style access for S3 requests (required for LocalStack and some S3-compatible services).")
+  :type :boolean
+  :visibility :admin
+  :default (when is-not-prod? true)
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)

enterprise/backend/src/metabase_enterprise/transforms/util.clj

@@ -1,12 +1,16 @@
 (ns metabase-enterprise.transforms.util
   (:require
+   [clojure.string :as str]
    [java-time.api :as t]
    [metabase.driver :as driver]
+   [metabase.lib.schema.common :as lib.schema.common]
    [metabase.query-processor.compile :as qp.compile]
    [metabase.sync.core :as sync]
    [metabase.util :as u]
    [metabase.util.date-2 :as u.date]
    [metabase.util.log :as log]
+   [metabase.util.malli :as mu]
+   [metabase.util.malli.registry :as mr]
    [toucan2.core :as t2])
   (:import
    (java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)
@@ -21,10 +25,22 @@
     (keyword schema name)
     (keyword name)))
 
+(defn query-transform?
+  "Check if this is a query transform: native query / mbql query."
+  [transform]
+  (= :query (-> transform :source :type keyword)))
+
+(defn python-transform?
+  "Check if this is a Python transform."
+  [transform]
+  (= :python (-> transform :source :type keyword)))
+
 (defn target-table-exists?
   "Test if the target table of a transform already exists."
   [{:keys [source target] :as _transform}]
-  (let [db-id (-> source :query :database)
+  (let [db-id (or (-> source :query :database)
+                  ;; python transform target
+                  (-> target :database))
         {driver :engine :as database} (t2/select-one :model/Database db-id)]
     (driver/table-exists? driver database target)))
 
@@ -66,7 +82,9 @@
   [{:keys [id target source], :as _transform}]
   (when target
     (let [target (update target :type keyword)
-          database-id (-> source :query :database)
+          database-id (or (-> source :query :database)
+                          ;; python transform target
+                          (-> target :database))
           {driver :engine :as database} (t2/select-one :model/Database database-id)]
       (driver/drop-transform-target! driver database target)
       (log/info "Deactivating  target " (pr-str target) "for transform" id)
@@ -122,3 +140,60 @@
   (-> run
       (u/update-some :start_time utc-timestamp-string)
       (u/update-some :end_time   utc-timestamp-string)))
+
+(mr/def ::column-definition
+  [:map
+   [:name :string]
+   [:type ::lib.schema.common/base-type]
+   [:nullable? {:optional true} :boolean]])
+
+(mr/def ::table-definition
+  [:map
+   [:name :keyword]
+   [:columns [:sequential ::column-definition]]
+   [:primary-key {:optional true} [:sequential :string]]])
+
+(defn dtype->base-type
+  "Maps pandas dtype strings directly to Metabase base types in the type hierarchy."
+  [dtype-str]
+  (cond
+    (str/starts-with? dtype-str "int") :type/Integer
+    (str/starts-with? dtype-str "float") :type/Float
+    (str/starts-with? dtype-str "bool") :type/Boolean
+    ;; datetime64[ns, timezone] indicates timezone-aware datetime
+    (str/starts-with? dtype-str "datetime64[ns, ") :type/DateTimeWithTZ
+    (str/starts-with? dtype-str "datetime") :type/DateTime
+    ;; this is not a real dtype, pandas uses 'object', but we override it if there's source or custom field metadata
+    (str/starts-with? dtype-str "date") :type/Date
+    :else :type/Text))
+
+(mu/defn create-table-from-schema!
+  "Create a table from a table-schema"
+  [driver :- :keyword
+   database-id :- pos-int?
+   table-schema :- ::table-definition]
+  (let [{:keys [columns] table-name :name} table-schema
+        column-definitions (into {} (map (fn [{:keys [name type database-type]}]
+                                           (let [database-type (when-not (and (= driver :bigquery-cloud-sdk)
+                                                                              (#{"ARRAY" "RECORD"} database-type))
+                                                                 database-type)
+                                                 db-type (if database-type
+                                                           [[:raw database-type]]
+                                                           (driver/type->database-type driver type))]
+                                             [name db-type])))
+                                 columns)
+        primary-key-opts (select-keys table-schema [:primary-key])]
+    (log/infof "Creating table %s with %d columns" table-name (count columns))
+    (driver/create-table! driver database-id table-name column-definitions primary-key-opts)))
+
+(defn rename-table!
+  "kename a table in the database."
+  [driver database-id old-table-name new-table-name]
+  (log/infof "Renaming table %s to %s" old-table-name new-table-name)
+  (driver/rename-table! driver database-id old-table-name new-table-name))
+
+(defn drop-table!
+  "Drop a table in the database."
+  [driver database-id table-name]
+  (log/infof "Dropping table %s" table-name)
+  (driver/drop-table! driver database-id table-name))

enterprise/backend/test/metabase_enterprise/python_transform/api_test.clj

@@ -0,0 +1,89 @@
+(ns metabase-enterprise.python-transform.api-test
+  "Tests for /api/ee/python-transform endpoints."
+  (:require
+   [clojure.test :refer :all]
+   [metabase-enterprise.python-transform.models.python-library :as python-library]
+   [metabase.test :as mt]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(deftest get-library-path-test
+  (testing "GET /api/ee/python-transform/library/:path"
+    (mt/with-premium-features #{:transforms}
+      (testing "requires superuser permissions"
+        (is (= "You don't have permissions to do that."
+               (mt/user-http-request :rasta :get 403 "ee/python-transform/library/common"))))
+
+      (testing "returns nil when no library exists"
+        (t2/delete! :model/PythonLibrary)
+        (is (nil? (mt/user-http-request :crowberto :get 204 "ee/python-transform/library/common"))))
+
+      (testing "returns existing library"
+        (t2/delete! :model/PythonLibrary)
+        (python-library/update-python-library-source! "common" "def my_function():\n    return 42")
+        (is (=? {:source "def my_function():\n    return 42"
+                 :path "common"
+                 :created_at some?
+                 :updated_at some?}
+                (mt/user-http-request :crowberto :get 200 "ee/python-transform/library/common"))))
+
+      (testing "rejects invalid paths"
+        (is (= "Invalid library path. Only 'common' is currently supported."
+               (:message (mt/user-http-request :crowberto :get 400 "ee/python-transform/library/invalid-path"))))))))
+
+(deftest put-library-path-test
+  (testing "PUT /api/ee/python-transform/library/:path"
+    (mt/with-premium-features #{:transforms}
+      (testing "requires superuser permissions"
+        (is (= "You don't have permissions to do that."
+               (mt/user-http-request :rasta :put 403 "ee/python-transform/library/common"
+                                     {:source "def test(): pass"}))))
+
+      (testing "validates request body"
+        (testing "requires source field"
+          (is (=? {:errors
+                   {:source "string"}}
+                  (mt/user-http-request :crowberto :put 400 "ee/python-transform/library/common"
+                                        {}))))
+
+        (testing "allows empty string"
+          (t2/delete! :model/PythonLibrary)
+          (is (=? {:source ""
+                   :path "common"
+                   :id integer?
+                   :created_at some?
+                   :updated_at some?}
+                  (mt/user-http-request :crowberto :put 200 "ee/python-transform/library/common"
+                                        {:source ""})))))
+
+      (testing "creates new library when none exists"
+        (t2/delete! :model/PythonLibrary)
+        (is (=? {:source "def new_function():\n    return 100"
+                 :path "common"
+                 :id integer?
+                 :created_at some?
+                 :updated_at some?}
+                (mt/user-http-request :crowberto :put 200 "ee/python-transform/library/common"
+                                      {:source "def new_function():\n    return 100"})))
+        (is (= "def new_function():\n    return 100"
+               (t2/select-one-fn :source :model/PythonLibrary))))
+
+      (testing "updates existing library"
+        (t2/delete! :model/PythonLibrary)
+        (python-library/update-python-library-source! "common" "def old_function():\n    return 1")
+        (is (=? {:source "def updated_function():\n    return 2"
+                 :path "common"
+                 :id integer?
+                 :created_at some?
+                 :updated_at some?}
+                (mt/user-http-request :crowberto :put 200 "ee/python-transform/library/common"
+                                      {:source "def updated_function():\n    return 2"})))
+        (is (= "def updated_function():\n    return 2"
+               (t2/select-one-fn :source :model/PythonLibrary)))
+        (is (= 1 (t2/count :model/PythonLibrary)) "Should not create duplicate records"))
+
+      (testing "rejects invalid paths"
+        (is (= "Invalid library path. Only 'common' is currently supported."
+               (:message (mt/user-http-request :crowberto :put 400 "ee/python-transform/library/invalid-path"
+                                               {:source "def test(): pass"}))))))))

enterprise/backend/test/metabase_enterprise/python_transform/models/python_library_test.clj

@@ -0,0 +1,56 @@
+(ns metabase-enterprise.python-transform.models.python-library-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase-enterprise.python-transform.models.python-library :as python-library]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(deftest update-python-library-source-test
+  (testing "update-python-library-source!"
+    (testing "creates new record when none exists"
+      (t2/delete! :model/PythonLibrary)
+      (is (=? {:source "def new_func(): return 1"
+               :path "common"
+               :id integer?
+               :created_at some?
+               :updated_at some?}
+              (python-library/update-python-library-source! "common" "def new_func(): return 1")))
+      (is (= 1 (t2/count :model/PythonLibrary)))
+      (is (= "def new_func(): return 1"
+             (t2/select-one-fn :source :model/PythonLibrary))))
+
+    (testing "updates existing record"
+      (is (= 1 (t2/count :model/PythonLibrary)))
+      (is (=? {:source "def updated_func(): return 2"
+               :path "common"
+               :id integer?
+               :created_at some?
+               :updated_at some?}
+              (python-library/update-python-library-source! "common" "def updated_func(): return 2")))
+      (is (= 1 (t2/count :model/PythonLibrary)) "Should not create duplicate")
+      (is (= "def updated_func(): return 2"
+             (t2/select-one-fn :source :model/PythonLibrary))))
+
+    (testing "rejects invalid paths"
+      (is (thrown-with-msg? clojure.lang.ExceptionInfo
+                            #"Invalid library path"
+                            (python-library/update-python-library-source! "invalid-path" "def test(): pass")))
+      (is (= 1 (t2/count :model/PythonLibrary)) "Should not create library with invalid path"))))
+
+(deftest get-python-library-by-path-test
+  (testing "get-python-library-by-path"
+    (testing "returns library when path is valid"
+      (t2/delete! :model/PythonLibrary)
+      (python-library/update-python-library-source! "common" "def test(): pass")
+      (is (=? {:source "def test(): pass"
+               :path "common"
+               :id integer?
+               :created_at some?
+               :updated_at some?}
+              (python-library/get-python-library-by-path "common"))))
+
+    (testing "rejects invalid paths"
+      (is (thrown-with-msg? clojure.lang.ExceptionInfo
+                            #"Invalid library path"
+                            (python-library/get-python-library-by-path "invalid-path"))))))

enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/row_level_restrictions_test.clj

@@ -186,13 +186,12 @@
                    :query {:source-query {:source-table $$checkins
                                           :fields [$id $date $user_id $venue_id]
                                           :filter [:and
-                                                                          ;; This still gets :default bucketing!
-                                                                          ;; auto-bucket-datetimes puts :day bucketing
-                                                                          ;; on both parts of this filter, since it's
-                                                                          ;; matching a YYYY-mm-dd string. Then
-                                                                          ;; optimize-temporal-filters sees that the
-                                                                          ;; :type/Date column already has :day
-                                                                          ;; granularity, and switches both to :default
+                                                   ;; This still gets :default bucketing! auto-bucket-datetimes
+                                                   ;; puts :day bucketing on both parts of this filter, since it's
+                                                   ;; matching a YYYY-mm-dd string. Then optimize-temporal-filters
+                                                   ;; sees that the
+                                                   ;; :type/Date column already has :day
+                                                   ;; granularity, and switches both to :default
                                                    [:> !default.date [:absolute-datetime
                                                                       #t "2014-01-01"
                                                                       :default]]

enterprise/backend/test/metabase_enterprise/semantic_search/api_test.clj

@@ -3,31 +3,20 @@
    [clojure.core.memoize :as memoize]
    [clojure.test :refer :all]
    [metabase-enterprise.semantic-search.api :as semantic.api]
-   [metabase-enterprise.semantic-search.embedding :as semantic.embedding]
    [metabase-enterprise.semantic-search.env :as semantic.env]
    [metabase-enterprise.semantic-search.index :as semantic.index]
    [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
    [metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]
    [metabase-enterprise.semantic-search.test-util :as semantic.tu]
    [metabase.search.ingestion :as search.ingestion]
-   [metabase.test :as mt]
-   [metabase.test.fixtures :as fixtures]))
+   [metabase.test :as mt]))
 
-(use-fixtures :once (compose-fixtures
-                     (fixtures/initialize :db)
-                     #'semantic.tu/once-fixture))
-
-(defmacro with-mock-index-metadata!
-  [& body]
-  `(with-redefs [semantic.env/get-pgvector-datasource! (constantly semantic.tu/db)
-                 semantic.embedding/get-configured-model (constantly semantic.tu/mock-embedding-model)
-                 semantic.env/get-index-metadata (constantly semantic.tu/mock-index-metadata)]
-     ~@body))
+(use-fixtures :once #'semantic.tu/once-fixture)
 
 (deftest status-endpoint-test
   (testing "GET /api/ee/semantic-search/status"
     (mt/with-premium-features #{:semantic-search}
-      (with-mock-index-metadata!
+      (semantic.tu/with-test-db! {:mode :mock-initialized}
         (with-open [index-ref (semantic.tu/open-temp-index!)]
           (with-redefs [semantic.index-metadata/get-active-index-state (fn [_ _]
                                                                          {:index @index-ref
@@ -71,17 +60,18 @@
 (deftest re-init-endpoint-test
   (testing "POST /api/search/re-init with semantic search support"
     (mt/with-premium-features #{:semantic-search}
-      (semantic.tu/with-index!
+      (semantic.tu/with-test-db! {:mode :mock-indexed}
         (let [original-index      semantic.tu/mock-index
               original-table-name (:table-name original-index)
               new-index           (with-redefs [semantic.index/model-table-suffix (constantly 345)]
                                     (#'semantic.pgvector-api/fresh-index semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model :force-reset? true))
-              new-table-name      (:table-name new-index)]
+              new-table-name      (:table-name new-index)
+              pgvector (semantic.env/get-pgvector-datasource!)]
 
           (is (semantic.tu/table-exists-in-db? original-table-name))
           (is (not (semantic.tu/table-exists-in-db? new-table-name)))
 
-          (let [best-index (semantic.index-metadata/find-compatible-index! semantic.tu/db semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model)]
+          (let [best-index (semantic.index-metadata/find-compatible-index! pgvector semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model)]
             (is (=? original-index (:index best-index)))
             (is (:active best-index)))
 
@@ -96,7 +86,7 @@
 
             (is (zero? (semantic.tu/index-count new-index))))
 
-          (let [best-index (semantic.index-metadata/find-compatible-index! semantic.tu/db semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model)]
+          (let [best-index (semantic.index-metadata/find-compatible-index! pgvector semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model)]
             (is (=? new-index (:index best-index)))
             (is (:active best-index)))
 

enterprise/backend/test/metabase_enterprise/semantic_search/core_test.clj

@@ -2,6 +2,7 @@
   (:require
    [clojure.test :refer :all]
    [metabase-enterprise.semantic-search.core :as semantic.core]
+   [metabase-enterprise.semantic-search.env :as semantic.env]
    [metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]
    [metabase-enterprise.semantic-search.test-util :as semantic.tu]
    [metabase.app-db.core :as mdb]
@@ -10,19 +11,17 @@
    [metabase.search.in-place.legacy :as in-place.legacy]
    [metabase.search.in-place.scoring :as in-place.scoring]
    [metabase.search.settings :as search.settings]
-   [metabase.test :as mt]
-   [metabase.test.fixtures :as fixtures]))
+   [metabase.test :as mt]))
 
-(use-fixtures :once (compose-fixtures
-                     (fixtures/initialize :db)
-                     #'semantic.tu/once-fixture))
+(use-fixtures :once #'semantic.tu/once-fixture)
 
 (deftest fallback-engine-available-with-semantic-test
   (mt/with-premium-features #{:semantic-search}
     (mt/with-temporary-setting-values [search.settings/search-engine "semantic"]
       (with-open [_ (semantic.tu/open-temp-index!)]
-        (semantic.tu/cleanup-index-metadata! semantic.tu/db semantic.tu/mock-index-metadata)
-        (semantic.tu/with-index!
+        (semantic.tu/cleanup-index-metadata! (semantic.env/get-pgvector-datasource!)
+                                             semantic.tu/mock-index-metadata)
+        (semantic.tu/with-test-db! {:mode :mock-indexed}
           (is (search.engine/supported-engine? (if (= :mysql (mdb/db-type))
                                                  ;; appdb is not supported on :mysql, should fallback to in-place
                                                  :search.engine/in-place
@@ -56,7 +55,9 @@
                           in-place.legacy/results
                           (fn [_]
                             (reset! legacy-called? true)
-                            [{:id 2 :name "legacy-result" :model "card" :collection_id 1 :score 0.7}])]
+                            (reify clojure.lang.IReduceInit
+                              (reduce [_this rf init]
+                                (reduce rf init [{:id 2 :name "legacy-result" :model "card" :collection_id 1 :score 0.7}]))))]
               (testing "API defaults to semantic engine when configured"
                 (let [response (mt/user-http-request :crowberto :get 200 "search" :q "test")]
                   (is @semantic-called? "Semantic search should be called by default")
@@ -97,11 +98,15 @@
                                       {:result (dissoc result :score)
                                        :score  (:score result)})
                 search.engine/results (fn [ctx]
-                                        (case (:search-engine ctx)
-                                          :search.engine/semantic (semantic.core/results ctx)
-                                          (if (fn? fallback-results)
-                                            (fallback-results ctx)
-                                            fallback-results)))]
+                                        (let [fallback-results* #(if (fn? fallback-results)
+                                                                   (fallback-results ctx)
+                                                                   fallback-results)]
+                                          (case (:search-engine ctx)
+                                            :search.engine/semantic (semantic.core/results ctx)
+                                            :search.engine/in-place (reify clojure.lang.IReduceInit
+                                                                      (reduce [_this rf init]
+                                                                        (reduce rf init (fallback-results*))))
+                                            (fallback-results*))))]
     (thunk)))
 
 (defn- make-card-result

enterprise/backend/test/metabase_enterprise/semantic_search/dlq_test.clj

@@ -3,6 +3,7 @@
    [clojure.test :refer :all]
    [honey.sql :as sql]
    [metabase-enterprise.semantic-search.dlq :as semantic.dlq]
+   [metabase-enterprise.semantic-search.env :as semantic.env]
    [metabase-enterprise.semantic-search.gate :as semantic.gate]
    [metabase-enterprise.semantic-search.index :as semantic.index]
    [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
@@ -46,7 +47,7 @@
 
 (deftest dlq-table-creation-test
   (testing "DLQ table creation and existence checks"
-    (let [pgvector       semantic.tu/db
+    (let [pgvector       (semantic.env/get-pgvector-datasource!)
           index-metadata (semantic.tu/unique-index-metadata)
           index-id       42]
       (with-open [_ (semantic.tu/open-metadata! pgvector index-metadata)]
@@ -58,7 +59,7 @@
 
 (deftest dlq-entry-management-test
   (testing "DLQ entry add, query, and delete operations"
-    (let [pgvector       semantic.tu/db
+    (let [pgvector       (semantic.env/get-pgvector-datasource!)
           index-metadata (semantic.tu/unique-index-metadata)
           index-id       42]
       (with-open [_ (semantic.tu/open-metadata! pgvector index-metadata)
@@ -97,7 +98,7 @@
 
 (deftest dlq-entry-upsert-test
   (testing "DLQ entry upsert behavior on conflict"
-    (let [pgvector       semantic.tu/db
+    (let [pgvector       (semantic.env/get-pgvector-datasource!)
           index-metadata (semantic.tu/unique-index-metadata)
           index-id       42]
       (with-open [_ (semantic.tu/open-metadata! pgvector index-metadata)
@@ -129,7 +130,7 @@
             (is (= t2 (:error_gated_at (first results))))))))))
 
 (deftest dlq-poll-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         model          semantic.tu/mock-embedding-model
         index          (semantic.index-metadata/qualify-index (semantic.index/default-index model) index-metadata)
@@ -194,7 +195,7 @@
           (is (= 3 (count (semantic.dlq/poll pgvector index-metadata @index-id-ref 10)))))))))
 
 (deftest dlq-retry-loop-test
-  (let [pgvector         semantic.tu/db
+  (let [pgvector         (semantic.env/get-pgvector-datasource!)
         index-metadata   (semantic.tu/unique-index-metadata)
         model            semantic.tu/mock-embedding-model
         index            (semantic.index-metadata/qualify-index (semantic.index/default-index model) index-metadata)
@@ -296,7 +297,7 @@
                 (is (pos? (:failure-count result)))))))))))
 
 (deftest try-batch-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         model          semantic.tu/mock-embedding-model
         index          (semantic.index-metadata/qualify-index (semantic.index/default-index model) index-metadata)

enterprise/backend/test/metabase_enterprise/semantic_search/gate_test.clj

@@ -4,6 +4,7 @@
    [clojure.test :refer :all]
    [clojure.walk :as walk]
    [honey.sql :as sql]
+   [metabase-enterprise.semantic-search.env :as semantic.env]
    [metabase-enterprise.semantic-search.gate :as semantic.gate]
    [metabase-enterprise.semantic-search.index :as semantic.index]
    [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
@@ -89,7 +90,7 @@
   (:ts (jdbc/execute-one! pgvector ["select clock_timestamp() ts"])))
 
 (deftest gate-documents!-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         t0             (ts "2025-01-01T00:00:00Z")
         t1             (ts "2025-01-01T00:01:00Z")
@@ -168,7 +169,7 @@
                (comparable-gate-row (dissoc (get-gate-row! pgvector index-metadata (:id (version c3 t3))) :gated_at))))))))
 
 (deftest poll-test
-  (let [pgvector        semantic.tu/db
+  (let [pgvector        (semantic.env/get-pgvector-datasource!)
         index-metadata  (semantic.tu/unique-index-metadata)
         epoch-watermark {:last-poll Instant/EPOCH :last-seen Instant/EPOCH}
         c1              {:model "card" :id "123" :searchable_text "a"}
@@ -275,7 +276,7 @@
              (:last-seen watermark))))))
 
 (deftest flush-watermark!-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         model1         {:provider "foo" :model-name "m1" :vector-dimensions 42}
         model2         {:provider "foo" :model-name "m2" :vector-dimensions 42}
@@ -330,7 +331,7 @@
 
 (deftest gate-documents-metrics-test
   (mt/with-prometheus-system! [_ system]
-    (let [pgvector       semantic.tu/db
+    (let [pgvector       (semantic.env/get-pgvector-datasource!)
           index-metadata (semantic.tu/unique-index-metadata)
           t1             (ts "2025-01-01T00:01:00Z")
           c1             {:model "card" :id "123" :searchable_text "Dog Training Guide"}

enterprise/backend/test/metabase_enterprise/semantic_search/index_metadata_test.clj

@@ -1,6 +1,7 @@
 (ns metabase-enterprise.semantic-search.index-metadata-test
   (:require
    [clojure.test :refer :all]
+   [metabase-enterprise.semantic-search.env :as semantic.env]
    [metabase-enterprise.semantic-search.index :as semantic.index]
    [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
    [metabase-enterprise.semantic-search.test-util :as semantic.tu]
@@ -23,7 +24,7 @@
                                             semantic.index/hnsw-index-name)))))))
 
 (deftest create-tables-if-not-exists!-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         ;; could use open-tables here but I chose to be explicit
         sut            #(semantic.tu/closeable
@@ -46,7 +47,7 @@
    (fn [_] (semantic.tu/cleanup-index-metadata! pgvector index-metadata))))
 
 (deftest drop-tables-if-exists!-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         sut            semantic.index-metadata/drop-tables-if-exists!]
     (with-open [_ (open-tables! pgvector index-metadata)]
@@ -59,7 +60,7 @@
         (is (= now-expected (semantic.tu/get-table-names pgvector)))))))
 
 (deftest ensure-control-row-exists!-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         sut            semantic.index-metadata/ensure-control-row-exists!]
     (with-open [_ (open-tables! pgvector index-metadata)]
@@ -74,7 +75,7 @@
           (is (= control-snap (semantic.tu/get-control-rows pgvector index-metadata))))))))
 
 (deftest activate-index!-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         un-index       (semantic.index/default-index semantic.tu/mock-embedding-model)
         index1         (semantic.index-metadata/qualify-index (assoc un-index :table-name "i1") index-metadata)
@@ -98,7 +99,7 @@
             (is (=? [{:active_id index-id2}] (semantic.tu/get-control-rows pgvector index-metadata)))))))))
 
 (deftest get-active-index-state-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         un-index       (semantic.index/default-index semantic.tu/mock-embedding-model)
         index          (semantic.index-metadata/qualify-index un-index index-metadata)
@@ -134,7 +135,7 @@
   (run! #(add-index! pgvector index-metadata %) inactive))
 
 (deftest find-compatible-index!-test
-  (let [pgvector         semantic.tu/db
+  (let [pgvector         (semantic.env/get-pgvector-datasource!)
         embedding-model1 semantic.tu/mock-embedding-model
         embedding-model2 (assoc semantic.tu/mock-embedding-model
                                 :model-name "mock2")
@@ -190,7 +191,7 @@
                   (is (nil? (sut' model))))))))))))
 
 (deftest create-new-index-spec-test
-  (let [pgvector         semantic.tu/db
+  (let [pgvector         (semantic.env/get-pgvector-datasource!)
         index-metadata   (semantic.tu/unique-index-metadata)
         embedding-model  semantic.tu/mock-embedding-model
         sut              semantic.index-metadata/create-new-index-spec]
@@ -202,7 +203,7 @@
                 result))))))
 
 (deftest record-new-index-table!-test
-  (let [pgvector        semantic.tu/db
+  (let [pgvector        (semantic.env/get-pgvector-datasource!)
         index-metadata  (semantic.tu/unique-index-metadata)
         embedding-model semantic.tu/mock-embedding-model
         index           (default-index embedding-model index-metadata)

enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj

@@ -3,6 +3,7 @@
    [clojure.string :as str]
    [clojure.test :refer :all]
    [metabase-enterprise.semantic-search.embedding :as semantic.embedding]
+   [metabase-enterprise.semantic-search.env :as semantic.env]
    [metabase-enterprise.semantic-search.index :as semantic.index]
    [metabase-enterprise.semantic-search.test-util :as semantic.tu]
    [metabase.analytics.core :as analytics]
@@ -18,14 +19,14 @@
   (mt/with-premium-features #{:semantic-search}
     (with-open [index-ref (semantic.tu/open-temp-index!)]
       ;; open-temp-index-table! creates the temp table, so drop it in order to test create!.
-      (semantic.index/drop-index-table! semantic.tu/db semantic.tu/mock-index)
+      (semantic.index/drop-index-table! (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index)
       (testing "index table is not present before create!"
         (is (not (semantic.tu/table-exists-in-db? (:table-name @index-ref))))
         (is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref))))
         (is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/fts-index-name @index-ref))))
         (is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/fts-native-index-name @index-ref)))))
       (testing "index table is present after create!"
-        (semantic.index/create-index-table-if-not-exists! semantic.tu/db semantic.tu/mock-index {:force-reset? false})
+        (semantic.index/create-index-table-if-not-exists! (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index {:force-reset? false})
         (is (semantic.tu/table-exists-in-db? (:table-name @index-ref)))
         (is (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref)))
         (is (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/fts-index-name @index-ref)))
@@ -37,7 +38,7 @@
       (testing "index table is present before drop!"
         (is (semantic.tu/table-exists-in-db? (:table-name @index-ref))))
       (testing "index table is not present after drop!"
-        (semantic.index/drop-index-table! semantic.tu/db semantic.tu/mock-index)
+        (semantic.index/drop-index-table! (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index)
         (is (not (semantic.tu/table-exists-in-db? (:table-name @index-ref))))))))
 
 (deftest upsert-index!-test
@@ -343,7 +344,7 @@
             (testing "Permission filtering metrics"
               (reset! analytics-calls [])
               (mt/with-test-user :crowberto
-                (semantic.index/query-index semantic.tu/db semantic.tu/mock-index
+                (semantic.index/query-index (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index
                                             {:search-string "dog training"}))
 
               (let [permission-calls (filter #(= :metabase-search/semantic-permission-filter-ms (first %)) @analytics-calls)]
@@ -355,7 +356,7 @@
             (testing "Semantic search timing metrics"
               (reset! analytics-calls [])
               (mt/with-test-user :crowberto
-                (semantic.index/query-index semantic.tu/db semantic.tu/mock-index
+                (semantic.index/query-index (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index
                                             {:search-string "elephant migration"
                                              :filter-items-in-personal-collection "only"}))
 

enterprise/backend/test/metabase_enterprise/semantic_search/indexer_test.clj

@@ -3,6 +3,7 @@
    [clojure.test :refer :all]
    [honey.sql :as sql]
    [metabase-enterprise.semantic-search.dlq :as semantic.dlq]
+   [metabase-enterprise.semantic-search.env :as semantic.env]
    [metabase-enterprise.semantic-search.gate :as semantic.gate]
    [metabase-enterprise.semantic-search.index :as semantic.index]
    [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
@@ -33,7 +34,7 @@
   (Timestamp/from (Instant/parse s)))
 
 (deftest indexing-step-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         model          semantic.tu/mock-embedding-model
         index          (semantic.index-metadata/qualify-index (semantic.index/default-index model) index-metadata)
@@ -214,7 +215,7 @@
            (log/fatal "Indexing loop thread not exiting during test!")))))))
 
 (deftest indexing-loop-thread-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         index          semantic.tu/mock-index
         metadata-row   {:indexer_last_poll Instant/EPOCH
@@ -283,7 +284,7 @@
             (is (.join thread (Duration/ofSeconds 1)))))))))
 
 (deftest indexing-loop-exit-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         model          semantic.tu/mock-embedding-model
         index          (semantic.index-metadata/qualify-index (semantic.index/default-index model) index-metadata)
@@ -371,7 +372,7 @@
    (fn [_] (semantic.dlq/drop-dlq-table-if-exists! pgvector index-metadata index-id))))
 
 (deftest quartz-job-run!-test
-  (let [pgvector        semantic.tu/db
+  (let [pgvector        (semantic.env/get-pgvector-datasource!)
         index-metadata  (semantic.tu/unique-index-metadata)
         open-job-thread (fn [& args]
                           (let [caught-ex (volatile! nil)]
@@ -458,7 +459,7 @@
 
 (deftest dlq-step-test
   (mt/with-prometheus-system! [_ system]
-    (let [pgvector       semantic.tu/db
+    (let [pgvector       (semantic.env/get-pgvector-datasource!)
           index-metadata (semantic.tu/unique-index-metadata)
           model          semantic.tu/mock-embedding-model
           index          (semantic.index-metadata/qualify-index (semantic.index/default-index model) index-metadata)
@@ -541,7 +542,7 @@
 
 (deftest indexer-stall-and-recovery-test
   (mt/with-prometheus-system! [_ system]
-    (let [pgvector             semantic.tu/db
+    (let [pgvector             (semantic.env/get-pgvector-datasource!)
           index-metadata       (semantic.tu/unique-index-metadata)
           model                semantic.tu/mock-embedding-model
           index                (semantic.index-metadata/qualify-index (semantic.index/default-index model) index-metadata)
@@ -660,7 +661,7 @@
                         (mt/metric-value system :metabase-search/semantic-indexer-poll-to-poll-interval-ms)))))))))))
 
 (deftest dlq-integration-with-indexer-loop-test
-  (let [pgvector         semantic.tu/db
+  (let [pgvector         (semantic.env/get-pgvector-datasource!)
         index-metadata   (semantic.tu/unique-index-metadata)
         model            semantic.tu/mock-embedding-model
         index            (semantic.index-metadata/qualify-index (semantic.index/default-index model) index-metadata)
@@ -776,7 +777,7 @@
 
 (deftest indexer-loop-metric-test
   (mt/with-prometheus-system! [_ system]
-    (let [pgvector semantic.tu/db
+    (let [pgvector (semantic.env/get-pgvector-datasource!)
           index-metadata (semantic.tu/unique-index-metadata)
           metadata-row {:id                42
                         :indexer_last_poll (ts "2025-01-01T12:00:00Z")

enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj

@@ -9,6 +9,7 @@
    [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
    [metabase-enterprise.semantic-search.db.migration :as semantic.db.migration]
    [metabase-enterprise.semantic-search.db.migration.impl :as semantic.db.migration.impl]
+   [metabase-enterprise.semantic-search.env :as semantic.env]
    [metabase-enterprise.semantic-search.index :as semantic.index]
    [metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]
    [metabase-enterprise.semantic-search.test-util :as semantic.tu]
@@ -21,11 +22,10 @@
 (set! *warn-on-reflection* true)
 
 (use-fixtures :once #'semantic.tu/once-fixture)
-(use-fixtures :each #'semantic.tu/ensure-no-migration-table-fixture)
 
 (deftest migration-table-versions-test
   (mt/with-premium-features #{:semantic-search}
-    (semantic.tu/with-test-db! semantic.tu/default-test-db
+    (semantic.tu/with-test-db-defaults!
       (letfn [(migrate-and-get-db-version
                 [attempted-version]
                 (with-redefs [semantic.db.migration.impl/schema-version attempted-version
@@ -56,15 +56,14 @@
 
 (deftest migration-lock-coordination-test
   (mt/with-premium-features #{:semantic-search}
-    (semantic.tu/with-test-db! semantic.tu/default-test-db
+    (semantic.tu/with-test-db-defaults!
       (testing "Migration of simultaneous init attempt is blocked"
         (let [original-write-fn @#'semantic.db.migration/write-successful-migration!
               original-migrate-fn @#'semantic.db.connection/do-with-migrate-tx
               original-maybe-migrate @#'semantic.db.migration/maybe-migrate!
               results (atom {:executed-migrations 0
                              :log []})]
-          (with-redefs-fn {;; TODO: why in ci init index won't succeed -- http on embedding
-                           #'semantic.pgvector-api/index-documents! (constantly nil)
+          (with-redefs-fn {#'semantic.pgvector-api/index-documents! (constantly nil)
                            #'semantic.db.connection/do-with-migrate-tx
                            (fn [& args]
                              (let [tid (.getId (Thread/currentThread))]
@@ -121,12 +120,12 @@
 (deftest expected-db-schema-after-migration-test
   (try
     (mt/with-premium-features #{:semantic-search}
-      (semantic.tu/with-test-db! semantic.tu/default-test-db
+      (semantic.tu/with-test-db-defaults!
         (with-redefs [semantic.pgvector-api/index-documents! (constantly nil)]
           (semantic.core/init! (semantic.tu/mock-documents) nil)
           (testing "migration table has expected columns"
             (is (map-contains-keys?
-                 (jdbc/execute-one! (semantic.db.datasource/ensure-initialized-data-source!)
+                 (jdbc/execute-one! (semantic.env/get-pgvector-datasource!)
                                     (sql/format {:select [:*]
                                                  :from [:migration]}))
                  (qualify :migration [:migrated_at
@@ -134,7 +133,7 @@
                                       :version]))))
           (testing "control table has expected columns"
             (is (map-contains-keys?
-                 (jdbc/execute-one! (semantic.db.datasource/ensure-initialized-data-source!)
+                 (jdbc/execute-one! (semantic.env/get-pgvector-datasource!)
                                     (sql/format {:select [:*]
                                                  :from [:index_control]}))
                  (qualify :index_control [:active_id
@@ -143,7 +142,7 @@
                                           :version]))))
           (testing "metadata table has expected columns"
             (is  (map-contains-keys?
-                  (jdbc/execute-one! semantic.tu/db
+                  (jdbc/execute-one! (semantic.env/get-pgvector-datasource!)
                                      (sql/format {:select [:*]
                                                   :from [:index_metadata]}))
                   (qualify :index_metadata [:id
@@ -190,7 +189,7 @@
                         "text_search_with_native_query_vector"
                         "verified"
                         "view_count"}
-                      (->>  (jdbc/execute! semantic.tu/db
+                      (->>  (jdbc/execute! (semantic.env/get-pgvector-datasource!)
                                            (sql/format {:select [:column_name]
                                                         :from [:information_schema.columns]
                                                         :where [[:= :table_name [:inline index-table]]]}))
@@ -198,7 +197,7 @@
                             set)))))
           (testing "index table has expected columns"
             (is (= ["document" "document_hash" "gated_at" "id" "model" "model_id" "updated_at"]
-                   (->> (jdbc/execute! semantic.tu/db
+                   (->> (jdbc/execute! (semantic.env/get-pgvector-datasource!)
                                        (sql/format {:select [:column_name]
                                                     :from [:information_schema.columns]
                                                     :where [[:= :table_name [:inline "index_gate"]]]}))
@@ -219,7 +218,7 @@
 
 (deftest dynamic-schema-migration-test
   (mt/with-premium-features #{:semantic-search}
-    (semantic.tu/with-test-db! semantic.tu/default-test-db
+    (semantic.tu/with-test-db-defaults!
       (with-redefs [semantic.pgvector-api/index-documents! (constantly nil)]
         (semantic.core/init! (semantic.tu/mock-documents) nil)
              ;; add column to index table

enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj

@@ -3,6 +3,7 @@
    [clojure.test :refer :all]
    [honey.sql :as sql]
    [metabase-enterprise.semantic-search.dlq :as semantic.dlq]
+   [metabase-enterprise.semantic-search.env :as semantic.env]
    [metabase-enterprise.semantic-search.index :as semantic.index]
    [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
    [metabase-enterprise.semantic-search.indexer :as semantic.indexer]
@@ -21,12 +22,11 @@
 (set! *warn-on-reflection* true)
 
 (use-fixtures :once #'semantic.tu/once-fixture)
-(use-fixtures :each #'semantic.tu/ensure-no-migration-table-fixture)
 
 ;; NOTE: isolation tests are absent, in prod there is only one index-metadata
 
 (deftest init-semantic-search!-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         model1         semantic.tu/mock-embedding-model
         model2         (assoc semantic.tu/mock-embedding-model :model-name "embed-harder")
@@ -69,7 +69,7 @@
                       (semantic.index-metadata/find-compatible-index! pgvector index-metadata model2))))))))))
 
 (deftest init-recreates-missing-index-table-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         model1         semantic.tu/mock-embedding-model
         sut*           semantic.pgvector-api/init-semantic-search!
@@ -98,7 +98,7 @@
     (is (thrown? Exception (apply sut args)))))
 
 (deftest index-documents!-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         model1         semantic.tu/mock-embedding-model
         model2         (assoc semantic.tu/mock-embedding-model :model-name "embedagain")
@@ -125,7 +125,7 @@
                        @calls))))))))))
 
 (deftest delete-documents!-test
-  (let [pgvector       semantic.tu/db
+  (let [pgvector       (semantic.env/get-pgvector-datasource!)
         index-metadata (semantic.tu/unique-index-metadata)
         dash           "dashboard"
         card           "card"
@@ -166,7 +166,7 @@
 (deftest query-test
   (semantic.tu/with-indexable-documents!
     (mt/with-dynamic-fn-redefs [semantic.index/model-table-suffix semantic.tu/mock-table-suffix]
-      (let [pgvector       semantic.tu/db
+      (let [pgvector       (semantic.env/get-pgvector-datasource!)
             index-metadata (semantic.tu/unique-index-metadata)
             model1         semantic.tu/mock-embedding-model
             model2         (assoc semantic.tu/mock-embedding-model :model-name "judge-embedd")
@@ -218,7 +218,7 @@
 
 (deftest e2e-index-a-sample-db-with-gate-test
   (let [docs            (mt/dataset test-data (vec (search.ingestion/searchable-documents)))
-        pgvector        semantic.tu/db
+        pgvector        (semantic.env/get-pgvector-datasource!)
         index-metadata  (semantic.tu/unique-index-metadata)
         embedding-model semantic.tu/mock-embedding-model
         open-job-thread (fn [& args]

enterprise/backend/test/metabase_enterprise/semantic_search/query_test.clj

@@ -1,7 +1,6 @@
 (ns metabase-enterprise.semantic-search.query-test
   (:require
    [clojure.test :refer :all]
-   [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
    [metabase-enterprise.semantic-search.index :as semantic.index]
    [metabase-enterprise.semantic-search.test-util :as semantic.tu]
    [metabase.permissions.models.data-permissions :as data-perms]
@@ -12,22 +11,21 @@
 
 (use-fixtures :once #'semantic.tu/once-fixture)
 
-(deftest database-initialised-test
-  (is (some? @semantic.db.datasource/data-source))
-  (is (= {:test 1} (semantic.db.datasource/test-connection!))))
-
 (deftest basic-query-test
   (testing "Simple queries with no filters"
     (mt/with-premium-features #{:semantic-search}
       (mt/as-admin
-        (semantic.tu/with-index!
-          (testing "Dog-related query finds dog content"
-            (let [results (semantic.tu/query-index {:search-string "puppy"})]
-              (is (= "Dog Training Guide" (-> results first :name)))))
+        (semantic.tu/with-test-db! {:mode :mock-indexed}
+          (semantic.tu/with-only-semantic-weights
+            (testing "Dog-related query finds dog content"
+              (let [results (-> (semantic.tu/query-index {:search-string "puppy"})
+                                semantic.tu/filter-for-mock-embeddings)]
+                (is (= "Dog Training Guide" (-> results first :name)))))
 
-          (testing "Bird-related query finds bird content"
-            (let [results (semantic.tu/query-index {:search-string "avian"})]
-              (is (= "Bird Watching Tips" (-> results first :name))))))))))
+            (testing "Bird-related query finds bird content"
+              (let [results (-> (semantic.tu/query-index {:search-string "avian"})
+                                semantic.tu/filter-for-mock-embeddings)]
+                (is (= "Bird Watching Tips" (-> results first :name)))))))))))
 
 (defn- index-of-name
   "Return the index of the item with :name `name` in `coll`"
@@ -41,75 +39,76 @@
   (testing "Tests for :search-native-query"
     (mt/with-premium-features #{:semantic-search}
       (mt/as-admin
-        (semantic.tu/with-index!
-          (let [not-top-10? #(or (nil? %) (< 10 %))]
-            (doseq [{:keys [name desc search-string search-native-query? expected-index?]}
-                    [{:name "Dog Training Guide"
-                      :desc "contains"
-                      :search-string "AVG(tricks)"
-                      :search-native-query? false
-                      :expected-index? not-top-10?}
-                     {:name "Dog Training Guide"
-                      :desc "contains"
-                      :search-string "AVG(tricks)"
-                      :search-native-query? true
-                      :expected-index? zero?}
-                     {:name "Dog Training Guide"
-                      :desc "contains"
-                      :search-string "GROUP BY breed"
-                      :search-native-query? false
-                      :expected-index? not-top-10?}
-                     {:name "Dog Training Guide"
-                      :desc "contains"
-                      :search-string "GROUP BY breed"
-                      :search-native-query? true
-                      :expected-index? zero?}
-                     {:name "Dog Training Guide"
-                      :desc "does not contain"
-                      :search-string "SUM(tricks)"
-                      :search-native-query? false
-                      :expected-index? not-top-10?}
-                     {:name "Dog Training Guide"
-                      :desc "does not contain"
-                      :search-string "SUM(tricks)"
-                      :search-native-query? true
-                      :expected-index? not-top-10?}
-                     {:name "Bird Watching Tips"
-                      :desc "does not contain (no native query)"
-                      :search-string "AVG(tricks)"
-                      :search-native-query? true
-                      :expected-index? not-top-10?}
-                     {:name "Dog Training Guide"
-                      :desc "semantic search"
-                      :search-string "puppy"
-                      :search-native-query? true
-                      :expected-index? zero?}
-                     {:name "Bird Watching Tips"
-                      :desc "semantic search"
-                      :search-string "avian"
-                      :search-native-query? true
-                      :expected-index? zero?}
-                     {:name "Dog Training Guide"
-                      :desc "non-native keyword query"
-                      :search-string "Training"
-                      :search-native-query? true
-                      :expected-index? zero?}
-                     {:name "Bird Watching Tips"
-                      :desc "non-native keyword query"
-                      :search-string "Watching"
-                      :search-native-query? true
-                      :expected-index? zero?}]]
-              (testing (format "\n%s %s %s %s" name desc search-string search-native-query?)
-                (is (-> (semantic.tu/query-index {:search-string search-string
-                                                  :search-native-query search-native-query?})
-                        (index-of-name name)
-                        expected-index?))))))))))
+        (semantic.tu/with-test-db! {:mode :mock-indexed}
+          (semantic.tu/with-only-semantic-weights
+            (let [not-top-10? #(or (nil? %) (< 10 %))]
+              (doseq [{:keys [name desc search-string search-native-query? expected-index?]}
+                      [{:name "Dog Training Guide"
+                        :desc "contains"
+                        :search-string "AVG(tricks)"
+                        :search-native-query? false
+                        :expected-index? not-top-10?}
+                       {:name "Dog Training Guide"
+                        :desc "contains"
+                        :search-string "AVG(tricks)"
+                        :search-native-query? true
+                        :expected-index? zero?}
+                       {:name "Dog Training Guide"
+                        :desc "contains"
+                        :search-string "GROUP BY breed"
+                        :search-native-query? false
+                        :expected-index? not-top-10?}
+                       {:name "Dog Training Guide"
+                        :desc "contains"
+                        :search-string "GROUP BY breed"
+                        :search-native-query? true
+                        :expected-index? zero?}
+                       {:name "Dog Training Guide"
+                        :desc "does not contain"
+                        :search-string "SUM(tricks)"
+                        :search-native-query? false
+                        :expected-index? not-top-10?}
+                       {:name "Dog Training Guide"
+                        :desc "does not contain"
+                        :search-string "SUM(tricks)"
+                        :search-native-query? true
+                        :expected-index? not-top-10?}
+                       {:name "Bird Watching Tips"
+                        :desc "does not contain (no native query)"
+                        :search-string "AVG(tricks)"
+                        :search-native-query? true
+                        :expected-index? not-top-10?}
+                       {:name "Dog Training Guide"
+                        :desc "semantic search"
+                        :search-string "puppy"
+                        :search-native-query? true
+                        :expected-index? zero?}
+                       {:name "Bird Watching Tips"
+                        :desc "semantic search"
+                        :search-string "avian"
+                        :search-native-query? true
+                        :expected-index? zero?}
+                       {:name "Dog Training Guide"
+                        :desc "non-native keyword query"
+                        :search-string "Training"
+                        :search-native-query? true
+                        :expected-index? zero?}
+                       {:name "Bird Watching Tips"
+                        :desc "non-native keyword query"
+                        :search-string "Watching"
+                        :search-native-query? true
+                        :expected-index? zero?}]]
+                (testing (format "\n%s %s %s %s" name desc search-string search-native-query?)
+                  (is (-> (semantic.tu/query-index {:search-string search-string
+                                                    :search-native-query search-native-query?})
+                          (index-of-name name)
+                          expected-index?)))))))))))
 
 (deftest model-filtering-test
   (testing "Filter results by model type"
     (mt/with-premium-features #{:semantic-search}
       (mt/as-admin
-        (semantic.tu/with-index!
+        (semantic.tu/with-test-db! {:mode :mock-indexed}
           (testing "Filter for cards only"
             (let [card-results (semantic.tu/query-index {:search-string "avian" :models ["card"]})
                   filtered-results (semantic.tu/filter-for-mock-embeddings card-results)]
@@ -132,7 +131,7 @@
   (testing "Filter results by archived status"
     (mt/with-premium-features #{:semantic-search}
       (mt/as-admin
-        (semantic.tu/with-index!
+        (semantic.tu/with-test-db! {:mode :mock-indexed}
           (testing "Include only non-archived items"
             (let [active-results (semantic.tu/query-index {:search-string "feline" :archived? false})]
               (is (every? #(not (:archived %)) active-results))))
@@ -152,7 +151,7 @@
   (testing "Filter results by creator"
     (mt/with-premium-features #{:semantic-search}
       (mt/as-admin
-        (semantic.tu/with-index!
+        (semantic.tu/with-test-db! {:mode :mock-indexed}
           (testing "Filter by single creator"
             (let [crowberto-results (semantic.tu/query-index {:search-string "aquatic" :created-by [(mt/user->id :crowberto)]})
                   filtered-results (semantic.tu/filter-for-mock-embeddings crowberto-results)]
@@ -169,7 +168,7 @@
   (testing "Filter results by verified status"
     (mt/with-premium-features #{:semantic-search}
       (mt/as-admin
-        (semantic.tu/with-index!
+        (semantic.tu/with-test-db! {:mode :mock-indexed}
           (testing "Include only verified items"
             (let [verified-results (semantic.tu/query-index {:search-string "puppy" :verified true})
                   filtered-results (semantic.tu/filter-for-mock-embeddings verified-results)]
@@ -185,7 +184,7 @@
   (testing "Complex filters with multiple criteria"
     (mt/with-premium-features #{:semantic-search}
       (mt/as-admin
-        (semantic.tu/with-index!
+        (semantic.tu/with-test-db! {:mode :mock-indexed}
           (testing "Non-archived cards by specific creator"
             (let [results (semantic.tu/query-index {:search-string "equine"
                                                     :models ["card"]
@@ -219,7 +218,7 @@
 
 (deftest permissions-test
   (mt/with-premium-features #{:semantic-search}
-    (semantic.tu/with-index!
+    (semantic.tu/with-test-db! {:mode :mock-indexed}
       (let [monsters-table (t2/select-one-pk :model/Table :name "Monsters Table")
             q (fn [model s] (map :name (semantic.tu/query-index {:search-string s, :models [model]})))
             all-users (perms-group/all-users)
@@ -255,11 +254,14 @@
     (mt/with-premium-features #{:semantic-search}
       (mt/as-admin
         (binding [semantic.index/*batch-size* 1]
-          (semantic.tu/with-index!
-            (testing "Horse-related query finds horse content"
-              (let [results (semantic.tu/query-index {:search-string "marine mammal"})]
-                (is (= "Whale Communication" (-> results first :name)))))
+          (semantic.tu/with-test-db! {:mode :mock-indexed}
+            (semantic.tu/with-only-semantic-weights
+              (testing "Horse-related query finds horse content"
+                (let [results (-> (semantic.tu/query-index {:search-string "marine mammal"})
+                                  semantic.tu/filter-for-mock-embeddings)]
+                  (is (= "Whale Communication" (-> results first :name)))))
 
-            (testing "Tiger-related query finds tiger content"
-              (let [results (semantic.tu/query-index {:search-string "endangered species"})]
-                (is (= "Tiger Conservation" (-> results first :name)))))))))))
+              (testing "Tiger-related query finds tiger content"
+                (let [results (-> (semantic.tu/query-index {:search-string "endangered species"})
+                                  semantic.tu/filter-for-mock-embeddings)]
+                  (is (= "Tiger Conservation" (-> results first :name))))))))))))

enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.clj

@@ -5,6 +5,7 @@
    [honey.sql :as sql]
    [java-time.api :as t]
    [metabase-enterprise.semantic-search.core :as semantic.core]
+   [metabase-enterprise.semantic-search.env :as semantic.env]
    [metabase-enterprise.semantic-search.repair :as semantic.repair]
    [metabase-enterprise.semantic-search.test-util :as semantic.tu]
    [metabase-enterprise.semantic-search.util :as semantic.util]
@@ -47,13 +48,21 @@
   [gate-contents id]
   (some #(when (= (:id %) id) %) gate-contents))
 
+(defn- clear-gate-table!
+  "Clear all entries from the gate table to ensure clean test state"
+  [pgvector gate-table-name]
+  (jdbc/execute! pgvector
+                 (-> {:delete-from [(keyword gate-table-name)]}
+                     (sql/format :quoted true))))
+
 (deftest repair-index-integration-test
   (testing "repair-index! properly handles document additions and deletions via gate table"
     (mt/with-premium-features #{:semantic-search}
-      (semantic.tu/with-index!
-        (let [pgvector       semantic.tu/db
-              index-metadata semantic.tu/mock-index-metadata
+      (semantic.tu/with-test-db! {:mode :mock-indexed}
+        (let [pgvector       (semantic.env/get-pgvector-datasource!)
+              index-metadata (semantic.env/get-index-metadata)
               gate-table     (:gate-table-name index-metadata)
+              _              (clear-gate-table! pgvector gate-table)
               initial-docs   [(create-test-document "card" 1 "Dog Training Guide")
                               (create-test-document "card" 2 "Cat Behavior Study")
                               (create-test-document "dashboard" 3 "Animal Stats")]]
@@ -83,21 +92,21 @@
 (deftest repair-table-cleanup-test
   (testing "The repair table gets cleaned up properly at the end of a repair-index! job"
     (mt/with-premium-features #{:semantic-search}
-      (semantic.tu/with-index!
-        (let [pgvector       semantic.tu/db
+      (semantic.tu/with-test-db! {:mode :mock-indexed}
+        (let [pgvector       (semantic.env/get-pgvector-datasource!)
               index-metadata semantic.tu/mock-index-metadata
               gate-table     (:gate-table-name index-metadata)
-              initial-docs   [(create-test-document "card" 1 "Dog Training Guide")]]
+              initial-docs   [(create-test-document "card" 6 "Dog Training Guide")]]
           (semantic.core/update-index! initial-docs)
           (semantic.tu/index-all!)
 
           (testing "repair table is cleaned up after successful repair"
             (let [test-repair-table-name "repair_table_cleanup_test"]
               (with-redefs [semantic.repair/repair-table-name (constantly test-repair-table-name)]
-                (semantic.core/repair-index! [(create-test-document "card" 2 "New Test Card")])
+                (semantic.core/repair-index! [(create-test-document "card" 7 "New Test Card")])
 
                 (let [gate-contents (gate-table-contents pgvector gate-table)]
-                  (is (tombstone? (gate-entry-by-id gate-contents "card_1")))
-                  (is (some? (gate-entry-by-id gate-contents "card_2")))
+                  (is (tombstone? (gate-entry-by-id gate-contents "card_6")))
+                  (is (some? (gate-entry-by-id gate-contents "card_7")))
                   (is (not (semantic.util/table-exists? pgvector test-repair-table-name))
                       "Repair table should be cleaned up after repair-index! completes"))))))))))

enterprise/backend/test/metabase_enterprise/semantic_search/scoring_test.clj

@@ -6,7 +6,6 @@
    [metabase-enterprise.semantic-search.scoring :as semantic.scoring]
    [metabase-enterprise.semantic-search.test-util :as semantic.tu]
    [metabase.app-db.core :as mdb]
-   [metabase.search.appdb.scoring-test :refer [with-weights]]
    [metabase.search.config :as search.config]
    [metabase.test :as mt])
   (:import
@@ -34,7 +33,7 @@
   "Populate the index with the given documents."
   {:style/indent :defn}
   [documents & body]
-  `(with-open [_# (semantic.tu/open-temp-index!)]
+  `(semantic.tu/with-test-db! {:mode :mock-initialized}
      (semantic.tu/upsert-index! (map add-doc-defaults ~documents))
      ~@body))
 
@@ -56,13 +55,17 @@
 (defn search-results
   "Like search-results* but with a sanity check that search without weights returns a different result."
   [ranker-key search-string & {:as raw-ctx}]
-  (let [result   (with-weights {ranker-key 1} (search-results* search-string raw-ctx))
-        inverted (with-weights {ranker-key -1} (search-results* search-string raw-ctx))]
+  (let [result   (semantic.tu/with-weights {ranker-key  1} (search-results* search-string raw-ctx))
+        inverted (semantic.tu/with-weights {ranker-key -1} (search-results* search-string raw-ctx))]
     ;; note that this may not be a strict reversal, due to ties.
     (is (not= inverted result)
         "sanity check: search-no-weights should be different")
     result))
 
+;;
+;; index-based scorers
+;;
+
 (deftest rrf-test
   (mt/with-premium-features #{:semantic-search}
     (with-index-contents!
@@ -143,7 +146,7 @@
         (is (= [["dataset" 1 "card ancient"]
                 ["metric"  3 "card old"]
                 ["card"    2 "card recent"]]
-               (with-weights {:model 1.0 :model/dataset 1.0}
+               (semantic.tu/with-weights {:model 1.0 :model/dataset 1.0}
                  (search-results* "card"))))))))
 
 (deftest recency-test
@@ -181,9 +184,9 @@
          {:model "dashboard" :id 2 :name "view dashboard" :view_count 0}
          {:model "dataset"   :id 3 :name "view dataset"   :view_count 0}]
         ;; fix some test flakes where dataset 3 exists and has some sort of recent views
-        (with-weights (assoc (search.config/weights :default)
-                             :user-recency 0
-                             :rrf 0)
+        (semantic.tu/with-weights (assoc (search.config/weights :default)
+                                         :user-recency 0
+                                         :rrf 0)
           (is (=? [{:model "dashboard", :id 2, :name "view dashboard"}
                    {:model "card",      :id 1, :name "view card"}
                    {:model "dataset",   :id 3, :name "view dataset"}]
@@ -202,8 +205,8 @@
 (defn indifferent?
   "Check that the results and their order do not depend on the given ranker."
   [ranker-key search-string & {:as raw-ctx}]
-  (= (with-weights {ranker-key 1} (search-results* search-string raw-ctx))
-     (with-weights {ranker-key -1} (search-results* search-string raw-ctx))))
+  (= (semantic.tu/with-weights {ranker-key  1} (search-results* search-string raw-ctx))
+     (semantic.tu/with-weights {ranker-key -1} (search-results* search-string raw-ctx))))
 
 (deftest dashboard-count-test-2
   (mt/with-premium-features #{:semantic-search}
@@ -213,6 +216,44 @@
          {:model "card" :id 2 :name "card" :dashboardcard_count 201}]
         (is (indifferent? :dashboard "card"))))))
 
+;;;
+;;; "premium" index-based scorers
+;;;
+
+(deftest official-collection-test
+  (with-index-contents!
+    [{:model "collection" :id 1 :name "collection normal" :official_collection false}
+     {:model "collection" :id 2 :name "collection official" :official_collection true}]
+    (testing "official collections has higher rank"
+      (mt/with-premium-features #{:semantic-search :official-collections}
+        (is (= [["collection" 2 "collection official"]
+                ["collection" 1 "collection normal"]]
+               (search-results :official-collection "collection")))))
+    (testing "only if feature is enabled"
+      (mt/with-premium-features #{:semantic-search}
+        (is (= [["collection" 1 "collection normal"]
+                ["collection" 2 "collection official"]]
+               (search-results* "collection")))))))
+
+(deftest verified-test
+  (with-index-contents!
+    [{:model "card" :id 1 :name "card normal" :verified false}
+     {:model "card" :id 2 :name "card verified" :verified true}]
+    (testing "verified items have higher rank"
+      (mt/with-premium-features #{:semantic-search :content-verification}
+        (is (= [["card" 2 "card verified"]
+                ["card" 1 "card normal"]]
+               (search-results :verified "card")))))
+    (testing "only if feature is enabled"
+      (mt/with-premium-features #{:semantic-search}
+        (is (= [["card" 1 "card normal"]
+                ["card" 2 "card verified"]]
+               (search-results* "card")))))))
+
+;;
+;; appdb-based scorers
+;;
+
 (deftest bookmark-test
   (mt/with-premium-features #{:semantic-search}
     ;; TODO (BOT-360) enable for :mysql

enterprise/backend/test/metabase_enterprise/semantic_search/task/index_cleanup_test.clj

@@ -69,7 +69,7 @@
 
 (deftest stale-index-cleanup-test
   (mt/with-premium-features #{:semantic-search}
-    (let [pgvector semantic.tu/db
+    (let [pgvector (semantic.env/get-pgvector-datasource!)
           index-metadata (semantic.tu/unique-index-metadata)
           retention-hours 24
           old-time (t/minus (t/offset-date-time) (t/hours (inc retention-hours)))
@@ -133,7 +133,7 @@
 
 (deftest tombstone-cleanup-test
   (mt/with-premium-features #{:semantic-search}
-    (let [pgvector semantic.tu/db
+    (let [pgvector (semantic.env/get-pgvector-datasource!)
           index-metadata (semantic.tu/unique-index-metadata)
           retention-hours (semantic.settings/tombstone-retention-hours)
           old-time (t/minus (t/offset-date-time) (t/hours (inc retention-hours)))
@@ -228,7 +228,7 @@
 
 (deftest repair-table-cleanup-test
   (mt/with-premium-features #{:semantic-search}
-    (let [pgvector semantic.tu/db
+    (let [pgvector (semantic.env/get-pgvector-datasource!)
           retention-hours 2
           old-time (t/minus (t/instant) (t/hours (inc retention-hours))) ; 3 hours ago
           recent-time (t/minus (t/instant) (t/hours 1))]                 ; 1 hour ago

enterprise/backend/test/metabase_enterprise/semantic_search/task/metric_collector_test.clj

@@ -67,11 +67,10 @@
 (deftest metric-collector-test
   (mt/with-premium-features #{:semantic-search}
     (mt/with-prometheus-system! [_ system]
-      (let [pgvector       semantic.tu/db
+      (let [pgvector       (semantic.env/get-pgvector-datasource!)
             index-metadata (semantic.tu/unique-index-metadata)
             model semantic.tu/mock-embedding-model]
-        (with-redefs [semantic.env/get-pgvector-datasource! (fn [] pgvector)
-                      semantic.env/get-index-metadata (fn [] index-metadata)
+        (with-redefs [semantic.env/get-index-metadata (fn [] index-metadata)
                       semantic.env/get-configured-embedding-model (fn [] model)]
           (testing "Missing tables are handled gracefully"
             (let [result (try

enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj

@@ -15,13 +15,13 @@
    [metabase-enterprise.semantic-search.indexer :as semantic.indexer]
    [metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]
    [metabase-enterprise.semantic-search.util :as semantic.util]
+   [metabase.search.config :as search.config]
    [metabase.search.ingestion :as search.ingestion]
    [metabase.test :as mt]
    [metabase.util :as u]
    [metabase.util.json :as json]
    [metabase.util.log :as log]
    [next.jdbc :as jdbc]
-   [next.jdbc.protocols :as jdbc.protocols]
    [next.jdbc.result-set :as jdbc.rs])
   (:import
    (clojure.lang IDeref)
@@ -31,7 +31,11 @@
 
 (set! *warn-on-reflection* true)
 
-;; If I won't find any use for following muted code in follow-up tasks I'll delete it -- lbrdnk
+;; Purpose of this fixure is to block running tests if db-url is not set. That's true for enterprise app-db tests in CI.
+(defn once-fixture
+  [f]
+  (when semantic.db.datasource/db-url
+    (f)))
 
 (def default-test-db "my_test_db")
 
@@ -60,54 +64,116 @@
   [db-name & body]
   `(do-with-temp-datasource! ~db-name (fn [] ~@body)))
 
+(defmulti do-with-setup-test-db!
+  "Setup pgvector database for tests."
+  {:arglists '([mode thunk])}
+  (fn [mode _thunk] mode))
+
+(defmethod do-with-setup-test-db! :blank
+  [_mode thunk]
+  (thunk))
+
+(declare mock-embedding-model
+         mock-index-metadata
+         mock-table-suffix)
+
+(defmethod do-with-setup-test-db! :mock-initialized
+  [_mode thunk]
+  (with-redefs [semantic.embedding/get-configured-model        (fn [] mock-embedding-model)
+                semantic.index-metadata/default-index-metadata mock-index-metadata
+                semantic.index/model-table-suffix              mock-table-suffix]
+    (let [pgvector (semantic.env/get-pgvector-datasource!)
+          index-metadata (semantic.env/get-index-metadata)
+          embedding-model (semantic.env/get-configured-embedding-model)]
+      (semantic.pgvector-api/init-semantic-search! pgvector index-metadata embedding-model)
+      (thunk))))
+
+;; TODO: declare with macro (the do- less version) throws weird errors -- investigate!
+(declare do-with-indexable-documents!)
+
+(defmethod do-with-setup-test-db! :mock-gated
+  [mode thunk]
+  ((get-method do-with-setup-test-db! :mock-initialized)
+   mode
+   (fn []
+     (do-with-indexable-documents!
+      (fn []
+        (let [pgvector (semantic.env/get-pgvector-datasource!)
+              index-metadata (semantic.env/get-index-metadata)]
+          (semantic.pgvector-api/gate-updates! pgvector
+                                               index-metadata
+                                               (search.ingestion/searchable-documents))
+          (thunk)))))))
+
+(declare index-all!)
+
+(defmethod do-with-setup-test-db! :mock-indexed
+  [mode thunk]
+  ((get-method do-with-setup-test-db! :mock-gated)
+   mode
+   (fn []
+     (index-all!)
+     (thunk))))
+
+;; Reminder: this can be adjusted so (1) each database is unique and (2) redefs are thread local (latter is not simple
+;; but possible I believe), so we can take advantage of parallel tests.
 (defn do-with-test-db!
   "Impl [[with-test-db]]"
-  [db-name thunk]
+  [{:keys [dbname mode cleanup]
+    :or {dbname default-test-db
+         mode :blank
+         cleanup :before}
+    :as _opts}
+   thunk]
   (with-temp-datasource! "postgres"
     (try
-      (jdbc/execute! (semantic.db.datasource/ensure-initialized-data-source!)
-                     [(str "DROP DATABASE IF EXISTS " db-name " (FORCE)")])
-      (log/fatal "creating database")
-      (jdbc/execute! (semantic.db.datasource/ensure-initialized-data-source!)
-                     [(str "CREATE DATABASE " db-name)])
-      (log/fatal "created database")
+      (when (#{:before :both} cleanup)
+        (jdbc/execute! (semantic.env/get-pgvector-datasource!)
+                       [(str "DROP DATABASE IF EXISTS " dbname " (FORCE)")]))
+      (log/debugf "Creating database %s" dbname)
+      (jdbc/execute! (semantic.env/get-pgvector-datasource!)
+                     [(str "CREATE DATABASE " dbname)])
+      (log/debugf "Created test pgvector database %s" dbname)
       (catch java.sql.SQLException e
-        (log/fatal "creation failed")
+        (log/debugf "Creation of test pgvector database %s failed" dbname)
         (throw e))))
-  (with-temp-datasource! db-name
-    (thunk)))
 
+  (with-temp-datasource! dbname
+    (do-with-setup-test-db! mode thunk))
+
+  (when (#{:after :both} cleanup)
+    (with-temp-datasource! "postgres"
+      (try
+        (jdbc/execute! (semantic.env/get-pgvector-datasource!)
+                       [(str "DROP DATABASE IF EXISTS " dbname " (FORCE)")])
+        (catch java.sql.SQLException e
+          (log/debugf "Test pgvector database teardown %s failed" dbname)
+          (throw e))))))
+
+;; TODO: When we are parallelizing tests, we'd have to make
+;;       - `with-test-db!` use thread-safe version of with-redefs or similar,
+;;       - instead of `default-test-db` we could make unique database per test by means of eg. some counter.
 (defmacro with-test-db!
   "Drop, create database dbname on pgvector and redefine datasource accordingly. Not thread safe."
-  [db-name & body]
-  `(do-with-test-db! ~db-name (fn [] ~@body)))
+  [opts & body]
+  `(do-with-test-db! ~opts (fn [] ~@body)))
 
-(def ^:private init-delay
-  (delay
-    (when-not @semantic.db.datasource/data-source
-      (semantic.db.datasource/init-db!))))
+(defmacro with-test-db-defaults!
+  "Tiny wrapper to avoid at this point redundant {} arg of [[with-test-db!]]."
+  [& body]
+  `(with-test-db! {} ~@body))
 
-(defn once-fixture [f]
-  (when semantic.db.datasource/db-url
-    @init-delay
-    (f)))
+(defmacro with-weights
+  "Execute `body` overriding search weights with `weight-map`."
+  [weight-map & body]
+  `(mt/with-dynamic-fn-redefs [search.config/weights (constantly ~weight-map)]
+     ~@body))
 
-(declare db)
-
-#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
-(defn ensure-no-migration-table-fixture [f]
-  (semantic.db.migration/drop-migration-table! db)
-  (f)
-  (semantic.db.migration/drop-migration-table! db))
-
-(def db
-  "Proxies the semantic.db.datasource/data-source, avoids the deref and prettifies a little"
-  ;; proxy because semantic.db.datasource/data-source is not initialised until the fixture runs
-  (reify jdbc.protocols/Sourceable
-    (get-datasource [_] (jdbc.protocols/get-datasource @semantic.db.datasource/data-source))))
-
-(comment
-  (jdbc/execute! db ["select 1"]))
+(defmacro with-only-semantic-weights
+  "Execute `body` with only the semantic search hybrid scorer weights active."
+  [& body]
+  `(with-weights {:rrf 1}
+     ~@body))
 
 (def mock-embeddings
   "Static mapping from strings to (made-up) 4-dimensional embedding vectors for testing. Each pair of strings represents a
@@ -183,14 +249,16 @@
 (defmethod semantic.embedding/get-embeddings-batch "mock" [_ texts] (get-mock-embeddings-batch texts))
 (defmethod semantic.embedding/pull-model           "mock" [_])
 
+#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
 (defn query-index [search-context]
-  (:results (semantic.index/query-index db mock-index search-context)))
+  (:results (semantic.index/query-index (semantic.env/get-pgvector-datasource!) mock-index search-context)))
 
+;; TODO: this should go!!!
 (defn upsert-index! [documents & {:keys [index] :or {index mock-index} :as opts}]
-  (semantic.index/upsert-index! db index documents opts))
+  (semantic.index/upsert-index! (semantic.env/get-pgvector-datasource!) index documents opts))
 
 (defn delete-from-index! [model ids]
-  (semantic.index/delete-from-index! db mock-index model ids))
+  (semantic.index/delete-from-index! (semantic.env/get-pgvector-datasource!) mock-index model ids))
 
 (defn dog-training-native-query []
   (mt/native-query {:query "SELECT AVG(tricks) FROM dogs WHERE age > 7 GROUP BY breed"}))
@@ -235,74 +303,116 @@
 
 (defn open-temp-index! ^Closeable [& {:keys [index] :or {index mock-index}}]
   (closeable
-   (do (semantic.index/create-index-table-if-not-exists! db index {:force-reset? true})
+   (do (semantic.index/create-index-table-if-not-exists! (semantic.env/get-pgvector-datasource!) index {:force-reset? true})
        index)
    (fn cleanup-temp-index-table! [{:keys [table-name] :as index}]
      (try
-       (semantic.index/drop-index-table! db index)
+       (semantic.index/drop-index-table! (semantic.env/get-pgvector-datasource!) index)
        (catch Exception e
          (log/error e "Warning: failed to clean up test table" table-name))))))
 
-(declare cleanup-index-metadata!)
+(defn do-with-indexable-documents!
+  "Wrap the thunk into with-temp, creating entities used throughout semantic search test.
 
-(defn open-temp-index-and-metadata! ^Closeable []
-  (closeable
-   (do (cleanup-index-metadata! db mock-index-metadata)
-       (semantic.pgvector-api/init-semantic-search! db mock-index-metadata mock-embedding-model)
-       mock-index-metadata)
-   (fn cleanup-temp-index-and-metadata! [index-metadata]
-     (try
-       (cleanup-index-metadata! db index-metadata)
-       (catch Exception e
-         (log/error e "Warning: failed to clean up index and metadata" index-metadata))))))
+  N.B. *disable-updates* is bound to avoid processing of those entities by logic in [[search.ingestion]] ns.
+  The processing is triggered by means :hook/search-index which :model/Card (and others) derive.
+
+  As of 2025-09-10, processing triggered by insertion, combined with manual gating of documents that callers
+  of this fn do, would result in duplicate processing and deletion of those entitities from index
+  due to [[search.ingestion/bulk-ingest!]].
+
+  For details see the https://metaboat.slack.com/archives/C07SJT1P0ET/p1757452434713309?thread_ts=1757410361.879029&cid=C07SJT1P0ET"
+  [thunk]
+  ;; NB: *disable-updates*
+  (binding [search.ingestion/*disable-updates* true]
+    (mt/dataset
+      test-data
+      (mt/with-temp [:model/Collection
+                     {col1 :id}
+                     {:name "Wildlife Collection" :archived false}
+
+                     :model/Collection
+                     {col2 :id}
+                     {:name "Archived Animals" :archived true}
+
+                     :model/Collection
+                     {col3 :id}
+                     {:name "Cryptozoology", :archived false}
+
+                     :model/Card
+                     {card1 :id}
+                     {:name "Dog Training Guide" :collection_id col1 :creator_id (mt/user->id :crowberto)
+                      :archived false :query_type "native" :dataset_query (dog-training-native-query)}
+
+                     :model/Card
+                     _
+                     {:name "Bird Watching Tips" :collection_id col1 :creator_id (mt/user->id :rasta) :archived false}
+
+                     :model/Card
+                     _
+                     {:name "Cat Behavior Study" :collection_id col2 :creator_id (mt/user->id :crowberto) :archived true}
+
+                     :model/Card
+                     _
+                     {:name "Horse Racing Analysis" :collection_id col1 :creator_id (mt/user->id :rasta) :archived false}
+
+                     :model/Card
+                     _
+                     {:name "Fish Tank Setup" :collection_id col2 :creator_id (mt/user->id :crowberto) :archived true}
+
+                     :model/Card
+                     _
+                     {:name "Bigfoot Sightings" :collection_id col3 :creator_id (mt/user->id :crowberto) :archived false}
+
+                     :model/ModerationReview
+                     _
+                     {:moderated_item_type "card"
+                      :moderated_item_id card1
+                      :moderator_id (mt/user->id :crowberto)
+                      :status "verified"
+                      :most_recent true}
+
+                     :model/Dashboard
+                     _
+                     {:name "Elephant Migration" :collection_id col1 :creator_id (mt/user->id :rasta) :archived false}
+
+                     :model/Dashboard
+                     _
+                     {:name "Lion Pride Dynamics" :collection_id col1 :creator_id (mt/user->id :crowberto) :archived false}
+
+                     :model/Dashboard
+                     _
+                     {:name "Penguin Colony Study" :collection_id col2 :creator_id (mt/user->id :rasta) :archived true}
+
+                     :model/Dashboard
+                     _
+                     {:name "Whale Communication" :collection_id col1 :creator_id (mt/user->id :crowberto) :archived false}
+
+                     :model/Dashboard
+                     _
+                     {:name "Tiger Conservation" :collection_id col2 :creator_id (mt/user->id :rasta) :archived true}
+
+                     :model/Dashboard
+                     _
+                     {:name "Loch Ness Stuff" :collection_id col3 :creator_id (mt/user->id :crowberto), :archived false}
+
+                     :model/Database
+                     {db-id :id}
+                     {:name "Animal Database"}
+
+                     :model/Table
+                     _
+                     {:name "Species Table", :db_id db-id}
+
+                     :model/Table
+                     _
+                     {:name "Monsters Table", :db_id db-id, :active true}]
+        (thunk)))))
 
 (defmacro with-indexable-documents!
-  "Add a collection of test documents to that can be indexed to the appdb."
+  "Wrapper for [[do-with-indexable-documents!]]."
   [& body]
-  `(mt/dataset ~(symbol "test-data")
-     (mt/with-temp [:model/Collection       {col1# :id}  {:name "Wildlife Collection" :archived false}
-
-                    :model/Collection       {col2# :id}  {:name "Archived Animals" :archived true}
-
-                    :model/Collection       {col3# :id}  {:name "Cryptozoology", :archived false}
-
-                    :model/Card             {card1# :id} {:name "Dog Training Guide" :collection_id col1# :creator_id (mt/user->id :crowberto) :archived false
-                                                          :query_type "native" :dataset_query (dog-training-native-query)}
-
-                    :model/Card             {}           {:name "Bird Watching Tips" :collection_id col1# :creator_id (mt/user->id :rasta) :archived false}
-
-                    :model/Card             {}           {:name "Cat Behavior Study" :collection_id col2# :creator_id (mt/user->id :crowberto) :archived true}
-
-                    :model/Card             {}           {:name "Horse Racing Analysis" :collection_id col1# :creator_id (mt/user->id :rasta) :archived false}
-
-                    :model/Card             {}           {:name "Fish Tank Setup" :collection_id col2# :creator_id (mt/user->id :crowberto) :archived true}
-
-                    :model/Card             {}           {:name "Bigfoot Sightings" :collection_id col3# :creator_id (mt/user->id :crowberto), :archived false}
-
-                    :model/ModerationReview {}           {:moderated_item_type "card"
-                                                          :moderated_item_id card1#
-                                                          :moderator_id (mt/user->id :crowberto)
-                                                          :status "verified"
-                                                          :most_recent true}
-
-                    :model/Dashboard        {}           {:name "Elephant Migration" :collection_id col1# :creator_id (mt/user->id :rasta) :archived false}
-
-                    :model/Dashboard        {}           {:name "Lion Pride Dynamics" :collection_id col1# :creator_id (mt/user->id :crowberto) :archived false}
-
-                    :model/Dashboard        {}           {:name "Penguin Colony Study" :collection_id col2# :creator_id (mt/user->id :rasta) :archived true}
-
-                    :model/Dashboard        {}           {:name "Whale Communication" :collection_id col1# :creator_id (mt/user->id :crowberto) :archived false}
-
-                    :model/Dashboard        {}           {:name "Tiger Conservation" :collection_id col2# :creator_id (mt/user->id :rasta) :archived true}
-
-                    :model/Dashboard        {}           {:name "Loch Ness Stuff" :collection_id col3# :creator_id (mt/user->id :crowberto), :archived false}
-
-                    :model/Database         {db-id# :id} {:name "Animal Database"}
-
-                    :model/Table            {}           {:name "Species Table", :db_id db-id#}
-
-                    :model/Table            {}           {:name "Monsters Table", :db_id db-id#, :active true}]
-       ~@body)))
+  `(do-with-indexable-documents! (fn [] ~@body)))
 
 (defn index-all!
   "Run indexer synchonously until we've exhausted polling all documents"
@@ -310,44 +420,25 @@
   (let [metadata-row   {:indexer_last_poll Instant/EPOCH
                         :indexer_last_seen Instant/EPOCH}
         indexing-state (semantic.indexer/init-indexing-state metadata-row)
-        step (fn [] (semantic.indexer/indexing-step db mock-index-metadata mock-index indexing-state))]
+        pgvector (semantic.env/get-pgvector-datasource!)
+        step (fn [] (semantic.indexer/indexing-step pgvector mock-index-metadata mock-index indexing-state))]
     (while (do (step) (pos? (:last-novel-count @indexing-state))))))
 
-(defmacro blocking-index!
-  "Execute body ensuring [[index-all!]] is invoked at the end"
-  [& body]
-  `(let [ret# (do ~@body)]
-     (index-all!)
-     ret#))
-
-(defmacro with-index!
-  "Ensure a clean, small index for testing populated with a few collections, cards, and dashboards."
-  [& body]
-  `(with-indexable-documents!
-     (with-redefs [semantic.embedding/get-configured-model        (fn [] mock-embedding-model)
-                   semantic.index-metadata/default-index-metadata mock-index-metadata
-                   semantic.index/model-table-suffix              mock-table-suffix]
-       (with-open [_# (open-temp-index-and-metadata!)]
-         (binding [search.ingestion/*force-sync* true]
-           (blocking-index!
-            (semantic.pgvector-api/gate-updates! (semantic.env/get-pgvector-datasource!)
-                                                 mock-index-metadata
-                                                 (search.ingestion/searchable-documents)))
-           ~@body)))))
-
+#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
 (defn table-exists-in-db?
   "Check if a table actually exists in the database"
   [table-name]
   (when table-name
     (try
-      (semantic.util/table-exists? db (name table-name))
+      (semantic.util/table-exists? (semantic.env/get-pgvector-datasource!) (name table-name))
       (catch Exception _ false))))
 
+#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
 (defn table-has-index?
   [table-name index-name]
   (when table-name
     (try
-      (let [result (jdbc/execute! db
+      (let [result (jdbc/execute! (semantic.env/get-pgvector-datasource!)
                                   ["SELECT EXISTS (SELECT 1 FROM pg_indexes WHERE tablename = ? AND indexname = ?)"
                                    (name table-name)
                                    (name index-name)])]
@@ -422,7 +513,7 @@
 (defn index-count
   "Count the number of documents in the index."
   [index]
-  (let [result (jdbc/execute-one! db
+  (let [result (jdbc/execute-one! (semantic.env/get-pgvector-datasource!)
                                   (-> (sql.helpers/select [:%count.* :count])
                                       (sql.helpers/from (keyword (:table-name index)))
                                       semantic.index/sql-format-quoted)
@@ -434,17 +525,18 @@
   "Query the full index table and return all documents with decoded embeddings.
   Not used in tests, but useful for debugging."
   []
-  (->> (jdbc/execute! db
+  (->> (jdbc/execute! (semantic.env/get-pgvector-datasource!)
                       (-> (sql.helpers/select :model :model_id :content :creator_id :embedding)
                           (sql.helpers/from (keyword (:table-name mock-index)))
                           semantic.index/sql-format-quoted)
                       {:builder-fn jdbc.rs/as-unqualified-lower-maps})
        (mapv decode-embedding)))
 
+#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
 (defn query-embeddings
   "Query the `mock-index` table and return the decoded `:embedding`s for the given `model`"
   [{:keys [model model_id]}]
-  (->> (jdbc/execute! db
+  (->> (jdbc/execute! (semantic.env/get-pgvector-datasource!)
                       (-> (sql.helpers/select :model :model_id :content :creator_id :embedding)
                           (sql.helpers/from (keyword (:table-name mock-index)))
                           (sql.helpers/where :and
@@ -454,10 +546,11 @@
                       {:builder-fn jdbc.rs/as-unqualified-lower-maps})
        (mapv decode-embedding)))
 
+#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
 (defn query-tsvectors
   "Query the `mock-index` table and return the unwrapped tsvector columns for the given `model`"
   [{:keys [model model_id]}]
-  (->> (jdbc/execute! db
+  (->> (jdbc/execute! (semantic.env/get-pgvector-datasource!)
                       (-> (sql.helpers/select :model :model_id :content :creator_id
                                               :text_search_vector :text_search_with_native_query_vector)
                           (sql.helpers/from (keyword (:table-name mock-index)))
@@ -480,10 +573,11 @@
            (query-embeddings {:model "dashboard"
                               :model_id "456"})))))
 
+#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
 (defn check-index-has-no-mock-docs []
   (let [{:keys [table-name]}     mock-index
         table-exists-sql         "select exists(select * from information_schema.tables where table_name = ?) table_exists"
-        [{:keys [table_exists]}] (jdbc/execute! db [table-exists-sql table-name])]
+        [{:keys [table_exists]}] (jdbc/execute! (semantic.env/get-pgvector-datasource!) [table-exists-sql table-name])]
     (when table_exists
       (check-index-has-no-mock-card)
       (check-index-has-no-mock-dashboard))))

enterprise/backend/test/metabase_enterprise/semantic_search/without_init_test.clj

@@ -17,14 +17,15 @@
 (defn- without-init! [test-func]
   (mt/with-premium-features #{:semantic-search}
     (try
-      (semantic.tu/cleanup-index-metadata! semantic.tu/db semantic.tu/mock-index-metadata)
+      (semantic.tu/cleanup-index-metadata! (semantic.env/get-pgvector-datasource!)
+                                           semantic.tu/mock-index-metadata)
       (with-redefs [semantic.db.datasource/data-source          (atom nil)
                     semantic.env/get-index-metadata             (constantly semantic.tu/mock-index-metadata)
                     semantic.env/get-configured-embedding-model (constantly semantic.tu/mock-embedding-model)
                     semantic.index/model-table-suffix           semantic.tu/mock-table-suffix]
         (test-func))
       (finally
-        (semantic.tu/cleanup-index-metadata! semantic.tu/db semantic.tu/mock-index-metadata)))))
+        (semantic.tu/cleanup-index-metadata! (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index-metadata)))))
 
 (deftest update-index!-without-init!-test
   (without-init!

enterprise/backend/test/metabase_enterprise/transforms/api_test.clj

@@ -8,8 +8,9 @@
    [metabase-enterprise.transforms.models.transform-transform-tag]
    [metabase-enterprise.transforms.query-test-util :as query-test-util]
    [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
-   [metabase-enterprise.transforms.test-util :refer [parse-instant with-transform-cleanup! utc-timestamp]]
+   [metabase-enterprise.transforms.test-util :as transforms.tu :refer [parse-instant with-transform-cleanup! utc-timestamp]]
    [metabase-enterprise.transforms.util :as transforms.util]
+   [metabase.driver :as driver]
    [metabase.legacy-mbql.normalize :as mbql.normalize]
    [metabase.lib.convert :as lib.convert]
    [metabase.lib.core :as lib]
@@ -72,6 +73,26 @@
                                             :schema schema
                                             :name   table-name}})))))))
 
+(deftest create-python-transform-test
+  (mt/with-premium-features #{:transforms}
+    (mt/dataset transforms-dataset/transforms-test
+      (with-transform-cleanup! [table-name "gadget_products"]
+        (let [schema    (get-test-schema)
+              transform-payload {:name   "My beautiful python runner"
+                                 :source {:type            "python"
+                                          :body            "print('hello world')"
+                                          :source-tables   {}}
+                                 :target {:type     "table"
+                                          :schema   schema
+                                          :name     table-name
+                                          :database (mt/id)}}
+              transform (mt/user-http-request :crowberto :post 200 "ee/transform"
+                                              transform-payload)]
+          (is (= "print('hello chris')"
+                 (-> (mt/user-http-request :crowberto :put 200 (format "ee/transform/%s" (:id transform))
+                                           (assoc-in transform-payload [:source :body] "print('hello chris')"))
+                     :source :body))))))))
+
 (deftest list-transforms-test
   (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
     (testing "Can list without query parameters"
@@ -319,6 +340,32 @@
                 (is (true? (transforms.util/target-table-exists? updated)))
                 (check-query-results table2-name [2 3 4 13] "Doohickey")))))))))
 
+(deftest execute-python-transform-test
+  (testing "transform execution with :transforms/table target"
+    (mt/test-drivers #{:h2 :postgres}
+      (mt/with-premium-features #{:transforms}
+        (mt/dataset transforms-dataset/transforms-test
+          (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
+            (with-transform-cleanup! [{table-name :name :as target} {:type   "table"
+                                                                     :schema schema
+                                                                     :name   "target_table"}]
+              (let [original           {:name   "Gadget Products"
+                                        :source {:type  "python"
+                                                 :source-database (mt/id)
+                                                 :source-tables {"transforms_customers" (mt/id :transforms_customers)}
+                                                 :body  (str "import pandas as pd\n"
+                                                             "\n"
+                                                             "def transform():\n"
+                                                             "    return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}
+                                        :target  (assoc target :database (mt/id))}
+                    {transform-id :id} (mt/user-http-request :crowberto :post 200 "ee/transform"
+                                                             original)]
+                (test-run transform-id)
+                (wait-for-table table-name 5000)
+                (is (true? (driver/table-exists? driver/*driver* (mt/db) target)))
+                (is (= [["Alice" 25] ["Bob" 30]]
+                       (transforms.tu/table-rows table-name)))))))))))
+
 (deftest get-runs-filter-by-single-transform-id-test
   (testing "GET /api/ee/transform/run - filter by single transform ID"
     (mt/with-premium-features #{:transforms}

enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj

@@ -5,7 +5,8 @@
    [metabase-enterprise.transforms.execute :as transforms.execute]
    [metabase-enterprise.transforms.query-test-util :as query-test-util]
    [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
-   [metabase-enterprise.transforms.test-util :refer [with-transform-cleanup!]]
+   [metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]
+   [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.driver :as driver]
    [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
    [metabase.lib.core :as lib]
@@ -13,6 +14,7 @@
    [metabase.query-processor :as qp]
    [metabase.sync.core :as sync]
    [metabase.test :as mt]
+   [metabase.test.data.sql :as sql.tx]
    [toucan2.core :as t2]
    [toucan2.util :as u]))
 
@@ -130,6 +132,24 @@
                           ["2024-01-23T00:00:00Z" 14.99]]
                          query-result)))))))))))
 
+(deftest create-table-from-schema!-test
+  (mt/test-drivers (mt/normal-drivers-with-feature :uploads)
+    (let [driver       driver/*driver*
+          db-id        (mt/id)
+          table-name   (mt/random-name)
+          schema-name  (sql.tx/session-schema driver)
+          table-schema {:name    (if schema-name
+                                   (keyword schema-name table-name)
+                                   (keyword table-name))
+                        :columns [{:name "id" :type :type/Integer :nullable? false}
+                                  {:name "name" :type :type/Text :nullable? true}]}]
+      (mt/as-admin
+        (testing "create-table-from-schema! should create the table successfully"
+          (transforms.util/create-table-from-schema! driver db-id table-schema)
+          (let [table-exists? (driver/table-exists? driver db-id {:schema schema-name :name table-name})]
+            (is (some? table-exists?) "Table should exist in the database schema")
+            (driver/drop-table! driver db-id (:name table-schema))))))))
+
 (deftest transform-schema-created-if-needed-test
   (mt/test-drivers (mt/normal-driver-select {:+features [:transforms/table :schemas]})
     (mt/dataset transforms-dataset/transforms-test
@@ -224,3 +244,98 @@
             (driver/execute-raw-queries! driver/*driver* spec
                                          [[(format "DROP OWNED BY %s;" no-schema-user)]
                                           [(format "DROP ROLE IF EXISTS %s;" no-schema-user)]])))))))
+
+(deftest atomic-python-transform-swap-test
+  (testing "Python transform execution with atomic table swap"
+    (mt/test-drivers #{:h2 :postgres}
+      (mt/with-premium-features #{:transforms}
+        (mt/dataset transforms-dataset/transforms-test
+          (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
+            (with-transform-cleanup! [{table-name :name :as target} {:type   "table"
+                                                                     :schema schema
+                                                                     :name   "atomic_python_swap"}]
+              (let [initial-transform {:name   "Python Transform Initial"
+                                       :source {:type  "python"
+                                                :source-tables {}
+                                                :body  (str "import pandas as pd\n"
+                                                            "\n"
+                                                            "def transform():\n"
+                                                            "    return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}
+                                       :target (assoc target :database (mt/id))}]
+                (mt/with-temp [:model/Transform transform initial-transform]
+                  (transforms.execute/execute-python-transform! transform {:run-method :manual})
+                  (wait-for-table table-name 10000)
+
+                  (let [initial-rows (transforms.tu/table-rows table-name)]
+                    (is (= [["Alice" 25] ["Bob" 30]] initial-rows) "Initial data should be Alice and Bob")
+
+                    (t2/update! :model/Transform (:id transform)
+                                {:source {:type "python"
+                                          :source-tables {}
+                                          :body (str "import pandas as pd\n"
+                                                     "\n"
+                                                     "def transform():\n"
+                                                     "    return pd.DataFrame({'name': ['Charlie', 'Diana', 'Eve'], 'age': [35, 40, 45]})")}}))
+
+                  ;; Test atomic swap by intercepting drop operation
+                  (let [drop-latch (java.util.concurrent.CountDownLatch. 1)
+                        original-drop-table! transforms.util/drop-table!]
+                    (with-redefs [transforms.util/drop-table! (fn [driver db-id table-name]
+                                                                ;; Wait for latch before proceeding with drop
+                                                                (.await drop-latch)
+                                                                (original-drop-table! driver db-id table-name))]
+                      ;; Start transform in background
+                      (let [transform-future (future
+                                               (transforms.execute/execute-python-transform!
+                                                (t2/select-one :model/Transform (:id transform))
+                                                {:run-method :manual}))]
+                        ;; Give transform time to create temp table and reach the drop operation
+                        (Thread/sleep 1000)
+                        ;; Verify old table is still accessible during transform
+                        (is (= [["Alice" 25] ["Bob" 30]]
+                               (transforms.tu/table-rows table-name))
+                            "Original data should still be accessible during transform")
+                        ;; Release the latch to allow drop and complete transform
+                        (.countDown drop-latch)
+                        ;; Wait for transform to complete
+                        @transform-future
+                        ;; Verify final state
+                        (is (= [["Charlie" 35] ["Diana" 40] ["Eve" 45]]
+                               (transforms.tu/table-rows table-name))
+                            "Table should now contain Charlie, Diana, and Eve")))))))))))))
+
+(deftest python-transform-temp-table-cleanup-test
+  (testing "Python transform cleans up temp tables on success"
+    (mt/test-drivers #{:h2 :postgres}
+      (mt/with-premium-features #{:transforms}
+        (mt/dataset transforms-dataset/transforms-test
+          (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
+            (with-transform-cleanup! [{table-name :name :as target} {:type   "table"
+                                                                     :schema schema
+                                                                     :name   "python_cleanup_test"}]
+              (let [transform-def {:name   "Python Transform Cleanup"
+                                   :source {:type  "python"
+                                            :source-tables {}
+                                            :body  (str "import pandas as pd\n"
+                                                        "\n"
+                                                        "def transform():\n"
+                                                        "    return pd.DataFrame({'col1': [1, 2, 3], 'col2': ['a', 'b', 'c']})")}
+                                   :target (assoc target :database (mt/id))}]
+                (mt/with-temp [:model/Transform transform transform-def]
+                  ;; Run initial transform
+                  (transforms.execute/execute-python-transform! transform {:run-method :manual})
+                  (wait-for-table table-name 10000)
+
+                  ;; Run again to trigger temp table creation and swap
+                  (transforms.execute/execute-python-transform! transform {:run-method :manual})
+
+                  ;; Check that no temp tables remain
+                  (let [db-id (mt/id)
+                        tables (t2/select :model/Table :db_id db-id :active true)
+                        temp-table-pattern (re-pattern (str ".*" table-name "_temp_.*"))]
+                    (is (not-any? #(re-matches temp-table-pattern (:name %)) tables)
+                        "No temp tables should remain after successful Python transform")
+
+                    ;; Verify the final data is correct
+                    (is (= [[1 "a"] [2 "b"] [3 "c"]] (transforms.tu/table-rows table-name))
+                        "Table should contain the expected data after swap")))))))))))

enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj

@@ -1,25 +1,46 @@
 (ns ^:mb/driver-tests metabase-enterprise.transforms.ordering-test
   (:require
    [clojure.test :refer :all]
+   [metabase-enterprise.transforms.execute :as transforms.execute]
    [metabase-enterprise.transforms.ordering :as ordering]
+   [metabase.driver :as driver]
+   [metabase.driver.sql :as driver.sql]
    [metabase.query-processor.compile :as qp.compile]
    [metabase.test :as mt]
+   [metabase.test.data.sql :as sql.tx]
    [toucan2.core :as t2]))
 
-(defn- make-transform [{:keys [query name]}]
-  {:source {:type "query",
-            :query query}
-   :name "transform1"
-   :target {:schema "public"
-            :name (or name "orders_2")
-            :type "table"}})
+(defn- make-transform [query & [name schema]]
+  (let [name (or name (mt/random-name))
+        default-schema (some-> driver/*driver* driver.sql/default-schema)
+        schema (or schema default-schema "public")]
+    {:source {:type :query
+              :query query}
+     :name (str "transform_" name)
+     :target {:schema schema
+              :name name
+              :type :table}}))
+
+(defn- make-python-transform [source-tables & [name schema target-db]]
+  (let [name (or name (mt/random-name))
+        default-schema (some-> driver/*driver* driver.sql/default-schema)
+        schema (or schema default-schema "public")
+        target-db (or target-db (mt/id))]
+    {:source {:type "python"
+              :source-tables source-tables
+              :body "# Python code here\nresult = df"}
+     :name (str "python_transform_" name)
+     :target {:database target-db
+              :schema schema
+              :name name
+              :type "table"}}))
 
 (deftest basic-ordering-test
   (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
     (mt/with-temp [:model/Transform {t1 :id} (make-transform
-                                              {:query {:database (mt/id),
-                                                       :type "query",
-                                                       :query {:source-table (mt/id :orders)}}})]
+                                              {:database (mt/id),
+                                               :type "query",
+                                               :query {:source-table (mt/id :orders)}})]
       (is (= {t1 #{}}
              (ordering/transform-ordering (t2/select :model/Transform :id t1)))))))
 
@@ -30,14 +51,15 @@
                    :model/Field _ {:table_id table
                                    :name "foo"}
                    :model/Transform {parent :id} (make-transform
-                                                  {:query {:database (mt/id),
-                                                           :type "query",
-                                                           :query {:source-table (mt/id :orders)}}})
+                                                  {:database (mt/id),
+                                                   :type "query",
+                                                   :query {:source-table (mt/id :orders)}}
+                                                  "orders_2")
                    :model/Transform {child :id} (make-transform
-                                                 {:query {:database (mt/id)
-                                                          :type "query"
-                                                          :query {:source-table table}}
-                                                  :name "orders_3"})]
+                                                 {:database (mt/id)
+                                                  :type "query"
+                                                  :query {:source-table table}}
+                                                 "orders_3")]
       (is (= {parent #{}
               child #{parent}}
              (ordering/transform-ordering (t2/select :model/Transform :id [:in [parent child]])))))))
@@ -46,36 +68,134 @@
   (mt/with-metadata-provider (mt/id)
     (#'ordering/transform-deps transform)))
 
+(deftest not-run-transform-dependency-ordering-test
+  (mt/test-driver :postgres
+    (testing "dependencies are correctly identified when no transforms have been run yet"
+      (mt/with-temp [:model/Transform {t1 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type :native,
+                                                 :native {:query "SELECT * FROM orders LIMIT 100"}}
+                                                "orders_transform")
+                     :model/Transform {t2 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type :native,
+                                                 :native {:query "SELECT * FROM products LIMIT 100"}}
+                                                "products_transform")
+                     :model/Transform {t3 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type :native,
+                                                 :native {:query "SELECT * FROM orders_transform ot
+                                                                  LEFT JOIN products p
+                                                                  ON ot.product_id = p.id
+                                                                  LIMIT 100"}}
+                                                "orders_transform_products")
+                     :model/Transform {t4 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type :native,
+                                                 :native {:query "SELECT * FROM orders_transform ot
+                                                                  LEFT JOIN products_transform pt
+                                                                  ON ot.product_id = pt.id
+                                                                  LIMIT 100"}}
+                                                "orders_transform_products_transform")]
+        (is (= #{{:transform t1} {:transform t2}}
+               (transform-deps-for-db (t2/select-one :model/Transform  t4))))
+        (is (= {t1 #{}
+                t2 #{}
+                t3 #{t1}
+                t4 #{t1 t2}}
+               (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3 t4]]))))))
+    (testing "dependencies are correctly identified when some transform have been run and some haven't"
+      (mt/with-temp [:model/Transform {t1 :id :as transform1} (make-transform
+                                                               {:database (mt/id),
+                                                                :type :native,
+                                                                :native {:query "SELECT * FROM checkins LIMIT 100"}}
+                                                               "checkins_transform")
+                     :model/Transform {t2 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type :native,
+                                                 :native {:query "SELECT * FROM venues LIMIT 100"}}
+                                                "venues_transform")
+                     :model/Transform {t3 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type :native,
+                                                 :native {:query "SELECT * FROM venues_transform vt
+                                                                  LEFT JOIN checkins_transform ct
+                                                                  ON vt.id = ct.venue_id
+                                                                  LIMIT 100"}}
+                                                "venues_transform_2")]
+        (try
+          (transforms.execute/run-mbql-transform! transform1 {:run-method :manual})
+          (let [table1 (t2/select-one-pk :model/Table :name "checkins_transform")]
+            (is (= #{{:table table1} {:transform t2}}
+                   (transform-deps-for-db (t2/select-one :model/Transform  t3)))))
+          (is (= {t1 #{}
+                  t2 #{}
+                  t3 #{t1 t2}}
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]]))))
+          (finally
+            (t2/delete! :model/Table :name "checkins_transform")))))))
+
+(deftest bigquery-native-transform-dependency-ordering-test
+  (mt/test-driver :bigquery-cloud-sdk
+    (let [[dataset] (sql.tx/qualified-name-components :bigquery-cloud-sdk nil)]
+      (testing "dependencies are correctly identified when some transform have been run and some haven't"
+        (mt/with-temp [:model/Transform {t1 :id} (make-transform
+                                                  {:database (mt/id),
+                                                   :type :native,
+                                                   :native {:query (format "SELECT * FROM `%s.checkins` LIMIT 100" dataset)}}
+                                                  "checkins_transform" dataset)
+                       :model/Table {table1 :id} {:schema dataset
+                                                  :name "checkins_transform"}
+                       :model/Transform {t2 :id} (make-transform
+                                                  {:database (mt/id),
+                                                   :type :native,
+                                                   :native {:query (format "SELECT * FROM `%s.venues` LIMIT 100" dataset)}}
+                                                  "venues_transform" dataset)
+                       :model/Transform {t3 :id} (make-transform
+                                                  {:database (mt/id),
+                                                   :type :native,
+                                                   :native {:query (format "SELECT * FROM `%s.venues_transform` vt
+                                                                            LEFT JOIN `%s.checkins_transform` ct
+                                                                            ON vt.id = ct.venue_id
+                                                                            LIMIT 100" dataset dataset)}}
+                                                  "venues_transform_2" dataset)]
+          (is (= #{{:table table1} {:transform t2}}
+                 (transform-deps-for-db (t2/select-one :model/Transform  t3))))
+          (is (= {t1 #{}
+                  t2 #{}
+                  t3 #{t1 t2}}
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]])))))))))
+
 (deftest basic-dependencies-test
   (mt/with-temp [:model/Transform {t1 :id} (make-transform
-                                            {:query {:database (mt/id),
-                                                     :type "query",
-                                                     :query {:source-table (mt/id :orders)}}})]
-    (is (= #{(mt/id :orders)}
+                                            {:database (mt/id),
+                                             :type "query",
+                                             :query {:source-table (mt/id :orders)}})]
+    (is (= #{{:table (mt/id :orders)}}
            (transform-deps-for-db (t2/select-one :model/Transform :id t1))))))
 
 (deftest joined-dependencies-test
   (mt/with-temp [:model/Transform {t1 :id} (make-transform
-                                            {:query {:database (mt/id),
-                                                     :type "query",
-                                                     :query {:source-table (mt/id :orders),
-                                                             :joins
-                                                             [{:fields "all",
-                                                               :strategy "left-join",
-                                                               :alias "Products",
-                                                               :condition
-                                                               [:=
-                                                                [:field
-                                                                 (mt/id :orders :product_id)
-                                                                 {:base-type "type/Integer"}]
-                                                                [:field
-                                                                 (mt/id :products :id)
-                                                                 {:base-type "type/Integer",
-                                                                  :join-alias "Products"}]],
-                                                               :source-table (mt/id :products)}]},
-                                                     :parameters []}})]
-    (is (= #{(mt/id :orders)
-             (mt/id :products)}
+                                            {:database (mt/id),
+                                             :type "query",
+                                             :query {:source-table (mt/id :orders),
+                                                     :joins
+                                                     [{:fields "all",
+                                                       :strategy "left-join",
+                                                       :alias "Products",
+                                                       :condition
+                                                       [:=
+                                                        [:field
+                                                         (mt/id :orders :product_id)
+                                                         {:base-type "type/Integer"}]
+                                                        [:field
+                                                         (mt/id :products :id)
+                                                         {:base-type "type/Integer",
+                                                          :join-alias "Products"}]],
+                                                       :source-table (mt/id :products)}]},
+                                             :parameters []})]
+    (is (= #{{:table (mt/id :orders)}
+             {:table (mt/id :products)}}
            (transform-deps-for-db (t2/select-one :model/Transform :id t1))))))
 
 (deftest card-dependencies-test
@@ -99,27 +219,27 @@
                                                                      :source-table (mt/id :products)}]},
                                                            :parameters []}}
                    :model/Transform {t1 :id} (make-transform
-                                              {:query {:database (mt/id),
-                                                       :type "query",
-                                                       :query
-                                                       {:aggregation [["count"]],
-                                                        :breakout [["field" "Products__category" {:base-type "type/Text"}]],
-                                                        :source-table (str "card__" card)},
-                                                       :parameters []}})]
-      (is (= #{(mt/id :orders)
-               (mt/id :products)}
+                                              {:database (mt/id),
+                                               :type "query",
+                                               :query
+                                               {:aggregation [["count"]],
+                                                :breakout [["field" "Products__category" {:base-type "type/Text"}]],
+                                                :source-table (str "card__" card)},
+                                               :parameters []})]
+      (is (= #{{:table (mt/id :orders)}
+               {:table (mt/id :products)}}
              (transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))
 
 (deftest native-dependencies-test
   (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
     (mt/with-temp [:model/Transform {t1 :id} (make-transform
-                                              {:query {:database (mt/id),
-                                                       :type :native
-                                                       :native (qp.compile/compile
-                                                                {:database (mt/id),
-                                                                 :type "query",
-                                                                 :query {:source-table (mt/id :orders)}})}})]
-      (is (= #{(mt/id :orders)}
+                                              {:database (mt/id),
+                                               :type :native
+                                               :native (qp.compile/compile
+                                                        {:database (mt/id),
+                                                         :type "query",
+                                                         :query {:source-table (mt/id :orders)}})})]
+      (is (= #{{:table (mt/id :orders)}}
              (transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))
 
 (deftest native-card-dependencies-test
@@ -145,21 +265,20 @@
                                                                      :source-table (mt/id :products)}]},
                                                            :parameters []}}
                    :model/Transform {t1 :id} (make-transform
-                                              {:query {:database (mt/id),
-                                                       :type :native,
-                                                       :native {:query
-                                                                (mt/native-query-with-card-template-tag
-                                                                 (:engine (mt/db))
-                                                                 (str "#" card)),
-
-                                                                :template-tags
-                                                                {(keyword (str "#" card))
-                                                                 {:type "card"
-                                                                  :name "card"
-                                                                  :display-name "card"
-                                                                  :card-id card}}}}})]
-      (is (= #{(mt/id :orders)
-               (mt/id :products)}
+                                              {:database (mt/id),
+                                               :type :native,
+                                               :native {:query
+                                                        (mt/native-query-with-card-template-tag
+                                                         (:engine (mt/db))
+                                                         (str "#" card)),
+                                                        :template-tags
+                                                        {(keyword (str "#" card))
+                                                         {:type "card"
+                                                          :name "card"
+                                                          :display-name "card"
+                                                          :card-id card}}}})]
+      (is (= #{{:table (mt/id :orders)}
+               {:table (mt/id :products)}}
              (transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))
 
 (defn- rotations
@@ -181,3 +300,173 @@
         cycles (rotations [3 4 6 7])]
     (is (contains? cycles
                    (ordering/find-cycle test-graph)))))
+
+(deftest get-transform-cycle-test
+  (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+    (testing "cycle is detected in transform referencing itself"
+      (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                :name "table_1"}
+                     :model/Field _ {:table_id table1
+                                     :name "foo"}
+                     :model/Transform {t1 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type "query",
+                                                 :query {:source-table table1}}
+                                                "table_1")]
+        (is (= {:cycle-str "transform_table_1"
+                :cycle [t1]}
+               (ordering/get-transform-cycle (t2/select-one :model/Transform :id t1))))))
+    (testing "cycle is caught in 2 transforms referencing each other"
+      (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                :name "table_1"}
+                     :model/Field _ {:table_id table1
+                                     :name "foo"}
+                     :model/Table {table2 :id} {:schema "public"
+                                                :name "table_2"}
+                     :model/Field _ {:table_id table2
+                                     :name "foo"}
+                     :model/Transform {t1 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type "query",
+                                                 :query {:source-table table1}}
+                                                "table_2")
+                     :model/Transform {t2 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type "query",
+                                                 :query {:source-table table2}}
+                                                "table_1")]
+        (is (= {:cycle-str "transform_table_2 -> transform_table_1",
+                :cycle [t1 t2]}
+               (ordering/get-transform-cycle (t2/select-one :model/Transform :id t1))))))
+    (testing "cycle is detected in 3 transforms referencing each other"
+      (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                :name "table_1"}
+                     :model/Field _ {:table_id table1
+                                     :name "foo"}
+                     :model/Table {table2 :id} {:schema "public"
+                                                :name "table_2"}
+                     :model/Field _ {:table_id table2
+                                     :name "foo"}
+                     :model/Table {table3 :id} {:schema "public"
+                                                :name "table_3"}
+                     :model/Field _ {:table_id table3
+                                     :name "foo"}
+                     :model/Transform {t1 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type "query",
+                                                 :query {:source-table table1}}
+                                                "table_2")
+                     :model/Transform {t2 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type "query",
+                                                 :query {:source-table table2}}
+                                                "table_3")
+                     :model/Transform {t3 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type "query",
+                                                 :query {:source-table table3}}
+                                                "table_1")]
+        (is (= {:cycle-str "transform_table_2 -> transform_table_1 -> transform_table_3",
+                :cycle [t1 t3 t2]}
+               (ordering/get-transform-cycle (t2/select-one :model/Transform :id t1)))))))
+
+  (deftest python-transform-basic-dependencies-test
+    (testing "Python transforms with source-tables dependencies are extracted correctly"
+      (mt/with-temp [:model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)
+                                                                       "products" (mt/id :products)})]
+        (is (= #{{:table (mt/id :orders)}
+                 {:table (mt/id :products)}}
+               (transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))
+
+  (deftest python-transform-ordering-test
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+      (testing "Python transforms are ordered correctly based on source-tables dependencies"
+        (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                  :name "output_1"}
+                       :model/Field _ {:table_id table1
+                                       :name "foo"}
+                       :model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)} "output_1")
+                       :model/Transform {t2 :id} (make-python-transform {"output_1" table1} "output_2")]
+          (is (= {t1 #{}
+                  t2 #{t1}}
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2]]))))))))
+
+  (deftest python-transform-multiple-dependencies-test
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+      (testing "Python transforms with multiple table dependencies"
+        (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                  :name "output_1"}
+                       :model/Field _ {:table_id table1
+                                       :name "foo"}
+                       :model/Table {table2 :id} {:schema "public"
+                                                  :name "output_2"}
+                       :model/Field _ {:table_id table2
+                                       :name "bar"}
+                       :model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)} "output_1")
+                       :model/Transform {t2 :id} (make-python-transform {"products" (mt/id :products)} "output_2")
+                       :model/Transform {t3 :id} (make-python-transform {"output_1" table1
+                                                                         "output_2" table2} "final_output")]
+          (is (= {t1 #{}
+                  t2 #{}
+                  t3 #{t1 t2}}
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]]))))))))
+
+  (deftest mixed-transform-ordering-test
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+      (testing "Python and query transforms are ordered together correctly"
+        (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                  :name "sql_output"}
+                       :model/Field _ {:table_id table1
+                                       :name "foo"}
+                       :model/Table {table2 :id} {:schema "public"
+                                                  :name "python_output"}
+                       :model/Field _ {:table_id table2
+                                       :name "bar"}
+                       ;; SQL transform that depends on orders table
+                       :model/Transform {t1 :id} (make-transform
+                                                  {:database (mt/id),
+                                                   :type "query",
+                                                   :query {:source-table (mt/id :orders)}}
+                                                  "sql_output")
+                       ;; Python transform that depends on the SQL transform's output
+                       :model/Transform {t2 :id} (make-python-transform {"sql_output" table1} "python_output")
+                       ;; Another SQL transform that depends on the Python transform's output
+                       :model/Transform {t3 :id} (make-transform
+                                                  {:database (mt/id)
+                                                   :type "query"
+                                                   :query {:source-table table2}}
+                                                  "final_output")]
+          (is (= {t1 #{}
+                  t2 #{t1}
+                  t3 #{t2}}
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]])))))))))
+
+#_(deftest mixed-database-transform-ordering-test
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+      (testing "Python and query transforms are ordered together correctly, across multiple databases"
+      ;; orders (db-1) -> sql_output (db-1) -> python_output (db-2) -> final_output (db-2)
+        (mt/with-temp [:model/Database db-2    {}
+                       :model/Table    table-1 {:db_id (mt/id),    :schema "public", :name "sql_output"}
+                       :model/Table    table-2 {:db_id (:id db-2), :schema "public", :name "python_output"}
+                       :model/Field    _       {:table_id (:id table-1), :name "foo"}
+                       :model/Field    _       {:table_id (:id table-2), :name "bar"}
+
+                     ;; SQL transform that depends on orders table
+                       :model/Transform {t1 :id} (make-transform
+                                                  {:database (mt/id)
+                                                   :type     "query"
+                                                   :query    {:source-table (mt/id :orders)}}
+                                                  (:name table-1))
+                     ;; Python transform that depends on the SQL transform's output
+                       :model/Transform {t2 :id} (make-python-transform {"sql_output" (:id table-1)} (:name table-2))
+                     ;; Another SQL transform that depends on the Python transform's output
+                       :model/Transform {t3 :id} (make-transform
+                                                  {:database (:id db-2)
+                                                   :type     "query"
+                                                   :query    {:source-table (:id table-2)}}
+                                                  "final_output")]
+          (is (= {t1 #{}
+                  t2 #{t1}
+                  t3 #{t2}}
+
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]]))))))))

enterprise/backend/test/metabase_enterprise/transforms/python_runner_test.clj

@@ -0,0 +1,488 @@
+(ns ^:mb/driver-tests metabase-enterprise.transforms.python-runner-test
+  (:require
+   [clojure.core.async :as a]
+   [clojure.data.csv :as csv]
+   [clojure.java.jdbc :as jdbc]
+   [clojure.string :as str]
+   [clojure.test :refer :all]
+   [java-time.api :as t]
+   [metabase-enterprise.transforms.python-runner :as python-runner]
+   [metabase-enterprise.transforms.schedule :as transforms.schedule]
+   [metabase-enterprise.transforms.settings :as transforms.settings]
+   [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
+   [metabase-enterprise.transforms.test-util :as transforms.tu]
+   [metabase-enterprise.transforms.util :as transforms.util]
+   [metabase.driver :as driver]
+   [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
+   [metabase.sync.core :as sync]
+   [metabase.task.core :as task]
+   [metabase.test :as mt]
+   [metabase.test.data.interface :as tx]
+   [metabase.test.data.sql :as sql.tx]
+   [metabase.util :as u]
+   [metabase.util.json :as json]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(defn template->regex
+  "Convert a template string with $var$ placeholders to a regex pattern.
+   Example: template->regex 'File ___PATH___/script.py, line ___LINE___'
+   will match 'File /tmp/unique-path/script.py, line 123'"
+  [template]
+  (-> template
+      (str/replace #"(?m)^\d{13}\s*" "")
+      (str/replace "\\" "\\\\")
+      (str/replace "." "\\.")
+      (str/replace "*" "\\*")
+      (str/replace "+" "\\+")
+      (str/replace "?" "\\?")
+      (str/replace "^" "\\^")
+      (str/replace "$" "\\$")
+      (str/replace "{" "\\{")
+      (str/replace "}" "\\}")
+      (str/replace "(" "\\(")
+      (str/replace ")" "\\)")
+      (str/replace "|" "\\|")
+      (str/replace "[" "\\[")
+      (str/replace "]" "\\]")
+      (str/replace #"___PATH___" "(/[\\\\w-]+)+")
+      (str/replace #"___LINE___" "\\\\d+")
+      re-pattern))
+
+(def ^:private test-id 1)
+
+(defn- execute [{:keys [code tables]}]
+  (with-open [shared-storage-ref (python-runner/open-s3-shared-storage! (or tables {}))]
+    (let [server-url     (transforms.settings/python-execution-server-url)
+          cancel-chan    (a/promise-chan)
+          table-name->id (or tables {})
+          _              (python-runner/copy-tables-to-s3! {:run-id         test-id
+                                                            :shared-storage @shared-storage-ref
+                                                            :table-name->id table-name->id
+                                                            :cancel-chan    cancel-chan})
+          response       (python-runner/execute-python-code-http-call! {:server-url     server-url
+                                                                        :code           code
+                                                                        :run-id         test-id
+                                                                        :table-name->id table-name->id
+                                                                        :shared-storage @shared-storage-ref})
+          {:keys [output output-manifest events]} (python-runner/read-output-objects @shared-storage-ref)]
+      ;; not sure about munging this all together but its what tests expect for now
+      (merge (:body response)
+             {:output          output
+              :output-manifest output-manifest
+              :stdout          (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
+              :stderr          (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))}))))
+
+(deftest ^:parallel transform-function-basic-test
+  (testing "executes transform function and returns CSV output"
+    (let [transform-code (str "import pandas as pd\n"
+                              "\n"
+                              "def transform():\n"
+                              "    return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")
+          result         (execute {:code transform-code})]
+      (is (=? {:output "name,age\nAlice,25\nBob,30\n"
+               :stdout "Successfully saved 2 rows to S3\nSuccessfully saved output manifest with 2 fields"
+               :stderr ""}
+              result)))))
+
+(deftest ^:parallel transform-function-missing-test
+  (testing "handles missing transform function"
+    (let [result (execute {:code (str "import pandas as pd\n"
+                                      "\n"
+                                      "# No transform function defined")})]
+      (is (=? {:exit_code 1
+               :stderr    "ERROR: User script must define a 'transform()' function"
+               :stdout    ""}
+              result)))))
+
+(deftest ^:parallel transform-function-wrong-return-type-test
+  (testing "handles transform function returning non-DataFrame"
+    (let [result (execute {:code (str "def transform():\n"
+                                      "    return 'not a dataframe'")})]
+      (is (=? {:exit_code 1
+               :stderr    "ERROR: Transform function must return a pandas DataFrame, got <class 'str'>"
+               :stdout    ""}
+              result)))))
+
+(deftest ^:parallel transform-function-error-test
+  (testing "handles transform function with error"
+    (let [result            (execute {:code (str "def transform():\n"
+                                                 "    raise ValueError('Something went wrong')")})
+          expected-template (str "ERROR: Transform function failed: Something went wrong\n"
+                                 "Traceback (most recent call last):\n"
+                                 "  File \"/app/external/transform_runner.py\", line ___LINE___, in main\n"
+                                 "    result = script.transform()\n"
+                                 "             ^^^^^^^^^^^^^^^^^^\n"
+                                 "  File \"___PATH___/script.py\", line 2, in transform\n"
+                                 "    raise ValueError('Something went wrong')\n"
+                                 "ValueError: Something went wrong")
+          stderr-pattern    (template->regex expected-template)]
+      (is (=? {:exit_code 1
+               :stderr    #(re-matches stderr-pattern %)
+               :stdout    ""
+               :timeout   false}
+              result)))))
+
+(deftest ^:parallel transform-function-complex-dataframe-test
+  (testing "can create complex DataFrames with transform"
+    (let [transform-code (str "import pandas as pd\n"
+                              "\n"
+                              "def transform():\n"
+                              "    data = {'x': [1, 2, 3], 'y': [10, 20, 30], 'z': ['a', 'b', 'c']}\n"
+                              "    return pd.DataFrame(data)")
+          result         (execute {:code transform-code})]
+      (is (=? {:output "x,y,z\n1,10,a\n2,20,b\n3,30,c\n"
+               :stdout "Successfully saved 3 rows to S3\nSuccessfully saved output manifest with 3 fields"
+               :stderr ""}
+              result)))))
+
+(deftest ^:parallel transform-function-with-db-parameter-test
+  (mt/test-drivers #{:postgres}
+    (testing "transform function can accept db parameter for forward compatibility"
+      (let [transform-code (str "import pandas as pd\n"
+                                "\n"
+                                "def transform():\n"
+                                "    data = {'name': ['Charlie', 'Dana'], 'score': [85, 92]}\n"
+                                "    return pd.DataFrame(data)")
+            result         (execute {:code transform-code})]
+        (is (=? {:output "name,score\nCharlie,85\nDana,92\n"
+                 :stdout "Successfully saved 2 rows to S3\nSuccessfully saved output manifest with 2 fields"
+                 :stderr ""}
+                result))))))
+
+(deftest transform-function-with-pass-thru
+  (testing "transform function successfully connects to PostgreSQL database and reads data"
+    (mt/test-drivers #{:postgres}
+      (mt/with-empty-db
+        (let [db-spec        (sql-jdbc.conn/db->pooled-connection-spec (mt/db))
+              _              (jdbc/execute! db-spec ["DROP TABLE IF EXISTS students"])
+              _              (jdbc/execute! db-spec ["CREATE TABLE students (id INTEGER PRIMARY KEY, name VARCHAR(100), score INTEGER)"])
+              _              (jdbc/execute! db-spec ["INSERT INTO students (id, name, score) VALUES (1, 'Alice', 85), (2, 'Bob', 92), (3, 'Charlie', 88), (4, 'Dana', 90)"])
+
+              _              (sync/sync-database! (mt/db))
+
+              transform-code (str "import pandas as pd\n"
+                                  "\n"
+                                  "def transform(students):\n"
+                                  "    return students")
+              result         (execute {:code   transform-code
+                                       :tables {"students" (mt/id :students)}})]
+
+          (is (=? {:output   "id,name,score\n1,Alice,85\n2,Bob,92\n3,Charlie,88\n4,Dana,90\n"
+                   :output-manifest {:fields [{:base_type "Integer",
+                                               :database_type "int4",
+                                               :dtype "int64",
+                                               :effective_type "Integer",
+                                               :name "id",
+                                               :root_type "Integer",
+                                               :semantic_type "PK"}
+                                              {:base_type "Text",
+                                               :database_type "varchar",
+                                               :dtype "object",
+                                               :effective_type "Text",
+                                               :name "name",
+                                               :root_type "Text",
+                                               :semantic_type "Name"}
+                                              {:base_type "Integer",
+                                               :database_type "int4",
+                                               :dtype "int64",
+                                               :effective_type "Integer",
+                                               :name "score",
+                                               :root_type "Integer",
+                                               :semantic_type "Score"}],
+                                     :source_metadata {:fields [{:base_type "Integer",
+                                                                 :database_type "int4",
+                                                                 :dtype "int64",
+                                                                 :effective_type "Integer",
+                                                                 :name "id",
+                                                                 :root_type "Integer",
+                                                                 :semantic_type "PK"}
+                                                                {:base_type "Text",
+                                                                 :database_type "varchar",
+                                                                 :dtype "object",
+                                                                 :effective_type "Text",
+                                                                 :name "name",
+                                                                 :root_type "Text",
+                                                                 :semantic_type "Name"}
+                                                                {:base_type "Integer",
+                                                                 :database_type "int4",
+                                                                 :dtype "int64",
+                                                                 :effective_type "Integer",
+                                                                 :name "score",
+                                                                 :root_type "Integer",
+                                                                 :semantic_type "Score"}],
+                                                       :table_metadata {:table_id (mt/malli=? int?)},
+                                                       :version "0.1.0"},
+                                     :version "0.1.0"}
+                   :stdout   (str "Successfully saved 4 rows to S3\n"
+                                  "Successfully saved output manifest with 3 fields")
+                   :stderr   ""}
+                  result)))))))
+
+(deftest transform-function-with-working-database-test
+  (testing "transform function successfully connects to PostgreSQL database and reads data"
+    (mt/test-drivers #{:postgres}
+      (mt/with-empty-db
+        (let [db-spec        (sql-jdbc.conn/db->pooled-connection-spec (mt/db))
+              _              (jdbc/execute! db-spec ["DROP TABLE IF EXISTS students"])
+              _              (jdbc/execute! db-spec ["CREATE TABLE students (id INTEGER PRIMARY KEY, name VARCHAR(100), score INTEGER)"])
+              _              (jdbc/execute! db-spec ["INSERT INTO students (id, name, score) VALUES (1, 'Alice', 85), (2, 'Bob', 92), (3, 'Charlie', 88), (4, 'Dana', 90)"])
+
+              _              (sync/sync-database! (mt/db))
+
+              transform-code (str "import pandas as pd\n"
+                                  "\n"
+                                  "def transform(students):\n"
+                                  "    # Calculate average score\n"
+                                  "    avg_score = students['score'].mean()\n"
+                                  "    result = pd.DataFrame({\n"
+                                  "        'student_count': [len(students)],\n"
+                                  "        'average_score': [round(avg_score, 2)]\n"
+                                  "    })\n"
+                                  "    return result")
+              result         (execute {:code   transform-code
+                                       :tables {"students" (mt/id :students)}})]
+
+          (is (=? {:output   "student_count,average_score\n4,88.75\n"
+                   :output-manifest {:version "0.1.0",
+                                     :fields  [{:name "student_count", :dtype "int64"}
+                                               {:name "average_score", :dtype "float64"}]}
+                   :stdout   (str "Successfully saved 1 rows to S3\n"
+                                  "Successfully saved output manifest with 2 fields")
+                   :stderr   ""}
+                  result)))))))
+
+(defn- datetime-equal?
+  [expected-iso-str actual-pandas-str]
+  (let [formatter   (t/formatter "yyyy-MM-dd HH:mm:ssXXX")
+        expected-dt (t/zoned-date-time expected-iso-str)
+        actual-dt   (t/offset-date-time formatter actual-pandas-str)]
+    (= (t/to-millis-from-epoch expected-dt)
+       (t/to-millis-from-epoch actual-dt))))
+
+(deftest transform-type-test
+  (mt/test-drivers #{:postgres}
+    (mt/with-empty-db
+      (let [db-spec (sql-jdbc.conn/db->pooled-connection-spec (mt/db))
+            _ (jdbc/execute! db-spec ["DROP TABLE IF EXISTS sample_table"])
+            _ (jdbc/execute! db-spec ["CREATE TABLE sample_table (
+                                                        id BIGSERIAL PRIMARY KEY,
+                                                        name VARCHAR(255),
+                                                        description TEXT,
+                                                        count BIGINT,
+                                                        price FLOAT,
+                                                        is_active BOOLEAN,
+                                                        created_date DATE,
+                                                        updated_at TIMESTAMP,
+                                                        scheduled_for TIMESTAMP WITH TIME ZONE
+                                                    )"])
+            _ (jdbc/execute! db-spec ["INSERT INTO sample_table
+                                                        (name, description, count, price, is_active, created_date, updated_at, scheduled_for)
+                                                      VALUES
+                                                        ('Product A', 'A sample product description', 100, 29.99, true, '2024-01-15', '2024-01-15 10:30:00', '2024-01-16 14:00:00+00'),
+                                                        ('Product B', 'Another product with longer description text', 50, 15.50, false, '2024-02-01', '2024-02-01 09:15:30', '2024-02-02 16:30:00-05'),
+                                                        ('Product C', NULL, 0, 0.0, true, '2024-03-10', '2024-03-10 18:45:15', '2024-03-11 08:00:00+02')"])
+
+            _ (sync/sync-database! (mt/db))
+
+            transform-code (str "import pandas as pd\n"
+                                "\n"
+                                "def transform(sample_table):\n"
+                                "    df = sample_table.copy()\n"
+                                "    return df")
+            result (execute {:code transform-code
+                             :tables {"sample_table" (mt/id :sample_table)}})
+            csv-data (csv/read-csv (:output result))
+            headers (first csv-data)
+            rows (rest csv-data)
+            [row1 row2 row3] rows
+            header-to-index (zipmap headers (range))
+            get-col (fn [row col-name] (nth row (header-to-index col-name)))
+            metadata (:output-manifest result)]
+
+        (is (= (set ["id" "name" "description" "count" "price" "is_active" "created_date" "updated_at" "scheduled_for"])
+               (set headers)))
+
+        (is (= "1" (get-col row1 "id")))
+        (is (= "Product A" (get-col row1 "name")))
+        (is (datetime-equal? "2024-01-16T14:00:00Z" (get-col row1 "scheduled_for")))
+
+        (is (= "2" (get-col row2 "id")))
+        (is (= "Product B" (get-col row2 "name")))
+        (is (datetime-equal? "2024-02-02T21:30:00Z" (get-col row2 "scheduled_for")))
+
+        (is (= "3" (get-col row3 "id")))
+        (is (= "Product C" (get-col row3 "name")))
+        (is (datetime-equal? "2024-03-11T06:00:00Z" (get-col row3 "scheduled_for")))
+        (testing "dtypes are preserved correctly"
+          (is (= {"id"            :type/Integer
+                  "name"          :type/Text
+                  "description"   :type/Text
+                  "count"         :type/Integer
+                  "price"         :type/Float
+                  "is_active"     :type/Boolean
+                  ;; Our hack works
+                  "created_date"  :type/Date
+                  "updated_at"    :type/DateTime
+                  "scheduled_for" :type/DateTimeWithTZ}
+                 (u/for-map [{:keys [name dtype]} (:fields metadata)]
+                   [name (transforms.util/dtype->base-type dtype)]))))))))
+
+(deftest python-transform-scheduled-job-test
+  (mt/test-helpers-set-global-values!
+    (mt/with-temp-scheduler!
+      (task/init! ::transforms.schedule/RunTransform)
+      (mt/test-drivers #{:h2 :postgres}
+        (mt/with-premium-features #{:transforms}
+          (mt/dataset transforms-dataset/transforms-test
+            (transforms.tu/with-transform-cleanup! [target {:type   "table"
+                                                            :schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))
+                                                            :name   "target_table"}]
+              (mt/with-temp
+                [:model/TransformTag {tag-id :id}       {:name "every second"}
+                 :model/Transform    {transform-id :id} {:name   "Gadget Products"
+                                                         :source {:type  "python"
+                                                                  :source-database (mt/id)
+                                                                  :source-tables {"transforms_customers" (mt/id :transforms_customers)}
+                                                                  :body  (str "import pandas as pd\n"
+                                                                              "\n"
+                                                                              "def transform():\n"
+                                                                              "    return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}
+                                                         :target  (assoc target :database (mt/id))}
+                 :model/TransformTransformTag   _  {:transform_id transform-id :tag_id tag-id :position 0}
+                 :model/TransformJob {job-id :id :as job} {:schedule "* * * * * ? *"}
+                 :model/TransformJobTransformTag _ {:job_id job-id :tag_id tag-id :position 0}]
+                (transforms.schedule/initialize-job! job)
+                (transforms.schedule/update-job! job-id "* * * * * ? *")
+                (is (true? (u/poll {:thunk   (fn [] (driver/table-exists? driver/*driver* (mt/db) target))
+                                    :done?   true?
+                                    :timeout 30000})))
+                (is (true? (t2/exists? :model/Table :name (:name target))))))))))))
+
+(deftest transform-function-with-library-test
+  (testing "transform function can use libraries"
+    (mt/test-drivers #{:postgres}
+      (mt/with-temp [:model/PythonLibrary _ {:path   "circle"
+                                             :source "import math\n\ndef calculate_circle_area(radius):\n    return math.pi * radius ** 2"}
+                     :model/PythonLibrary _ {:path   "utils"
+                                             :source "def format_currency(amount):\n    return f\"${amount:,.2f}\""}]
+        (let [transform-code (str "import pandas as pd\n"
+                                  "from circle import calculate_circle_area\n"
+                                  "from utils import format_currency\n"
+                                  "\n"
+                                  "def transform():\n"
+                                  "    data = [\n"
+                                  "        {'radius': 5, 'area': calculate_circle_area(5), 'price': format_currency(78.54)},\n"
+                                  "        {'radius': 10, 'area': calculate_circle_area(10), 'price': format_currency(314.16)}\n"
+                                  "    ]\n"
+                                  "    return pd.DataFrame(data)")
+              result         (execute {:code transform-code})]
+          (is (=? {:output #(and (str/includes? % "radius,area,price")
+                                 (str/includes? % "5,78.5")
+                                 (str/includes? % "$78.54")
+                                 (str/includes? % "10,314.1")
+                                 (str/includes? % "$314.16"))
+                   :stdout "Successfully saved 2 rows to S3\nSuccessfully saved output manifest with 3 fields"
+                   :stderr ""}
+                  result)))))))
+
+(deftest transform-function-without-libraries-test
+  (testing "transform function works when no libraries exist"
+    (mt/test-drivers #{:postgres}
+      (with-redefs [t2/select-fn->fn (fn [k v model]
+                                       (when (and (= k :path)
+                                                  (= v :source)
+                                                  (= model :model/PythonLibrary))
+                                         {}))]
+        (let [transform-code (str "import pandas as pd\n"
+                                  "\n"
+                                  "def transform():\n"
+                                  "    return pd.DataFrame({'status': ['ok']})")
+              result         (execute {:code transform-code})]
+          (is (=? {:output "status\nok\n"
+                   :stdout "Successfully saved 1 rows to CSV\nSuccessfully saved output manifest with 1 fields"
+                   :stderr ""}
+                  result)))))))
+
+(deftest transform-function-library-import-error-test
+  (testing "transform function handles missing library gracefully"
+    (mt/test-drivers #{:postgres}
+      (with-redefs [t2/select-fn->fn (fn [k v model]
+                                       (when (and (= k :path)
+                                                  (= v :source)
+                                                  (= model :model/PythonLibrary))
+                                         {"utils" "def helper():\n    return 42"}))]
+        (let [transform-code (str "import pandas as pd\n"
+                                  "from common import some_function  # This library doesn't exist\n"
+                                  "\n"
+                                  "def transform():\n"
+                                  "    return pd.DataFrame({'value': [some_function()]})")
+              result         (execute {:code transform-code})]
+          (is (=? {:error     "Execution failed"
+                   :exit-code 1
+                   :stderr    #(str/includes? % "No module named 'common'")}
+                  result)))))))
+
+(deftest transform-type-roundtrip-test
+  (mt/test-drivers #{:postgres :h2 :mysql :bigquery-cloud-sdk :redshift :snowflake :sqlserver :clickhouse}
+    (mt/with-empty-db
+      (let [driver       driver/*driver*
+            db-id        (mt/id)
+            table-name   (if (= driver :redshift)
+                           (tx/db-qualified-table-name (get-in (mt/db) [:settings :database-source-dataset-name]) (mt/random-name))
+                           (mt/random-name))
+            schema-name  (sql.tx/session-schema driver)
+            qualified-table-name (if schema-name
+                                   (keyword schema-name table-name)
+                                   (keyword table-name))
+            table-schema {:name    qualified-table-name
+                          :columns [{:name "id"            :type :type/Integer  :nullable? false}
+                                    {:name "price"         :type :type/Float    :nullable? true}
+                                    {:name "active"        :type :type/Boolean  :nullable? true}
+                                    {:name "created_tz"    :type :type/DateTimeWithTZ :nullable? true}
+                                    {:name "created_at"    :type :type/DateTime :nullable? true}
+                                    {:name "created_date"  :type :type/Date     :nullable? true}
+                                    {:name "description"   :type :type/Text     :nullable? true}]}
+            _ (mt/as-admin
+                (transforms.util/create-table-from-schema! driver db-id table-schema))
+
+            row-values   [[1
+                           19.99
+                           (if (= driver/*driver* :sqlserver) 1 true)
+                           (if (= driver/*driver* :mysql) "2024-01-01 12:00:00" "2024-01-01T12:00:00Z")
+                           (if (= driver/*driver* :mysql) "2024-01-01 12:00:00" "2024-01-01T12:00:00")
+                           "2024-01-01"
+                           "Sample product"]]
+            _ (driver/insert-from-source! driver db-id qualified-table-name
+                                          (mapv :name (:columns table-schema))
+                                          {:type :rows :data row-values})
+
+            _ (sync/sync-database! (mt/db) {:scan :schema})
+
+            transform-code (str "import pandas as pd\n"
+                                "\n"
+                                "def transform(" table-name "):\n"
+                                "    return " table-name)
+            result (execute {:code transform-code
+                             :tables {table-name (mt/id qualified-table-name)}})
+
+            metadata (:output-manifest result)]
+
+        (testing "All expected columns are present"
+          (is (= #{"id" "price" "active" "created_tz" "created_at" "created_date" "description"}
+                 (set (map :name (:fields metadata))))))
+
+        (testing "dtypes are preserved correctly"
+          (is (= {"id"           :type/Integer
+                  "price"        :type/Float
+                  "active"       :type/Boolean
+                  "created_tz"   :type/DateTimeWithTZ
+                  "created_at"   :type/DateTime
+                  "created_date" :type/Date
+                  "description"  :type/Text}
+                 (u/for-map [{:keys [name dtype]} (:fields metadata)]
+                   [name (transforms.util/dtype->base-type dtype)]))))
+
+        ;; cleanup
+        (driver/drop-table! driver db-id qualified-table-name)))))

enterprise/backend/test/metabase_enterprise/transforms/test_util.clj

@@ -52,6 +52,12 @@
     `(mt/with-model-cleanup [:model/Transform]
        ~@body)))
 
+(defn table-rows
+  [table-name]
+  (mt/rows (mt/process-query {:database (mt/id)
+                              :query    {:source-table (t2/select-one-pk :model/Table :name table-name)}
+                              :type     :query})))
+
 (defn parse-timestamp
   "Parse a local datetime and convert it to a ZonedDateTime in the default timezone."
   ^ZonedDateTime [timestamp-string]

enterprise/frontend/src/.eslintrc.js

@@ -58,6 +58,10 @@ module.exports = {
             ...baseRestrictredConfig,
             paths: [
               ...baseRestrictredConfig.paths,
+              {
+                name: "zod",
+                message: "Please import from `zod/mini` instead.",
+              },
               {
                 name: "metabase/lib/redux",
                 importNames: ["useStore", "useDispatch"],

enterprise/frontend/src/embedding-sdk-bundle/components/public/CollectionBrowser/CollectionBrowser.schema.ts

@@ -0,0 +1,30 @@
+import {
+  any,
+  function as functionSchema,
+  optional,
+  strictObject,
+} from "zod/mini";
+import type { infer as zInfer } from "zod/v4/core/core";
+
+import type { ValidateInferredSchema } from "embedding-sdk-bundle/types/schema";
+
+import type { CollectionBrowserProps } from "./CollectionBrowser";
+
+const rawPropsSchema = strictObject({
+  EmptyContentComponent: optional(any()),
+  className: optional(any()),
+  collectionId: optional(any()),
+  onClick: optional(any()),
+  pageSize: optional(any()),
+  style: optional(any()),
+  visibleColumns: optional(any()),
+  visibleEntityTypes: optional(any()),
+});
+const propsSchema: ValidateInferredSchema<
+  CollectionBrowserProps,
+  zInfer<typeof rawPropsSchema>
+> = rawPropsSchema;
+
+export const collectionBrowserPropsSchema = functionSchema({
+  input: [propsSchema],
+});

enterprise/frontend/src/embedding-sdk-bundle/components/public/CollectionBrowser/CollectionBrowser.tsx

@@ -22,6 +22,8 @@ import CollectionBreadcrumbs from "metabase/nav/containers/CollectionBreadcrumbs
 import { Stack } from "metabase/ui";
 import type { CollectionId, CollectionItemModel } from "metabase-types/api";
 
+import { collectionBrowserPropsSchema } from "./CollectionBrowser.schema";
+
 const USER_FACING_ENTITY_NAMES = [
   "collection",
   "dashboard",
@@ -200,6 +202,7 @@ const CollectionBrowserWrapper = ({
   return <CollectionBrowserInner collectionId={collectionId} {...restProps} />;
 };
 
-export const CollectionBrowser = withPublicComponentWrapper(
-  CollectionBrowserWrapper,
+export const CollectionBrowser = Object.assign(
+  withPublicComponentWrapper(CollectionBrowserWrapper),
+  { schema: collectionBrowserPropsSchema },
 );

enterprise/frontend/src/embedding-sdk-bundle/components/public/CreateDashboardModal/CreateDashboardModal.schema.ts

@@ -0,0 +1,27 @@
+import {
+  any,
+  function as functionSchema,
+  nonoptional,
+  optional,
+  strictObject,
+} from "zod/mini";
+import type { infer as zInfer } from "zod/v4/core/core";
+
+import type { ValidateInferredSchema } from "embedding-sdk-bundle/types/schema";
+
+import type { CreateDashboardModalProps } from "./CreateDashboardModal";
+
+const rawPropsSchema = strictObject({
+  initialCollectionId: optional(any()),
+  isOpen: optional(any()),
+  onClose: optional(any()),
+  onCreate: nonoptional(any()),
+});
+const propsSchema: ValidateInferredSchema<
+  CreateDashboardModalProps,
+  zInfer<typeof rawPropsSchema>
+> = rawPropsSchema;
+
+export const createDashboardModalSchema = functionSchema({
+  input: [propsSchema],
+});

enterprise/frontend/src/embedding-sdk-bundle/components/public/CreateDashboardModal/CreateDashboardModal.tsx

@@ -8,6 +8,8 @@ import { useCollectionQuery, useLocale } from "metabase/common/hooks";
 import { CreateDashboardModal as CreateDashboardModalCore } from "metabase/dashboard/containers/CreateDashboardModal";
 import { useSelector } from "metabase/lib/redux";
 
+import { createDashboardModalSchema } from "./CreateDashboardModal.schema";
+
 /**
  * @expand
  * @category CreateDashboardModal
@@ -68,6 +70,7 @@ const CreateDashboardModalInner = ({
   );
 };
 
-export const CreateDashboardModal = withPublicComponentWrapper(
-  CreateDashboardModalInner,
+export const CreateDashboardModal = Object.assign(
+  withPublicComponentWrapper(CreateDashboardModalInner),
+  { schema: createDashboardModalSchema },
 );

enterprise/frontend/src/embedding-sdk-bundle/components/public/InteractiveQuestion/InteractiveQuestion.schema.ts

@@ -0,0 +1,56 @@
+import {
+  any,
+  function as functionSchema,
+  nonoptional,
+  optional,
+  strictObject,
+  type infer as zInfer,
+} from "zod/mini";
+
+import type { ValidateInferredSchema } from "embedding-sdk-bundle/types/schema";
+
+import type { InteractiveQuestionProps } from "./InteractiveQuestion";
+
+const rawPropsSchema = strictObject({
+  children: optional(any()),
+  className: optional(any()),
+  componentPlugins: optional(
+    strictObject({
+      mapQuestionClickActions: optional(any()),
+      dashboard: optional(any()),
+    }),
+  ),
+  deserializedCard: optional(any()),
+  entityTypes: optional(any()),
+  height: optional(any()),
+  initialSqlParameters: optional(any()),
+  isSaveEnabled: optional(any()),
+  onBeforeSave: optional(any()),
+  onNavigateBack: optional(any()),
+  onRun: optional(any()),
+  onSave: optional(any()),
+  options: optional(any()),
+  plugins: optional(
+    strictObject({
+      mapQuestionClickActions: optional(any()),
+      dashboard: optional(any()),
+    }),
+  ),
+  questionId: nonoptional(any()),
+  style: optional(any()),
+  targetCollection: optional(any()),
+  targetDashboardId: optional(any()),
+  title: optional(any()),
+  width: optional(any()),
+  withChartTypeSelector: optional(any()),
+  withDownloads: optional(any()),
+  withResetButton: optional(any()),
+});
+const propsSchema: ValidateInferredSchema<
+  InteractiveQuestionProps,
+  zInfer<typeof rawPropsSchema>
+> = rawPropsSchema;
+
+export const interactiveQuestionSchema = functionSchema({
+  input: [propsSchema],
+});

enterprise/frontend/src/embedding-sdk-bundle/components/public/InteractiveQuestion/InteractiveQuestion.tsx

@@ -26,6 +26,8 @@ import {
   type SdkQuestionProps,
 } from "embedding-sdk-bundle/components/public/SdkQuestion/SdkQuestion";
 
+import { interactiveQuestionSchema } from "./InteractiveQuestion.schema";
+
 /**
  * @interface
  * @expand
@@ -100,4 +102,5 @@ const subComponents: InteractiveQuestionComponents = {
 export const InteractiveQuestion = Object.assign(
   _InteractiveQuestion,
   subComponents,
+  { schema: interactiveQuestionSchema },
 );

enterprise/frontend/src/embedding-sdk-bundle/components/public/MetabotQuestion/MetabotChatEmbedding.module.css

@@ -18,7 +18,7 @@
   position: relative;
   max-width: 41.5rem;
   width: 100%;
-  box-shadow: 0 2px 13.8px -1px #1871bf52;
+  border: 1px solid var(--mb-color-brand);
   border-radius: 22px;
 }
 

enterprise/frontend/src/embedding-sdk-bundle/components/public/MetabotQuestion/MetabotChatEmbedding.tsx

@@ -1,5 +1,5 @@
 import cx from "classnames";
-import { useCallback, useEffect, useState } from "react";
+import { useEffect, useState } from "react";
 import { t } from "ttag";
 
 import { useSdkDispatch } from "embedding-sdk-bundle/store";
@@ -12,7 +12,6 @@ import {
   Tooltip,
   UnstyledButton,
 } from "metabase/ui";
-import { MetabotIcon } from "metabase-enterprise/metabot/components/MetabotIcon";
 import {
   useMetabotAgent,
   useMetabotChatHandlers,
@@ -30,10 +29,15 @@ export const MetabotChatEmbedding = () => {
   const metabot = useMetabotAgent();
   const { handleSubmitInput, handleResetInput } = useMetabotChatHandlers();
 
-  const resetInput = useCallback(() => {
+  const resetInput = () => {
     handleResetInput();
     setInputExpanded(false);
-  }, [handleResetInput]);
+  };
+
+  const startNewConversation = () => {
+    resetInput();
+    metabot.resetConversation();
+  };
 
   const [inputExpanded, setInputExpanded] = useState(false);
   const handleMaybeExpandInput = () => {
@@ -66,6 +70,9 @@ export const MetabotChatEmbedding = () => {
     dispatch(resetConversationId());
   }, [dispatch]);
 
+  const isOngoingConversation = metabot.messages.length > 0;
+  const isPromptDefined = metabot.prompt.length > 0;
+
   return (
     <Box className={Styles.container} data-testid="metabot-chat">
       <Flex
@@ -74,20 +81,20 @@ export const MetabotChatEmbedding = () => {
           metabot.isDoingScience && Styles.innerContainerLoading,
           inputExpanded && Styles.innerContainerExpanded,
         )}
-        gap="sm"
       >
         <Flex
-          w="33px"
-          h="24px"
+          w="16px"
+          h="16px"
           style={{
             flexShrink: 0,
           }}
           justify="center"
+          align="center"
         >
           {metabot.isDoingScience ? (
             <Loader size="sm" />
           ) : (
-            <MetabotIcon isLoading={false} />
+            <Icon name="ai" c="var(--mb-color-brand)" size="1rem" />
           )}
         </Flex>
         <Textarea
@@ -123,7 +130,7 @@ export const MetabotChatEmbedding = () => {
             }
           }}
         />
-        {metabot.isDoingScience ? (
+        {metabot.isDoingScience && (
           <UnstyledButton
             h="1rem"
             data-testid="metabot-cancel-request"
@@ -133,17 +140,40 @@ export const MetabotChatEmbedding = () => {
               <Icon name="stop" c="var(--mb-color-text-primary)" size="1rem" />
             </Tooltip>
           </UnstyledButton>
-        ) : (
-          <UnstyledButton
-            h="1rem"
-            onClick={resetInput}
-            data-testid="metabot-close-chat"
-            style={{
-              visibility: metabot.prompt.length > 0 ? "visible" : "hidden",
-            }}
-          >
-            <Icon name="close" c="var(--mb-color-text-primary)" size="1rem" />
-          </UnstyledButton>
+        )}
+
+        {!metabot.isDoingScience && (
+          <>
+            {isOngoingConversation && !isPromptDefined && (
+              <UnstyledButton
+                h="1rem"
+                onClick={startNewConversation}
+                data-testid="metabot-new-conversation"
+              >
+                <Tooltip label={t`Start new chat`}>
+                  <Icon
+                    name="edit_document_outlined"
+                    c="var(--mb-color-text-primary)"
+                    size="1rem"
+                  />
+                </Tooltip>
+              </UnstyledButton>
+            )}
+
+            {isPromptDefined && (
+              <UnstyledButton
+                h="1rem"
+                onClick={resetInput}
+                data-testid="metabot-close-chat"
+              >
+                <Icon
+                  name="close"
+                  c="var(--mb-color-text-primary)"
+                  size="1rem"
+                />
+              </UnstyledButton>
+            )}
+          </>
         )}
       </Flex>
     </Box>

enterprise/frontend/src/embedding-sdk-bundle/components/public/MetabotQuestion/MetabotQuestion.schema.ts

@@ -0,0 +1,5 @@
+import { function as functionSchema } from "zod/mini";
+
+export const metabotQuestionSchema = functionSchema({
+  input: [],
+});

enterprise/frontend/src/embedding-sdk-bundle/components/public/MetabotQuestion/MetabotQuestion.tsx

@@ -19,6 +19,7 @@ import {
 import { useMetabotReactions } from "metabase-enterprise/metabot/hooks/use-metabot-reactions";
 
 import { MetabotChatEmbedding } from "./MetabotChatEmbedding";
+import { metabotQuestionSchema } from "./MetabotQuestion.schema";
 import { QuestionDetails } from "./QuestionDetails";
 import { QuestionTitle } from "./QuestionTitle";
 
@@ -109,6 +110,7 @@ const MetabotQuestionWrapped = () => {
   );
 };
 
-export const MetabotQuestion = withPublicComponentWrapper(
-  MetabotQuestionWrapped,
+export const MetabotQuestion = Object.assign(
+  withPublicComponentWrapper(MetabotQuestionWrapped),
+  { schema: metabotQuestionSchema },
 );