Parametrise PythonLibraryEditor

- python-runner no longer simulates a single http api that does
everything (incl s3 interaction)
- Separates file lifetimes (shared storage) from execution
- Easier to see how to avoid reading into strings (e.g. for piping into
tmp files, follow up)

Earlier on we had a service that managed much of the I/O, metabase now
coordinates it so it makes little sense pretending there is still
an overall http api.

.clj-kondo/config.edn

@@ -1,7 +1,7 @@
 ;; -*- comment-column: 100; -*-
 {:config-paths ["macros" "src" "config"]
  :linters
- { ;;
+ {;;
   ;; enabled optional linters (these are disabled by default)
   ;;
 
@@ -186,7 +186,7 @@
   :metabase/disallow-hardcoded-driver-names-in-tests
   ;; don't include `:h2` since that will probably lead to a mountain of tests where we are specifically testing
   ;; middleware or compilation behavior with just H2 since it's the default.
-  {:drivers #{ ;; core drivers
+  {:drivers #{;; core drivers
               #_:h2
               :postgres
               :mysql
@@ -418,6 +418,7 @@
   metabase-enterprise.transforms.api-test/with-transform-tags                           clojure.core/let
   metabase-enterprise.transforms.api-test/with-worker-runs                              clojure.core/let
   metabase-enterprise.transforms.test-util/with-transform-cleanup!                      clojure.core/let
+  metabase-enterprise.transforms.execute/with-transform-lifecycle                       clojure.core/let
   metabase.actions.test-util/with-actions                                               clojure.core/let
   metabase.api.common/let-404                                                           clojure.core/let
   metabase.app-db.custom-migrations.util/with-temp-scheduler!                           clojure.core/fn
@@ -734,7 +735,9 @@
   ;; `metabase-enterprise.sandbox.query-processor*-test`
   ;;
   {:pattern "^metabase(?:((?:-enterprise\\.sandbox)?\\.query-processor)|(?:\\.driver\\.sql(?!server)(?!ite))).*-test$"
-   :name    general-qp-and-driver-test-namespaces}]
+   :name    general-qp-and-driver-test-namespaces}
+  {:pattern "^metabase\\.python-runner\\..*"
+   :name    python-runner-namespaces}]
 
  :config-in-ns
  {test-namespaces

.clj-kondo/config/modules/config.edn

@@ -718,6 +718,7 @@
            metabase.native-query-snippets.core}
    :uses #{api
            collections
+           lib
            models
            permissions
            premium-features
@@ -886,6 +887,19 @@
            enterprise/advanced-permissions
            enterprise/sandbox}}
 
+  python-runner
+  {:team "Workflows"
+   :api  #{metabase.python-runner.api
+           metabase.python-runner.core
+           metabase.python-runner.init}
+   :uses #{api
+           driver
+           models
+           query-processor
+           request
+           util
+           enterprise/transforms}}
+
   queries
   {:team "Querying"
    :api  #{metabase.queries.api
@@ -1947,8 +1961,11 @@
   {:team "Querying"
    :api  #{metabase-enterprise.transforms.api
            metabase-enterprise.transforms.core
-           metabase-enterprise.transforms.init}
-   :uses #{api
+           metabase-enterprise.transforms.init
+           ;; TODO: Remove this from API once we remove the python-runner callback to metabase
+           metabase-enterprise.transforms.settings}
+   :uses #{analytics
+           api
            app-db
            config
            driver
@@ -1964,6 +1981,8 @@
            sync
            task
            task-history
+           ;; TODO: Remove this once we're using a better file format than CSV
+           upload
            util}}
 
   enterprise/upload-management

.github/actions/await-port/action.yml

@@ -11,6 +11,18 @@ runs:
     - name: Wait for port ${{ inputs.port }} to be ready
       shell: bash
       run: |
+        echo "Waiting for port ${{ inputs.port }} to be ready..."
+        timeout=300  # 5 minutes timeout
+        elapsed=0
         while ! nc -z localhost ${{ inputs.port }}; do
-          sleep 0.1;
+          if [ $elapsed -ge $timeout ]; then
+            echo "❌ Timeout waiting for port ${{ inputs.port }} after ${timeout} seconds"
+            exit 1
+          fi
+          sleep 1
+          elapsed=$((elapsed + 1))
+          if [ $((elapsed % 10)) -eq 0 ]; then
+            echo "  Still waiting for port ${{ inputs.port }}... ($elapsed seconds)"
+          fi
         done
+        echo "✅ Port ${{ inputs.port }} is ready!"

.github/scripts/get-issue-data.js

@@ -0,0 +1,84 @@
+const githubSlackMap = require("../../release/github-slack-map.json");
+
+async function getIssueData({ context, github, manualIssueNumber }) {
+  let issueNumber;
+  let issueData;
+  let prReviews = [];
+  const triggerType = context.eventName;
+
+  if (context.eventName === "issues") {
+    issueNumber = context.payload.issue.number;
+    issueData = context.payload.issue;
+  } else if (context.eventName === "workflow_dispatch") {
+    issueNumber = parseInt(manualIssueNumber);
+    const { data } = await github.rest.issues.get({
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      issue_number: issueNumber,
+    });
+    issueData = data;
+  } else if (context.eventName === "pull_request_review") {
+    const prNumber = context.payload.pull_request.number;
+    const { data: prData } = await github.rest.pulls.get({
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      pull_number: prNumber,
+    });
+
+    // Extract issue number from branch name (e.g., "claude-fix/issue-12345-some-title")
+    const branchMatch = prData.head.ref.match(/claude-fix\/issue-(\d+)/);
+    if (!branchMatch) {
+      throw new Error(
+        `Could not extract issue number from branch name: ${prData.head.ref}`,
+      );
+    }
+
+    issueNumber = parseInt(branchMatch[1]);
+
+    const { data } = await github.rest.issues.get({
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      issue_number: issueNumber,
+    });
+    issueData = data;
+
+    const { data: reviews } = await github.rest.pulls.listReviews({
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      pull_number: prNumber,
+    });
+
+    prReviews = reviews.filter(
+      (review) =>
+        review.state === "CHANGES_REQUESTED" &&
+        isMetabaseMember(review.user.login),
+    );
+  }
+
+  const { data: comments } = await github.rest.issues.listComments({
+    owner: context.repo.owner,
+    repo: context.repo.repo,
+    issue_number: issueNumber,
+  });
+
+  const employeeComments = comments.filter((comment) =>
+    isMetabaseMember(comment.user.login),
+  );
+
+  return {
+    issueNumber,
+    issueData,
+    employeeComments,
+    prReviews,
+    triggerType,
+    isIssueAuthorEmployee: isMetabaseMember(issueData.user.login),
+  };
+}
+
+function isMetabaseMember(username) {
+  return username in githubSlackMap;
+}
+
+module.exports = {
+  getIssueData,
+};

.github/workflows/backend.yml

@@ -195,6 +195,7 @@ jobs:
       - name: Run cljfmt
         run: ./bin/mage cljfmt-all --force-check
 
+
   be-tests-result:
     needs:
       - be-linter-eastwood

.github/workflows/claude-code-bug-fix.yml

@@ -4,97 +4,22 @@ on:
   issues:
     types:
       - labeled
+  pull_request_review:
+    types:
+      - submitted
   workflow_dispatch:
     inputs:
       issue_number:
         description: "Issue number to fix"
         required: true
         type: string
+      override_employee_check:
+        description: "Override employee check for issue creator"
+        required: false
+        default: false
+        type: boolean
 
 jobs:
-  check-employee-and-issue:
-    runs-on: ubuntu-latest
-    timeout-minutes: 2
-    permissions:
-      contents: read
-      issues: read
-    if: |
-      (github.event_name == 'issues' && github.event.label.name == 'Claude Bug Fix' && github.event.issue.state == 'open') ||
-      (github.event_name == 'workflow_dispatch')
-    outputs:
-      is_employee: ${{ steps.check_employee.outputs.is_employee }}
-      issue_number: ${{ steps.get_issue.outputs.issue_number }}
-      issue_data: ${{ steps.get_issue.outputs.issue_data }}
-    steps:
-      - name: Generate GitHub App Token
-        uses: actions/create-github-app-token@v1
-        id: app-token
-        with:
-          app-id: ${{ secrets.METABASE_BOT_APP_ID }}
-          private-key: ${{ secrets.METABASE_BOT_APP_PRIVATE_KEY }}
-
-      - name: Get issue data
-        id: get_issue
-        uses: actions/github-script@v7
-        with:
-          script: |
-            let issueNumber, issueData;
-
-            if (context.eventName === 'issues') {
-              // Original issue label event
-              issueNumber = context.payload.issue.number;
-              issueData = context.payload.issue;
-            } else if (context.eventName === 'workflow_dispatch') {
-              // Manual trigger with issue number input
-              issueNumber = parseInt('${{ github.event.inputs.issue_number }}');
-              const { data } = await github.rest.issues.get({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                issue_number: issueNumber
-              });
-              issueData = data;
-            }
-
-            core.setOutput('issue_number', issueNumber);
-            core.setOutput('issue_data', JSON.stringify(issueData));
-
-            console.log(`Issue #${issueNumber}: ${issueData.title}`);
-            console.log(`Issue author: ${issueData.user.login}`);
-
-      - name: Check if issue creator is a Metabase employee
-        id: check_employee
-        uses: actions/github-script@v7
-        env:
-          ISSUE_DATA: ${{ steps.get_issue.outputs.issue_data }}
-        with:
-          github-token: ${{ steps.app-token.outputs.token }}
-          script: |
-            const issueData = JSON.parse(process.env.ISSUE_DATA);
-            const issueAuthor = issueData.user.login;
-
-            try {
-              // Check if user is a member of the metabase organization (including private members)
-              const { data: membership } = await github.rest.orgs.getMembershipForUser({
-                org: 'metabase',
-                username: issueAuthor
-              });
-
-              if (membership.state === 'active') {
-                console.log(`User ${issueAuthor} is an active member of the Metabase organization. Proceeding with bug fix automation.`);
-                core.setOutput('is_employee', 'true');
-              } else {
-                console.log(`User ${issueAuthor} is not an active member of the Metabase organization. Skipping bug fix automation.`);
-                core.setOutput('is_employee', 'false');
-              }
-            } catch (error) {
-              if (error.status === 404) {
-                console.log(`User ${issueAuthor} is not a member of the Metabase organization. Skipping bug fix automation.`);
-              } else {
-                console.error('Error checking organization membership:', error);
-              }
-              core.setOutput('is_employee', 'false');
-            }
-
   auto-fix-bug:
     runs-on: ubuntu-latest
     timeout-minutes: 30
@@ -103,8 +28,10 @@ jobs:
       issues: write
       pull-requests: write
       id-token: write
-    needs: check-employee-and-issue
-    if: needs.check-employee-and-issue.outputs.is_employee == 'true'
+    if: |
+      (github.event_name == 'issues' && github.event.label.name == 'Claude Bug Fix' && github.event.issue.state == 'open') ||
+      (github.event_name == 'workflow_dispatch') ||
+      (github.event_name == 'pull_request_review' && github.event.review.state == 'changes_requested' && startsWith(github.event.pull_request.head.ref, 'claude-fix/'))
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -112,67 +39,95 @@ jobs:
           fetch-depth: 0
           token: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Get issue data and check employee status
+        id: get_issue
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const { getIssueData } = require('${{ github.workspace }}/.github/scripts/get-issue-data.js');
+
+            try {
+              const result = await getIssueData({
+                context,
+                github,
+                manualIssueNumber: '${{ github.event.inputs.issue_number }}'
+              });
+
+              const { issueNumber, issueData, employeeComments, prReviews, triggerType, isIssueAuthorEmployee } = result;
+
+              // Check employee status and fail if not authorized
+              if (!result.isIssueAuthorEmployee && '${{ github.event.inputs.override_employee_check }}' !== 'true') {
+                  core.setFailed(`User ${result.issueData.user.login} is not a Metabase employee. Skipping bug fix automation.`);
+              }
+
+              core.setOutput('issue_number', issueNumber);
+              core.setOutput('issue_data', JSON.stringify(issueData));
+              core.setOutput('issue_comments', JSON.stringify(employeeComments));
+              core.setOutput('pr_reviews', JSON.stringify(prReviews));
+              core.setOutput('trigger_type', triggerType);
+            } catch (error) {
+              core.setFailed(error.message);
+            }
+
       - name: Set up git configuration
         run: |
           git config --global user.name "Claude Code[bot]"
           git config --global user.email "noreply@anthropic.com"
 
-      - name: Prepare dynamic issue info
-        id: prepare_issue_info
+      - name: Prepare issue context and bug fix prompt
+        id: prepare_prompt
         uses: actions/github-script@v7
         env:
-          ISSUE_DATA: ${{ needs.check-employee-and-issue.outputs.issue_data }}
+          ISSUE_DATA: ${{ steps.get_issue.outputs.issue_data }}
+          ISSUE_COMMENTS: ${{ steps.get_issue.outputs.issue_comments }}
+          PR_REVIEWS: ${{ steps.get_issue.outputs.pr_reviews }}
+          TRIGGER_TYPE: ${{ steps.get_issue.outputs.trigger_type }}
         with:
           script: |
             const issueData = JSON.parse(process.env.ISSUE_DATA);
-            const issueNumber = '${{ needs.check-employee-and-issue.outputs.issue_number }}';
+            const issueComments = JSON.parse(process.env.ISSUE_COMMENTS);
+            const prReviews = JSON.parse(process.env.PR_REVIEWS);
+            const triggerType = process.env.TRIGGER_TYPE;
+            const issueNumber = '${{ steps.get_issue.outputs.issue_number }}';
 
-            // Convert issue title to kebab case
+            // Convert issue title to kebab case and create branch name
             const issueTitle = issueData.title || 'untitled';
             const kebabTitle = issueTitle
               .toLowerCase()
               .replace(/[^a-z0-9]+/g, '-')
               .replace(/^-+|-+$/g, '')
               .replace(/-+/g, '-')
-              .substring(0, 50); // Limit length
+              .substring(0, 50);
+            const branchName = `claude-fix/issue-${issueNumber}-${kebabTitle}`;
 
-            // Create base branch name
-            const baseBranchName = `claude-fix/issue-${issueNumber}-${kebabTitle}`;
-            core.exportVariable('BASE_BRANCH_NAME', baseBranchName);
+            // Format data for bash environment
+            const formattedComments = issueComments.length > 0
+              ? issueComments.map(comment =>
+                  `**Comment by @${comment.user.login} on ${comment.created_at}:**\n${comment.body}`
+                ).join('\n\n---\n\n')
+              : '';
+            const formattedReviews = prReviews.length > 0
+              ? prReviews.map(review =>
+                  `**Review by @${review.user.login} on ${review.submitted_at} (Changes Requested):**\n${review.body || 'No review comment provided'}`
+                ).join('\n\n---\n\n')
+              : '';
 
-            // Export issue info for the prompt
+            core.exportVariable('BRANCH_NAME', branchName);
             core.exportVariable('ISSUE_NUMBER', issueNumber);
             core.exportVariable('ISSUE_TITLE', issueData.title);
             core.exportVariable('ISSUE_AUTHOR', issueData.user.login);
             core.exportVariable('ISSUE_CREATED', issueData.created_at);
             core.exportVariable('ISSUE_BODY', issueData.body || 'No description provided');
+            core.exportVariable('ISSUE_COMMENTS_FORMATTED', formattedComments);
+            core.exportVariable('PR_REVIEWS_FORMATTED', formattedReviews);
+            core.exportVariable('TRIGGER_TYPE', triggerType);
 
-      - name: Determine branch strategy
+      - name: Generate bug fix prompt
         run: |
-          # Use the base branch name (no counters)
-          BRANCH_NAME="${{ env.BASE_BRANCH_NAME }}"
-
-          # Check if this exact branch already exists
-          if git branch -r --list "origin/${BRANCH_NAME}" | grep -q .; then
-            echo "Branch already exists: $BRANCH_NAME"
-            echo "Will push to existing branch and skip PR creation"
-            echo "BRANCH_EXISTS=true" >> $GITHUB_ENV
-          else
-            echo "Creating new branch: $BRANCH_NAME"
-            echo "Will create new PR"
-            echo "BRANCH_EXISTS=false" >> $GITHUB_ENV
-          fi
-
-          # Export the branch name
-          echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
-
-      - name: Prepare bug fix prompt
-        id: prepare_prompt
-        run: |
-          # Prepare the prompt for Claude
           cat << 'EOF' >> $GITHUB_ENV
           BUG_FIX_PROMPT<<PROMPT_END
-          You are helping to automatically fix a bug reported in GitHub issue #${{ env.ISSUE_NUMBER }}.
+          You are helping to fix a bug reported in GitHub issue #${{ env.ISSUE_NUMBER }}.
 
           ## Bug Report Details
           **Title:** ${{ env.ISSUE_TITLE }}
@@ -182,8 +137,27 @@ jobs:
           **Description:**
           ${{ env.ISSUE_BODY }}
 
+          **Comments:**
+          ${{ env.ISSUE_COMMENTS_FORMATTED }}
+          EOF
+
+          # Add review feedback section only if this is a review trigger
+          if [ "${{ env.TRIGGER_TYPE }}" = "pull_request_review" ]; then
+            cat << 'EOF' >> $GITHUB_ENV
+
+          ## Previous Fix Attempt - Review Feedback (MUST ADDRESS)
+          Your previous automated fix attempt received the following feedback that needs to be addressed:
+
+          ${{ env.PR_REVIEWS_FORMATTED }}
+          EOF
+          fi
+
+          # Add the rest of the prompt (always the same)
+          cat << 'EOF' >> $GITHUB_ENV
+
           ## Instructions
-          1. **Analyze the bug report** - Understand what the issue is describing
+          1. **Analyze the issue description and comments** - Understand what the issue is describing
+             a. Pay particular attention to reproduction steps and if there's an agreed upon fix.
           2. **Search the codebase** - Use Grep and Glob tools to find relevant files
           3. **Implement a fix** - Follow existing code patterns and conventions
           4. **Test the fix** - Run appropriate tests using the commands in CLAUDE.md:
@@ -205,15 +179,26 @@ jobs:
           - Only modify actual production code in the main repository directories
           - The scratchpad will be cleaned up automatically - nothing from it will be committed
 
-          ## If You Cannot Fix The Bug
-          If you determine that you cannot create a definitive fix, create a file called `FAILED_FIX_ANALYSIS.md` with:
-          1. **Why No Fix** - Specific reason why you couldn't implement a fix
-          2. **Tool Limitations** - If you needed additional tools/access that weren't available
-
           Start by understanding the bug and then systematically work toward a solution.
           PROMPT_END
           EOF
 
+      - name: Determine branch strategy
+        run: |
+          BRANCH_NAME="${{ env.BRANCH_NAME }}"
+          if git branch -r --list "origin/${BRANCH_NAME}" | grep -q .; then
+            echo "Branch already exists: $BRANCH_NAME"
+            echo "Will push to existing branch and skip PR creation"
+            echo "BRANCH_EXISTS=true" >> $GITHUB_ENV
+          else
+            echo "Creating new branch: $BRANCH_NAME"
+            echo "Will create new PR"
+            echo "BRANCH_EXISTS=false" >> $GITHUB_ENV
+          fi
+
+          # Export the branch name
+          echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
+
       - name: Create scratchpad directory for Claude
         run: |
           # Create a scratchpad directory for Claude to use for temporary files
@@ -249,20 +234,6 @@ jobs:
       - name: Check for changes and analysis
         id: check_changes
         run: |
-          # Check if Claude created an analysis file first and store its content
-          if [ -f "FAILED_FIX_ANALYSIS.md" ]; then
-            echo "has_analysis=true" >> $GITHUB_OUTPUT
-            echo "Analysis file found"
-            # Store analysis content in a GitHub output (base64 encoded to handle multiline)
-            ANALYSIS_CONTENT=$(base64 -w 0 FAILED_FIX_ANALYSIS.md)
-            echo "analysis_content=$ANALYSIS_CONTENT" >> $GITHUB_OUTPUT
-            # Remove analysis file so it doesn't count as a code change
-            rm -f FAILED_FIX_ANALYSIS.md
-          else
-            echo "has_analysis=false" >> $GITHUB_OUTPUT
-            echo "No analysis file found"
-          fi
-
           # Now check for actual code changes (excluding analysis file)
           if [ -n "$(git status --porcelain)" ]; then
             echo "has_changes=true" >> $GITHUB_OUTPUT
@@ -278,25 +249,16 @@ jobs:
         id: prepare_reviewers
         uses: actions/github-script@v7
         env:
-          ISSUE_DATA: ${{ needs.check-employee-and-issue.outputs.issue_data }}
+          ISSUE_DATA: ${{ steps.get_issue.outputs.issue_data }}
         with:
           script: |
-            // Get existing team labels from the issue
             const issueData = JSON.parse(process.env.ISSUE_DATA);
             const issueLabels = issueData.labels.map(label => label.name);
             const teamLabels = issueLabels.filter(label => label.startsWith('.Team/'));
-
-            console.log('Issue labels:', issueLabels);
-            console.log('Team labels found:', teamLabels);
-
-            // Add issue creator as reviewer
             const issueCreator = issueData.user.login;
-
             core.setOutput('team_labels', teamLabels.join(','));
             core.setOutput('reviewers', issueCreator);
 
-            console.log('Reviewer:', issueCreator);
-
       - name: Create Pull Request
         if: steps.check_changes.outputs.has_changes == 'true'
         uses: peter-evans/create-pull-request@v6
@@ -363,25 +325,3 @@ jobs:
         with:
           script: |
             console.log('🤖 No code changes were made by Claude.');
-
-            // Check if Claude provided a detailed analysis
-            if ('${{ steps.check_changes.outputs.has_analysis }}' === 'true') {
-              try {
-                // Decode the base64-encoded analysis content
-                const analysisBase64 = '${{ steps.check_changes.outputs.analysis_content }}';
-                const analysisContent = Buffer.from(analysisBase64, 'base64').toString('utf8');
-                console.log('📋 Claude provided detailed analysis:');
-                console.log('='.repeat(50));
-                console.log(analysisContent);
-                console.log('='.repeat(50));
-              } catch (error) {
-                console.log('❌ Could not decode analysis content:', error);
-              }
-            } else {
-              console.log('ℹ️  No detailed analysis file was created by Claude.');
-              console.log('This could be because:');
-              console.log('- The issue requires more context or clarification');
-              console.log('- The fix is complex and requires human judgment');
-              console.log('- The issue is not a straightforward code bug');
-              console.log('- Claude couldn\'t locate the relevant code to modify');
-            }

.github/workflows/drivers.yml

@@ -338,6 +338,12 @@ jobs:
     name: H2
     steps:
     - uses: actions/checkout@v4
+      with:
+        submodules: true
+        token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+    - name: Start python-runner
+      working-directory: python-runner
+      run: make run-all
     - name: Test H2 driver
       uses: ./.github/actions/test-driver
       with:
@@ -442,6 +448,12 @@ jobs:
     name: "${{ matrix.version.name }} ${{ matrix.job.name }}"
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
+          token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      - name: Start python-runner
+        working-directory: python-runner
+        run: make run-all
       - name: Test ${{ matrix.version.name }}
         uses: ./.github/actions/test-driver
         with:
@@ -745,6 +757,12 @@ jobs:
           POSTGRES_HOST_AUTH_METHOD: trust
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
+          token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      - name: Start python-runner
+        working-directory: python-runner
+        run: make run-all
       - name: Test ${{ matrix.version.name }} (${{ matrix.job.name }})
         uses: ./.github/actions/test-driver
         with:

.gitignore

@@ -163,3 +163,4 @@ CLAUDE.local.md
 
 /logs
 /.clojure-mcp
+__pycache__/

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "python-runner"]
+	path = python-runner
+	url = https://github.com/metabase/python-runner-container.git
+	branch = prototype

deps.edn

@@ -36,7 +36,7 @@
   com.clearspring.analytics/stream          {:mvn/version "2.9.8"               ; Various sketching algorithms
                                              :exclusions  [it.unimi.dsi/fastutil
                                                            org.slf4j/slf4j-api]}
-  com.clojure-goes-fast/jvm-alloc-rate-meter{:mvn/version "0.1.4"}              ; Monitor heap allocation rate.
+  com.clojure-goes-fast/jvm-alloc-rate-meter {:mvn/version "0.1.4"}              ; Monitor heap allocation rate.
   com.clojure-goes-fast/jvm-hiccup-meter    {:mvn/version "0.1.1"}              ; Monitor GC pauses and other system-induced pauses.
   com.draines/postal                        {:mvn/version "2.0.5"               ; SMTP library
                                              :exclusions [com.sun.mail/jakarta.mail]}
@@ -45,7 +45,7 @@
   com.github.albfernandez/juniversalchardet {:mvn/version "2.5.0"}
   com.github.blagerweij/liquibase-sessionlock {:mvn/version "1.6.9"}            ; session lock for liquibase migrations
   com.github.jknack/handlebars              ^:antq/exclude                      ; 4.4.0 requires java 17+
-                                            {:mvn/version "4.3.1"}              ; templating engine
+  {:mvn/version "4.3.1"}              ; templating engine
   com.github.oliyh/martian-clj-http         {:mvn/version "0.1.31"}             ; openapi client
   com.github.pangloss/transducers           {:git/sha "c91d39046c8c64dc31b84ebb27154a8084fcda4c" ; like medley for transducers
                                              :git/url "https://github.com/pangloss/transducers"}
@@ -55,14 +55,14 @@
   com.github.vertical-blank/sql-formatter   {:mvn/version "2.0.5"}              ; Java SQL formatting library https://github.com/vertical-blank/sql-formatter
   com.google.guava/guava                    {:mvn/version "33.4.8-jre"}         ; dep for BigQuery, Spark, and GA. Require here rather than letting different dep versions stomp on each other — see comments on #9697
   com.h2database/h2                         ^:antq/exclude                      ; https://metaboat.slack.com/archives/CKZEMT1MJ/p1727191010259979
-                                            {:mvn/version "2.1.214"}            ; embedded SQL database
+  {:mvn/version "2.1.214"}            ; embedded SQL database
   com.ibm.icu/icu4j                         {:mvn/version "77.1"}               ; Used to transliterate Unicode characters into Latin characters
   com.knuddels/jtokkit                      {:mvn/version "1.1.0"}              ; Java tokenizer library for OpenAI models, used for token counting in semantic search
   com.mchange/c3p0                          {:mvn/version "0.10.1"}             ; DB connection pool
   com.snowplowanalytics/snowplow-java-tracker ^:antq/exclude                    ; 2+ depend on apache httpclient 5.x, but saml and clj-http depend on 4.x
-                                            {:mvn/version "1.0.1"}              ; Snowplow analytics
+  {:mvn/version "1.0.1"}              ; Snowplow analytics
   com.sun.mail/jakarta.mail                 ^:antq/exclude                      ; 2+ changes imports from javax.mail to jakarta.mail
-                                            {:mvn/version "1.6.8"}              ; SMTP lib, required by postal
+  {:mvn/version "1.6.8"}              ; SMTP lib, required by postal
   com.taoensso/nippy                        {:mvn/version "3.6.0"}              ; Fast serialization (i.e., GZIP) library for Clojure
   com.vladsch.flexmark/flexmark             {:mvn/version "0.64.8"}             ; Markdown parsing
   com.vladsch.flexmark/flexmark-ext-autolink
@@ -92,7 +92,7 @@
   ;; TODO: replace back to normal artifact
   io.github.metabase/macaw                  {:mvn/version "0.2.27"}             ; Parse native SQL queries
   io.github.resilience4j/resilience4j-retry ^:antq/exclude                      ; 2.x do not support Java 8
-                                            {:mvn/version "1.7.1"}              ; Support for retrying operations
+  {:mvn/version "1.7.1"}              ; Support for retrying operations
   io.prometheus/simpleclient_hotspot        {:mvn/version "0.16.0"}             ; prometheus jvm collector
   junegunn/grouper                          {:mvn/version "0.1.1"}              ; Batch processing helper
   kixi/stats                                {:mvn/version "0.5.7"               ; Various statistic measures implemented as transducers
@@ -135,7 +135,7 @@
                                              :exclusions  [org.bouncycastle/bcpkix-jdk15on
                                                            org.bouncycastle/bcprov-jdk15on]}
   org.apache.poi/poi-ooxml-full             {:mvn/version "5.4.1"}
-  org.apache.sshd/sshd-core                 {:mvn/version "2.15.0"              ; ssh tunneling and test server
+  org.apache.sshd/sshd-core                 {:mvn/version "2.16.0"              ; ssh tunneling and test server
                                              :exclusions  [org.slf4j/slf4j-api
                                                            org.slf4j/jcl-over-slf4j]}
   org.apache.tika/tika-core                 {:mvn/version "3.2.2"}
@@ -147,7 +147,7 @@
   org.clj-commons/hickory                   {:mvn/version "0.7.7"               ; Parse HTML into Clojure data structures
                                              :exclusions [org.jsoup/jsoup]}
   org.clojars.pntblnk/clj-ldap              {:mvn/version "0.0.17"}             ; LDAP client
-  org.clojure/clojure                       {:mvn/version "1.12.1"}
+  org.clojure/clojure                       {:mvn/version "1.12.2"}
   org.clojure/core.async                    {:mvn/version "1.7.701"
                                              :exclusions  [org.clojure/tools.reader]}
   org.clojure/core.logic                    {:mvn/version "1.1.0"}              ; optimized pattern matching library for Clojure
@@ -198,6 +198,8 @@
                                                            org.eclipse.jetty.ee9/jetty-ee9-servlet
                                                            org.eclipse.jetty.ee9.websocket/jetty-ee9-websocket-jetty-server]}
   slingshot/slingshot                       {:mvn/version "0.12.2"}             ; enhanced throw/catch, used by other deps
+  software.amazon.awssdk/s3                 {:mvn/version "2.33.0"}
+  software.amazon.awssdk/sts                {:mvn/version "2.33.0"}           ; AWS SDK v2 for S3
   stencil/stencil                           {:mvn/version "0.5.0"}              ; Mustache templates for Clojure
   user-agent/user-agent                     {:mvn/version "0.1.1"}              ; User-Agent string parser, for Login History page & elsewhere
   weavejester/dependency                    {:mvn/version "1.0.0"}              ; Dependency graphs and topological sorting
@@ -253,7 +255,7 @@
     {:mvn/version "0.4.0"}                     ; Enables easy memory measurement
     com.gfredericks/test.chuck   {:mvn/version "0.2.15"}                    ; generating strings from regexes (useful with malli)
     criterium/criterium          {:mvn/version "0.4.6"}                     ; benchmarking library
-    djblue/portal                {:mvn/version "0.60.2"}                    ; ui for inspecting values
+    djblue/portal                {:mvn/version "0.61.0"}                    ; ui for inspecting values
     io.github.camsaul/humane-are {:mvn/version "1.0.2"}                     ; cleaner test output
     io.github.metabase/hawk      {:mvn/version "1.0.11"}                    ; test runner
     jonase/eastwood              {:mvn/version "1.4.3"                      ; inspects namespaces and reports possible problems using tools.analyzer
@@ -836,8 +838,13 @@
   :test/lib
   {:exec-args {:only ["test/metabase/lib"
                       "test/metabase/lib_be"]}}
+  ;; test the python transforms
+  ;;    clj -X:dev:ee:ee-dev:driver:test:test/python-transform
+  :test/python-transform
+  {:exec-args {:only [metabase-enterprise.transforms.api-test
+                      "enterprise/backend/test/metabase_enterprise/transforms"]}}
 
-  ;; test the Query Processor
+;; test the Query Processor
   ;;
   ;;    clj -X:dev:ee:ee-dev:test:test/qp
   :test/qp

docs/ai/metabot.md

@@ -68,7 +68,9 @@ You can also ask Metabot to tell you about specific tables in your database.
 
 ## How Metabot uses the query builder
 
-When you ask Metabot to create a chart from natural language, it first looks for existing questions that might answer your request. If it finds a relevant question, it'll point you to that instead of creating something new. Otherwise, Metabot will use the [query builder](../questions/query-builder/editor.md) to create a new chart for you. Keep in mind that Metabot can only handle basic query builder operations. It's limited to single-level aggregation and grouping, so if you need more complex analysis, you can take over and refine the query yourself, or switch to the native editor.
+When you ask Metabot to create a chart from natural language, it first looks for existing questions that might answer your request. If it finds a relevant question, it'll point you to that instead of creating something new. Otherwise, Metabot will use the [query builder](../questions/query-builder/editor.md) to create a new chart for you. 
+
+Keep in mind that Metabot is still learning the query builder. Metabot can only handle basic query builder operations, and it lacks access to the library of [custom expressions](../questions/query-builder/expressions-list.md). Metabot is also limited to single-level aggregation and grouping, so if you need more complex analysis, you can take over and refine the query yourself, or switch to the SQL editor.
 
 ## Metabot in the native editor
 

docs/api.html

@@ -12771,7 +12771,7 @@ redirect_from:
                       },
                       "schema-filters-type" : {
                         "type" : "string",
-                        "enum" : [ "include", "exclude", "all" ]
+                        "enum" : [ "inclusion", "exclusion", "all" ]
                       }
                     },
                     "required" : [ "schema-filters-type", "schema-filters-patterns" ]
@@ -12849,7 +12849,7 @@ redirect_from:
                       },
                       "schema-filters-type" : {
                         "type" : "string",
-                        "enum" : [ "include", "exclude", "all" ]
+                        "enum" : [ "inclusion", "exclusion", "all" ]
                       }
                     },
                     "required" : [ "schema-filters-type", "schema-filters-patterns" ]
@@ -15864,7 +15864,35 @@ redirect_from:
       "get" : {
         "summary" : "GET /api/ee/transform-job/",
         "description" : "Get all transform jobs.",
-        "parameters" : [ ],
+        "parameters" : [ {
+          "in" : "query",
+          "name" : "last_run_start_time",
+          "required" : false,
+          "schema" : {
+            "type" : "string",
+            "minLength" : 1
+          }
+        }, {
+          "in" : "query",
+          "name" : "next_run_start_time",
+          "required" : false,
+          "schema" : {
+            "type" : "string",
+            "minLength" : 1
+          }
+        }, {
+          "in" : "query",
+          "name" : "transform_tag_ids",
+          "required" : false,
+          "schema" : {
+            "type" : "array",
+            "items" : {
+              "description" : "value must be an integer greater or equal to than zero.",
+              "type" : "integer",
+              "minimum" : 0
+            }
+          }
+        } ],
         "responses" : {
           "2XX" : {
             "description" : "Successful response"
@@ -16018,6 +16046,34 @@ redirect_from:
         "tags" : [ "/api/ee/transform-job" ]
       }
     },
+    "/api/ee/transform-job/{job-id}/transforms" : {
+      "get" : {
+        "summary" : "GET /api/ee/transform-job/{job-id}/transforms",
+        "description" : "Get the transforms of job specified by the job's ID.",
+        "parameters" : [ {
+          "in" : "path",
+          "name" : "job-id",
+          "required" : true,
+          "schema" : {
+            "type" : "integer",
+            "minimum" : 1
+          },
+          "description" : "value must be an integer greater than zero."
+        } ],
+        "responses" : {
+          "2XX" : {
+            "description" : "Successful response"
+          },
+          "4XX" : {
+            "description" : "Client error response"
+          },
+          "5XX" : {
+            "description" : "Server error response"
+          }
+        },
+        "tags" : [ "/api/ee/transform-job" ]
+      }
+    },
     "/api/ee/transform-tag/" : {
       "post" : {
         "summary" : "POST /api/ee/transform-tag/",
@@ -16448,6 +16504,34 @@ redirect_from:
         "tags" : [ "/api/ee/transform" ]
       }
     },
+    "/api/ee/transform/{id}/dependencies" : {
+      "get" : {
+        "summary" : "GET /api/ee/transform/{id}/dependencies",
+        "description" : "Get the dependencies of a specific transform.",
+        "parameters" : [ {
+          "in" : "path",
+          "name" : "id",
+          "required" : true,
+          "schema" : {
+            "type" : "integer",
+            "minimum" : 1
+          },
+          "description" : "value must be an integer greater than zero."
+        } ],
+        "responses" : {
+          "2XX" : {
+            "description" : "Successful response"
+          },
+          "4XX" : {
+            "description" : "Client error response"
+          },
+          "5XX" : {
+            "description" : "Server error response"
+          }
+        },
+        "tags" : [ "/api/ee/transform" ]
+      }
+    },
     "/api/ee/transform/{id}/run" : {
       "post" : {
         "summary" : "POST /api/ee/transform/{id}/run",

docs/configuring-metabase/config-template.md

@@ -117,6 +117,7 @@ config:
     db-connection-timeout-ms: 10000
     db-query-timeout-minutes: 20
     default-maps-enabled: true
+    disable-cors-on-localhost: false
     download-row-limit: null
     email-from-address: notifications@metabase.com
     email-from-address-override: notifications@metabase.com
@@ -134,7 +135,7 @@ config:
     email-smtp-username: null
     email-smtp-username-override: null
     embedding-app-origins-interactive: null
-    embedding-app-origins-sdk: ""
+    embedding-app-origins-sdk: ''
     embedding-homepage: hidden
     embedding-secret-key: null
     enable-embedding-interactive: false

docs/configuring-metabase/environment-variables.md

@@ -393,6 +393,15 @@ Timeout in minutes for databases query execution, both Metabase application data
 
 Whether or not the default GeoJSON maps are enabled.
 
+### `MB_DISABLE_CORS_ON_LOCALHOST`
+
+- Type: boolean
+- Default: `false`
+- [Exported as](../installation-and-operation/serialization.md): `disable-cors-on-localhost`.
+- [Configuration file name](./config-file.md): `disable-cors-on-localhost`
+
+Prevents the server from sending CORS headers for requests originating from localhost.
+
 ### `MB_DOWNLOAD_ROW_LIMIT`
 
 - Type: positive-integer
@@ -562,7 +571,7 @@ Allow these space delimited origins to embed Metabase interactive.
 > Only available on Metabase [Pro](https://www.metabase.com/product/pro) and [Enterprise](https://www.metabase.com/product/enterprise) plans.
 
 - Type: string
-- Default: `""`
+- Default: ``
 - [Configuration file name](./config-file.md): `embedding-app-origins-sdk`
 
 Allow Metabase SDK access to these space delimited origins.

docs/developers-guide/driver-changelog.md

@@ -15,10 +15,16 @@ title: Driver interface changelog
 - Added metabase.driver/compile-transform, metabase.driver/compile-drop-table, metabase.driver/execute-raw-queries!,
   metabase.driver/run-transform!, metabase.driver/drop-transform-target!, metabase.driver/native-query-deps,
   metabase.driver/connection-spec, metabase.driver/table-exists?, metabase.driver.sql/normalize-name,
-  metabase.driver.sql/default-schema, and metabase.driver.sql/find-table to implement sql transforms.
+  and metabase.driver.sql/default-schema to implement sql transforms.
 
 - Added the driver multi-methods `metabase.driver/schema-exists?` and `metabase.driver/create-schema-if-needed!` which should be implemented by drivers that support `:schemas` and `:transforms/table`.
 
+- Added `metabase.driver/type->database-type` multimethod that returns the database type for a given Metabase type (from the type hierarchy) as a HoneySQL spec. Unlike `upload-type->database-type` which is specific to upload types, this method handles general Metabase base types.
+
+- Added `metabase.driver/rename-table!` multimethod to rename a table from one name to another. Both table names may be qualified by schema (e.g., `schema.table`). This is currently used for transform operations and uploads.
+
+- Added `metabase.driver/insert-from-source!` multimethod that abstracts data insertion from various sources into existing tables. This multimethod dispatches on both the driver and the data source type (`:rows` or `:csv-file`). It allows drivers to optimize based on the data source type and returns the number of rows inserted. Default implementations are provided for both `:rows` and `:csv-file` source types.
+
 - `metabase.driver/humanize-connection-error-message` now takes a list of all messages in the exception chain,
   instead of just the top-level message as a string.
 

docs/installation-and-operation/development-instance.md

@@ -11,7 +11,7 @@ summary: Create development instances of Metabase for testing without paying per
 
 On Pro and Enterprise plans, you can purchase a development instance or development token as an add-on to your plan.
 
-- **Self-hosted**: you can purchase a dev token that you can use for one or more self-hosted development instances (meaning: you can reuse the same token for multiple dev instances at no additional cost). The development token will have access to the same set of paid features as your production token.
+- **Self-hosted**: you can purchase a dev token so you don't get billed for every test user acccount. You can use the same dev token for one or more self-hosted development instances (meaning: you can reuse the same token for multiple dev instances at no additional cost). The development token will have access to the same set of paid features as your production token.
 - **Cloud**: you can purchase a development instance hosted on [Metabase Cloud](https://www.metabase.com/cloud/). The development instance will have access to the same set of paid features as your production token.
 
 Dev instances are only available to Pro and Enterprise plans; they're not included in the base plans. Dev instances must be purchased as an add-on to your plan, after you've completed your trial and made your first payment.
@@ -22,6 +22,10 @@ That's the basic value proposition. Unlike production instances where you pay pe
 
 Once you're happy with the changes in your dev instance, you can [export your changes and import them](./serialization.md) into your production Metabase.
 
+## Even if you self-host, dev instance tokens can save you money
+
+While it's true that you can use your production token to test multiple Metabases when self-hosting, you'll still get charged for test user accounts in each of those Metabases. In that case, it might make sense to get a dev instance token so you're only charged a flat rate for as many dev instances as you like, with as many test users as you need.
+
 ## You can't use development instances in production
 
 The UI for development instances, including the UI for all its exports and notifications, is watermarked.

e2e/support/helpers/api/createSnippet.ts

@@ -1,12 +1,27 @@
-export function createSnippet({
-  name,
-  content,
-}: {
-  name: string;
+import type {
+  NativeQuerySnippet,
+  RegularCollectionId,
+} from "metabase-types/api";
+
+export type NativeQuerySnippetDetails = {
+  name?: string;
+  description?: string | null;
   content: string;
-}): Cypress.Chainable<Cypress.Response<NativeQuerySnippet>> {
-  cy.request("POST", "/api/native-query-snippet", {
+  collection_id?: RegularCollectionId | null;
+};
+
+export function createSnippet({
+  name = "Test snippet",
+  description = null,
+  content,
+  collection_id = null,
+}: NativeQuerySnippetDetails): Cypress.Chainable<
+  Cypress.Response<NativeQuerySnippet>
+> {
+  return cy.request("POST", "/api/native-query-snippet", {
     name,
+    description,
     content,
+    collection_id,
   });
 }

e2e/support/helpers/e2e-misc-helpers.js

@@ -281,6 +281,7 @@ export function interceptIfNotPreviouslyDefined({ method, url, alias } = {}) {
  * @param {boolean=} [options.addToDashboard]
  * @param {boolean=} [options.wrapId]
  * @param {string=} [options.idAlias]
+ * @param {Object=} [pickEntityOptions]
  */
 export function saveQuestion(
   name,

e2e/test/scenarios/admin-2/people.cy.spec.js

@@ -767,7 +767,7 @@ describe("scenarios > admin > people > group managers", () => {
       cy.url().should("match", /\/$/);
     });
 
-    it("can manage members from the people page", () => {
+    it("can manage members from the people page", { tags: "@flaky" }, () => {
       // Open membership select for a user
       // eslint-disable-next-line no-unscoped-text-selectors -- deprecated usage
       cy.findByText(noCollectionUserName)

e2e/test/scenarios/admin/transforms.cy.spec.ts

@@ -701,37 +701,43 @@ H.describeWithSnowplowEE("scenarios > admin > transforms", () => {
       assertTableDoesNotExistError();
     });
 
-    it("should be able to delete the target and restore the same target back", () => {
-      cy.log("create and run a transform");
-      createMbqlTransform({ visitTransform: true });
-      runTransformAndWaitForSuccess();
+    it(
+      "should be able to delete the target and restore the same target back",
+      { tags: "@flaky" },
+      () => {
+        cy.log("create and run a transform");
+        createMbqlTransform({ visitTransform: true });
+        runTransformAndWaitForSuccess();
 
-      cy.log("delete the old target without creating the new one");
-      getTransformPage().button("Change target").click();
-      H.modal().within(() => {
-        cy.findByLabelText("Table name").clear().type(TARGET_TABLE_2);
-        cy.findByLabelText("Delete transform_table").click();
-        cy.button("Change target and delete the old one").click();
-        cy.wait("@deleteTransformTable");
-        cy.wait("@updateTransform");
-      });
+        cy.log("delete the old target without creating the new one");
+        getTransformPage().button("Change target").click();
+        H.modal().within(() => {
+          cy.findByLabelText("Table name").clear().type(TARGET_TABLE_2);
+          cy.findByLabelText("Delete transform_table").click();
+          cy.button("Change target and delete the old one").click();
+          cy.wait("@deleteTransformTable");
+          cy.wait("@updateTransform");
+        });
 
-      cy.log("change the target back to the original one");
-      getTransformPage().button("Change target").click();
-      H.modal().within(() => {
-        cy.findByLabelText("Table name").clear().type(TARGET_TABLE);
-        cy.button("Change target").click();
-        cy.wait("@updateTransform");
-      });
+        cy.log("change the target back to the original one");
+        getTransformPage().button("Change target").click();
+        H.modal().within(() => {
+          cy.findByLabelText("Table name").clear().type(TARGET_TABLE);
+          cy.button("Change target").click();
+          cy.wait("@updateTransform");
+        });
 
-      cy.log("run the transform to re-create the original target");
-      runTransformAndWaitForSuccess();
+        cy.log("run the transform to re-create the original target");
+        runTransformAndWaitForSuccess();
 
-      cy.log("verify the target is available");
-      getTableLink().click();
-      H.queryBuilderHeader().findByText("Transform Table").should("be.visible");
-      H.assertQueryBuilderRowCount(3);
-    });
+        cy.log("verify the target is available");
+        getTableLink().click();
+        H.queryBuilderHeader()
+          .findByText("Transform Table")
+          .should("be.visible");
+        H.assertQueryBuilderRowCount(3);
+      },
+    );
 
     it("should not allow to overwrite an existing table when changing the target", () => {
       createMbqlTransform({ visitTransform: true });
@@ -2239,8 +2245,8 @@ function assertOptionSelected(name: string) {
 
 function assertOptionNotSelected(name: string) {
   cy.findByRole("option", { name }).should(
-    "not.have.attr",
+    "have.attr",
     "aria-selected",
-    "true",
+    "false",
   );
 }

e2e/test/scenarios/collections/uploads.cy.spec.js

@@ -15,80 +15,84 @@ H.describeWithSnowplow(
       cy.intercept("POST", "/api/table/*/replace-csv").as("replaceCSV");
     });
 
-    it("Can upload a CSV file to an empty postgres schema", () => {
-      const testFile = VALID_CSV_FILES[0];
-      const EMPTY_SCHEMA_NAME = "empty_uploads";
+    it(
+      "Can upload a CSV file to an empty postgres schema",
+      { tags: "@flaky" },
+      () => {
+        const testFile = VALID_CSV_FILES[0];
+        const EMPTY_SCHEMA_NAME = "empty_uploads";
 
-      cy.intercept("PUT", "/api/setting/*").as("saveSettings");
-      cy.intercept("GET", "/api/database").as("databaseList");
+        cy.intercept("PUT", "/api/setting/*").as("saveSettings");
+        cy.intercept("GET", "/api/database").as("databaseList");
 
-      H.restore("postgres-writable");
-      cy.signInAsAdmin();
+        H.restore("postgres-writable");
+        cy.signInAsAdmin();
 
-      H.queryWritableDB(
-        "DROP SCHEMA IF EXISTS empty_uploads CASCADE;",
-        "postgres",
-      );
-      H.queryWritableDB(
-        "CREATE SCHEMA IF NOT EXISTS empty_uploads;",
-        "postgres",
-      );
-
-      cy.request("POST", "/api/collection", {
-        name: "Uploads Collection",
-        parent_id: null,
-      }).then(({ body: { id: collectionId } }) => {
-        cy.wrap(collectionId).as("collectionId");
-      });
-      cy.visit("/admin/settings/uploads");
-
-      cy.findByLabelText("Upload Settings Form")
-        .findByPlaceholderText("Select a database")
-        .click();
-      H.popover().findByText("Writable Postgres12").click();
-      cy.findByLabelText("Upload Settings Form")
-        .findByPlaceholderText("Select a schema")
-        .click();
-
-      H.popover().findByText(EMPTY_SCHEMA_NAME).click();
-
-      cy.findByLabelText("Upload Settings Form")
-        .button("Enable uploads")
-        .click();
-
-      cy.wait(["@saveSettings", "@databaseList"]);
-
-      uploadFileToCollection(testFile);
-
-      const tableQuery = `SELECT * FROM information_schema.tables WHERE table_name LIKE '%${testFile.tableName}_%' ORDER BY table_name DESC LIMIT 1;`;
-
-      H.queryWritableDB(tableQuery, "postgres").then((result) => {
-        expect(result.rows.length).to.equal(1);
-        const tableName = result.rows[0].table_name;
         H.queryWritableDB(
-          `SELECT count(*) FROM ${EMPTY_SCHEMA_NAME}.${tableName};`,
+          "DROP SCHEMA IF EXISTS empty_uploads CASCADE;",
           "postgres",
-        ).then((result) => {
-          expect(Number(result.rows[0].count)).to.equal(testFile.rowCount);
+        );
+        H.queryWritableDB(
+          "CREATE SCHEMA IF NOT EXISTS empty_uploads;",
+          "postgres",
+        );
+
+        cy.request("POST", "/api/collection", {
+          name: "Uploads Collection",
+          parent_id: null,
+        }).then(({ body: { id: collectionId } }) => {
+          cy.wrap(collectionId).as("collectionId");
         });
-      });
+        cy.visit("/admin/settings/uploads");
 
-      cy.log(
-        "Ensure that table is visible in admin without refreshing (metabase#38041)",
-      );
+        cy.findByLabelText("Upload Settings Form")
+          .findByPlaceholderText("Select a database")
+          .click();
+        H.popover().findByText("Writable Postgres12").click();
+        cy.findByLabelText("Upload Settings Form")
+          .findByPlaceholderText("Select a schema")
+          .click();
 
-      cy.findByTestId("app-bar")
-        .findByRole("button", { name: "Settings" })
-        .click();
-      H.popover().findByText("Admin settings").click();
+        H.popover().findByText(EMPTY_SCHEMA_NAME).click();
 
-      cy.findByRole("link", { name: "Table Metadata" }).click();
+        cy.findByLabelText("Upload Settings Form")
+          .button("Enable uploads")
+          .click();
 
-      H.DataModel.TablePicker.getDatabase("Writable Postgres12").click();
-      H.DataModel.TablePicker.getSchema(EMPTY_SCHEMA_NAME).click();
-      H.DataModel.TablePicker.getTables().should("have.length", 1);
-      H.DataModel.TablePicker.getTable("Dog Breeds").should("be.visible");
-    });
+        cy.wait(["@saveSettings", "@databaseList"]);
+
+        uploadFileToCollection(testFile);
+
+        const tableQuery = `SELECT * FROM information_schema.tables WHERE table_name LIKE '%${testFile.tableName}_%' ORDER BY table_name DESC LIMIT 1;`;
+
+        H.queryWritableDB(tableQuery, "postgres").then((result) => {
+          expect(result.rows.length).to.equal(1);
+          const tableName = result.rows[0].table_name;
+          H.queryWritableDB(
+            `SELECT count(*) FROM ${EMPTY_SCHEMA_NAME}.${tableName};`,
+            "postgres",
+          ).then((result) => {
+            expect(Number(result.rows[0].count)).to.equal(testFile.rowCount);
+          });
+        });
+
+        cy.log(
+          "Ensure that table is visible in admin without refreshing (metabase#38041)",
+        );
+
+        cy.findByTestId("app-bar")
+          .findByRole("button", { name: "Settings" })
+          .click();
+        H.popover().findByText("Admin settings").click();
+
+        cy.findByRole("link", { name: "Table Metadata" }).click();
+
+        H.DataModel.TablePicker.getDatabase("Writable Postgres12").click();
+        H.DataModel.TablePicker.getSchema(EMPTY_SCHEMA_NAME).click();
+        H.DataModel.TablePicker.getTables().should("have.length", 1);
+        H.DataModel.TablePicker.getTable("Dog Breeds").should("be.visible");
+      },
+    );
 
     ["postgres", "mysql"].forEach((dialect) => {
       describe(`CSV Uploading (${dialect})`, () => {

e2e/test/scenarios/embedding/embedding-reproductions.cy.spec.js

@@ -957,20 +957,24 @@ describe("issue 40660", () => {
     });
   });
 
-  it("static dashboard content shouldn't overflow its container (metabase#40660)", () => {
-    H.openStaticEmbeddingModal({
-      activeTab: "parameters",
-      previewMode: "preview",
-    });
+  it(
+    "static dashboard content shouldn't overflow its container (metabase#40660)",
+    { tags: "@flaky" },
+    () => {
+      H.openStaticEmbeddingModal({
+        activeTab: "parameters",
+        previewMode: "preview",
+      });
 
-    H.getIframeBody().within(() => {
-      cy.findByTestId("embed-frame").scrollTo("bottom");
+      H.getIframeBody().within(() => {
+        cy.findByTestId("embed-frame").scrollTo("bottom");
 
-      cy.findByRole("link", { name: "Powered by Metabase" }).should(
-        "be.visible",
-      );
-    });
-  });
+        cy.findByRole("link", { name: "Powered by Metabase" }).should(
+          "be.visible",
+        );
+      });
+    },
+  );
 });
 
 // Skipped since it does not make sense when CSP is disabled

e2e/test/scenarios/models/models-list-view.cy.spec.js

@@ -123,6 +123,17 @@ describe("scenarios > models list view", () => {
         cy.findByText("2.07").should("not.exist");
       });
 
+      cy.log("Verify that empty column preview displays placeholder value");
+      cy.get("@rightColumns").within(() => {
+        cy.get("input").type("DISC");
+      });
+      H.popover().within(() => {
+        cy.findByText("DISCOUNT").click();
+      });
+      cy.get("@listPreview").within(() => {
+        cy.findByText("$123.46").should("be.visible");
+      });
+
       cy.log("Update list item icon");
       cy.get("@listPreview").within(() => {
         cy.findByRole("img").should("have.attr", "aria-label", "document icon");

e2e/test/scenarios/models/reproductions.cy.spec.ts

@@ -143,31 +143,35 @@ describe("issue 35711", () => {
     cy.signInAsAdmin();
   });
 
-  it("can edit metadata of a model with a custom column (metabase#35711)", () => {
-    H.createQuestion(
-      {
-        type: "model",
-        query: {
-          "source-table": ORDERS_ID,
-          expressions: {
-            "Custom column": ["-", DISCOUNT_FIELD_REF, 1],
+  it(
+    "can edit metadata of a model with a custom column (metabase#35711)",
+    { tags: "@flaky" },
+    () => {
+      H.createQuestion(
+        {
+          type: "model",
+          query: {
+            "source-table": ORDERS_ID,
+            expressions: {
+              "Custom column": ["-", DISCOUNT_FIELD_REF, 1],
+            },
+            limit: 5, // optimization
           },
-          limit: 5, // optimization
         },
-      },
-      { visitQuestion: true },
-    );
+        { visitQuestion: true },
+      );
 
-    H.openQuestionActions();
-    H.popover().findByText("Edit metadata").click();
-    H.waitForLoaderToBeRemoved();
+      H.openQuestionActions();
+      H.popover().findByText("Edit metadata").click();
+      H.waitForLoaderToBeRemoved();
 
-    reorderTaxAndTotalColumns();
-    assertNoError();
+      reorderTaxAndTotalColumns();
+      assertNoError();
 
-    cy.findByTestId("editor-tabs-query-name").click();
-    assertNoError();
-  });
+      cy.findByTestId("editor-tabs-query-name").click();
+      assertNoError();
+    },
+  );
 });
 
 describe("issues 25884 and 34349", () => {
@@ -977,20 +981,26 @@ describe("issue 39993", () => {
     cy.intercept("PUT", "/api/card/*").as("updateModel");
   });
 
-  it("should preserve viz settings for models with custom expressions (metabase#39993)", () => {
-    H.createQuestion(modelDetails).then(({ body: card }) =>
-      H.visitModel(card.id),
-    );
-    H.openQuestionActions();
-    H.popover().findByText("Edit metadata").click();
-    H.waitForLoaderToBeRemoved();
-    cy.log("drag & drop the custom column 100 px to the left");
-    H.moveDnDKitElement(H.tableHeaderColumn(columnName), { horizontal: -100 });
-    cy.button("Save changes").click();
-    cy.wait("@updateModel");
-    cy.findAllByTestId("header-cell").eq(0).should("have.text", "Exp");
-    cy.findAllByTestId("header-cell").eq(1).should("have.text", "ID");
-  });
+  it(
+    "should preserve viz settings for models with custom expressions (metabase#39993)",
+    { tags: "@flaky" },
+    () => {
+      H.createQuestion(modelDetails).then(({ body: card }) =>
+        H.visitModel(card.id),
+      );
+      H.openQuestionActions();
+      H.popover().findByText("Edit metadata").click();
+      H.waitForLoaderToBeRemoved();
+      cy.log("drag & drop the custom column 100 px to the left");
+      H.moveDnDKitElement(H.tableHeaderColumn(columnName), {
+        horizontal: -100,
+      });
+      cy.button("Save changes").click();
+      cy.wait("@updateModel");
+      cy.findAllByTestId("header-cell").eq(0).should("have.text", "Exp");
+      cy.findAllByTestId("header-cell").eq(1).should("have.text", "ID");
+    },
+  );
 });
 
 describe("issue 34574", () => {
@@ -1154,36 +1164,40 @@ describe("issue 36161", () => {
     cy.intercept("POST", "/api/dataset").as("dataset");
   });
 
-  it("should allow to override metadata for custom columns (metabase#36161)", () => {
-    H.visitModel(ORDERS_MODEL_ID);
-    cy.wait("@dataset");
+  it(
+    "should allow to override metadata for custom columns (metabase#36161)",
+    { tags: "@flaky" },
+    () => {
+      H.visitModel(ORDERS_MODEL_ID);
+      cy.wait("@dataset");
 
-    H.openQuestionActions("Edit query definition");
-    H.getNotebookStep("data").button("Pick columns").click();
-    H.popover().findByText("Select all").click();
-    H.getNotebookStep("data").button("Custom column").click();
-    H.enterCustomColumnDetails({ formula: "[ID]", name: "ID2" });
-    H.popover().button("Done").click();
-    H.getNotebookStep("expression").icon("add").click();
-    H.enterCustomColumnDetails({ formula: "[ID]", name: "ID3" });
-    H.popover().button("Done").click();
-    H.runButtonOverlay().click();
-    cy.wait("@dataset");
-    cy.findByTestId("editor-tabs-columns-name").click();
-    H.openColumnOptions("ID2");
-    H.renameColumn("ID2", "ID2 custom");
-    H.openColumnOptions("ID3");
-    H.renameColumn("ID3", "ID3 custom");
-    H.saveMetadataChanges();
+      H.openQuestionActions("Edit query definition");
+      H.getNotebookStep("data").button("Pick columns").click();
+      H.popover().findByText("Select all").click();
+      H.getNotebookStep("data").button("Custom column").click();
+      H.enterCustomColumnDetails({ formula: "[ID]", name: "ID2" });
+      H.popover().button("Done").click();
+      H.getNotebookStep("expression").icon("add").click();
+      H.enterCustomColumnDetails({ formula: "[ID]", name: "ID3" });
+      H.popover().button("Done").click();
+      H.runButtonOverlay().click();
+      cy.wait("@dataset");
+      cy.findByTestId("editor-tabs-columns-name").click();
+      H.openColumnOptions("ID2");
+      H.renameColumn("ID2", "ID2 custom");
+      H.openColumnOptions("ID3");
+      H.renameColumn("ID3", "ID3 custom");
+      H.saveMetadataChanges();
 
-    H.openNotebook();
-    H.getNotebookStep("data").button("Filter").click();
-    H.popover().within(() => {
-      cy.findByText("ID").should("be.visible");
-      cy.findByText("ID2 custom").should("be.visible");
-      cy.findByText("ID3 custom").should("be.visible");
-    });
-  });
+      H.openNotebook();
+      H.getNotebookStep("data").button("Filter").click();
+      H.popover().within(() => {
+        cy.findByText("ID").should("be.visible");
+        cy.findByText("ID2 custom").should("be.visible");
+        cy.findByText("ID3 custom").should("be.visible");
+      });
+    },
+  );
 });
 
 describe("issue 34514", () => {

e2e/test/scenarios/native/snippet-tags.cy.spec.ts

@@ -0,0 +1,404 @@
+const { H } = cy;
+
+import { ORDERS_QUESTION_ID } from "e2e/support/cypress_sample_instance_data";
+
+describe("scenarios > native > snippet tags", () => {
+  beforeEach(() => {
+    H.restore();
+    cy.signInAsAdmin();
+  });
+
+  it("should be able to create a snippet with variable tags", () => {
+    H.startNewNativeQuestion();
+    H.NativeEditor.type("select id from products where ");
+
+    cy.log("create a snippet");
+    getEditorTopBar().icon("snippet").click();
+    getEditorSidebar().findByText("Create snippet").click();
+    H.modal().within(() => {
+      getSnippetContentInput().type("category = {{category}}", {
+        parseSpecialCharSequences: false,
+      });
+      getSnippetNameInput().type("variable-snippet");
+      cy.button("Save").click();
+    });
+
+    cy.log("verify that the snippet was inserted");
+    H.NativeEditor.get().should(
+      "contain",
+      "select id from products where {{snippet: variable-snippet}}",
+    );
+
+    cy.log("verify that the query can be run");
+    getEditorTopBar().findByPlaceholderText("Category").type("Widget");
+    H.runNativeQuery();
+    H.assertQueryBuilderRowCount(54);
+  });
+
+  it("should be able to create a snippet with card tags", () => {
+    H.startNewNativeQuestion();
+    H.NativeEditor.type("select * from ");
+
+    cy.log("create a snippet");
+    getEditorTopBar().icon("snippet").click();
+    getEditorSidebar().findByText("Create snippet").click();
+    H.modal().within(() => {
+      getSnippetContentInput().type(`{{#${ORDERS_QUESTION_ID}}}`, {
+        parseSpecialCharSequences: false,
+      });
+      getSnippetNameInput().type("card-snippet");
+      cy.button("Save").click();
+    });
+
+    cy.log("verify that the snippet was inserted");
+    H.NativeEditor.get().should(
+      "contain",
+      "select * from {{snippet: card-snippet}}",
+    );
+
+    cy.log("verify that the query can be run");
+    H.runNativeQuery();
+    H.tableInteractive().should("be.visible");
+  });
+
+  it("should be able to create a snippet with snippet tags", () => {
+    H.startNewNativeQuestion();
+    H.NativeEditor.type("select * from ");
+
+    cy.log("create a snippet");
+    getEditorTopBar().icon("snippet").click();
+    getEditorSidebar().findByText("Create snippet").click();
+    H.modal().within(() => {
+      getSnippetContentInput().type("category = {{category}}", {
+        parseSpecialCharSequences: false,
+      });
+      getSnippetNameInput().type("snippet1");
+      cy.button("Save").click();
+    });
+
+    cy.log("create a snippet that uses the previous snippet");
+    getEditorTopBar().icon("snippet").click();
+    getEditorSidebar().icon("add").click();
+    H.popover().findByText("New snippet").click();
+    H.modal().within(() => {
+      getSnippetContentInput().type("{{snippet: snippet1}}", {
+        parseSpecialCharSequences: false,
+      });
+      getSnippetNameInput().type("snippet2");
+      cy.button("Save").click();
+    });
+
+    cy.log("verify that the snippet can used");
+    H.NativeEditor.clear();
+    H.NativeEditor.type("select id from products where {{snippet: snippet2}}");
+    getEditorTopBar().findByLabelText("Category").type("Gizmo");
+    H.runNativeQuery();
+    H.tableInteractive().should("be.visible");
+  });
+
+  it("should be able to create a tag with the same name as the inner snippet tag", () => {
+    createQuestionAndSnippet().then(({ card }) => {
+      H.visitQuestion(card.id);
+    });
+
+    cy.log("add a local tag with the same name");
+    getEditorVisibilityToggler().click();
+    H.NativeEditor.type(" and category = {{filter}}");
+
+    cy.log("verify that the query can be run");
+    getEditorTopBar().findByPlaceholderText("Filter").type("Widget");
+    H.runNativeQuery();
+    H.assertQueryBuilderRowCount(54);
+  });
+
+  it("should be able to use snippets where tags overlap", () => {
+    cy.log("create snippets with overlapping tags");
+    H.createSnippet({
+      name: "snippet1",
+      content: "category = {{category1}} or category = {{category2}}",
+    });
+    H.createSnippet({
+      name: "snippet2",
+      content: "category = {{category2}}",
+    });
+
+    cy.log("use both snippets in a query");
+    H.startNewNativeQuestion();
+    H.NativeEditor.type(
+      "select id from products where {{snippet: snippet1}} or {{snippet: snippet2}}",
+    );
+
+    cy.log("verify that the query can be run");
+    getEditorTopBar().findByPlaceholderText("Category1").type("Widget");
+    getEditorTopBar().findByPlaceholderText("Category2").type("Gadget");
+    H.runNativeQuery();
+    H.assertQueryBuilderRowCount(107);
+  });
+
+  it("should be able to update a snippet and change tags", () => {
+    createQuestionAndSnippet().then(({ card }) => {
+      H.visitQuestion(card.id);
+    });
+
+    cy.log("update the snippet");
+    getEditorVisibilityToggler().click();
+    getEditorTopBar().icon("snippet").click();
+    getEditorSidebar().within(() => {
+      cy.icon("chevrondown").click({ force: true });
+      cy.button(/Edit/).click();
+    });
+    H.modal().within(() => {
+      getSnippetContentInput()
+        .clear()
+        .type("ean = {{ean}} or vendor = {{vendor}}", {
+          parseSpecialCharSequences: false,
+        });
+      cy.button("Save").click();
+    });
+
+    cy.log("verify that the tags in the query were updated");
+    getEditorTopBar().within(() => {
+      cy.findByPlaceholderText("Filter").should("not.exist");
+      cy.findByPlaceholderText("Ean").type("1018947080336");
+      cy.findByPlaceholderText("Vendor").type("Balistreri-Ankunding");
+    });
+    H.runNativeQuery();
+    H.assertTableRowsCount(2);
+  });
+
+  it("should be able to change the inner tag type to Number", () => {
+    createQuestionAndSnippet({ snippetContent: "ID = {{filter}}" }).then(
+      ({ card }) => {
+        H.visitQuestion(card.id);
+      },
+    );
+
+    cy.log("change the type");
+    getEditorVisibilityToggler().click();
+    getEditorTopBar().icon("variable").click();
+    getVariableTypeSelect().click();
+    H.popover().findByText("Number").click();
+
+    cy.log("verify that the parameter can be used");
+    getEditorTopBar().findByPlaceholderText("Filter").type("10");
+    H.runNativeQuery();
+    H.assertQueryBuilderRowCount(1);
+  });
+
+  it("should be able to change the inner tag type to Field Filter", () => {
+    createQuestionAndSnippet({ snippetContent: "{{filter}}" }).then(
+      ({ card }) => {
+        H.visitQuestion(card.id);
+      },
+    );
+
+    cy.log("change the type");
+    getEditorVisibilityToggler().click();
+    getEditorTopBar().icon("variable").click();
+    getVariableTypeSelect().click();
+    H.popover().findByText("Field Filter").click();
+    H.popover().findByText("Products").click();
+    H.popover().findByText("Category").click();
+
+    cy.log("verify that the parameter can be used");
+    H.filterWidget().click();
+    H.popover().within(() => {
+      cy.findByText("Gadget").click();
+      cy.button("Add filter").click();
+    });
+    H.runNativeQuery();
+    H.assertQueryBuilderRowCount(53);
+  });
+
+  it("should be able to open a question with an old snippet name", () => {
+    createQuestionAndSnippet().then(({ card, snippet }) => {
+      cy.request("PUT", `/api/native-query-snippet/${snippet.id}`, {
+        name: "category-snippet",
+      });
+      cy.visit(`/question/${card.id}`);
+    });
+
+    cy.log("verify that the parameter can be used");
+    getEditorTopBar().findByPlaceholderText("Filter").type("Widget");
+    H.queryBuilderHeader().icon("play").click();
+    H.assertQueryBuilderRowCount(54);
+  });
+
+  it("should not be able to create card cycles via snippets", () => {
+    cy.log("create a card and a snippet referencing this card");
+    H.createNativeQuestion({
+      native: {
+        query: "select 1",
+      },
+    }).then(({ body: card }) => {
+      H.createSnippet({
+        name: "cycle-snippet",
+        content: `{{#${card.id}}}`,
+      });
+      H.visitQuestion(card.id);
+    });
+
+    cy.log("create a cycle and try to save the card");
+    getEditorVisibilityToggler().click();
+    H.NativeEditor.clear();
+    H.NativeEditor.type("select * from {{snippet: cycle-snippet}}");
+    H.queryBuilderHeader().button("Save").click();
+    H.modal().within(() => {
+      cy.button("Save").click();
+      cy.findByText("Cannot save card with cycles.").should("be.visible");
+    });
+  });
+
+  it("should not be able to create snippet cycles via snippets", () => {
+    cy.log("create two snippets, one using another");
+    H.createSnippet({
+      name: "snippet1",
+      content: "1",
+    });
+    H.createSnippet({
+      name: "snippet2",
+      content: "{{snippet: snippet1}}",
+    });
+
+    cy.log("open the modal for the first snippet");
+    H.startNewNativeQuestion();
+    getEditorTopBar().icon("snippet").click();
+    getEditorSidebar().within(() => {
+      cy.icon("chevrondown")
+        .should("have.length", 2)
+        .first()
+        .click({ force: true });
+      cy.button(/Edit/).click();
+    });
+
+    cy.log("create a cycle and save the first snippet");
+    H.modal().within(() => {
+      getSnippetContentInput()
+        .clear()
+        .type("{{snippet: snippet2}}", { parseSpecialCharSequences: false })
+        .click();
+      cy.button("Save").click();
+    });
+
+    cy.log("run the query and verify that we detect a cycle");
+    H.NativeEditor.type("select {{snippet: snippet1}}");
+    H.runNativeQuery();
+    H.queryBuilderMain()
+      .findByText(
+        'This query has circular referencing sub-queries. The snippet "snippet2" and the snippet "snippet1" seem to be part of the problem.',
+      )
+      .should("be.visible");
+
+    cy.log(
+      "try to save a question with a snippet cycle and verify we detect it",
+    );
+    H.queryBuilderHeader().findByText("Save").click();
+    H.modal().within(() => {
+      cy.findByLabelText("Name").type("SQL");
+      cy.button("Save").click();
+      cy.findByText("Cannot save card with cycles.").should("be.visible");
+      cy.button("Cancel").click();
+    });
+
+    cy.log(
+      "remove a cycle, save the card, add a cycle again and verify that we detect it on update",
+    );
+    H.NativeEditor.clear().type("select 1");
+    H.saveQuestion("SQL", undefined, {
+      tab: "Browse",
+      path: ["Our analytics"],
+    });
+    H.NativeEditor.clear().type("select {{snippet: snippet1}}");
+    H.queryBuilderHeader().button("Save").click();
+    H.modal().within(() => {
+      cy.button("Save").click();
+      cy.findByText("Cannot save card with cycles.").should("be.visible");
+    });
+  });
+
+  it("should work for a public or embedded question", () => {
+    createQuestionAndSnippet().then(({ card }) => {
+      cy.log("public question - no parameter value");
+      H.visitPublicQuestion(card.id);
+      H.assertTableRowsCount(200);
+
+      cy.log("public question - with parameter value");
+      H.filterWidget().findByLabelText("Filter").type("Gadget{enter}");
+      H.assertTableRowsCount(53);
+
+      cy.log("embedded question - no parameter value");
+      H.visitEmbeddedPage({
+        resource: { question: card.id },
+        params: {},
+      });
+      H.assertTableRowsCount(200);
+
+      cy.log("embedded question - with parameter value");
+      H.filterWidget().findByLabelText("Filter").type("Gadget{enter}");
+      H.assertTableRowsCount(53);
+    });
+  });
+});
+
+function getEditorSidebar() {
+  return cy.findByTestId("sidebar-right");
+}
+
+function getEditorTopBar() {
+  return cy.findByTestId("native-query-top-bar");
+}
+
+function getEditorVisibilityToggler() {
+  return cy.findByTestId("visibility-toggler");
+}
+
+function getVariableTypeSelect() {
+  return cy.findByTestId("variable-type-select");
+}
+
+function getSnippetNameInput() {
+  return cy.findByLabelText("Give your snippet a name");
+}
+
+function getSnippetContentInput() {
+  return cy.findByLabelText("Enter some SQL here so you can reuse it later");
+}
+
+function createQuestionAndSnippet({
+  snippetContent = "category = {{filter}}",
+}: {
+  snippetContent?: string;
+} = {}) {
+  return H.createSnippet({
+    name: "filter-snippet",
+    content: snippetContent,
+  }).then(({ body: snippet }) => {
+    return H.createNativeQuestion({
+      native: {
+        query: "select id from products [[where {{snippet: filter-snippet}}]]",
+        "template-tags": {
+          "snippet: filter-snippet": {
+            id: "4b77cc1f-ea70-4ef6-84db-58432fce6928",
+            name: "snippet: filter-snippet",
+            "display-name": "snippet: filter-snippet",
+            type: "snippet",
+            "snippet-id": snippet.id,
+            "snippet-name": snippet.name,
+          },
+          filter: {
+            id: "4b77cc1f-ea70-4ef6-84db-58432fce6929",
+            name: "filter",
+            "display-name": "Filter",
+            type: "text",
+          },
+        },
+      },
+      enable_embedding: true,
+      embedding_params: {
+        filter: "enabled",
+      },
+    }).then(({ body: card }) => {
+      return { card, snippet };
+    });
+  });
+}

e2e/test/scenarios/onboarding/setup/setup-embedding.cy.spec.ts

@@ -75,195 +75,201 @@ H.describeWithSnowplowEE("scenarios > setup embedding (EMB-477)", () => {
       .should("be.visible");
   });
 
-  it("[cloud-hosted] should allow users to go through the embedding setup and onboarding flow", () => {
-    // mock `is-hosted`, we can't set the token before the user is created, we'll set it right after that
-    H.activateToken("pro-cloud");
+  it(
+    "[cloud-hosted] should allow users to go through the embedding setup and onboarding flow",
+    { tags: "@flaky" },
+    () => {
+      // mock `is-hosted`, we can't set the token before the user is created, we'll set it right after that
+      H.activateToken("pro-cloud");
 
-    mockCloudHosted();
+      mockCloudHosted();
 
-    cy.visit(
-      "/setup/embedding?first_name=Firstname&last_name=Lastname&email=testy@metabase.test&site_name=Epic Team",
-    );
-
-    cy.log("0: Welcome step");
-    assertEmbeddingOnboardingPageLoaded();
-
-    H.expectUnstructuredSnowplowEvent({
-      event: "embedding_setup_step_seen",
-      event_detail: "welcome",
-    });
-
-    step().within(() => {
-      cy.findByText(
-        /Let's get you up and running with a starting setup for embedded analytics/,
-      ).should("be.visible");
-      cy.button("Start").should("be.visible").click();
-    });
-
-    cy.log("1: User creation step");
-    sidebar().within(() => {
-      cy.findByRole("listitem", { current: "step" }).should(
-        "have.text",
-        "Set up your account",
-      );
-    });
-    step().within(() => {
-      cy.findByRole("heading", { name: "What should we call you?" }).should(
-        "be.visible",
+      cy.visit(
+        "/setup/embedding?first_name=Firstname&last_name=Lastname&email=testy@metabase.test&site_name=Epic Team",
       );
 
+      cy.log("0: Welcome step");
+      assertEmbeddingOnboardingPageLoaded();
+
       H.expectUnstructuredSnowplowEvent({
         event: "embedding_setup_step_seen",
-        event_detail: "user-creation",
+        event_detail: "welcome",
       });
 
-      cy.findByLabelText("First name").should("have.value", "Firstname");
-      cy.findByLabelText("Last name").should("have.value", "Lastname");
-      cy.findByLabelText("Email").should("have.value", "testy@metabase.test");
-      cy.findByLabelText("Company or team name").should(
-        "have.value",
-        "Epic Team",
-      );
+      step().within(() => {
+        cy.findByText(
+          /Let's get you up and running with a starting setup for embedded analytics/,
+        ).should("be.visible");
+        cy.button("Start").should("be.visible").click();
+      });
 
-      fillOutUserForm();
+      cy.log("1: User creation step");
+      sidebar().within(() => {
+        cy.findByRole("listitem", { current: "step" }).should(
+          "have.text",
+          "Set up your account",
+        );
+      });
+      step().within(() => {
+        cy.findByRole("heading", { name: "What should we call you?" }).should(
+          "be.visible",
+        );
 
-      cy.intercept("POST", "/api/setup").as("setup");
-      cy.button("Next").should("be.enabled").click();
-    });
-
-    cy.log(
-      "Now we have a user we simulate being on cloud by setting a Metatabase Token",
-    );
-    cy.wait("@setup");
-
-    // set a real token so we have real token-features for the rest of the flow
-    H.activateToken("pro-cloud");
-
-    cy.log("2: Data connection step");
-    sidebar().within(() => {
-      cy.findByRole("listitem", { current: "step" }).should(
-        "have.text",
-        "Connect to your data",
-      );
-    });
-    step().within(() => {
-      cy.findByRole("heading", { name: "Connect to your data" }).should(
-        "be.visible",
-      );
-
-      cy.findByRole("option", { name: "PostgreSQL" }).click();
-      fillOutDatabaseForm();
-
-      cy.log("simulate database that takes some time to fully sync");
-      const NO_OF_MIN_RETRY = 4;
-      let currentRetry = 0;
-      cy.intercept("GET", "api/database", (req) => {
-        req.on("response", (res) => {
-          currentRetry++;
-          const [postgresDb] = res.body.data;
-          if (
-            postgresDb.initial_sync_status !== "complete" ||
-            currentRetry < NO_OF_MIN_RETRY
-          ) {
-            // Simulate syncing not complete until we reach a certain number of retries
-            postgresDb.initial_sync_status = "incomplete";
-          }
+        H.expectUnstructuredSnowplowEvent({
+          event: "embedding_setup_step_seen",
+          event_detail: "user-creation",
         });
+
+        cy.findByLabelText("First name").should("have.value", "Firstname");
+        cy.findByLabelText("Last name").should("have.value", "Lastname");
+        cy.findByLabelText("Email").should("have.value", "testy@metabase.test");
+        cy.findByLabelText("Company or team name").should(
+          "have.value",
+          "Epic Team",
+        );
+
+        fillOutUserForm();
+
+        cy.intercept("POST", "/api/setup").as("setup");
+        cy.button("Next").should("be.enabled").click();
       });
-      cy.button("Connect database").should("be.enabled").click();
-    });
 
-    cy.log("3: Content generation step");
-    sidebar().within(() => {
-      cy.findByRole("listitem", { current: "step" }).should(
-        "have.text",
-        "Generate starter content",
+      cy.log(
+        "Now we have a user we simulate being on cloud by setting a Metatabase Token",
       );
-    });
+      cy.wait("@setup");
 
-    const SECOND = 1000;
-    // Waiting for the database sync to complete
-    step()
-      .findByRole("heading", {
-        name: "Select Tables to Embed",
-        timeout: 10 * SECOND,
-      })
-      .should("be.visible");
+      // set a real token so we have real token-features for the rest of the flow
+      H.activateToken("pro-cloud");
 
-    expectNoDatabaseStatus();
-
-    step().within(() => {
-      cy.findByRole("checkbox", { name: "feedback" })
-        .should("be.enabled")
-        .check();
-      cy.findByRole("checkbox", { name: "orders" })
-        .should("be.enabled")
-        .check();
-      cy.findByRole("checkbox", { name: "products" })
-        .should("be.enabled")
-        .check();
-
-      cy.button("Continue").should("be.enabled").click();
-    });
-
-    cy.log("4: Processing step");
-
-    step().within(() => {
-      cy.findByRole("heading", { name: "Setting Up Your Analytics" }).should(
-        "be.visible",
-      );
-      /**
-       * Since this loader will go away automatically it's crucial to wait for it to go away
-       * before start asserting the next steps.
-       */
-      cy.findByText("Creating dashboards...").should("not.exist");
-    });
-
-    cy.log("5: Embed in your app step");
-    sidebar().within(() => {
-      cy.findByRole("listitem", { current: "step" }).should(
-        "have.text",
-        "Add to your app",
-      );
-    });
-
-    step().within(() => {
-      cy.findByRole("heading", { name: "Add to your app" }).should(
-        "be.visible",
-      );
-      cy.findByRole("tab", { name: 'A look at "Feedback"' }).click();
-      H.getIframeBody().within(() => {
-        cy.findByText('A look at "Feedback"').should("be.visible");
+      cy.log("2: Data connection step");
+      sidebar().within(() => {
+        cy.findByRole("listitem", { current: "step" }).should(
+          "have.text",
+          "Connect to your data",
+        );
       });
-      cy.findByRole("code").contains(
-        new RegExp(
-          `<iframe src="${Cypress.config("baseUrl")}/dashboard/\\d+" width="800px" height="500px" />`,
-        ),
+      step().within(() => {
+        cy.findByRole("heading", { name: "Connect to your data" }).should(
+          "be.visible",
+        );
+
+        cy.findByRole("option", { name: "PostgreSQL" }).click();
+        fillOutDatabaseForm();
+
+        cy.log("simulate database that takes some time to fully sync");
+        const NO_OF_MIN_RETRY = 4;
+        let currentRetry = 0;
+        cy.intercept("GET", "api/database", (req) => {
+          req.on("response", (res) => {
+            currentRetry++;
+            const [postgresDb] = res.body.data;
+            if (
+              postgresDb.initial_sync_status !== "complete" ||
+              currentRetry < NO_OF_MIN_RETRY
+            ) {
+              // Simulate syncing not complete until we reach a certain number of retries
+              postgresDb.initial_sync_status = "incomplete";
+            }
+          });
+        });
+        cy.button("Connect database").should("be.enabled").click();
+      });
+
+      cy.log("3: Content generation step");
+      sidebar().within(() => {
+        cy.findByRole("listitem", { current: "step" }).should(
+          "have.text",
+          "Generate starter content",
+        );
+      });
+
+      const SECOND = 1000;
+      // Waiting for the database sync to complete
+      step()
+        .findByRole("heading", {
+          name: "Select Tables to Embed",
+          timeout: 10 * SECOND,
+        })
+        .should("be.visible");
+
+      expectNoDatabaseStatus();
+
+      step().within(() => {
+        cy.findByRole("checkbox", { name: "feedback" })
+          .should("be.enabled")
+          .check();
+        cy.findByRole("checkbox", { name: "orders" })
+          .should("be.enabled")
+          .check();
+        cy.findByRole("checkbox", { name: "products" })
+          .should("be.enabled")
+          .check();
+
+        cy.button("Continue").should("be.enabled").click();
+      });
+
+      cy.log("4: Processing step");
+
+      step().within(() => {
+        cy.findByRole("heading", { name: "Setting Up Your Analytics" }).should(
+          "be.visible",
+        );
+        /**
+         * Since this loader will go away automatically it's crucial to wait for it to go away
+         * before start asserting the next steps.
+         */
+        cy.findByText("Creating dashboards...").should("not.exist");
+      });
+
+      cy.log("5: Embed in your app step");
+      sidebar().within(() => {
+        cy.findByRole("listitem", { current: "step" }).should(
+          "have.text",
+          "Add to your app",
+        );
+      });
+
+      step().within(() => {
+        cy.findByRole("heading", { name: "Add to your app" }).should(
+          "be.visible",
+        );
+        cy.findByRole("tab", { name: 'A look at "Feedback"' }).click();
+        H.getIframeBody().within(() => {
+          cy.findByText('A look at "Feedback"').should("be.visible");
+        });
+        cy.findByRole("code").contains(
+          new RegExp(
+            `<iframe src="${Cypress.config("baseUrl")}/dashboard/\\d+" width="800px" height="500px" />`,
+          ),
+        );
+
+        cy.button("I see Metabase").click();
+
+        cy.findByRole("heading", { name: "You're on your way!" }).should(
+          "be.visible",
+        );
+        cy.findByRole("link", { name: "Take me to Metabase" }).click();
+      });
+
+      cy.log("5: Metabase");
+      cy.button("New").should("be.visible");
+
+      cy.log(
+        "Only one 'Automatically Generated Dashboards' collection should be created",
       );
-
-      cy.button("I see Metabase").click();
-
-      cy.findByRole("heading", { name: "You're on your way!" }).should(
-        "be.visible",
-      );
-      cy.findByRole("link", { name: "Take me to Metabase" }).click();
-    });
-
-    cy.log("5: Metabase");
-    cy.button("New").should("be.visible");
-
-    cy.log(
-      "Only one 'Automatically Generated Dashboards' collection should be created",
-    );
-    sidebar().within(() => {
-      cy.findByText("Automatically Generated Dashboards").should("be.visible");
-      cy.findByText("Automatically Generated Dashboards").should(
-        "have.length",
-        1,
-      );
-    });
-    expectNoDatabaseStatus();
-  });
+      sidebar().within(() => {
+        cy.findByText("Automatically Generated Dashboards").should(
+          "be.visible",
+        );
+        cy.findByText("Automatically Generated Dashboards").should(
+          "have.length",
+          1,
+        );
+      });
+      expectNoDatabaseStatus();
+    },
+  );
 });
 
 /**

e2e/test/scenarios/permissions/admin-permissions.cy.spec.js

@@ -310,7 +310,7 @@ describe("scenarios > admin > permissions", { tags: "@OSS" }, () => {
     });
 
     context("group focused view", () => {
-      it("shows filterable list of groups", () => {
+      it("shows filterable list of groups", { tags: "@flaky" }, () => {
         cy.visit("/admin/permissions");
 
         // no groups selected initially and it shows an empty state

e2e/test/scenarios/permissions/sandboxing/sandboxing-via-ui.cy.spec.ts

@@ -78,12 +78,16 @@ describe(
       H.restore("sandboxing-snapshot" as any);
     });
 
-    it("shows all data before sandboxing policy is applied", () => {
-      signInAs(gizmoViewer);
-      assertNoResultsOrValuesAreSandboxed(dashboard, sandboxableQuestions);
-      signInAs(widgetViewer);
-      assertNoResultsOrValuesAreSandboxed(dashboard, sandboxableQuestions);
-    });
+    it(
+      "shows all data before sandboxing policy is applied",
+      { tags: "@flaky" },
+      () => {
+        signInAs(gizmoViewer);
+        assertNoResultsOrValuesAreSandboxed(dashboard, sandboxableQuestions);
+        signInAs(widgetViewer);
+        assertNoResultsOrValuesAreSandboxed(dashboard, sandboxableQuestions);
+      },
+    );
 
     describe("we can apply a sandbox policy", () => {
       beforeEach(() => {

e2e/test/scenarios/question-reproductions/reproductions-3.cy.spec.js

@@ -1955,7 +1955,7 @@ describe("issue 45063", { tags: "@flaky" }, () => {
       });
     });
 
-    it("should work with models", () => {
+    it("should work with models", { tags: "@flaky" }, () => {
       createGuiModel({ sourceTableId: ORDERS_ID });
       verifyRemappedFilter({
         visitCard: () => cy.get("@modelId").then(H.visitModel),

e2e/test/scenarios/question-reproductions/reproductions.cy.spec.ts

@@ -1202,3 +1202,91 @@ describe("issue 52872", () => {
       });
   });
 });
+
+describe("issue 35561", () => {
+  beforeEach(() => {
+    H.restore();
+    cy.signInAsAdmin();
+  });
+
+  it("should display a functional fk drill for both null and non-null values", () => {
+    H.createNativeQuestion(
+      {
+        name: "Model",
+        native: {
+          query: `SELECT 1 AS ID, CAST(NULL AS INT) AS USER_ID
+UNION ALL
+SELECT 2 AS ID, 3 AS USER_ID
+`,
+        },
+        type: "model",
+      },
+      {},
+    ).then(async ({ body: model }) => {
+      await cy.request("POST", `/api/card/${model.id}/query`);
+      await H.setModelMetadata(model.id, (field) => {
+        if (field.name === "USER_ID") {
+          return {
+            ...field,
+            display_name: "User ID",
+            semantic_type: "type/FK",
+            fk_target_field_id: ORDERS.ID,
+          };
+        }
+        return field;
+      });
+      H.createQuestion(
+        {
+          name: "Question",
+          query: {
+            "source-table": `card__${model.id}`,
+          },
+          display: "table",
+        },
+        { wrapId: true, visitQuestion: true },
+      );
+    });
+
+    cy.findByTestId("table-root")
+      .findAllByRole("row")
+      .first()
+      .within(() => {
+        cy.get("[data-column-id='USER_ID']").click();
+      });
+    H.popover().within(() => {
+      cy.findByText("View Models with no User").should("be.visible").click();
+    });
+    cy.findByTestId("table-body").within(() => {
+      cy.findAllByRole("row").should("have.length", 1);
+      cy.findAllByRole("row")
+        .first()
+        .within(() => {
+          cy.get("[data-column-id='USER_ID']").within(() => {
+            cy.get("[data-testid=cell-data]").should("not.exist");
+          });
+        });
+    });
+
+    H.visitQuestion("@questionId");
+
+    cy.findByTestId("table-root")
+      .findAllByRole("row")
+      .eq(1)
+      .within(() => {
+        cy.get("[data-column-id='USER_ID']").click();
+      });
+    H.popover().within(() => {
+      cy.findByText("View this User's Models").should("be.visible").click();
+    });
+    cy.findByTestId("table-body").within(() => {
+      cy.findAllByRole("row").should("have.length", 1);
+      cy.findAllByRole("row")
+        .first()
+        .within(() => {
+          cy.get("[data-column-id='USER_ID']").within(() => {
+            cy.get("[data-testid=cell-data]").should("have.text", "3");
+          });
+        });
+    });
+  });
+});

e2e/test/scenarios/table-editing/table-editing.cy.spec.ts

@@ -304,15 +304,20 @@ describe("scenarios > table-editing", () => {
           cy.findAllByRole("button").last().click();
         });
 
-        cy.wait("@updateTableData").then(({ response }) => {
+        cy.wait("@updateTableData").then(({ response, request }) => {
           const targetDate = dayjs(
             new Date(2024, 4, day, hour, minute, 0),
           ).format("YYYY-MM-DDTHH:mm:ss");
-          const savedDate = response?.body.outputs[0].row.CREATED_AT;
 
-          expect(response?.body.outputs[0].op).to.equal("updated");
-          // Omit timezone offset
-          expect(savedDate.startsWith(targetDate)).to.be.true;
+          const requestDate = request.body.params.CREATED_AT;
+          const responseDate = response?.body.outputs[0].row.CREATED_AT;
+
+          // Check if request date matches the response date.
+          // In theory FE should preserve the initial input date timezone offset (might be based on the CI environment)
+          expect(requestDate).to.equal(responseDate);
+
+          // Check if date param matches the input date (without timezone offset at offset 19)
+          expect(requestDate.slice(0, 19)).to.equal(targetDate);
         });
 
         H.undoToast().findByText("Successfully updated").should("be.visible");

enterprise/backend/src/metabase_enterprise/semantic_search/repair.clj

@@ -13,7 +13,7 @@
    [medley.core :as m]
    [metabase.util :as u]
    [metabase.util.log :as log]
-   [nano-id.core :as nano-id]
+   [metabase.util.string :as u.str]
    [next.jdbc :as jdbc]
    [next.jdbc.result-set :as jdbc.rs]))
 
@@ -93,7 +93,7 @@
   Format: repair_<millis-since-epoch>_<short-id>"
   []
   (let [millis-since-epoch (t/to-millis-from-epoch (t/instant))
-        short-id           (u/lower-case-en (subs (nano-id/nano-id) 0 6))]
+        short-id           (u/lower-case-en (u.str/random-string 6))]
     (format "repair_%d_%s" millis-since-epoch short-id)))
 
 (defn with-repair-table!

enterprise/backend/src/metabase_enterprise/transforms/api.clj

@@ -1,7 +1,10 @@
 (ns metabase-enterprise.transforms.api
   (:require
+   [clojure.core.async :as a]
+   [clojure.string :as str]
    [metabase-enterprise.transforms.api.transform-job]
    [metabase-enterprise.transforms.api.transform-tag]
+   [metabase-enterprise.transforms.canceling :as trasnforms.canceling]
    [metabase-enterprise.transforms.execute :as transforms.execute]
    [metabase-enterprise.transforms.models.transform :as transform.model]
    [metabase-enterprise.transforms.models.transform-run :as transform-run]
@@ -29,12 +32,21 @@
 (set! *warn-on-reflection* true)
 
 (mr/def ::transform-source
-  [:map
-   [:type [:= "query"]]
-   [:query [:map [:database :int]]]])
+  [:multi {:dispatch (comp keyword :type)}
+   [:query
+    [:map
+     [:type [:= "query"]]
+     [:query [:map [:database :int]]]]]
+   [:python
+    [:map {:closed true}
+     [:source-database {:optional true} :int]
+     [:source-tables   [:map-of :string :int]]
+     [:type [:= "python"]]
+     [:body :string]]]])
 
 (mr/def ::transform-target
   [:map
+   [:database {:optional true} :int]
    [:type [:enum "table"]]
    [:schema {:optional true} [:or ms/NonBlankString :nil]]
    [:name :string]])
@@ -57,19 +69,22 @@
 
 (defn- source-database-id
   [transform]
-  (-> transform :source :query :database))
+  (if (transforms.util/python-transform? transform)
+    (-> transform :source :source-database)
+    (-> transform :source :query :database)))
 
 (defn- check-database-feature
   [transform]
-  (let [database (api/check-400 (t2/select-one :model/Database (source-database-id transform))
-                                (deferred-tru "The source database cannot be found."))
-        feature (transforms.util/required-database-feature transform)]
-    (api/check-400 (not (:is_sample database))
-                   (deferred-tru "Cannot run transforms on the sample database."))
-    (api/check-400 (not (:is_audit database))
-                   (deferred-tru "Cannot run transforms on audit databases."))
-    (api/check-400 (driver.u/supports? (:engine database) feature database)
-                   (deferred-tru "The database does not support the requested transform target type."))))
+  (when (transforms.util/query-transform? transform)
+    (let [database (api/check-400 (t2/select-one :model/Database (source-database-id transform))
+                                  (deferred-tru "The source database cannot be found."))
+          feature (transforms.util/required-database-feature transform)]
+      (api/check-400 (not (:is_sample database))
+                     (deferred-tru "Cannot run transforms on the sample database."))
+      (api/check-400 (not (:is_audit database))
+                     (deferred-tru "Cannot run transforms on audit databases."))
+      (api/check-400 (driver.u/supports? (:engine database) feature database)
+                     (deferred-tru "The database does not support the requested transform target type.")))))
 
 (api.macros/defendpoint :get "/"
   "Get a list of transforms."
@@ -172,9 +187,10 @@
           new (merge old body)
           target-fields #(-> % :target (select-keys [:schema :name]))]
       (check-database-feature new)
-      (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]
-        (throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)
-                        {:status-code 400})))
+      (when (transforms.util/query-transform? old)
+        (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]
+          (throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)
+                          {:status-code 400}))))
       (api/check (not (and (not= (target-fields old) (target-fields new))
                            (transforms.util/target-table-exists? new)))
                  403
@@ -207,11 +223,14 @@
 (api.macros/defendpoint :post "/:id/cancel"
   "Cancel the current run for a given transform."
   [{:keys [id]} :- [:map
-                    [:id :string]]]
+                    [:id ms/PositiveInt]]]
   (log/info "canceling transform " id)
   (api/check-superuser)
-  (let [run (api/check-404 (transform-run/running-run-for-run-id id))]
-    (transform-run-cancelation/mark-cancel-started-run! (:id run)))
+  (let [transform (api/check-404 (t2/select-one :model/Transform id))
+        run (api/check-404 (transform-run/running-run-for-transform-id id))]
+    (transform-run-cancelation/mark-cancel-started-run! (:id run))
+    (when (transforms.util/python-transform? transform)
+      (trasnforms.canceling/cancel-run! (:id run))))
   nil)
 
 (api.macros/defendpoint :post "/:id/run"
@@ -222,9 +241,16 @@
   (api/check-superuser)
   (let [transform (api/check-404 (t2/select-one :model/Transform id))
         start-promise (promise)]
-    (u.jvm/in-virtual-thread*
-     (transforms.execute/run-mbql-transform! transform {:start-promise start-promise
-                                                        :run-method :manual}))
+    (if (transforms.util/python-transform? transform)
+      (u.jvm/in-virtual-thread*
+       (transforms.execute/execute-python-transform! transform {:start-promise start-promise
+                                                                :message-log   (atom {:pre-python  []
+                                                                                      :python      nil
+                                                                                      :post-python []})
+                                                                :run-method :manual}))
+      (u.jvm/in-virtual-thread*
+       (transforms.execute/run-mbql-transform! transform {:start-promise start-promise
+                                                          :run-method :manual})))
     (when (instance? Throwable @start-promise)
       (throw @start-promise))
     (let [result @start-promise
@@ -234,6 +260,53 @@
                               :run_id run-id})
           (assoc :status 202)))))
 
+;; hack
+(def ^:private python-test-run-id Integer/MAX_VALUE)
+
+(api.macros/defendpoint :post "/test-python"
+  "Test Python code execution without creating a transform."
+  [_route-params
+   _query-params
+   body :- [:map
+            [:code :string]
+            [:tables [:map-of :string :int]]]]
+  (log/info "test python code execution")
+  (api/check-superuser)
+  (let [run-id python-test-run-id
+        cancel-chan (a/promise-chan)]
+    (trasnforms.canceling/chan-start-run! run-id cancel-chan)
+    (try
+      (let [{:keys [response output events]} (transforms.execute/test-python-transform! (:code body) (:tables body) run-id cancel-chan)
+            {:keys [body status]} response]
+        (if (= status 200)
+          (do
+            (log/info "Python test execution succeeded")
+            (-> (response/response {:message (deferred-tru "Python code executed successfully")
+                                    :result  {:body (assoc body :output output)}})
+                (assoc :status 200)))
+          (do
+            (log/error "Error executing Python test code")
+            (-> (response/response {:message   (deferred-tru "Python code execution failed")
+                                    :error     body
+                                    :stdout    (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
+                                    :stderr    (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))
+                                    :exit_code (:exit_code (:exit_code body))})
+                (assoc :status status)))))
+      (finally
+        (trasnforms.canceling/chan-end-run! run-id)))))
+
+(api.macros/defendpoint :post "/test-python/cancel"
+  "Cancel the current test-python execution."
+  [_route-params
+   _query-params]
+  (log/info "canceling test python execution")
+  (api/check-superuser)
+  (if (trasnforms.canceling/chan-signal-cancel! python-test-run-id)
+    (-> (response/response {:message (deferred-tru "Python test canceled")})
+        (assoc :status 200))
+    (-> (response/response {:message (deferred-tru "No running Python test to cancel")})
+        (assoc :status 404))))
+
 (def ^{:arglists '([request respond raise])} routes
   "`/api/ee/transform` routes."
   (handlers/routes

enterprise/backend/src/metabase_enterprise/transforms/canceling.clj

@@ -49,7 +49,9 @@
    (chan-signal-cancel! run-id)
    (transform-run/timeout-run! run-id)))
 
-(defn- cancel-run! [run-id]
+(defn cancel-run!
+  "Cancel run"
+  [run-id]
   (when (chan-signal-cancel! run-id)
     (transform-run/cancel-run! run-id)))
 

enterprise/backend/src/metabase_enterprise/transforms/execute.clj

@@ -1,17 +1,27 @@
 (ns metabase-enterprise.transforms.execute
   (:require
    [clojure.core.async :as a]
+   [clojure.java.io :as io]
+   [clojure.string :as str]
    [metabase-enterprise.transforms.canceling :as canceling]
+   [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
    [metabase-enterprise.transforms.models.transform-run :as transform-run]
+   [metabase-enterprise.transforms.python-runner :as python-runner]
    [metabase-enterprise.transforms.settings :as transforms.settings]
    [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.driver :as driver]
    [metabase.driver.util :as driver.u]
    [metabase.lib.schema.common :as schema.common]
    [metabase.query-processor.pipeline :as qp.pipeline]
+   [metabase.util :as u]
+   [metabase.util.jvm :as u.jvm]
    [metabase.util.log :as log]
    [metabase.util.malli.registry :as mr]
-   [toucan2.core :as t2]))
+   [toucan2.core :as t2])
+  (:import
+   (clojure.lang IDeref)
+   (java.io Closeable File)
+   (java.time Duration)))
 
 (set! *warn-on-reflection* true)
 
@@ -37,24 +47,38 @@
    (log/info "Syncing target" (pr-str target) "for transform")
    (transforms.util/activate-table-and-mark-computed! database target)))
 
-(defn run-transform!
+(defn run-cancelable-transform!
   "Run a compiled transform"
-  [run-id driver {:keys [db-id conn-spec output-schema] :as transform-details} opts]
-  ;; local run is responsible for status
+  [run-id driver {:keys [db conn-spec output-schema]} run-transform!]
+  ;; local run is responsible for status, using canceling lifecycle
   (try
-    (when-not (driver/schema-exists? driver db-id output-schema)
-      (driver/create-schema-if-needed! driver conn-spec output-schema))
+    (when (driver.u/supports? driver :schemas db)
+      (when-not (driver/schema-exists? driver (:id db) output-schema)
+        (driver/create-schema-if-needed! driver conn-spec output-schema)))
     (canceling/chan-start-timeout-vthread! run-id (transforms.settings/transform-timeout))
-    (binding [qp.pipeline/*canceled-chan* (a/promise-chan)]
-      (canceling/chan-start-run! run-id qp.pipeline/*canceled-chan*)
-      (driver/run-transform! driver transform-details opts))
-    (transform-run/succeed-started-run! run-id)
+    (let [cancel-chan (a/promise-chan)
+          ret (binding [qp.pipeline/*canceled-chan* cancel-chan]
+                (canceling/chan-start-run! run-id cancel-chan)
+                (run-transform! cancel-chan))]
+      (transform-run/succeed-started-run! run-id)
+      ret)
     (catch Throwable t
       (transform-run/fail-started-run! run-id {:message (.getMessage t)})
       (throw t))
     (finally
       (canceling/chan-end-run! run-id))))
 
+(defn- try-start-unless-already-running [id run-method]
+  (try
+    (transform-run/start-run! id {:run_method run-method})
+    (catch java.sql.SQLException e
+      (if (= (.getSQLState e) "23505")
+        (throw (ex-info "Transform is already running"
+                        {:error :already-running
+                         :transform-id id}
+                        e))
+        (throw e)))))
+
 (defn run-mbql-transform!
   "Run `transform` and sync its target table.
 
@@ -67,7 +91,7 @@
      (let [db (get-in source [:query :database])
            {driver :engine :as database} (t2/select-one :model/Database db)
            feature (transforms.util/required-database-feature transform)
-           transform-details {:db-id db
+           transform-details {:db db
                               :transform-type (keyword (:type target))
                               :conn-spec (driver/connection-spec driver database)
                               :query (transforms.util/compile-source source)
@@ -78,20 +102,13 @@
          (throw (ex-info "The database does not support the requested transform target type."
                          {:driver driver, :database database, :feature feature})))
        ;; mark the execution as started and notify any observers
-       (let [{run-id :id} (try
-                            (transform-run/start-run! id {:run_method run-method})
-                            (catch java.sql.SQLException e
-                              (if (= (.getSQLState e) "23505")
-                                (throw (ex-info "Transform is already running"
-                                                {:error :already-running
-                                                 :transform-id id}
-                                                e))
-                                (throw e))))]
+       (let [{run-id :id} (try-start-unless-already-running id run-method)]
          (when start-promise
            (deliver start-promise [:started run-id]))
          (log/info "Executing transform" id "with target" (pr-str target))
-         (run-transform! run-id driver transform-details opts)
-         (sync-target! target database run-id)))
+         (run-cancelable-transform! run-id driver transform-details (fn [_cancel-chan] (driver/run-transform! driver transform-details opts)))
+         (transforms.instrumentation/with-stage-timing [run-id :sync :table-sync]
+           (sync-target! target database run-id))))
      (catch Throwable t
        (log/error t "Error executing transform")
        (when start-promise
@@ -99,3 +116,236 @@
          ;; but we assume nobody would catch the exception anyway
          (deliver start-promise t))
        (throw t)))))
+
+(defn- log! [message-log s]
+  (when message-log
+    (swap! message-log (fn [m] (update m (if (:python m) :post-python :pre-python) conj s))))
+  nil)
+
+(defn- replace-python-logs! [message-log events]
+  (when message-log (swap! message-log assoc :python events))
+  nil)
+
+(defn- save-log-as-message! [run-id message-log]
+  (when message-log
+    (let [{:keys [pre-python python post-python]} @message-log]
+      (t2/update! :model/TransformRun
+                  :id run-id
+                  {:message (str/join "\n" (concat pre-python
+                                                   (for [{:keys [stream message]} python]
+                                                     (if (= "stdout" stream)
+                                                       (str "\033[32m" message "\033[0m")
+                                                       (str "\033[31m" message "\033[0m")))
+                                                   post-python))}))))
+
+(defn- log-loop! [run-id message-log]
+  (let [poll #(python-runner/get-logs run-id)]
+    (try
+      (loop []
+        (if (.isInterrupted (Thread/currentThread))
+          (log/debug "Message update loop interrupted")
+          (do (Thread/sleep 1000)
+              (let [{:keys [status body]} (poll)]
+                (cond
+                  (<= 200 status 299)
+                  (let [{:keys [execution_id events]} body]
+                    (if-not (= run-id execution_id)
+                      (do (log/debugf "Run id did not match expected: %s actual: %s" run-id execution_id)
+                          (recur))
+                      (do
+                        (replace-python-logs! message-log events)
+                        (save-log-as-message! run-id message-log)
+                        (recur))))
+                  (= 404 status)
+                  (do
+                    (log/debugf "No logs yet (or run finished), run-id: %s" run-id)
+                    (recur))
+                  :else
+                  (do
+                    (log/warnf "Unexpected status polling for logs %s %s, run-id: %s" status body run-id)
+                    (log/debug "Exiting due to poll error")))))))
+      (catch InterruptedException _)
+      (catch Throwable e
+        (log/errorf e "An exception was caught during msg update loop, run-id: %s" run-id)))))
+
+(defn- open-log-thread! ^Closeable [run-id message-log]
+  (let [log-thread-promise (promise)
+        cleanup (fn []
+                  (let [^Thread log-thread (deref log-thread-promise 1000 nil)]
+                    (when-not log-thread (log/fatalf "Log thread reference not bound, run-id: %s" run-id))
+                    (when (.isAlive log-thread)
+                      (.interrupt log-thread)
+                      (when-not (.join log-thread (Duration/ofSeconds 10))
+                        (log/fatalf "Log thread could not be interrupted, run-id: %s" run-id)))))]
+    (u.jvm/in-virtual-thread*
+     (deliver log-thread-promise (Thread/currentThread))
+     (log-loop! run-id message-log))
+    (when-not (deref log-thread-promise 1000 nil) (log/fatalf "Log thread reference not bound, run-id: %s" run-id))
+    (reify
+      IDeref
+      (deref [_] @log-thread-promise)
+      Closeable
+      (close [_] (cleanup)))))
+
+(defn- debug-info-str [{:keys [exit-code events]}]
+  (str/join "\n"
+            ;; todo this was temporary before the log stuff, needs a rethink at some point
+            ["stdout"
+             "======"
+             (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
+             "stderr"
+             "======"
+             (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))
+             "======"
+             (format "exit code %d" exit-code)]))
+
+(defn- create-table-and-insert-data!
+  "Create a table from metadata and insert data from source."
+  [driver db-id table-name metadata data-source]
+  (let [table-schema {:name (if (keyword? table-name) table-name (keyword table-name))
+                      :columns (mapv (fn [{:keys [name dtype base_type database_type]}]
+                                       {:name name
+                                        :type (or (some->> base_type (keyword "type"))
+                                                  (transforms.util/dtype->base-type dtype))
+                                        :database-type database_type
+                                        :nullable? true})
+                                     (:fields metadata))}
+        data-source (assoc data-source :table-schema table-schema)]
+    (transforms.util/create-table-from-schema! driver db-id table-schema)
+    (driver/insert-from-source! driver db-id table-name (mapv :name (:columns table-schema)) data-source)))
+
+(defn- transfer-file-to-db [driver db {:keys [target] :as transform} metadata temp-file]
+  (let [table-name (transforms.util/qualified-table-name driver target)
+        table-exists? (transforms.util/target-table-exists? transform)
+        data-source {:type :csv-file
+                     :file temp-file}]
+    (if table-exists?
+      ;; Table exists - use temp table + atomic swap pattern
+      (let [temp-table-name (keyword (str (u/qualified-name table-name) "_temp_" (System/currentTimeMillis)))]
+        (log/info "Existing table deletected, Create then swap")
+        (try
+          (create-table-and-insert-data! driver (:id db) temp-table-name metadata data-source)
+          ;; TODO: These operations should be within a transaction for atomicity
+          (transforms.util/drop-table! driver (:id db) table-name)
+          (transforms.util/rename-table! driver (:id db) temp-table-name table-name)
+          (catch Exception e
+            (log/error e "Failed to transfer data to table")
+            (try
+              (transforms.util/drop-table! driver (:id db) temp-table-name)
+              (catch Exception _))
+            (throw e))))
+      ;; Table doesn't exist - create directly with target name
+      (do
+        (log/info "New table")
+        (create-table-and-insert-data! driver (:id db) table-name metadata data-source)))))
+
+(defn test-python-transform!
+  "Execute a transform in test mode (does not write result into a table)."
+  [code tables->id run-id cancel-chan]
+  (with-open [shared-storage-ref (python-runner/open-s3-shared-storage! tables->id)]
+    (let [server-url              (transforms.settings/python-execution-server-url)
+          _                       (python-runner/copy-tables-to-s3! {:run-id         run-id
+                                                                     :shared-storage @shared-storage-ref
+                                                                     :table-name->id tables->id
+                                                                     :cancel-chan    cancel-chan})
+          _                       (python-runner/open-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan
+          response
+          (python-runner/execute-python-code-http-call!
+           {:server-url     server-url
+            :code           code
+            :run-id         run-id
+            :table-name->id tables->id
+            :shared-storage @shared-storage-ref})
+          {:keys [output events]} (python-runner/read-output-objects @shared-storage-ref)]
+      {:response response
+       :events   events
+       :output   output})))
+
+(defn- run-python-transform! [{:keys [source] :as transform} db run-id cancel-chan message-log]
+  (with-open [log-thread-ref     (open-log-thread! run-id message-log)
+              shared-storage-ref (python-runner/open-s3-shared-storage! (:source-tables source))]
+    (let [driver     (:engine db)
+          server-url (transforms.settings/python-execution-server-url)
+          _          (python-runner/copy-tables-to-s3! {:run-id                  run-id
+                                                        :shared-storage @shared-storage-ref
+                                                        :table-name->id (:source-tables source)
+                                                        :cancel-chan    cancel-chan})
+          _          (python-runner/open-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan
+          {:keys [status body] :as response}
+          (python-runner/execute-python-code-http-call!
+           {:server-url     server-url
+            :code           (:body source)
+            :run-id         run-id
+            :table-name->id (:source-tables source)
+            :shared-storage @shared-storage-ref})
+          {:keys [exit_code]} body
+          ;; TODO temporary to keep more code stable while refactoring
+          ;; no need to materialize these early (i.e output we can stream directly into a tmp file or db if small)
+          {:keys [output output-manifest events]} (python-runner/read-output-objects @shared-storage-ref)]
+      (.close log-thread-ref)           ; early close to force any writes to flush
+      (when (seq events)
+        (replace-python-logs! message-log events))
+      (if (not= 200 status)
+        (throw (ex-info (debug-info-str {:exit-code exit_code :events events}) ;; todo do better here
+                        {:status-code     400
+                         :api-status-code status
+                         :body            body
+                         :events          events
+                         :error           (:error body)}))
+        (try
+          (let [temp-file (File/createTempFile "transform-output-" ".csv")]
+            (when-not (seq (:fields output-manifest))
+              (throw (ex-info "No fields in metadata"
+                              {:metadata output-manifest
+                               :raw-body body
+                               :events   events})))
+            (try
+              (with-open [writer (io/writer temp-file)]
+                (.write writer ^String output))
+              (let [file-size (.length temp-file)]
+                (transforms.instrumentation/with-stage-timing [run-id :data-transfer :file-to-dwh]
+                  (transfer-file-to-db driver db transform output-manifest temp-file))
+                (transforms.instrumentation/record-data-transfer! run-id :file-to-dwh file-size nil))
+              (finally
+                (.delete temp-file))))
+          response
+          (catch Exception e
+            (log/error e "Failed to to create resulting table")
+            (throw (ex-info "Failed to create the resulting table" {:error (.getMessage e)}))))))))
+
+(defn execute-python-transform!
+  "Execute a Python transform by calling the python runner.
+
+  This is executing synchronously, but supports being kicked off in the background
+  by delivering the `start-promise` just before the start when the beginning of the execution has been booked
+  in the database."
+  [transform {:keys [run-method start-promise message-log]}]
+  (when (transforms.util/python-transform? transform)
+    (try
+      (let [message-log (or message-log (atom {:pre-python  []
+                                               :python      nil
+                                               :post-python []}))
+            {:keys [target] transform-id :id} transform
+            {driver :engine :as db} (t2/select-one :model/Database (:database target))
+            {run-id :id} (try-start-unless-already-running transform-id run-method)]
+        (some-> start-promise (deliver [:started run-id]))
+        (log! message-log "Executing Python transform")
+        (log/info "Executing Python transform" transform-id "with target" (pr-str target))
+        (let [start-ms (u/start-timer)
+              transform-details
+              {:db db
+               :transform-type (keyword (:type target))
+               :conn-spec (driver/connection-spec driver db)
+               :output-schema (:schema target)
+               :output-table (transforms.util/qualified-table-name driver target)}
+              result   (run-cancelable-transform! run-id driver transform-details (fn [cancel-chan] (run-python-transform! transform db run-id cancel-chan message-log)))]
+          (transforms.instrumentation/with-stage-timing [run-id :sync :table-sync]
+            (sync-target! target db run-id))
+          (log! message-log (format "Python execution finished in %s" (Duration/ofMillis (u/since-ms start-ms))))
+          (save-log-as-message! run-id message-log)
+          {:run_id run-id
+           :result result}))
+      (catch Throwable t
+        (log/error t "Error executing Python transform")
+        (log! message-log "Error executing python transform")
+        (throw t)))))

enterprise/backend/src/metabase_enterprise/transforms/instrumentation.clj

@@ -0,0 +1,181 @@
+(ns metabase-enterprise.transforms.instrumentation
+  (:require
+   [metabase.analytics.prometheus :as prometheus]
+   [metabase.util :as u]
+   [metabase.util.log :as log]
+   [metabase.util.malli :as mu]
+   [metabase.util.malli.registry :as mr]))
+
+(set! *warn-on-reflection* true)
+
+(mr/def ::stage-type
+  [:enum :data-transfer :computation :sync])
+
+(mr/def ::stage-label
+  [:enum
+   :dwh-to-file
+   :file-to-s3
+   :file-to-dwh
+
+   :python-execution
+   :mbql-query
+   :table-sync])
+
+(mu/defn record-stage-start!
+  "Record the start of a transform stage."
+  [job-run-id :- pos-int?
+   stage-type :- ::stage-type
+   stage-label :- ::stage-label]
+  (log/infof "Transform stage started: run-id=%d type=%s label=%s" job-run-id (name stage-type) (name stage-label))
+  (prometheus/inc! :metabase-transforms/stage-started
+                   {:job-run-id (str job-run-id)
+                    :stage-type (name stage-type)
+                    :stage-label (name stage-label)}))
+
+(mu/defn record-stage-completion!
+  "Record the successful completion of a transform stage."
+  [job-run-id :- pos-int?
+   stage-type :- ::stage-type
+   stage-label :- ::stage-label
+   duration-ms :- int?]
+  (log/infof "Transform stage completed: run-id=%d type=%s label=%s duration=%dms" job-run-id (name stage-type) (name stage-label) duration-ms)
+  (let [labels {:job-run-id (str job-run-id)
+                :stage-type (name stage-type)
+                :stage-label (name stage-label)}]
+    (prometheus/inc! :metabase-transforms/stage-completed labels)
+    (prometheus/observe! :metabase-transforms/stage-duration-ms labels duration-ms)))
+
+(mu/defn record-stage-failure!
+  "Record the failure of a transform stage."
+  [job-run-id :- pos-int?
+   stage-type :- ::stage-type
+   stage-label :- ::stage-label
+   duration-ms :- [:maybe int?]]
+  (log/warnf "Transform stage failed: run-id=%d type=%s label=%s duration=%s" job-run-id (name stage-type) (name stage-label) (if duration-ms (str duration-ms "ms") "unknown"))
+  (let [labels {:job-run-id (str job-run-id)
+                :stage-type (name stage-type)
+                :stage-label (name stage-label)}]
+    (prometheus/inc! :metabase-transforms/stage-failed labels)
+    (when duration-ms
+      (prometheus/observe! :metabase-transforms/stage-duration-ms labels duration-ms))))
+
+(defn with-timing
+  "Generic timing function that executes body while recording metrics.
+
+   Takes a map of optional functions:
+   - :start-fn - function to call when starting
+   - :success-fn - function to call on success (receives duration-ms as last arg)
+   - :error-fn - function to call on error (receives duration-ms as last arg)
+
+   Usage:
+   (with-timing {:start-fn #(record-stage-start! %1 %2 %3)
+                 :success-fn #(record-stage-completion! %1 %2 %3 %4)
+                 :error-fn #(record-stage-failure! %1 %2 %3 %4)}
+                [job-run-id stage-type stage-label]
+                #(do-work)) "
+  [callbacks args body-fn]
+  (let [start-time (u/start-timer)]
+    (when-let [start-fn (:start-fn callbacks)]
+      (apply start-fn args))
+    (try
+      (let [result (body-fn)]
+        (when-let [success-fn (:success-fn callbacks)]
+          (apply success-fn (conj (vec args) (long (u/since-ms start-time)))))
+        result)
+      (catch Throwable t
+        (when-let [error-fn (:error-fn callbacks)]
+          (apply error-fn (conj (vec args) (long (u/since-ms start-time)))))
+        (throw t)))))
+
+(defmacro with-stage-timing
+  "Execute body while timing a transform stage. Automatically records start, completion or failure.
+
+   Usage:
+   (with-stage-timing [job-run-id :data-transfer :dwh-to-file]
+     ;; stage implementation
+     result)"
+  [args & body]
+  `(with-timing {:start-fn record-stage-start!
+                 :success-fn record-stage-completion!
+                 :error-fn record-stage-failure!}
+     ~args
+     (^:once fn* [] ~@body)))
+
+(mu/defn record-data-transfer!
+  "Record metrics about data transfer (size and rows)."
+  [job-run-id :- pos-int?
+   stage-label :- ::stage-label
+   bytes :- [:maybe int?]
+   rows :- [:maybe int?]]
+  (log/infof "Data transfer recorded: run-id=%d stage=%s%s%s"
+             job-run-id (name stage-label)
+             (if bytes (str " bytes=" bytes) "")
+             (if rows (str " rows=" rows) ""))
+  (let [labels {:job-run-id (str job-run-id)
+                :stage-label (name stage-label)}]
+    (when bytes
+      (prometheus/observe! :metabase-transforms/data-transfer-bytes labels bytes))
+    (when rows
+      (prometheus/observe! :metabase-transforms/data-transfer-rows labels rows))))
+
+(defn record-job-start!
+  "Record the start of a transform job run."
+  [job-id run-method]
+  (log/infof "Transform job started: job-id=%d run-method=%s" job-id (name run-method))
+  (prometheus/inc! :metabase-transforms/job-runs-total
+                   {:job-id (str job-id)
+                    :run-method (name run-method)}))
+
+(defn record-job-completion!
+  "Record the successful completion of a transform job run."
+  [job-id run-method duration-ms]
+  (log/infof "Transform job completed: job-id=%d run-method=%s duration=%dms" job-id (name run-method) duration-ms)
+  (let [labels {:job-id (str job-id)
+                :run-method (name run-method)}]
+    (prometheus/inc! :metabase-transforms/job-runs-completed labels)
+    (prometheus/observe! :metabase-transforms/job-run-duration-ms labels duration-ms)))
+
+(defn record-job-failure!
+  "Record the failure of a transform job run."
+  [job-id run-method duration-ms]
+  (log/warnf "Transform job failed: job-id=%d run-method=%s duration=%s" job-id (name run-method) (if duration-ms (str duration-ms "ms") "unknown"))
+  (let [labels {:job-id (str job-id)
+                :run-method (name run-method)}]
+    (prometheus/inc! :metabase-transforms/job-runs-failed labels)
+    (when duration-ms
+      (prometheus/observe! :metabase-transforms/job-run-duration-ms labels duration-ms))))
+
+(defmacro with-job-timing
+  "Execute body while timing a transform job run. Automatically records start, completion or failure.
+
+   Usage:
+   (with-job-timing [job-id run-method]
+     ;; job implementation
+     result)"
+  [[job-id run-method] & body]
+  `(with-timing {:start-fn record-job-start!
+                 :success-fn record-job-completion!
+                 :error-fn record-job-failure!}
+     [~job-id ~run-method]
+     (^:once fn* [] ~@body)))
+
+(mu/defn record-python-api-call!
+  "Record metrics about Python API calls."
+  [job-run-id :- pos-int?
+   duration-ms :- int?
+   status :- [:enum :success :error :timeout]]
+  (log/infof "Python API call %s: run-id=%d duration=%dms" (name status) job-run-id duration-ms)
+  (let [labels {:job-run-id (str job-run-id)}]
+    (prometheus/inc! :metabase-transforms/python-api-calls-total
+                     (assoc labels :status (name status)))
+    (prometheus/observe! :metabase-transforms/python-api-call-duration-ms labels duration-ms)))
+
+(defmacro with-python-api-timing
+  "Execute body while timing a Python API call."
+  [[job-run-id] & body]
+  `(with-timing {:success-fn (fn [job-run-id# duration-ms#]
+                               (record-python-api-call! job-run-id# duration-ms# :success))
+                 :error-fn (fn [job-run-id# duration-ms#]
+                             (record-python-api-call! job-run-id# duration-ms# :error))}
+     [~job-run-id]
+     (^:once fn* [] ~@body)))

enterprise/backend/src/metabase_enterprise/transforms/jobs.clj

@@ -6,10 +6,12 @@
    [clojurewerkz.quartzite.triggers :as triggers]
    [flatland.ordered.set :as ordered-set]
    [metabase-enterprise.transforms.execute :as transforms.execute]
+   [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
    [metabase-enterprise.transforms.models.job-run :as transforms.job-run]
    [metabase-enterprise.transforms.models.transform-run :as transform-run]
    [metabase-enterprise.transforms.ordering :as transforms.ordering]
    [metabase-enterprise.transforms.settings :as transforms.settings]
+   [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.task.core :as task]
    [metabase.util.log :as log]
    [toucan2.core :as t2]))
@@ -64,14 +66,21 @@
         (map transforms-by-id complete)))))
 
 (defn- run-transform! [run-id run-method {transform-id :id :as transform}]
-  (when (transform-run/running-run-for-run-id transform-id)
+  (when
+   (transform-run/running-run-for-transform-id transform-id)
     (log/warn "Transform" (pr-str transform-id) "already running, waiting")
     (loop []
       (Thread/sleep 2000)
-      (when (transform-run/running-run-for-run-id transform-id)
+      (when (transform-run/running-run-for-transform-id transform-id)
         (recur))))
+
   (log/info "Executing job transform" (pr-str transform-id))
-  (transforms.execute/run-mbql-transform! transform {:run-method run-method})
+  (if (transforms.util/python-transform? transform)
+    (transforms.instrumentation/with-stage-timing [run-id :computation :python-execution]
+      (transforms.execute/execute-python-transform! transform {:run-method run-method}))
+    (transforms.instrumentation/with-stage-timing [run-id :computation :mbql-query]
+      (transforms.execute/run-mbql-transform! transform {:run-method run-method})))
+
   (transforms.job-run/add-run-activity! run-id))
 
 (defn run-transforms!
@@ -110,12 +119,13 @@
     (let [transforms (job-transform-ids job-id)]
       (log/info "Executing transform job" (pr-str job-id) "with transforms" (pr-str transforms))
       (let [{run-id :id} (transforms.job-run/start-run! job-id run-method)]
-        (try
-          (run-transforms! run-id transforms opts)
-          (transforms.job-run/succeed-started-run! run-id)
-          (catch Throwable t
-            (transforms.job-run/fail-started-run! run-id {:message (.getMessage t)})
-            (throw t)))))))
+        (transforms.instrumentation/with-job-timing [job-id run-method]
+          (try
+            (run-transforms! run-id transforms opts)
+            (transforms.job-run/succeed-started-run! run-id)
+            (catch Throwable t
+              (transforms.job-run/fail-started-run! run-id {:message (.getMessage t)})
+              (throw t))))))))
 
 (def ^:private job-key "metabase-enterprise.transforms.jobs.timeout-job")
 

enterprise/backend/src/metabase_enterprise/transforms/models/transform_run.clj

@@ -147,11 +147,11 @@
                         :message   "Canceled by user but could not guarantee run stopped."})
     (cancel/delete-old-canceling-runs!)))
 
-(defn running-run-for-run-id
+(defn running-run-for-transform-id
   "Return a single active transform run or nil."
-  [id]
+  [transform-id]
   (t2/select-one :model/TransformRun
-                 :id id
+                 :transform_id transform-id
                  :is_active true))
 
 (defn- timestamp-constraint

enterprise/backend/src/metabase_enterprise/transforms/models/transform_run_cancelation.clj

@@ -14,14 +14,10 @@
   ([run-id]
    (mark-cancel-started-run! run-id {}))
   ([run-id properties]
-   (t2/insert! :model/TransformRunCancelation
-               (assoc properties
-                      :run_id run-id)
-               {:where [:exists {:select [1]
-                                 :from   [:transform_run]
-                                 :where  [:and
-                                          [:= :transform_run.id run-id]
-                                          :transform_run.is_active]}]})))
+   (when (t2/exists? :transform_run :id run-id :is_active true)
+     (t2/insert! :model/TransformRunCancelation
+                 (assoc properties
+                        :run_id run-id)))))
 
 (defn reducible-canceled-local-runs
   "Return a reducible sequence of local canceled runs."

enterprise/backend/src/metabase_enterprise/transforms/ordering.clj

@@ -16,23 +16,29 @@
 (set! *warn-on-reflection* true)
 
 (defn- transform-deps [transform]
-  (let [query (-> (get-in transform [:source :query])
-                  transforms.util/massage-sql-query
-                  qp.preprocess/preprocess
-                  ;; legacy usage -- don't do things like this going forward
-                  #_{:clj-kondo/ignore [:discouraged-var]}
-                  lib/->legacy-MBQL)]
-    (case (:type query)
-      :native (driver/native-query-deps (-> (qp.store/metadata-provider)
-                                            lib.metadata/database
-                                            :engine)
-                                        (get-in query [:native :query]))
-      :query (into #{}
-                   (keep #(clojure.core.match/match %
-                            [:source-table source] (when (int? source)
-                                                     source)
-                            _ nil))
-                   (tree-seq coll? seq query)))))
+  (if (transforms.util/python-transform? transform)
+    ;; Python transforms have dependencies via source-tables mapping
+    (into #{}
+          (map #(hash-map :table %))
+          (vals (get-in transform [:source :source-tables])))
+    ;; Query transforms have dependencies via SQL query analysis
+    (let [query (-> (get-in transform [:source :query])
+                    transforms.util/massage-sql-query
+                    qp.preprocess/preprocess
+                    ;; legacy usage -- don't do things like this going forward
+                    #_{:clj-kondo/ignore [:discouraged-var]}
+                    lib/->legacy-MBQL)]
+      (case (:type query)
+        :native (driver/native-query-deps (-> (qp.store/metadata-provider)
+                                              lib.metadata/database
+                                              :engine)
+                                          (get-in query [:native :query]))
+        :query (into #{}
+                     (keep #(clojure.core.match/match %
+                              [:source-table source] (when (int? source)
+                                                       {:table source})
+                              _ nil))
+                     (tree-seq coll? seq query))))))
 
 (defn- dependency-map [transforms]
   (into {}
@@ -56,20 +62,32 @@
 
   The result is a map of transform id -> #{transform ids the transform depends on}. Dependencies are limited to just
   the transforms in the original list -- if a transform depends on some transform not in the list, the 'extra'
-  dependency is ignored."
+  dependency is ignored. Both query and Python transforms can have dependencies on tables produced by other transforms."
   [transforms]
-  (let [transforms-by-db (->> transforms
+
+  (let [;; Group all transforms by their database
+        transforms-by-db (->> transforms
                               (map (fn [transform]
-                                     {(get-in transform [:source :query :database]) [transform]}))
+                                     (let [db-id (if (transforms.util/python-transform? transform)
+                                                   (get-in transform [:target :database])
+                                                   (get-in transform [:source :query :database]))]
+                                       {db-id [transform]})))
                               (apply merge-with into))
 
+        transform-ids (into #{} (map :id) transforms)
         {:keys [output-tables dependencies]} (->> transforms-by-db
                                                   (map (fn [[db-id db-transforms]]
                                                          (qp.store/with-metadata-provider db-id
                                                            {:output-tables (output-table-map db-transforms)
                                                             :dependencies (dependency-map db-transforms)})))
-                                                  (apply merge-with merge))]
-    (update-vals dependencies #(into #{} (keep output-tables) %))))
+                                                  (apply merge-with merge))
+
+        all-deps (update-vals dependencies #(into #{}
+                                                  (keep (fn [{:keys [table transform]}]
+                                                          (or (output-tables table)
+                                                              (transform-ids transform))))
+                                                  %))]
+    all-deps))
 
 (defn find-cycle
   "Finds a path containing a cycle in the directed graph `node->children`.
@@ -115,9 +133,12 @@
                                transforms)
         db-id (get-in to-check [:source :query :database])]
     (qp.store/with-metadata-provider db-id
-      (let [output-tables (output-table-map (filter #(= (get-in % [:source :query :database]) db-id)
-                                                    transforms))
-            node->children #(->> % transforms-by-id transform-deps (keep output-tables))
+      (let [db-transforms (filter #(= (get-in % [:source :query :database]) db-id) transforms)
+            output-tables (output-table-map db-transforms)
+            transform-ids (into #{} (map :id) db-transforms)
+            node->children #(->> % transforms-by-id transform-deps (keep (fn [{:keys [table transform]}]
+                                                                           (or (output-tables table)
+                                                                               (transform-ids transform)))))
             id->name (comp :name transforms-by-id)
             cycle (find-cycle node->children [transform-id])]
         (when cycle

enterprise/backend/src/metabase_enterprise/transforms/python_runner.clj

@@ -0,0 +1,442 @@
+(ns metabase-enterprise.transforms.python-runner
+  (:require
+   [clj-http.client :as http]
+   [clojure.core.async :as a]
+   [clojure.java.io :as io]
+   [clojure.string :as str]
+   [medley.core :as m]
+   [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
+   [metabase-enterprise.transforms.settings :as transforms.settings]
+   [metabase.driver :as driver]
+   [metabase.query-processor.compile :as qp.compile]
+   [metabase.query-processor.pipeline :as qp.pipeline]
+   ^{:clj-kondo/ignore [:discouraged-namespace]}
+   [metabase.query-processor.store :as qp.store]
+   [metabase.util.json :as json]
+   [metabase.util.log :as log]
+   [toucan2.core :as t2])
+  (:import
+   (clojure.lang IDeref)
+   (java.io BufferedWriter Closeable File OutputStream OutputStreamWriter)
+   (java.net URI)
+   (java.nio.charset StandardCharsets)
+   (java.time Duration)
+   (software.amazon.awssdk.auth.credentials AwsBasicCredentials StaticCredentialsProvider)
+   (software.amazon.awssdk.auth.credentials DefaultCredentialsProvider)
+   (software.amazon.awssdk.core.sync RequestBody)
+   (software.amazon.awssdk.regions Region)
+   (software.amazon.awssdk.services.s3 S3Client S3ClientBuilder S3Configuration)
+   (software.amazon.awssdk.services.s3.model DeleteObjectRequest GetObjectRequest NoSuchKeyException PutObjectRequest)
+   (software.amazon.awssdk.services.s3.presigner S3Presigner S3Presigner$Builder)
+   (software.amazon.awssdk.services.s3.presigner.model GetObjectPresignRequest PresignedGetObjectRequest PresignedPutObjectRequest PutObjectPresignRequest)))
+
+(set! *warn-on-reflection* true)
+
+(defn- safe-delete
+  "Safely delete a file."
+  [^File file]
+  (try (.delete file) (catch Exception _)))
+
+(defn- write-to-stream! [^OutputStream os col-names reducible-rows]
+  (let [writer (-> os
+                   (OutputStreamWriter. StandardCharsets/UTF_8)
+                   (BufferedWriter.))]
+
+    (run! (fn [row]
+            (let [row-map (zipmap col-names row)]
+              (json/encode-to row-map writer {})
+              (.newLine writer)))
+          reducible-rows)
+
+    (doto writer
+      (.flush)
+      (.close))))
+
+(defn- execute-mbql-query
+  [driver db-id query respond cancel-chan]
+  (driver/with-driver driver
+    (let [native (qp.compile/compile {:type :query, :database db-id :query query})
+
+          query  {:database db-id
+                  :type     :native
+                  :native   native}]
+      (qp.store/with-metadata-provider db-id
+        (binding [qp.pipeline/*canceled-chan* cancel-chan]
+          (driver/execute-reducible-query driver query {:canceled-chan cancel-chan} respond))))))
+
+(defn- root-type
+  [base-type]
+  (when base-type
+    (some #(when (isa? base-type %) %)
+          [:type/Float
+           :type/Integer
+           :type/Boolean
+           :type/DateTimeWithTZ
+           :type/DateTime
+           :type/Dictionary
+           :type/JSON
+           :type/Date
+           :type/Text])))
+
+(defn- closest-ancestor [t pred]
+  (loop [remaining (conj clojure.lang.PersistentQueue/EMPTY [t])]
+    (when-let [t (first remaining)]
+      (if (pred t)
+        t
+        (recur (into (pop t) (parents t)))))))
+
+(defn- effective-semantic-type-i-think
+  "Kinda sketchy but maybe reasonable way to infer the effective-semantic-type"
+  [{:keys [base_type effective_type semantic_type]}]
+  (or semantic_type (closest-ancestor (or effective_type base_type) #(isa? % :Semantic/*))))
+
+(defn- generate-manifest
+  "Generate a metadata manifest for the table columns."
+  [table-id cols-meta]
+  {:version        "0.1.0"
+   :fields         (mapv (fn [col-meta]
+                           {:name           (:name col-meta)
+                            :base_type      (some-> (:base_type col-meta) name)
+                            :database_type  (some-> (:database_type col-meta) name)
+                            :root_type      (some-> (root-type (:base_type col-meta)) name)
+                            ;; replace nil values with values indicating how they behave in practice.
+                            ;; there may be better ways of doing this already, but i worry it's just implicit in QP
+                            :semantic_type  (some-> (effective-semantic-type-i-think col-meta) name)
+                            :effective_type (some-> (or (:effective_type col-meta) (:database_type col-meta)) name)
+                            ;; TODO get this passed through
+                            :field_id       (:id col-meta)})
+                         cols-meta)
+   :table_metadata {:table_id table-id}})
+
+(defn- maybe-fixup-v [col v]
+  ;; clickhouse returns bigdecimals for int64 values
+  (if (and (isa? (:base_type col) :type/Integer)
+           (or (instance? BigDecimal v)
+               (float? v)))
+    (bigint v)
+    v))
+
+(defn- write-table-data-to-file! [id temp-file cancel-chan]
+  (let [db-id           (t2/select-one-fn :db_id (t2/table-name :model/Table) :id id)
+        driver          (t2/select-one-fn :engine :model/Database db-id)
+        all-fields-meta (t2/select [:model/Field :id :name :base_type :effective_type :semantic_type :database_type :database_position :nfc_path :parent_id]
+                                   :table_id id
+                                   :active true
+                                   {:order-by [[:database_position :asc]]})
+        fields-meta (filter #(and (nil? (:parent_id %)) (nil? (:nfc_path %))) all-fields-meta)
+        query {:source-table id}
+        manifest (generate-manifest id fields-meta)]
+    (execute-mbql-query driver db-id query
+                        (fn [{cols-meta :cols} reducible-rows]
+                          (with-open [os (io/output-stream temp-file)]
+                            (let [filtered-col-meta (m/index-by :name fields-meta)
+                                  filtered-cols (filter #(contains? filtered-col-meta (:name %)) cols-meta)
+                                  col-name->index (into {} (map-indexed (fn [i col] [(:name col) i]) cols-meta))
+                                  filtered-indices (mapv (fn [col] [(col-name->index (:name col)) col]) filtered-cols)
+                                  filtered-rows (eduction (map (fn [row] (mapv (fn [[idx col]]
+                                                                                 (let [v (nth row idx)]
+                                                                                   (maybe-fixup-v col v))) filtered-indices)))
+                                                          reducible-rows)]
+                              (write-to-stream! os (mapv :name filtered-cols) filtered-rows))))
+                        cancel-chan)
+    manifest))
+
+(defn- maybe-with-endpoint-s3-client [^S3ClientBuilder builder endpoint]
+  (let [region (transforms.settings/python-storage-s-3-region)]
+    (case [(some? endpoint) (some? region)]
+      [true true]   (doto builder
+                      (.endpointOverride (URI/create endpoint))
+                      (.region (Region/of region)))
+      [true false]  (log/warn "Ignoring endpoint because region is not defined")
+      [false true]  (.region builder (Region/of region))
+      [false false] builder)))
+
+(defn- maybe-with-endpoint-s3-presigner [^S3Presigner$Builder builder endpoint]
+  (let [region (transforms.settings/python-storage-s-3-region)]
+    (case [(some? endpoint) (some? region)]
+      [true true]   (doto builder
+                      (.endpointOverride (URI/create endpoint))
+                      (.region (Region/of region)))
+      [true false]  (log/warn "Ignoring endpoint because region is not defined")
+      [false true]  (.region builder (Region/of region))
+      [false false] (.region builder (Region/US_EAST_1)))))
+
+(defn- create-s3-configuration
+  "Create S3Configuration with path-style access setting"
+  ^S3Configuration []
+  (-> (S3Configuration/builder)
+      (.pathStyleAccessEnabled (transforms.settings/python-storage-s-3-path-style-access))
+      (.build)))
+
+(defn- build-put-object-request
+  "Build a PutObjectRequest with bucket and key"
+  [^String bucket-name ^String key]
+  (-> (PutObjectRequest/builder)
+      (.bucket bucket-name)
+      (.key key)
+      (.build)))
+
+(defn- build-get-object-request
+  "Build a GetObjectRequest with bucket and key"
+  [^String bucket-name ^String key]
+  (-> (GetObjectRequest/builder)
+      (.bucket bucket-name)
+      (.key key)
+      (.build)))
+
+(defn- build-delete-object-request
+  "Build a DeleteObjectRequest with bucket and key"
+  [^String bucket-name ^String key]
+  (-> (DeleteObjectRequest/builder)
+      (.bucket bucket-name)
+      (.key key)
+      (.build)))
+
+(def ^:private ^Duration presigned-url-duration
+  "Default duration for presigned URLs"
+  (Duration/ofHours 1))
+
+(defn- maybe-with-credentials-s3-client [^S3ClientBuilder builder]
+  (let [access-key (transforms.settings/python-storage-s-3-access-key)
+        secret-key (transforms.settings/python-storage-s-3-secret-key)]
+    (if (or access-key secret-key)
+      (if-not (and access-key secret-key)
+        (do (log/warnf "Ignoring %s because %s is not defined"
+                       (if access-key "access-key" "secret-key")
+                       (if (not access-key) "access-key" "secret-key"))
+            (.credentialsProvider builder (DefaultCredentialsProvider/create)))
+        (.credentialsProvider builder
+                              (StaticCredentialsProvider/create
+                               (AwsBasicCredentials/create access-key secret-key))))
+      (.credentialsProvider builder (DefaultCredentialsProvider/create)))))
+
+(defn- maybe-with-credentials-s3-presigner [^S3Presigner$Builder builder]
+  (let [access-key (transforms.settings/python-storage-s-3-access-key)
+        secret-key (transforms.settings/python-storage-s-3-secret-key)]
+    (if (or access-key secret-key)
+      (if-not (and access-key secret-key)
+        (do (log/warnf "Ignoring %s because %s is not defined"
+                       (if access-key "access-key" "secret-key")
+                       (if (not access-key) "access-key" "secret-key"))
+            builder)
+        (.credentialsProvider builder
+                              (StaticCredentialsProvider/create
+                               (AwsBasicCredentials/create access-key secret-key))))
+      builder)))
+
+;; We just recreate the client every time, to keep things simple if config is changed.
+(defn- create-s3-client
+  "Create S3 client for host operations (uploads, reads)"
+  ^S3Client []
+  (let [^S3ClientBuilder builder (S3Client/builder)]
+    (doto builder
+      (maybe-with-endpoint-s3-client (transforms.settings/python-storage-s-3-endpoint))
+      maybe-with-credentials-s3-client
+      (.serviceConfiguration (create-s3-configuration)))
+    (.build builder)))
+
+(defn- create-s3-presigner-for-container
+  "Create S3 presigner for container operations (presigned URLs).
+  Uses container-endpoint if different from host endpoint."
+  ^S3Presigner []
+  (let [container-endpoint (transforms.settings/python-storage-s-3-container-endpoint)
+        builder (S3Presigner/builder)]
+    (doto builder
+      (maybe-with-endpoint-s3-presigner (or container-endpoint (transforms.settings/python-storage-s-3-endpoint)))
+      maybe-with-credentials-s3-presigner
+      (.serviceConfiguration (create-s3-configuration)))
+    (.build builder)))
+
+(defn- upload-file-to-s3
+  "Upload file using host client"
+  [^S3Client s3-client ^String bucket-name ^String key ^File file]
+  (let [^PutObjectRequest request (build-put-object-request bucket-name key)]
+    (.putObject s3-client request (RequestBody/fromFile file))))
+
+(defn- generate-presigned-get-url
+  "Generate GET URL using container presigner"
+  [^S3Presigner presigner ^String bucket-name ^String key]
+  (let [^GetObjectRequest get-request (build-get-object-request bucket-name key)
+        ^GetObjectPresignRequest presign-request (-> (GetObjectPresignRequest/builder)
+                                                     (.signatureDuration presigned-url-duration)
+                                                     (.getObjectRequest get-request)
+                                                     (.build))
+        presigned ^PresignedGetObjectRequest (.presignGetObject presigner presign-request)]
+    (.toString (.url presigned))))
+
+(defn- generate-presigned-put-url
+  "Generate PUT URL using container presigner"
+  [^S3Presigner presigner ^String bucket-name ^String key]
+  (let [^PutObjectRequest put-request (build-put-object-request bucket-name key)
+        ^PutObjectPresignRequest presign-request (-> (PutObjectPresignRequest/builder)
+                                                     (.signatureDuration presigned-url-duration)
+                                                     (.putObjectRequest put-request)
+                                                     (.build))
+        presigned ^PresignedPutObjectRequest (.presignPutObject presigner presign-request)]
+    (.toString (.url presigned))))
+
+(defn- delete-s3-object [^S3Client s3-client ^String bucket-name ^String key]
+  (try
+    (let [^DeleteObjectRequest request (build-delete-object-request bucket-name key)]
+      (.deleteObject s3-client request))
+    (catch Exception e
+      (log/debugf e "Error deleting s3 object %s" key)
+      ;; Ignore deletion errors - object might not exist, or we might not have permissions
+      ;; NOTE: we plan to put general retention on the bucket so that objects will eventually be deleted
+      nil)))
+
+(defn- cleanup-s3-objects [^S3Client s3-client bucket-name s3-keys]
+  (run! (partial delete-s3-object s3-client bucket-name) s3-keys))
+
+(defn- read-from-s3
+  ([s3-client bucket-name key] (read-from-s3 s3-client bucket-name key ::throw))
+  ([^S3Client s3-client ^String bucket-name ^String key not-found]
+   (try
+     (let [^GetObjectRequest request (build-get-object-request bucket-name key)
+           response                  (.getObject s3-client request)]
+       (slurp response))
+     (catch NoSuchKeyException e
+       (if (identical? ::throw not-found)
+         (throw e)
+         not-found)))))
+
+(defn get-logs
+  "Return the logs of the current running python process"
+  [run-id]
+  (let [server-url (transforms.settings/python-execution-server-url)]
+    (http/get (str server-url "/logs")
+              {:content-type     :json
+               :accept           :json
+               :throw-exceptions false
+               :as               :json
+               :query-params     {:request_id run-id}})))
+
+(defn- s3-shared-storage [table-name->id]
+  (let [prefix              (some-> (transforms.settings/python-storage-s-3-prefix) (str "/"))
+        work-dir-name       (str prefix "run-" (System/nanoTime) "-" (rand-int 10000))
+        container-presigner (create-s3-presigner-for-container)
+        bucket-name         (transforms.settings/python-storage-s-3-bucket)
+        loc
+        (fn [method relative-path]
+          ;; save a tracking root to a well known location
+          ;; later can ls these to collect those objects whose ttl has elapsed
+          ;; (OR setup a retention policy in s3?)
+          (let [path (str work-dir-name "/" relative-path)]
+            {:path   path
+             :method method
+             :url    (case method :put (generate-presigned-put-url container-presigner bucket-name path)
+                           :get (generate-presigned-get-url container-presigner bucket-name path))}))]
+    {:s3-client   (create-s3-client)                        ;; do not like mixing interactive things with descriptions, but its damn convenient to have it here for now
+     :bucket-name bucket-name
+     :objects
+     (into
+      {:output          (loc :put "output.csv")
+       :output-manifest (loc :put "output-manifest.json")
+       :events          (loc :put "events.jsonl")}
+      (for [[table-name id] table-name->id]
+        {[:table id :manifest] (loc :get (str "-table-" (name table-name) "-" id ".manifest.json"))
+         [:table id :data]     (loc :get (str "-table-" (name table-name) "-" id ".jsonl"))}))}))
+
+(defn open-s3-shared-storage!
+  "Returns a deref'able shared storage value, (.close) will optimistically delete any s3 objects named in storage (data files for tables, metadata files etc).
+  The intention is the bucket specifies a generic object retention policy to ensure objects are eventually deleted (e.g. because the process dies during writing and .close never gets called)"
+  ^Closeable [table-name->id]
+  (let [shared-storage (s3-shared-storage table-name->id)]
+    (reify IDeref
+      (deref [_] shared-storage)
+      Closeable
+      (close [_] (cleanup-s3-objects (:s3-client shared-storage) (:bucket-name shared-storage) (map :path (vals (:objects shared-storage))))))))
+
+(defn copy-tables-to-s3!
+  "Writes table content to their corresponding objects named in shared-storage, see (open-shared-storage!).
+  Blocks until all tables are fully written and committed to shared storage."
+  [{:keys [run-id
+           shared-storage
+           table-name->id
+           cancel-chan]}]
+  (doseq [id (vals table-name->id)
+          :let [{:keys [s3-client bucket-name objects]} shared-storage
+                {data-path :path}     (get objects [:table id :data])
+                {manifest-path :path} (get objects [:table id :manifest])]]
+    (let [temp-file       (File/createTempFile data-path "")
+          manifest-file   (File/createTempFile manifest-path "")]
+      (try
+        ;; Write table data to temporary file and get manifest
+        (let [manifest (transforms.instrumentation/with-stage-timing [run-id :data-transfer :dwh-to-file]
+                         (write-table-data-to-file! id temp-file cancel-chan))]
+          ;; Write manifest to file
+          (with-open [writer (io/writer manifest-file)]
+            (json/encode-to manifest writer {}))
+          (let [file-size (.length temp-file)
+                manifest-size (.length manifest-file)]
+            (transforms.instrumentation/record-data-transfer! run-id :dwh-to-file file-size nil)
+
+            ;; Upload both files to S3
+            (transforms.instrumentation/with-stage-timing [run-id :data-transfer :file-to-s3]
+              (upload-file-to-s3 s3-client bucket-name data-path temp-file)
+              (upload-file-to-s3 s3-client bucket-name manifest-path manifest-file))
+
+            (transforms.instrumentation/record-data-transfer! run-id :file-to-s3 (+ file-size manifest-size) nil)))
+        (finally
+          ;; Clean up temporary files
+          (safe-delete temp-file)
+          (safe-delete manifest-file))))))
+
+(defn execute-python-code-http-call!
+  "Calls the /execute endpoint of the python runner. Blocks until the run either succeeds or fails and returns
+  the response from the server."
+  [{:keys [server-url code run-id table-name->id shared-storage]}]
+  (let [{:keys [objects]} shared-storage
+        {:keys [output output-manifest events]} objects
+
+        table-name->url          (update-vals table-name->id (comp :url #(get objects [:table % :data])))
+        table-name->manifest-url (update-vals table-name->id (comp :url #(get objects [:table % :manifest])))
+
+        payload                  {:code                code
+                                  :timeout             30
+                                  :request_id          run-id
+                                  :output_url          (:url output)
+                                  :output_manifest_url (:url output-manifest)
+                                  :events_url          (:url events)
+                                  :table_mapping       table-name->url
+                                  :manifest_mapping    table-name->manifest-url}
+
+        response                 (transforms.instrumentation/with-python-api-timing [run-id]
+                                   (http/post (str server-url "/execute")
+                                              {:content-type     :json
+                                               :accept           :json
+                                               :body             (json/encode payload)
+                                               :throw-exceptions false
+                                               :as               :json}))]
+    ;; when a 500 is returned we observe a string in the body (despite the python returning json)
+    ;; always try to parse the returned string as json before yielding (could tighten this up at some point)
+    (update response :body (fn [string-if-error]
+                             (if (string? string-if-error)
+                               (try
+                                 (json/decode+kw string-if-error)
+                                 (catch Exception _
+                                   {:error string-if-error}))
+                               string-if-error)))))
+
+(defn open-cancellation-process!
+  "Starts a core.async process that optimistically sends a cancellation request to the python executor if cancel-chan receives a value.
+  Returns a channel that will receive either the async http call j.u.c.FutureTask in the case of cancellation, or nil when the cancel-chan is closed."
+  [server-url run-id cancel-chan]
+  (a/go (when (a/<! cancel-chan)
+          (http/post (str server-url "/cancel")
+                     {:content-type :json
+                      :body         (json/encode {:request_id run-id})
+                      :async?       true}
+                     identity identity))))
+
+;; temporary, we should not need to realize data/events files into memory longer term
+(defn read-output-objects
+  "Temporary function that strings/jsons stuff in S3 and returns it for compatibility."
+  [{:keys [s3-client bucket-name objects]}]
+  (let [{:keys [output output-manifest events]} objects
+        output-content          (read-from-s3 s3-client bucket-name (:path output) nil)
+        output-manifest-content (read-from-s3 s3-client bucket-name (:path output-manifest) "{}")
+        events-content          (read-from-s3 s3-client bucket-name (:path events))]
+    {:output output-content
+     :output-manifest (json/decode+kw output-manifest-content)
+     :events (mapv json/decode+kw (str/split-lines events-content))}))

enterprise/backend/src/metabase_enterprise/transforms/schedule.clj

@@ -101,6 +101,7 @@
         trigger (build-trigger job-id schedule)]
     (task/schedule-task! job trigger)))
 
+;; TODO: should this be part of transformjob hooks?
 (defn update-job!
   "Update the trigger for a transform job."
   [job-id schedule]

enterprise/backend/src/metabase_enterprise/transforms/settings.clj

@@ -1,17 +1,119 @@
 (ns metabase-enterprise.transforms.settings
   (:require
+   [metabase.config.core :as config]
    [metabase.settings.core :as setting]
    [metabase.util.i18n :refer [deferred-tru]]))
 
 (set! *warn-on-reflection* true)
 
+(def ^:private is-not-prod? (or config/is-dev? config/is-test?))
+
 (setting/defsetting transform-timeout
   (deferred-tru "The timeout for a transform job, in minutes.")
-  :type       :integer
+  :type :integer
   :visibility :internal
-  :default    (* 4 60)
-  :feature    :transforms
-  :doc        false
-  :export?    false
+  :default (* 4 60)
+  :feature :transforms
+  :doc false
+  :export? false
   :encryption :no
-  :audit      :getter)
+  :audit :getter)
+
+(setting/defsetting python-execution-server-url
+  (deferred-tru "URL for the Python execution server that runs transform functions.")
+  :type :string
+  :visibility :admin
+  :default (when is-not-prod? "http://localhost:5001")
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-endpoint
+  (deferred-tru "S3 endpoint URL for storing Python execution artifacts.")
+  :type :string
+  :visibility :admin
+  :default (when is-not-prod? "http://localhost:4566")
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-region
+  (deferred-tru "AWS region for S3 storage.")
+  :type :string
+  :visibility :admin
+  :default (when is-not-prod? "us-east-1")
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-bucket
+  (deferred-tru "S3 bucket name for storing Python execution artifacts.")
+  :type :string
+  :visibility :admin
+  :default (if is-not-prod? "metabase-python-runner" "metabase-cloud-python-runner-staging")
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-prefix
+  (deferred-tru "Prefix to use for S3 objects. Necessary in prod where we only have access to a particular prefix.")
+  :type :string
+  :visibility :admin
+  :feature :transforms
+  :default (if is-not-prod? "test-prefix" "hosting-4b9351bb22b84791")
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-access-key
+  (deferred-tru "AWS access key ID for S3 authentication.")
+  :type :string
+  :visibility :admin
+  :feature :transforms
+  :default (when is-not-prod? "test")
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-secret-key
+  (deferred-tru "AWS secret access key for S3 authentication.")
+  :type :string
+  :visibility :admin
+  :feature :transforms
+  :default (when is-not-prod? "test")
+  :doc false
+  :export? false
+  :encryption :when-encryption-key-set
+  :audit :never)
+
+(setting/defsetting python-storage-s-3-container-endpoint
+  (deferred-tru "Alternative S3 endpoint accessible from containers. Leave empty if same as main endpoint.")
+  :type :string
+  :visibility :admin
+  :default (when is-not-prod? "http://localstack:4566")
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-path-style-access
+  (deferred-tru "Use path-style access for S3 requests (required for LocalStack and some S3-compatible services).")
+  :type :boolean
+  :visibility :admin
+  :default (when is-not-prod? true)
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)

enterprise/backend/src/metabase_enterprise/transforms/util.clj

@@ -1,12 +1,16 @@
 (ns metabase-enterprise.transforms.util
   (:require
+   [clojure.string :as str]
    [java-time.api :as t]
    [metabase.driver :as driver]
+   [metabase.lib.schema.common :as lib.schema.common]
    [metabase.query-processor.compile :as qp.compile]
    [metabase.sync.core :as sync]
    [metabase.util :as u]
    [metabase.util.date-2 :as u.date]
    [metabase.util.log :as log]
+   [metabase.util.malli :as mu]
+   [metabase.util.malli.registry :as mr]
    [toucan2.core :as t2])
   (:import
    (java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)
@@ -21,10 +25,22 @@
     (keyword schema name)
     (keyword name)))
 
+(defn query-transform?
+  "Check if this is a query transform: native query / mbql query."
+  [transform]
+  (= :query (-> transform :source :type keyword)))
+
+(defn python-transform?
+  "Check if this is a Python transform."
+  [transform]
+  (= :python (-> transform :source :type keyword)))
+
 (defn target-table-exists?
   "Test if the target table of a transform already exists."
   [{:keys [source target] :as _transform}]
-  (let [db-id (-> source :query :database)
+  (let [db-id (or (-> source :query :database)
+                  ;; python transform target
+                  (-> target :database))
         {driver :engine :as database} (t2/select-one :model/Database db-id)]
     (driver/table-exists? driver database target)))
 
@@ -66,7 +82,9 @@
   [{:keys [id target source], :as _transform}]
   (when target
     (let [target (update target :type keyword)
-          database-id (-> source :query :database)
+          database-id (or (-> source :query :database)
+                          ;; python transform target
+                          (-> target :database))
           {driver :engine :as database} (t2/select-one :model/Database database-id)]
       (driver/drop-transform-target! driver database target)
       (log/info "Deactivating  target " (pr-str target) "for transform" id)
@@ -122,3 +140,60 @@
   (-> run
       (u/update-some :start_time utc-timestamp-string)
       (u/update-some :end_time   utc-timestamp-string)))
+
+(mr/def ::column-definition
+  [:map
+   [:name :string]
+   [:type ::lib.schema.common/base-type]
+   [:nullable? {:optional true} :boolean]])
+
+(mr/def ::table-definition
+  [:map
+   [:name :keyword]
+   [:columns [:sequential ::column-definition]]
+   [:primary-key {:optional true} [:sequential :string]]])
+
+(defn dtype->base-type
+  "Maps pandas dtype strings directly to Metabase base types in the type hierarchy."
+  [dtype-str]
+  (cond
+    (str/starts-with? dtype-str "int") :type/Integer
+    (str/starts-with? dtype-str "float") :type/Float
+    (str/starts-with? dtype-str "bool") :type/Boolean
+    ;; datetime64[ns, timezone] indicates timezone-aware datetime
+    (str/starts-with? dtype-str "datetime64[ns, ") :type/DateTimeWithTZ
+    (str/starts-with? dtype-str "datetime") :type/DateTime
+    ;; this is not a real dtype, pandas uses 'object', but we override it if there's source or custom field metadata
+    (str/starts-with? dtype-str "date") :type/Date
+    :else :type/Text))
+
+(mu/defn create-table-from-schema!
+  "Create a table from a table-schema"
+  [driver :- :keyword
+   database-id :- pos-int?
+   table-schema :- ::table-definition]
+  (let [{:keys [columns] table-name :name} table-schema
+        column-definitions (into {} (map (fn [{:keys [name type database-type]}]
+                                           (let [database-type (when-not (and (= driver :bigquery-cloud-sdk)
+                                                                              (#{"ARRAY" "RECORD"} database-type))
+                                                                 database-type)
+                                                 db-type (if database-type
+                                                           [[:raw database-type]]
+                                                           (driver/type->database-type driver type))]
+                                             [name db-type])))
+                                 columns)
+        primary-key-opts (select-keys table-schema [:primary-key])]
+    (log/infof "Creating table %s with %d columns" table-name (count columns))
+    (driver/create-table! driver database-id table-name column-definitions primary-key-opts)))
+
+(defn rename-table!
+  "kename a table in the database."
+  [driver database-id old-table-name new-table-name]
+  (log/infof "Renaming table %s to %s" old-table-name new-table-name)
+  (driver/rename-table! driver database-id old-table-name new-table-name))
+
+(defn drop-table!
+  "Drop a table in the database."
+  [driver database-id table-name]
+  (log/infof "Dropping table %s" table-name)
+  (driver/drop-table! driver database-id table-name))

enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj

@@ -60,6 +60,7 @@
       (testing "Migration of simultaneous init attempt is blocked"
         (let [original-write-fn @#'semantic.db.migration/write-successful-migration!
               original-migrate-fn @#'semantic.db.connection/do-with-migrate-tx
+              original-maybe-migrate @#'semantic.db.migration/maybe-migrate!
               results (atom {:executed-migrations 0
                              :log []})]
           (with-redefs-fn {;; TODO: why in ci init index won't succeed -- http on embedding
@@ -72,6 +73,12 @@
                                (apply original-migrate-fn args)
                                (swap! results update :log conj [tid :ended (t/local-date-time)]))
                              nil)
+                           ;; add delay into migration function to ensure that thread performing migration
+                           ;; finishes earlier
+                           #'semantic.db.migration/maybe-migrate!
+                           (fn [& args]
+                             (Thread/sleep 200)
+                             (apply original-maybe-migrate args))
                            #'semantic.db.migration/write-successful-migration!
                            (fn [& args]
                              (Thread/sleep 2000)

enterprise/backend/test/metabase_enterprise/semantic_search/query_test.clj

@@ -21,13 +21,16 @@
     (mt/with-premium-features #{:semantic-search}
       (mt/as-admin
         (semantic.tu/with-index!
-          (testing "Dog-related query finds dog content"
-            (let [results (semantic.tu/query-index {:search-string "puppy"})]
-              (is (= "Dog Training Guide" (-> results first :name)))))
+          (semantic.tu/with-only-semantic-weights
+            (testing "Dog-related query finds dog content"
+              (let [results (-> (semantic.tu/query-index {:search-string "puppy"})
+                                semantic.tu/filter-for-mock-embeddings)]
+                (is (= "Dog Training Guide" (-> results first :name)))))
 
-          (testing "Bird-related query finds bird content"
-            (let [results (semantic.tu/query-index {:search-string "avian"})]
-              (is (= "Bird Watching Tips" (-> results first :name))))))))))
+            (testing "Bird-related query finds bird content"
+              (let [results (-> (semantic.tu/query-index {:search-string "avian"})
+                                semantic.tu/filter-for-mock-embeddings)]
+                (is (= "Bird Watching Tips" (-> results first :name)))))))))))
 
 (defn- index-of-name
   "Return the index of the item with :name `name` in `coll`"
@@ -42,68 +45,69 @@
     (mt/with-premium-features #{:semantic-search}
       (mt/as-admin
         (semantic.tu/with-index!
-          (let [not-top-10? #(or (nil? %) (< 10 %))]
-            (doseq [{:keys [name desc search-string search-native-query? expected-index?]}
-                    [{:name "Dog Training Guide"
-                      :desc "contains"
-                      :search-string "AVG(tricks)"
-                      :search-native-query? false
-                      :expected-index? not-top-10?}
-                     {:name "Dog Training Guide"
-                      :desc "contains"
-                      :search-string "AVG(tricks)"
-                      :search-native-query? true
-                      :expected-index? zero?}
-                     {:name "Dog Training Guide"
-                      :desc "contains"
-                      :search-string "GROUP BY breed"
-                      :search-native-query? false
-                      :expected-index? not-top-10?}
-                     {:name "Dog Training Guide"
-                      :desc "contains"
-                      :search-string "GROUP BY breed"
-                      :search-native-query? true
-                      :expected-index? zero?}
-                     {:name "Dog Training Guide"
-                      :desc "does not contain"
-                      :search-string "SUM(tricks)"
-                      :search-native-query? false
-                      :expected-index? not-top-10?}
-                     {:name "Dog Training Guide"
-                      :desc "does not contain"
-                      :search-string "SUM(tricks)"
-                      :search-native-query? true
-                      :expected-index? not-top-10?}
-                     {:name "Bird Watching Tips"
-                      :desc "does not contain (no native query)"
-                      :search-string "AVG(tricks)"
-                      :search-native-query? true
-                      :expected-index? not-top-10?}
-                     {:name "Dog Training Guide"
-                      :desc "semantic search"
-                      :search-string "puppy"
-                      :search-native-query? true
-                      :expected-index? zero?}
-                     {:name "Bird Watching Tips"
-                      :desc "semantic search"
-                      :search-string "avian"
-                      :search-native-query? true
-                      :expected-index? zero?}
-                     {:name "Dog Training Guide"
-                      :desc "non-native keyword query"
-                      :search-string "Training"
-                      :search-native-query? true
-                      :expected-index? zero?}
-                     {:name "Bird Watching Tips"
-                      :desc "non-native keyword query"
-                      :search-string "Watching"
-                      :search-native-query? true
-                      :expected-index? zero?}]]
-              (testing (format "\n%s %s %s %s" name desc search-string search-native-query?)
-                (is (-> (semantic.tu/query-index {:search-string search-string
-                                                  :search-native-query search-native-query?})
-                        (index-of-name name)
-                        expected-index?))))))))))
+          (semantic.tu/with-only-semantic-weights
+            (let [not-top-10? #(or (nil? %) (< 10 %))]
+              (doseq [{:keys [name desc search-string search-native-query? expected-index?]}
+                      [{:name "Dog Training Guide"
+                        :desc "contains"
+                        :search-string "AVG(tricks)"
+                        :search-native-query? false
+                        :expected-index? not-top-10?}
+                       {:name "Dog Training Guide"
+                        :desc "contains"
+                        :search-string "AVG(tricks)"
+                        :search-native-query? true
+                        :expected-index? zero?}
+                       {:name "Dog Training Guide"
+                        :desc "contains"
+                        :search-string "GROUP BY breed"
+                        :search-native-query? false
+                        :expected-index? not-top-10?}
+                       {:name "Dog Training Guide"
+                        :desc "contains"
+                        :search-string "GROUP BY breed"
+                        :search-native-query? true
+                        :expected-index? zero?}
+                       {:name "Dog Training Guide"
+                        :desc "does not contain"
+                        :search-string "SUM(tricks)"
+                        :search-native-query? false
+                        :expected-index? not-top-10?}
+                       {:name "Dog Training Guide"
+                        :desc "does not contain"
+                        :search-string "SUM(tricks)"
+                        :search-native-query? true
+                        :expected-index? not-top-10?}
+                       {:name "Bird Watching Tips"
+                        :desc "does not contain (no native query)"
+                        :search-string "AVG(tricks)"
+                        :search-native-query? true
+                        :expected-index? not-top-10?}
+                       {:name "Dog Training Guide"
+                        :desc "semantic search"
+                        :search-string "puppy"
+                        :search-native-query? true
+                        :expected-index? zero?}
+                       {:name "Bird Watching Tips"
+                        :desc "semantic search"
+                        :search-string "avian"
+                        :search-native-query? true
+                        :expected-index? zero?}
+                       {:name "Dog Training Guide"
+                        :desc "non-native keyword query"
+                        :search-string "Training"
+                        :search-native-query? true
+                        :expected-index? zero?}
+                       {:name "Bird Watching Tips"
+                        :desc "non-native keyword query"
+                        :search-string "Watching"
+                        :search-native-query? true
+                        :expected-index? zero?}]]
+                (testing (format "\n%s %s %s %s" name desc search-string search-native-query?)
+                  (is (-> (semantic.tu/query-index {:search-string search-string
+                                                    :search-native-query search-native-query?})
+                          (index-of-name name)
+                          expected-index?)))))))))))
 
 (deftest model-filtering-test
   (testing "Filter results by model type"
@@ -256,10 +260,13 @@
       (mt/as-admin
         (binding [semantic.index/*batch-size* 1]
           (semantic.tu/with-index!
-            (testing "Horse-related query finds horse content"
-              (let [results (semantic.tu/query-index {:search-string "marine mammal"})]
-                (is (= "Whale Communication" (-> results first :name)))))
+            (semantic.tu/with-only-semantic-weights
+              (testing "Horse-related query finds horse content"
+                (let [results (-> (semantic.tu/query-index {:search-string "marine mammal"})
+                                  semantic.tu/filter-for-mock-embeddings)]
+                  (is (= "Whale Communication" (-> results first :name)))))
 
-            (testing "Tiger-related query finds tiger content"
-              (let [results (semantic.tu/query-index {:search-string "endangered species"})]
-                (is (= "Tiger Conservation" (-> results first :name)))))))))))
+              (testing "Tiger-related query finds tiger content"
+                (let [results (-> (semantic.tu/query-index {:search-string "endangered species"})
+                                  semantic.tu/filter-for-mock-embeddings)]
+                  (is (= "Tiger Conservation" (-> results first :name))))))))))))

enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.clj

@@ -47,6 +47,13 @@
   [gate-contents id]
   (some #(when (= (:id %) id) %) gate-contents))
 
+(defn- clear-gate-table!
+  "Clear all entries from the gate table to ensure clean test state"
+  [pgvector gate-table-name]
+  (jdbc/execute! pgvector
+                 (-> {:delete-from [(keyword gate-table-name)]}
+                     (sql/format :quoted true))))
+
 (deftest repair-index-integration-test
   (testing "repair-index! properly handles document additions and deletions via gate table"
     (mt/with-premium-features #{:semantic-search}
@@ -54,6 +61,7 @@
         (let [pgvector       semantic.tu/db
               index-metadata semantic.tu/mock-index-metadata
               gate-table     (:gate-table-name index-metadata)
+              _              (clear-gate-table! pgvector gate-table)
               initial-docs   [(create-test-document "card" 1 "Dog Training Guide")
                               (create-test-document "card" 2 "Cat Behavior Study")
                               (create-test-document "dashboard" 3 "Animal Stats")]]
@@ -87,17 +95,17 @@
         (let [pgvector       semantic.tu/db
               index-metadata semantic.tu/mock-index-metadata
               gate-table     (:gate-table-name index-metadata)
-              initial-docs   [(create-test-document "card" 1 "Dog Training Guide")]]
+              initial-docs   [(create-test-document "card" 6 "Dog Training Guide")]]
           (semantic.core/update-index! initial-docs)
           (semantic.tu/index-all!)
 
           (testing "repair table is cleaned up after successful repair"
             (let [test-repair-table-name "repair_table_cleanup_test"]
               (with-redefs [semantic.repair/repair-table-name (constantly test-repair-table-name)]
-                (semantic.core/repair-index! [(create-test-document "card" 2 "New Test Card")])
+                (semantic.core/repair-index! [(create-test-document "card" 7 "New Test Card")])
 
                 (let [gate-contents (gate-table-contents pgvector gate-table)]
-                  (is (tombstone? (gate-entry-by-id gate-contents "card_1")))
-                  (is (some? (gate-entry-by-id gate-contents "card_2")))
+                  (is (tombstone? (gate-entry-by-id gate-contents "card_6")))
+                  (is (some? (gate-entry-by-id gate-contents "card_7")))
                   (is (not (semantic.util/table-exists? pgvector test-repair-table-name))
                       "Repair table should be cleaned up after repair-index! completes"))))))))))

enterprise/backend/test/metabase_enterprise/semantic_search/scoring_test.clj

@@ -6,7 +6,6 @@
    [metabase-enterprise.semantic-search.scoring :as semantic.scoring]
    [metabase-enterprise.semantic-search.test-util :as semantic.tu]
    [metabase.app-db.core :as mdb]
-   [metabase.search.appdb.scoring-test :refer [with-weights]]
    [metabase.search.config :as search.config]
    [metabase.test :as mt])
   (:import
@@ -56,13 +55,17 @@
 (defn search-results
   "Like search-results* but with a sanity check that search without weights returns a different result."
   [ranker-key search-string & {:as raw-ctx}]
-  (let [result   (with-weights {ranker-key 1} (search-results* search-string raw-ctx))
-        inverted (with-weights {ranker-key -1} (search-results* search-string raw-ctx))]
+  (let [result   (semantic.tu/with-weights {ranker-key  1} (search-results* search-string raw-ctx))
+        inverted (semantic.tu/with-weights {ranker-key -1} (search-results* search-string raw-ctx))]
     ;; note that this may not be a strict reversal, due to ties.
     (is (not= inverted result)
         "sanity check: search-no-weights should be different")
     result))
 
+;;
+;; index-based scorers
+;;
+
 (deftest rrf-test
   (mt/with-premium-features #{:semantic-search}
     (with-index-contents!
@@ -143,7 +146,7 @@
         (is (= [["dataset" 1 "card ancient"]
                 ["metric"  3 "card old"]
                 ["card"    2 "card recent"]]
-               (with-weights {:model 1.0 :model/dataset 1.0}
+               (semantic.tu/with-weights {:model 1.0 :model/dataset 1.0}
                  (search-results* "card"))))))))
 
 (deftest recency-test
@@ -181,9 +184,9 @@
          {:model "dashboard" :id 2 :name "view dashboard" :view_count 0}
          {:model "dataset"   :id 3 :name "view dataset"   :view_count 0}]
         ;; fix some test flakes where dataset 3 exists and has some sort of recent views
-        (with-weights (assoc (search.config/weights :default)
-                             :user-recency 0
-                             :rrf 0)
+        (semantic.tu/with-weights (assoc (search.config/weights :default)
+                                         :user-recency 0
+                                         :rrf 0)
           (is (=? [{:model "dashboard", :id 2, :name "view dashboard"}
                    {:model "card",      :id 1, :name "view card"}
                    {:model "dataset",   :id 3, :name "view dataset"}]
@@ -202,8 +205,8 @@
 (defn indifferent?
   "Check that the results and their order do not depend on the given ranker."
   [ranker-key search-string & {:as raw-ctx}]
-  (= (with-weights {ranker-key 1} (search-results* search-string raw-ctx))
-     (with-weights {ranker-key -1} (search-results* search-string raw-ctx))))
+  (= (semantic.tu/with-weights {ranker-key  1} (search-results* search-string raw-ctx))
+     (semantic.tu/with-weights {ranker-key -1} (search-results* search-string raw-ctx))))
 
 (deftest dashboard-count-test-2
   (mt/with-premium-features #{:semantic-search}
@@ -213,6 +216,44 @@
          {:model "card" :id 2 :name "card" :dashboardcard_count 201}]
         (is (indifferent? :dashboard "card"))))))
 
+;;;
+;;; "premium" index-based scorers
+;;;
+
+(deftest official-collection-test
+  (with-index-contents!
+    [{:model "collection" :id 1 :name "collection normal" :official_collection false}
+     {:model "collection" :id 2 :name "collection official" :official_collection true}]
+    (testing "official collections has higher rank"
+      (mt/with-premium-features #{:semantic-search :official-collections}
+        (is (= [["collection" 2 "collection official"]
+                ["collection" 1 "collection normal"]]
+               (search-results :official-collection "collection")))))
+    (testing "only if feature is enabled"
+      (mt/with-premium-features #{:semantic-search}
+        (is (= [["collection" 1 "collection normal"]
+                ["collection" 2 "collection official"]]
+               (search-results* "collection")))))))
+
+(deftest verified-test
+  (with-index-contents!
+    [{:model "card" :id 1 :name "card normal" :verified false}
+     {:model "card" :id 2 :name "card verified" :verified true}]
+    (testing "verified items have higher rank"
+      (mt/with-premium-features #{:semantic-search :content-verification}
+        (is (= [["card" 2 "card verified"]
+                ["card" 1 "card normal"]]
+               (search-results :verified "card")))))
+    (testing "only if feature is enabled"
+      (mt/with-premium-features #{:semantic-search}
+        (is (= [["card" 1 "card normal"]
+                ["card" 2 "card verified"]]
+               (search-results* "card")))))))
+
+;;
+;; appdb-based scorers
+;;
+
 (deftest bookmark-test
   (mt/with-premium-features #{:semantic-search}
     ;; TODO (BOT-360) enable for :mysql

enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj

@@ -15,6 +15,7 @@
    [metabase-enterprise.semantic-search.indexer :as semantic.indexer]
    [metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]
    [metabase-enterprise.semantic-search.util :as semantic.util]
+   [metabase.search.config :as search.config]
    [metabase.search.ingestion :as search.ingestion]
    [metabase.test :as mt]
    [metabase.util :as u]
@@ -82,6 +83,18 @@
   [db-name & body]
   `(do-with-test-db! ~db-name (fn [] ~@body)))
 
+(defmacro with-weights
+  "Execute `body` overriding search weights with `weight-map`."
+  [weight-map & body]
+  `(mt/with-dynamic-fn-redefs [search.config/weights (constantly ~weight-map)]
+     ~@body))
+
+(defmacro with-only-semantic-weights
+  "Execute `body` with only the semantic search hybrid scorer weights active."
+  [& body]
+  `(with-weights {:rrf 1}
+     ~@body))
+
 (def ^:private init-delay
   (delay
     (when-not @semantic.db.datasource/data-source

enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj

@@ -963,6 +963,64 @@
                    :content "11 = 11"}
                   (t2/select-one :model/NativeQuerySnippet :entity_id (:entity_id snippet)))))))))
 
+(deftest snippet-template-tags-import-test
+  (testing "Template tags import preserves nil, empty, and populated values"
+
+    (testing "Missing template_tags field -> {} when selected"
+      (mt/with-empty-h2-app-db!
+        (let [snippet-data {:serdes/meta [{:model "NativeQuerySnippet"
+                                           :id    "test-entity-1"
+                                           :label "test_snippet_1"}]
+                            :name        "no-tags-field"
+                            :content     "WHERE id = {{id}}"
+                            :creator_id  "test@example.com"
+                            :entity_id   "test-entity-1"}
+              ingestion    (ingestion-in-memory [snippet-data])]
+          (serdes.load/load-metabase! ingestion)
+          (let [template-tags (t2/select-one-fn :template_tags :model/NativeQuerySnippet :entity_id "test-entity-1")]
+            ;; Toucan hooks compute the template-tags:
+            (is (=? {"id" {:type         :text
+                           :display-name "ID"
+                           :name         "id"}}
+                    template-tags))))))
+
+    (testing "Empty map template_tags -> preserved as empty map"
+      (mt/with-empty-h2-app-db!
+        (let [snippet-data {:serdes/meta   [{:model "NativeQuerySnippet"
+                                             :id    "test-entity-2"
+                                             :label "test_snippet_2"}]
+                            :name          "empty-tags"
+                            :content       "SELECT 1"
+                            :template_tags {}
+                            :creator_id    "test@example.com"
+                            :entity_id     "test-entity-2"}
+              ingestion    (ingestion-in-memory [snippet-data])]
+          (serdes.load/load-metabase! ingestion)
+          (let [template-tags (t2/select-one-fn :template_tags :model/NativeQuerySnippet :entity_id "test-entity-2")]
+            (is (= {} template-tags))))))
+
+    (testing "Snippet template tags get preserved rather than recalculated"
+      (mt/with-empty-h2-app-db!
+        (let [snippet-data {:serdes/meta   [{:model "NativeQuerySnippet"
+                                             :id    "test-entity-3"
+                                             :label "test_snippet_3"}]
+                            :name          "with-tags"
+                            :content       "WHERE id = {{snippet: id}}"
+                            :template_tags {"id" {:type         :snippet
+                                                  :name         "snippet: id"
+                                                  :snippet-name "id"
+                                                  ;; Definitely not calculated that way:
+                                                  :display-name "Snippet: WOOP"}}
+                            :creator_id    "test@example.com"
+                            :entity_id     "test-entity-3"}
+              ingestion    (ingestion-in-memory [snippet-data])]
+          (serdes.load/load-metabase! ingestion)
+          (let [template-tags (t2/select-one-fn :template_tags :model/NativeQuerySnippet :entity_id "test-entity-3")]
+            (is (=? {"snippet: id" {:type         :snippet
+                                    :name         "snippet: id"
+                                    :display-name "Snippet: WOOP"}}
+                    template-tags))))))))
+
 (deftest load-action-test
   (let [serialized (atom nil)
         eid (u/generate-nano-id)]

enterprise/backend/test/metabase_enterprise/transforms/api_test.clj

@@ -8,8 +8,9 @@
    [metabase-enterprise.transforms.models.transform-transform-tag]
    [metabase-enterprise.transforms.query-test-util :as query-test-util]
    [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
-   [metabase-enterprise.transforms.test-util :refer [parse-instant with-transform-cleanup! utc-timestamp]]
+   [metabase-enterprise.transforms.test-util :as transforms.tu :refer [parse-instant with-transform-cleanup! utc-timestamp]]
    [metabase-enterprise.transforms.util :as transforms.util]
+   [metabase.driver :as driver]
    [metabase.legacy-mbql.normalize :as mbql.normalize]
    [metabase.lib.convert :as lib.convert]
    [metabase.lib.core :as lib]
@@ -72,6 +73,26 @@
                                             :schema schema
                                             :name   table-name}})))))))
 
+(deftest create-python-transform-test
+  (mt/with-premium-features #{:transforms}
+    (mt/dataset transforms-dataset/transforms-test
+      (with-transform-cleanup! [table-name "gadget_products"]
+        (let [schema    (get-test-schema)
+              transform-payload {:name   "My beautiful python runner"
+                                 :source {:type            "python"
+                                          :body            "print('hello world')"
+                                          :source-tables   {}}
+                                 :target {:type     "table"
+                                          :schema   schema
+                                          :name     table-name
+                                          :database (mt/id)}}
+              transform (mt/user-http-request :crowberto :post 200 "ee/transform"
+                                              transform-payload)]
+          (is (= "print('hello chris')"
+                 (-> (mt/user-http-request :crowberto :put 200 (format "ee/transform/%s" (:id transform))
+                                           (assoc-in transform-payload [:source :body] "print('hello chris')"))
+                     :source :body))))))))
+
 (deftest list-transforms-test
   (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
     (testing "Can list without query parameters"
@@ -319,6 +340,32 @@
                 (is (true? (transforms.util/target-table-exists? updated)))
                 (check-query-results table2-name [2 3 4 13] "Doohickey")))))))))
 
+(deftest execute-python-transform-test
+  (testing "transform execution with :transforms/table target"
+    (mt/test-drivers #{:h2 :postgres}
+      (mt/with-premium-features #{:transforms}
+        (mt/dataset transforms-dataset/transforms-test
+          (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
+            (with-transform-cleanup! [{table-name :name :as target} {:type   "table"
+                                                                     :schema schema
+                                                                     :name   "target_table"}]
+              (let [original           {:name   "Gadget Products"
+                                        :source {:type  "python"
+                                                 :source-database (mt/id)
+                                                 :source-tables {"transforms_customers" (mt/id :transforms_customers)}
+                                                 :body  (str "import pandas as pd\n"
+                                                             "\n"
+                                                             "def transform():\n"
+                                                             "    return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}
+                                        :target  (assoc target :database (mt/id))}
+                    {transform-id :id} (mt/user-http-request :crowberto :post 200 "ee/transform"
+                                                             original)]
+                (test-run transform-id)
+                (wait-for-table table-name 5000)
+                (is (true? (driver/table-exists? driver/*driver* (mt/db) target)))
+                (is (= [["Alice" 25] ["Bob" 30]]
+                       (transforms.tu/table-rows table-name)))))))))))
+
 (deftest get-runs-filter-by-single-transform-id-test
   (testing "GET /api/ee/transform/run - filter by single transform ID"
     (mt/with-premium-features #{:transforms}

enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj

@@ -5,7 +5,8 @@
    [metabase-enterprise.transforms.execute :as transforms.execute]
    [metabase-enterprise.transforms.query-test-util :as query-test-util]
    [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
-   [metabase-enterprise.transforms.test-util :refer [with-transform-cleanup!]]
+   [metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]
+   [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.driver :as driver]
    [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
    [metabase.lib.core :as lib]
@@ -13,6 +14,7 @@
    [metabase.query-processor :as qp]
    [metabase.sync.core :as sync]
    [metabase.test :as mt]
+   [metabase.test.data.sql :as sql.tx]
    [toucan2.core :as t2]
    [toucan2.util :as u]))
 
@@ -130,6 +132,24 @@
                           ["2024-01-23T00:00:00Z" 14.99]]
                          query-result)))))))))))
 
+(deftest create-table-from-schema!-test
+  (mt/test-drivers (mt/normal-drivers-with-feature :uploads)
+    (let [driver       driver/*driver*
+          db-id        (mt/id)
+          table-name   (mt/random-name)
+          schema-name  (sql.tx/session-schema driver)
+          table-schema {:name    (if schema-name
+                                   (keyword schema-name table-name)
+                                   (keyword table-name))
+                        :columns [{:name "id" :type :type/Integer :nullable? false}
+                                  {:name "name" :type :type/Text :nullable? true}]}]
+      (mt/as-admin
+        (testing "create-table-from-schema! should create the table successfully"
+          (transforms.util/create-table-from-schema! driver db-id table-schema)
+          (let [table-exists? (driver/table-exists? driver db-id {:schema schema-name :name table-name})]
+            (is (some? table-exists?) "Table should exist in the database schema")
+            (driver/drop-table! driver db-id (:name table-schema))))))))
+
 (deftest transform-schema-created-if-needed-test
   (mt/test-drivers (mt/normal-driver-select {:+features [:transforms/table :schemas]})
     (mt/dataset transforms-dataset/transforms-test
@@ -224,3 +244,98 @@
             (driver/execute-raw-queries! driver/*driver* spec
                                          [[(format "DROP OWNED BY %s;" no-schema-user)]
                                           [(format "DROP ROLE IF EXISTS %s;" no-schema-user)]])))))))
+
+(deftest atomic-python-transform-swap-test
+  (testing "Python transform execution with atomic table swap"
+    (mt/test-drivers #{:h2 :postgres}
+      (mt/with-premium-features #{:transforms}
+        (mt/dataset transforms-dataset/transforms-test
+          (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
+            (with-transform-cleanup! [{table-name :name :as target} {:type   "table"
+                                                                     :schema schema
+                                                                     :name   "atomic_python_swap"}]
+              (let [initial-transform {:name   "Python Transform Initial"
+                                       :source {:type  "python"
+                                                :source-tables {}
+                                                :body  (str "import pandas as pd\n"
+                                                            "\n"
+                                                            "def transform():\n"
+                                                            "    return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}
+                                       :target (assoc target :database (mt/id))}]
+                (mt/with-temp [:model/Transform transform initial-transform]
+                  (transforms.execute/execute-python-transform! transform {:run-method :manual})
+                  (wait-for-table table-name 10000)
+
+                  (let [initial-rows (transforms.tu/table-rows table-name)]
+                    (is (= [["Alice" 25] ["Bob" 30]] initial-rows) "Initial data should be Alice and Bob")
+
+                    (t2/update! :model/Transform (:id transform)
+                                {:source {:type "python"
+                                          :source-tables {}
+                                          :body (str "import pandas as pd\n"
+                                                     "\n"
+                                                     "def transform():\n"
+                                                     "    return pd.DataFrame({'name': ['Charlie', 'Diana', 'Eve'], 'age': [35, 40, 45]})")}}))
+
+                  ;; Test atomic swap by intercepting drop operation
+                  (let [drop-latch (java.util.concurrent.CountDownLatch. 1)
+                        original-drop-table! transforms.util/drop-table!]
+                    (with-redefs [transforms.util/drop-table! (fn [driver db-id table-name]
+                                                                ;; Wait for latch before proceeding with drop
+                                                                (.await drop-latch)
+                                                                (original-drop-table! driver db-id table-name))]
+                      ;; Start transform in background
+                      (let [transform-future (future
+                                               (transforms.execute/execute-python-transform!
+                                                (t2/select-one :model/Transform (:id transform))
+                                                {:run-method :manual}))]
+                        ;; Give transform time to create temp table and reach the drop operation
+                        (Thread/sleep 1000)
+                        ;; Verify old table is still accessible during transform
+                        (is (= [["Alice" 25] ["Bob" 30]]
+                               (transforms.tu/table-rows table-name))
+                            "Original data should still be accessible during transform")
+                        ;; Release the latch to allow drop and complete transform
+                        (.countDown drop-latch)
+                        ;; Wait for transform to complete
+                        @transform-future
+                        ;; Verify final state
+                        (is (= [["Charlie" 35] ["Diana" 40] ["Eve" 45]]
+                               (transforms.tu/table-rows table-name))
+                            "Table should now contain Charlie, Diana, and Eve")))))))))))))
+
+(deftest python-transform-temp-table-cleanup-test
+  (testing "Python transform cleans up temp tables on success"
+    (mt/test-drivers #{:h2 :postgres}
+      (mt/with-premium-features #{:transforms}
+        (mt/dataset transforms-dataset/transforms-test
+          (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
+            (with-transform-cleanup! [{table-name :name :as target} {:type   "table"
+                                                                     :schema schema
+                                                                     :name   "python_cleanup_test"}]
+              (let [transform-def {:name   "Python Transform Cleanup"
+                                   :source {:type  "python"
+                                            :source-tables {}
+                                            :body  (str "import pandas as pd\n"
+                                                        "\n"
+                                                        "def transform():\n"
+                                                        "    return pd.DataFrame({'col1': [1, 2, 3], 'col2': ['a', 'b', 'c']})")}
+                                   :target (assoc target :database (mt/id))}]
+                (mt/with-temp [:model/Transform transform transform-def]
+                  ;; Run initial transform
+                  (transforms.execute/execute-python-transform! transform {:run-method :manual})
+                  (wait-for-table table-name 10000)
+
+                  ;; Run again to trigger temp table creation and swap
+                  (transforms.execute/execute-python-transform! transform {:run-method :manual})
+
+                  ;; Check that no temp tables remain
+                  (let [db-id (mt/id)
+                        tables (t2/select :model/Table :db_id db-id :active true)
+                        temp-table-pattern (re-pattern (str ".*" table-name "_temp_.*"))]
+                    (is (not-any? #(re-matches temp-table-pattern (:name %)) tables)
+                        "No temp tables should remain after successful Python transform")
+
+                    ;; Verify the final data is correct
+                    (is (= [[1 "a"] [2 "b"] [3 "c"]] (transforms.tu/table-rows table-name))
+                        "Table should contain the expected data after swap")))))))))))

enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj

@@ -1,25 +1,46 @@
 (ns ^:mb/driver-tests metabase-enterprise.transforms.ordering-test
   (:require
    [clojure.test :refer :all]
+   [metabase-enterprise.transforms.execute :as transforms.execute]
    [metabase-enterprise.transforms.ordering :as ordering]
+   [metabase.driver :as driver]
+   [metabase.driver.sql :as driver.sql]
    [metabase.query-processor.compile :as qp.compile]
    [metabase.test :as mt]
+   [metabase.test.data.sql :as sql.tx]
    [toucan2.core :as t2]))
 
-(defn- make-transform [{:keys [query name]}]
-  {:source {:type "query",
-            :query query}
-   :name "transform1"
-   :target {:schema "public"
-            :name (or name "orders_2")
-            :type "table"}})
+(defn- make-transform [query & [name schema]]
+  (let [name (or name (mt/random-name))
+        default-schema (some-> driver/*driver* driver.sql/default-schema)
+        schema (or schema default-schema "public")]
+    {:source {:type :query
+              :query query}
+     :name (str "transform_" name)
+     :target {:schema schema
+              :name name
+              :type :table}}))
+
+(defn- make-python-transform [source-tables & [name schema target-db]]
+  (let [name (or name (mt/random-name))
+        default-schema (some-> driver/*driver* driver.sql/default-schema)
+        schema (or schema default-schema "public")
+        target-db (or target-db (mt/id))]
+    {:source {:type "python"
+              :source-tables source-tables
+              :body "# Python code here\nresult = df"}
+     :name (str "python_transform_" name)
+     :target {:database target-db
+              :schema schema
+              :name name
+              :type "table"}}))
 
 (deftest basic-ordering-test
   (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
     (mt/with-temp [:model/Transform {t1 :id} (make-transform
-                                              {:query {:database (mt/id),
-                                                       :type "query",
-                                                       :query {:source-table (mt/id :orders)}}})]
+                                              {:database (mt/id),
+                                               :type "query",
+                                               :query {:source-table (mt/id :orders)}})]
       (is (= {t1 #{}}
              (ordering/transform-ordering (t2/select :model/Transform :id t1)))))))
 
@@ -30,14 +51,15 @@
                    :model/Field _ {:table_id table
                                    :name "foo"}
                    :model/Transform {parent :id} (make-transform
-                                                  {:query {:database (mt/id),
-                                                           :type "query",
-                                                           :query {:source-table (mt/id :orders)}}})
+                                                  {:database (mt/id),
+                                                   :type "query",
+                                                   :query {:source-table (mt/id :orders)}}
+                                                  "orders_2")
                    :model/Transform {child :id} (make-transform
-                                                 {:query {:database (mt/id)
-                                                          :type "query"
-                                                          :query {:source-table table}}
-                                                  :name "orders_3"})]
+                                                 {:database (mt/id)
+                                                  :type "query"
+                                                  :query {:source-table table}}
+                                                 "orders_3")]
       (is (= {parent #{}
               child #{parent}}
              (ordering/transform-ordering (t2/select :model/Transform :id [:in [parent child]])))))))
@@ -46,36 +68,134 @@
   (mt/with-metadata-provider (mt/id)
     (#'ordering/transform-deps transform)))
 
+(deftest not-run-transform-dependency-ordering-test
+  (mt/test-driver :postgres
+    (testing "dependencies are correctly identified when no transforms have been run yet"
+      (mt/with-temp [:model/Transform {t1 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type :native,
+                                                 :native {:query "SELECT * FROM orders LIMIT 100"}}
+                                                "orders_transform")
+                     :model/Transform {t2 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type :native,
+                                                 :native {:query "SELECT * FROM products LIMIT 100"}}
+                                                "products_transform")
+                     :model/Transform {t3 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type :native,
+                                                 :native {:query "SELECT * FROM orders_transform ot
+                                                                  LEFT JOIN products p
+                                                                  ON ot.product_id = p.id
+                                                                  LIMIT 100"}}
+                                                "orders_transform_products")
+                     :model/Transform {t4 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type :native,
+                                                 :native {:query "SELECT * FROM orders_transform ot
+                                                                  LEFT JOIN products_transform pt
+                                                                  ON ot.product_id = pt.id
+                                                                  LIMIT 100"}}
+                                                "orders_transform_products_transform")]
+        (is (= #{{:transform t1} {:transform t2}}
+               (transform-deps-for-db (t2/select-one :model/Transform  t4))))
+        (is (= {t1 #{}
+                t2 #{}
+                t3 #{t1}
+                t4 #{t1 t2}}
+               (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3 t4]]))))))
+    (testing "dependencies are correctly identified when some transform have been run and some haven't"
+      (mt/with-temp [:model/Transform {t1 :id :as transform1} (make-transform
+                                                               {:database (mt/id),
+                                                                :type :native,
+                                                                :native {:query "SELECT * FROM checkins LIMIT 100"}}
+                                                               "checkins_transform")
+                     :model/Transform {t2 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type :native,
+                                                 :native {:query "SELECT * FROM venues LIMIT 100"}}
+                                                "venues_transform")
+                     :model/Transform {t3 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type :native,
+                                                 :native {:query "SELECT * FROM venues_transform vt
+                                                                  LEFT JOIN checkins_transform ct
+                                                                  ON vt.id = ct.venue_id
+                                                                  LIMIT 100"}}
+                                                "venues_transform_2")]
+        (try
+          (transforms.execute/run-mbql-transform! transform1 {:run-method :manual})
+          (let [table1 (t2/select-one-pk :model/Table :name "checkins_transform")]
+            (is (= #{{:table table1} {:transform t2}}
+                   (transform-deps-for-db (t2/select-one :model/Transform  t3)))))
+          (is (= {t1 #{}
+                  t2 #{}
+                  t3 #{t1 t2}}
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]]))))
+          (finally
+            (t2/delete! :model/Table :name "checkins_transform")))))))
+
+(deftest bigquery-native-transform-dependency-ordering-test
+  (mt/test-driver :bigquery-cloud-sdk
+    (let [[dataset] (sql.tx/qualified-name-components :bigquery-cloud-sdk nil)]
+      (testing "dependencies are correctly identified when some transform have been run and some haven't"
+        (mt/with-temp [:model/Transform {t1 :id} (make-transform
+                                                  {:database (mt/id),
+                                                   :type :native,
+                                                   :native {:query (format "SELECT * FROM `%s.checkins` LIMIT 100" dataset)}}
+                                                  "checkins_transform" dataset)
+                       :model/Table {table1 :id} {:schema dataset
+                                                  :name "checkins_transform"}
+                       :model/Transform {t2 :id} (make-transform
+                                                  {:database (mt/id),
+                                                   :type :native,
+                                                   :native {:query (format "SELECT * FROM `%s.venues` LIMIT 100" dataset)}}
+                                                  "venues_transform" dataset)
+                       :model/Transform {t3 :id} (make-transform
+                                                  {:database (mt/id),
+                                                   :type :native,
+                                                   :native {:query (format "SELECT * FROM `%s.venues_transform` vt
+                                                                            LEFT JOIN `%s.checkins_transform` ct
+                                                                            ON vt.id = ct.venue_id
+                                                                            LIMIT 100" dataset dataset)}}
+                                                  "venues_transform_2" dataset)]
+          (is (= #{{:table table1} {:transform t2}}
+                 (transform-deps-for-db (t2/select-one :model/Transform  t3))))
+          (is (= {t1 #{}
+                  t2 #{}
+                  t3 #{t1 t2}}
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]])))))))))
+
 (deftest basic-dependencies-test
   (mt/with-temp [:model/Transform {t1 :id} (make-transform
-                                            {:query {:database (mt/id),
-                                                     :type "query",
-                                                     :query {:source-table (mt/id :orders)}}})]
-    (is (= #{(mt/id :orders)}
+                                            {:database (mt/id),
+                                             :type "query",
+                                             :query {:source-table (mt/id :orders)}})]
+    (is (= #{{:table (mt/id :orders)}}
            (transform-deps-for-db (t2/select-one :model/Transform :id t1))))))
 
 (deftest joined-dependencies-test
   (mt/with-temp [:model/Transform {t1 :id} (make-transform
-                                            {:query {:database (mt/id),
-                                                     :type "query",
-                                                     :query {:source-table (mt/id :orders),
-                                                             :joins
-                                                             [{:fields "all",
-                                                               :strategy "left-join",
-                                                               :alias "Products",
-                                                               :condition
-                                                               [:=
-                                                                [:field
-                                                                 (mt/id :orders :product_id)
-                                                                 {:base-type "type/Integer"}]
-                                                                [:field
-                                                                 (mt/id :products :id)
-                                                                 {:base-type "type/Integer",
-                                                                  :join-alias "Products"}]],
-                                                               :source-table (mt/id :products)}]},
-                                                     :parameters []}})]
-    (is (= #{(mt/id :orders)
-             (mt/id :products)}
+                                            {:database (mt/id),
+                                             :type "query",
+                                             :query {:source-table (mt/id :orders),
+                                                     :joins
+                                                     [{:fields "all",
+                                                       :strategy "left-join",
+                                                       :alias "Products",
+                                                       :condition
+                                                       [:=
+                                                        [:field
+                                                         (mt/id :orders :product_id)
+                                                         {:base-type "type/Integer"}]
+                                                        [:field
+                                                         (mt/id :products :id)
+                                                         {:base-type "type/Integer",
+                                                          :join-alias "Products"}]],
+                                                       :source-table (mt/id :products)}]},
+                                             :parameters []})]
+    (is (= #{{:table (mt/id :orders)}
+             {:table (mt/id :products)}}
            (transform-deps-for-db (t2/select-one :model/Transform :id t1))))))
 
 (deftest card-dependencies-test
@@ -99,27 +219,27 @@
                                                                      :source-table (mt/id :products)}]},
                                                            :parameters []}}
                    :model/Transform {t1 :id} (make-transform
-                                              {:query {:database (mt/id),
-                                                       :type "query",
-                                                       :query
-                                                       {:aggregation [["count"]],
-                                                        :breakout [["field" "Products__category" {:base-type "type/Text"}]],
-                                                        :source-table (str "card__" card)},
-                                                       :parameters []}})]
-      (is (= #{(mt/id :orders)
-               (mt/id :products)}
+                                              {:database (mt/id),
+                                               :type "query",
+                                               :query
+                                               {:aggregation [["count"]],
+                                                :breakout [["field" "Products__category" {:base-type "type/Text"}]],
+                                                :source-table (str "card__" card)},
+                                               :parameters []})]
+      (is (= #{{:table (mt/id :orders)}
+               {:table (mt/id :products)}}
              (transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))
 
 (deftest native-dependencies-test
   (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
     (mt/with-temp [:model/Transform {t1 :id} (make-transform
-                                              {:query {:database (mt/id),
-                                                       :type :native
-                                                       :native (qp.compile/compile
-                                                                {:database (mt/id),
-                                                                 :type "query",
-                                                                 :query {:source-table (mt/id :orders)}})}})]
-      (is (= #{(mt/id :orders)}
+                                              {:database (mt/id),
+                                               :type :native
+                                               :native (qp.compile/compile
+                                                        {:database (mt/id),
+                                                         :type "query",
+                                                         :query {:source-table (mt/id :orders)}})})]
+      (is (= #{{:table (mt/id :orders)}}
              (transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))
 
 (deftest native-card-dependencies-test
@@ -145,21 +265,20 @@
                                                                      :source-table (mt/id :products)}]},
                                                            :parameters []}}
                    :model/Transform {t1 :id} (make-transform
-                                              {:query {:database (mt/id),
-                                                       :type :native,
-                                                       :native {:query
-                                                                (mt/native-query-with-card-template-tag
-                                                                 (:engine (mt/db))
-                                                                 (str "#" card)),
-
-                                                                :template-tags
-                                                                {(keyword (str "#" card))
-                                                                 {:type "card"
-                                                                  :name "card"
-                                                                  :display-name "card"
-                                                                  :card-id card}}}}})]
-      (is (= #{(mt/id :orders)
-               (mt/id :products)}
+                                              {:database (mt/id),
+                                               :type :native,
+                                               :native {:query
+                                                        (mt/native-query-with-card-template-tag
+                                                         (:engine (mt/db))
+                                                         (str "#" card)),
+                                                        :template-tags
+                                                        {(keyword (str "#" card))
+                                                         {:type "card"
+                                                          :name "card"
+                                                          :display-name "card"
+                                                          :card-id card}}}})]
+      (is (= #{{:table (mt/id :orders)}
+               {:table (mt/id :products)}}
              (transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))
 
 (defn- rotations
@@ -181,3 +300,173 @@
         cycles (rotations [3 4 6 7])]
     (is (contains? cycles
                    (ordering/find-cycle test-graph)))))
+
+(deftest get-transform-cycle-test
+  (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+    (testing "cycle is detected in transform referencing itself"
+      (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                :name "table_1"}
+                     :model/Field _ {:table_id table1
+                                     :name "foo"}
+                     :model/Transform {t1 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type "query",
+                                                 :query {:source-table table1}}
+                                                "table_1")]
+        (is (= {:cycle-str "transform_table_1"
+                :cycle [t1]}
+               (ordering/get-transform-cycle (t2/select-one :model/Transform :id t1))))))
+    (testing "cycle is caught in 2 transforms referencing each other"
+      (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                :name "table_1"}
+                     :model/Field _ {:table_id table1
+                                     :name "foo"}
+                     :model/Table {table2 :id} {:schema "public"
+                                                :name "table_2"}
+                     :model/Field _ {:table_id table2
+                                     :name "foo"}
+                     :model/Transform {t1 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type "query",
+                                                 :query {:source-table table1}}
+                                                "table_2")
+                     :model/Transform {t2 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type "query",
+                                                 :query {:source-table table2}}
+                                                "table_1")]
+        (is (= {:cycle-str "transform_table_2 -> transform_table_1",
+                :cycle [t1 t2]}
+               (ordering/get-transform-cycle (t2/select-one :model/Transform :id t1))))))
+    (testing "cycle is detected in 3 transforms referencing each other"
+      (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                :name "table_1"}
+                     :model/Field _ {:table_id table1
+                                     :name "foo"}
+                     :model/Table {table2 :id} {:schema "public"
+                                                :name "table_2"}
+                     :model/Field _ {:table_id table2
+                                     :name "foo"}
+                     :model/Table {table3 :id} {:schema "public"
+                                                :name "table_3"}
+                     :model/Field _ {:table_id table3
+                                     :name "foo"}
+                     :model/Transform {t1 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type "query",
+                                                 :query {:source-table table1}}
+                                                "table_2")
+                     :model/Transform {t2 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type "query",
+                                                 :query {:source-table table2}}
+                                                "table_3")
+                     :model/Transform {t3 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type "query",
+                                                 :query {:source-table table3}}
+                                                "table_1")]
+        (is (= {:cycle-str "transform_table_2 -> transform_table_1 -> transform_table_3",
+                :cycle [t1 t3 t2]}
+               (ordering/get-transform-cycle (t2/select-one :model/Transform :id t1)))))))
+
+  (deftest python-transform-basic-dependencies-test
+    (testing "Python transforms with source-tables dependencies are extracted correctly"
+      (mt/with-temp [:model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)
+                                                                       "products" (mt/id :products)})]
+        (is (= #{{:table (mt/id :orders)}
+                 {:table (mt/id :products)}}
+               (transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))
+
+  (deftest python-transform-ordering-test
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+      (testing "Python transforms are ordered correctly based on source-tables dependencies"
+        (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                  :name "output_1"}
+                       :model/Field _ {:table_id table1
+                                       :name "foo"}
+                       :model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)} "output_1")
+                       :model/Transform {t2 :id} (make-python-transform {"output_1" table1} "output_2")]
+          (is (= {t1 #{}
+                  t2 #{t1}}
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2]]))))))))
+
+  (deftest python-transform-multiple-dependencies-test
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+      (testing "Python transforms with multiple table dependencies"
+        (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                  :name "output_1"}
+                       :model/Field _ {:table_id table1
+                                       :name "foo"}
+                       :model/Table {table2 :id} {:schema "public"
+                                                  :name "output_2"}
+                       :model/Field _ {:table_id table2
+                                       :name "bar"}
+                       :model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)} "output_1")
+                       :model/Transform {t2 :id} (make-python-transform {"products" (mt/id :products)} "output_2")
+                       :model/Transform {t3 :id} (make-python-transform {"output_1" table1
+                                                                         "output_2" table2} "final_output")]
+          (is (= {t1 #{}
+                  t2 #{}
+                  t3 #{t1 t2}}
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]]))))))))
+
+  (deftest mixed-transform-ordering-test
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+      (testing "Python and query transforms are ordered together correctly"
+        (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                  :name "sql_output"}
+                       :model/Field _ {:table_id table1
+                                       :name "foo"}
+                       :model/Table {table2 :id} {:schema "public"
+                                                  :name "python_output"}
+                       :model/Field _ {:table_id table2
+                                       :name "bar"}
+                       ;; SQL transform that depends on orders table
+                       :model/Transform {t1 :id} (make-transform
+                                                  {:database (mt/id),
+                                                   :type "query",
+                                                   :query {:source-table (mt/id :orders)}}
+                                                  "sql_output")
+                       ;; Python transform that depends on the SQL transform's output
+                       :model/Transform {t2 :id} (make-python-transform {"sql_output" table1} "python_output")
+                       ;; Another SQL transform that depends on the Python transform's output
+                       :model/Transform {t3 :id} (make-transform
+                                                  {:database (mt/id)
+                                                   :type "query"
+                                                   :query {:source-table table2}}
+                                                  "final_output")]
+          (is (= {t1 #{}
+                  t2 #{t1}
+                  t3 #{t2}}
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]])))))))))
+
+#_(deftest mixed-database-transform-ordering-test
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+      (testing "Python and query transforms are ordered together correctly, across multiple databases"
+      ;; orders (db-1) -> sql_output (db-1) -> python_output (db-2) -> final_output (db-2)
+        (mt/with-temp [:model/Database db-2    {}
+                       :model/Table    table-1 {:db_id (mt/id),    :schema "public", :name "sql_output"}
+                       :model/Table    table-2 {:db_id (:id db-2), :schema "public", :name "python_output"}
+                       :model/Field    _       {:table_id (:id table-1), :name "foo"}
+                       :model/Field    _       {:table_id (:id table-2), :name "bar"}
+
+                     ;; SQL transform that depends on orders table
+                       :model/Transform {t1 :id} (make-transform
+                                                  {:database (mt/id)
+                                                   :type     "query"
+                                                   :query    {:source-table (mt/id :orders)}}
+                                                  (:name table-1))
+                     ;; Python transform that depends on the SQL transform's output
+                       :model/Transform {t2 :id} (make-python-transform {"sql_output" (:id table-1)} (:name table-2))
+                     ;; Another SQL transform that depends on the Python transform's output
+                       :model/Transform {t3 :id} (make-transform
+                                                  {:database (:id db-2)
+                                                   :type     "query"
+                                                   :query    {:source-table (:id table-2)}}
+                                                  "final_output")]
+          (is (= {t1 #{}
+                  t2 #{t1}
+                  t3 #{t2}}
+
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]]))))))))

enterprise/backend/test/metabase_enterprise/transforms/python_runner_test.clj

@@ -0,0 +1,424 @@
+(ns ^:mb/driver-tests metabase-enterprise.transforms.python-runner-test
+  (:require
+   [clojure.core.async :as a]
+   [clojure.data.csv :as csv]
+   [clojure.java.jdbc :as jdbc]
+   [clojure.string :as str]
+   [clojure.test :refer :all]
+   [java-time.api :as t]
+   [metabase-enterprise.transforms.python-runner :as python-runner]
+   [metabase-enterprise.transforms.schedule :as transforms.schedule]
+   [metabase-enterprise.transforms.settings :as transforms.settings]
+   [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
+   [metabase-enterprise.transforms.test-util :as transforms.tu]
+   [metabase-enterprise.transforms.util :as transforms.util]
+   [metabase.driver :as driver]
+   [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
+   [metabase.sync.core :as sync]
+   [metabase.task.core :as task]
+   [metabase.test :as mt]
+   [metabase.test.data.interface :as tx]
+   [metabase.test.data.sql :as sql.tx]
+   [metabase.util :as u]
+   [metabase.util.json :as json]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(defn template->regex
+  "Convert a template string with $var$ placeholders to a regex pattern.
+   Example: template->regex 'File ___PATH___/script.py, line ___LINE___'
+   will match 'File /tmp/unique-path/script.py, line 123'"
+  [template]
+  (-> template
+      (str/replace #"(?m)^\d{13}\s*" "")
+      (str/replace "\\" "\\\\")
+      (str/replace "." "\\.")
+      (str/replace "*" "\\*")
+      (str/replace "+" "\\+")
+      (str/replace "?" "\\?")
+      (str/replace "^" "\\^")
+      (str/replace "$" "\\$")
+      (str/replace "{" "\\{")
+      (str/replace "}" "\\}")
+      (str/replace "(" "\\(")
+      (str/replace ")" "\\)")
+      (str/replace "|" "\\|")
+      (str/replace "[" "\\[")
+      (str/replace "]" "\\]")
+      (str/replace #"___PATH___" "(/[\\\\w-]+)+")
+      (str/replace #"___LINE___" "\\\\d+")
+      re-pattern))
+
+(def ^:private test-id 1)
+
+(defn- execute [{:keys [code tables]}]
+  (with-open [shared-storage-ref (python-runner/open-s3-shared-storage! (or tables {}))]
+    (let [server-url     (transforms.settings/python-execution-server-url)
+          cancel-chan    (a/promise-chan)
+          table-name->id (or tables {})
+          _              (python-runner/copy-tables-to-s3! {:run-id         test-id
+                                                            :shared-storage @shared-storage-ref
+                                                            :table-name->id table-name->id
+                                                            :cancel-chan    cancel-chan})
+          response       (python-runner/execute-python-code-http-call! {:server-url     server-url
+                                                                        :code           code
+                                                                        :run-id         test-id
+                                                                        :table-name->id table-name->id
+                                                                        :shared-storage @shared-storage-ref})
+          {:keys [output output-manifest events]} (python-runner/read-output-objects @shared-storage-ref)]
+      ;; not sure about munging this all together but its what tests expect for now
+      (merge (:body response)
+             {:output          output
+              :output-manifest output-manifest
+              :stdout          (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
+              :stderr          (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))}))))
+
+(deftest ^:parallel transform-function-basic-test
+  (testing "executes transform function and returns CSV output"
+    (let [transform-code (str "import pandas as pd\n"
+                              "\n"
+                              "def transform():\n"
+                              "    return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")
+          result         (execute {:code transform-code})]
+      (is (=? {:output "name,age\nAlice,25\nBob,30\n"
+               :stdout "Successfully saved 2 rows to S3\nSuccessfully saved output manifest with 2 fields"
+               :stderr ""}
+              result)))))
+
+(deftest ^:parallel transform-function-missing-test
+  (testing "handles missing transform function"
+    (let [result (execute {:code (str "import pandas as pd\n"
+                                      "\n"
+                                      "# No transform function defined")})]
+      (is (=? {:exit_code 1
+               :stderr    "ERROR: User script must define a 'transform()' function"
+               :stdout    ""}
+              result)))))
+
+(deftest ^:parallel transform-function-wrong-return-type-test
+  (testing "handles transform function returning non-DataFrame"
+    (let [result (execute {:code (str "def transform():\n"
+                                      "    return 'not a dataframe'")})]
+      (is (=? {:exit_code 1
+               :stderr    "ERROR: Transform function must return a pandas DataFrame, got <class 'str'>"
+               :stdout    ""}
+              result)))))
+
+(deftest ^:parallel transform-function-error-test
+  (testing "handles transform function with error"
+    (let [result            (execute {:code (str "def transform():\n"
+                                                 "    raise ValueError('Something went wrong')")})
+          expected-template (str "ERROR: Transform function failed: Something went wrong\n"
+                                 "Traceback (most recent call last):\n"
+                                 "  File \"/app/external/transform_runner.py\", line ___LINE___, in main\n"
+                                 "    result = script.transform()\n"
+                                 "             ^^^^^^^^^^^^^^^^^^\n"
+                                 "  File \"___PATH___/script.py\", line 2, in transform\n"
+                                 "    raise ValueError('Something went wrong')\n"
+                                 "ValueError: Something went wrong")
+          stderr-pattern    (template->regex expected-template)]
+      (is (=? {:exit_code 1
+               :stderr    #(re-matches stderr-pattern %)
+               :stdout    ""
+               :timeout   false}
+              result)))))
+
+(deftest ^:parallel transform-function-complex-dataframe-test
+  (testing "can create complex DataFrames with transform"
+    (let [transform-code (str "import pandas as pd\n"
+                              "\n"
+                              "def transform():\n"
+                              "    data = {'x': [1, 2, 3], 'y': [10, 20, 30], 'z': ['a', 'b', 'c']}\n"
+                              "    return pd.DataFrame(data)")
+          result         (execute {:code transform-code})]
+      (is (=? {:output "x,y,z\n1,10,a\n2,20,b\n3,30,c\n"
+               :stdout "Successfully saved 3 rows to S3\nSuccessfully saved output manifest with 3 fields"
+               :stderr ""}
+              result)))))
+
+(deftest ^:parallel transform-function-with-db-parameter-test
+  (mt/test-drivers #{:postgres}
+    (testing "transform function can accept db parameter for forward compatibility"
+      (let [transform-code (str "import pandas as pd\n"
+                                "\n"
+                                "def transform():\n"
+                                "    data = {'name': ['Charlie', 'Dana'], 'score': [85, 92]}\n"
+                                "    return pd.DataFrame(data)")
+            result         (execute {:code transform-code})]
+        (is (=? {:output "name,score\nCharlie,85\nDana,92\n"
+                 :stdout "Successfully saved 2 rows to S3\nSuccessfully saved output manifest with 2 fields"
+                 :stderr ""}
+                result))))))
+
+(deftest transform-function-with-pass-thru
+  (testing "transform function successfully connects to PostgreSQL database and reads data"
+    (mt/test-drivers #{:postgres}
+      (mt/with-empty-db
+        (let [db-spec        (sql-jdbc.conn/db->pooled-connection-spec (mt/db))
+              _              (jdbc/execute! db-spec ["DROP TABLE IF EXISTS students"])
+              _              (jdbc/execute! db-spec ["CREATE TABLE students (id INTEGER PRIMARY KEY, name VARCHAR(100), score INTEGER)"])
+              _              (jdbc/execute! db-spec ["INSERT INTO students (id, name, score) VALUES (1, 'Alice', 85), (2, 'Bob', 92), (3, 'Charlie', 88), (4, 'Dana', 90)"])
+
+              _              (sync/sync-database! (mt/db))
+
+              transform-code (str "import pandas as pd\n"
+                                  "\n"
+                                  "def transform(students):\n"
+                                  "    return students")
+              result         (execute {:code   transform-code
+                                       :tables {"students" (mt/id :students)}})]
+
+          (is (=? {:output   "id,name,score\n1,Alice,85\n2,Bob,92\n3,Charlie,88\n4,Dana,90\n"
+                   :output-manifest {:fields [{:base_type "Integer",
+                                               :database_type "int4",
+                                               :dtype "int64",
+                                               :effective_type "Integer",
+                                               :name "id",
+                                               :root_type "Integer",
+                                               :semantic_type "PK"}
+                                              {:base_type "Text",
+                                               :database_type "varchar",
+                                               :dtype "object",
+                                               :effective_type "Text",
+                                               :name "name",
+                                               :root_type "Text",
+                                               :semantic_type "Name"}
+                                              {:base_type "Integer",
+                                               :database_type "int4",
+                                               :dtype "int64",
+                                               :effective_type "Integer",
+                                               :name "score",
+                                               :root_type "Integer",
+                                               :semantic_type "Score"}],
+                                     :source_metadata {:fields [{:base_type "Integer",
+                                                                 :database_type "int4",
+                                                                 :dtype "int64",
+                                                                 :effective_type "Integer",
+                                                                 :name "id",
+                                                                 :root_type "Integer",
+                                                                 :semantic_type "PK"}
+                                                                {:base_type "Text",
+                                                                 :database_type "varchar",
+                                                                 :dtype "object",
+                                                                 :effective_type "Text",
+                                                                 :name "name",
+                                                                 :root_type "Text",
+                                                                 :semantic_type "Name"}
+                                                                {:base_type "Integer",
+                                                                 :database_type "int4",
+                                                                 :dtype "int64",
+                                                                 :effective_type "Integer",
+                                                                 :name "score",
+                                                                 :root_type "Integer",
+                                                                 :semantic_type "Score"}],
+                                                       :table_metadata {:table_id (mt/malli=? int?)},
+                                                       :version "0.1.0"},
+                                     :version "0.1.0"}
+                   :stdout   (str "Successfully saved 4 rows to S3\n"
+                                  "Successfully saved output manifest with 3 fields")
+                   :stderr   ""}
+                  result)))))))
+
+(deftest transform-function-with-working-database-test
+  (testing "transform function successfully connects to PostgreSQL database and reads data"
+    (mt/test-drivers #{:postgres}
+      (mt/with-empty-db
+        (let [db-spec        (sql-jdbc.conn/db->pooled-connection-spec (mt/db))
+              _              (jdbc/execute! db-spec ["DROP TABLE IF EXISTS students"])
+              _              (jdbc/execute! db-spec ["CREATE TABLE students (id INTEGER PRIMARY KEY, name VARCHAR(100), score INTEGER)"])
+              _              (jdbc/execute! db-spec ["INSERT INTO students (id, name, score) VALUES (1, 'Alice', 85), (2, 'Bob', 92), (3, 'Charlie', 88), (4, 'Dana', 90)"])
+
+              _              (sync/sync-database! (mt/db))
+
+              transform-code (str "import pandas as pd\n"
+                                  "\n"
+                                  "def transform(students):\n"
+                                  "    # Calculate average score\n"
+                                  "    avg_score = students['score'].mean()\n"
+                                  "    result = pd.DataFrame({\n"
+                                  "        'student_count': [len(students)],\n"
+                                  "        'average_score': [round(avg_score, 2)]\n"
+                                  "    })\n"
+                                  "    return result")
+              result         (execute {:code   transform-code
+                                       :tables {"students" (mt/id :students)}})]
+
+          (is (=? {:output   "student_count,average_score\n4,88.75\n"
+                   :output-manifest {:version "0.1.0",
+                                     :fields  [{:name "student_count", :dtype "int64"}
+                                               {:name "average_score", :dtype "float64"}]}
+                   :stdout   (str "Successfully saved 1 rows to S3\n"
+                                  "Successfully saved output manifest with 2 fields")
+                   :stderr   ""}
+                  result)))))))
+
+(defn- datetime-equal?
+  [expected-iso-str actual-pandas-str]
+  (let [formatter   (t/formatter "yyyy-MM-dd HH:mm:ssXXX")
+        expected-dt (t/zoned-date-time expected-iso-str)
+        actual-dt   (t/offset-date-time formatter actual-pandas-str)]
+    (= (t/to-millis-from-epoch expected-dt)
+       (t/to-millis-from-epoch actual-dt))))
+
+(deftest transform-type-test
+  (mt/test-drivers #{:postgres}
+    (mt/with-empty-db
+      (let [db-spec (sql-jdbc.conn/db->pooled-connection-spec (mt/db))
+            _ (jdbc/execute! db-spec ["DROP TABLE IF EXISTS sample_table"])
+            _ (jdbc/execute! db-spec ["CREATE TABLE sample_table (
+                                                        id BIGSERIAL PRIMARY KEY,
+                                                        name VARCHAR(255),
+                                                        description TEXT,
+                                                        count BIGINT,
+                                                        price FLOAT,
+                                                        is_active BOOLEAN,
+                                                        created_date DATE,
+                                                        updated_at TIMESTAMP,
+                                                        scheduled_for TIMESTAMP WITH TIME ZONE
+                                                    )"])
+            _ (jdbc/execute! db-spec ["INSERT INTO sample_table
+                                                        (name, description, count, price, is_active, created_date, updated_at, scheduled_for)
+                                                      VALUES
+                                                        ('Product A', 'A sample product description', 100, 29.99, true, '2024-01-15', '2024-01-15 10:30:00', '2024-01-16 14:00:00+00'),
+                                                        ('Product B', 'Another product with longer description text', 50, 15.50, false, '2024-02-01', '2024-02-01 09:15:30', '2024-02-02 16:30:00-05'),
+                                                        ('Product C', NULL, 0, 0.0, true, '2024-03-10', '2024-03-10 18:45:15', '2024-03-11 08:00:00+02')"])
+
+            _ (sync/sync-database! (mt/db))
+
+            transform-code (str "import pandas as pd\n"
+                                "\n"
+                                "def transform(sample_table):\n"
+                                "    df = sample_table.copy()\n"
+                                "    return df")
+            result (execute {:code transform-code
+                             :tables {"sample_table" (mt/id :sample_table)}})
+            csv-data (csv/read-csv (:output result))
+            headers (first csv-data)
+            rows (rest csv-data)
+            [row1 row2 row3] rows
+            header-to-index (zipmap headers (range))
+            get-col (fn [row col-name] (nth row (header-to-index col-name)))
+            metadata (:output-manifest result)]
+
+        (is (= (set ["id" "name" "description" "count" "price" "is_active" "created_date" "updated_at" "scheduled_for"])
+               (set headers)))
+
+        (is (= "1" (get-col row1 "id")))
+        (is (= "Product A" (get-col row1 "name")))
+        (is (datetime-equal? "2024-01-16T14:00:00Z" (get-col row1 "scheduled_for")))
+
+        (is (= "2" (get-col row2 "id")))
+        (is (= "Product B" (get-col row2 "name")))
+        (is (datetime-equal? "2024-02-02T21:30:00Z" (get-col row2 "scheduled_for")))
+
+        (is (= "3" (get-col row3 "id")))
+        (is (= "Product C" (get-col row3 "name")))
+        (is (datetime-equal? "2024-03-11T06:00:00Z" (get-col row3 "scheduled_for")))
+        (testing "dtypes are preserved correctly"
+          (is (= {"id"            :type/Integer
+                  "name"          :type/Text
+                  "description"   :type/Text
+                  "count"         :type/Integer
+                  "price"         :type/Float
+                  "is_active"     :type/Boolean
+                  ;; Our hack works
+                  "created_date"  :type/Date
+                  "updated_at"    :type/DateTime
+                  "scheduled_for" :type/DateTimeWithTZ}
+                 (u/for-map [{:keys [name dtype]} (:fields metadata)]
+                   [name (transforms.util/dtype->base-type dtype)]))))))))
+
+(deftest python-transform-scheduled-job-test
+  (mt/test-helpers-set-global-values!
+    (mt/with-temp-scheduler!
+      (task/init! ::transforms.schedule/RunTransform)
+      (mt/test-drivers #{:h2 :postgres}
+        (mt/with-premium-features #{:transforms}
+          (mt/dataset transforms-dataset/transforms-test
+            (transforms.tu/with-transform-cleanup! [target {:type   "table"
+                                                            :schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))
+                                                            :name   "target_table"}]
+              (mt/with-temp
+                [:model/TransformTag {tag-id :id}       {:name "every second"}
+                 :model/Transform    {transform-id :id} {:name   "Gadget Products"
+                                                         :source {:type  "python"
+                                                                  :source-database (mt/id)
+                                                                  :source-tables {"transforms_customers" (mt/id :transforms_customers)}
+                                                                  :body  (str "import pandas as pd\n"
+                                                                              "\n"
+                                                                              "def transform():\n"
+                                                                              "    return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}
+                                                         :target  (assoc target :database (mt/id))}
+                 :model/TransformTransformTag   _  {:transform_id transform-id :tag_id tag-id :position 0}
+                 :model/TransformJob {job-id :id :as job} {:schedule "* * * * * ? *"}
+                 :model/TransformJobTransformTag _ {:job_id job-id :tag_id tag-id :position 0}]
+                (transforms.schedule/initialize-job! job)
+                (transforms.schedule/update-job! job-id "* * * * * ? *")
+                (is (true? (u/poll {:thunk   (fn [] (driver/table-exists? driver/*driver* (mt/db) target))
+                                    :done?   true?
+                                    :timeout 30000})))
+                (is (true? (t2/exists? :model/Table :name (:name target))))))))))))
+
+(deftest transform-type-roundtrip-test
+  (mt/test-drivers #{:postgres :h2 :mysql :bigquery-cloud-sdk :redshift :snowflake :sqlserver :clickhouse}
+    (mt/with-empty-db
+      (let [driver       driver/*driver*
+            db-id        (mt/id)
+            table-name   (if (= driver :redshift)
+                           (tx/db-qualified-table-name (get-in (mt/db) [:settings :database-source-dataset-name]) (mt/random-name))
+                           (mt/random-name))
+            schema-name  (sql.tx/session-schema driver)
+            qualified-table-name (if schema-name
+                                   (keyword schema-name table-name)
+                                   (keyword table-name))
+            table-schema {:name    qualified-table-name
+                          :columns [{:name "id"            :type :type/Integer  :nullable? false}
+                                    {:name "price"         :type :type/Float    :nullable? true}
+                                    {:name "active"        :type :type/Boolean  :nullable? true}
+                                    {:name "created_tz"    :type :type/DateTimeWithTZ :nullable? true}
+                                    {:name "created_at"    :type :type/DateTime :nullable? true}
+                                    {:name "created_date"  :type :type/Date     :nullable? true}
+                                    {:name "description"   :type :type/Text     :nullable? true}]}
+            _ (mt/as-admin
+                (transforms.util/create-table-from-schema! driver db-id table-schema))
+
+            row-values   [[1
+                           19.99
+                           (if (= driver/*driver* :sqlserver) 1 true)
+                           (if (= driver/*driver* :mysql) "2024-01-01 12:00:00" "2024-01-01T12:00:00Z")
+                           (if (= driver/*driver* :mysql) "2024-01-01 12:00:00" "2024-01-01T12:00:00")
+                           "2024-01-01"
+                           "Sample product"]]
+            _ (driver/insert-from-source! driver db-id qualified-table-name
+                                          (mapv :name (:columns table-schema))
+                                          {:type :rows :data row-values})
+
+            _ (sync/sync-database! (mt/db) {:scan :schema})
+
+            transform-code (str "import pandas as pd\n"
+                                "\n"
+                                "def transform(" table-name "):\n"
+                                "    return " table-name)
+            result (execute {:code transform-code
+                             :tables {table-name (mt/id qualified-table-name)}})
+
+            metadata (:output-manifest result)]
+
+        (testing "All expected columns are present"
+          (is (= #{"id" "price" "active" "created_tz" "created_at" "created_date" "description"}
+                 (set (map :name (:fields metadata))))))
+
+        (testing "dtypes are preserved correctly"
+          (is (= {"id"           :type/Integer
+                  "price"        :type/Float
+                  "active"       :type/Boolean
+                  "created_tz"   :type/DateTimeWithTZ
+                  "created_at"   :type/DateTime
+                  "created_date" :type/Date
+                  "description"  :type/Text}
+                 (u/for-map [{:keys [name dtype]} (:fields metadata)]
+                   [name (transforms.util/dtype->base-type dtype)]))))
+
+        ;; cleanup
+        (driver/drop-table! driver db-id qualified-table-name)))))

enterprise/backend/test/metabase_enterprise/transforms/test_util.clj

@@ -52,6 +52,12 @@
     `(mt/with-model-cleanup [:model/Transform]
        ~@body)))
 
+(defn table-rows
+  [table-name]
+  (mt/rows (mt/process-query {:database (mt/id)
+                              :query    {:source-table (t2/select-one-pk :model/Table :name table-name)}
+                              :type     :query})))
+
 (defn parse-timestamp
   "Parse a local datetime and convert it to a ZonedDateTime in the default timezone."
   ^ZonedDateTime [timestamp-string]

enterprise/frontend/src/embedding-sdk-package/package.template.json

@@ -1,6 +1,6 @@
 {
   "name": "@metabase/embedding-sdk-react",
-  "version": "0.57.0-alpha.1",
+  "version": "0.57.0-alpha.2",
   "description": "Metabase Embedding SDK for React",
   "bin": "./dist/cli.js",
   "repository": {

enterprise/frontend/src/metabase-enterprise/api/python-transform-library.ts

@@ -0,0 +1,35 @@
+import { Api } from "metabase/api";
+
+export type PythonLibrary = {
+  source: string;
+};
+
+export type GetPythonLibraryRequest = {
+  name: string;
+};
+
+export type UpdatePythonLibraryRequest = {
+  name: string;
+  source: string;
+};
+
+export const pythonLibraryApi = Api.injectEndpoints({
+  endpoints: (builder) => ({
+    getPythonLibrary: builder.query<PythonLibrary, GetPythonLibraryRequest>({
+      query: ({ name }) => ({
+        url: `/api/ee/python-transform/user-modules/${name}`,
+        method: "GET",
+      }),
+    }),
+    updatePythonLibrary: builder.mutation<void, UpdatePythonLibraryRequest>({
+      query: ({ name, source }) => ({
+        url: `/api/ee/python-transform/user-modules/${name}`,
+        method: "PUT",
+        body: { source },
+      }),
+    }),
+  }),
+});
+
+export const { useGetPythonLibraryQuery, useUpdatePythonLibraryMutation } =
+  pythonLibraryApi;

enterprise/frontend/src/metabase-enterprise/api/transform.ts

@@ -108,6 +108,14 @@ export const transformApi = EnterpriseApi.injectEndpoints({
       invalidatesTags: (_, error) =>
         invalidateTags(error, [listTag("transform")]),
     }),
+    cancelTransform: builder.mutation<void, TransformId>({
+      query: (id) => ({
+        method: "POST",
+        url: `/api/ee/transform/${id}/cancel`,
+      }),
+      invalidatesTags: (_, error, id) =>
+        invalidateTags(error, [idTag("transform", id)]),
+    }),
     deleteTransformTarget: builder.mutation<void, TransformId>({
       query: (id) => ({
         method: "DELETE",
@@ -126,6 +134,7 @@ export const {
   useGetTransformQuery,
   useLazyGetTransformQuery,
   useRunTransformMutation,
+  useCancelTransformMutation,
   useCreateTransformMutation,
   useUpdateTransformMutation,
   useDeleteTransformMutation,

enterprise/frontend/src/metabase-enterprise/transforms/analytics.ts

@@ -34,7 +34,7 @@ export function trackTransformCreate({
   creationType,
 }: {
   triggeredFrom: "transform-page-create-menu";
-  creationType: "query" | "native" | "saved-question";
+  creationType: "query" | "native" | "python" | "saved-question";
 }) {
   trackSimpleEvent({
     event: "transform_create",

enterprise/frontend/src/metabase-enterprise/transforms/api/python-runner.ts

@@ -0,0 +1,68 @@
+import { Api } from "metabase/api";
+
+export interface ExecutePythonRequest {
+  code: string;
+  tables: Record<string, number>;
+}
+
+export interface ExecutePythonResponse {
+  output?: string;
+  stdout?: string;
+  stderr?: string;
+  error?: string;
+  exit_code?: number;
+  timeout?: boolean;
+}
+
+export const pythonRunnerApi = Api.injectEndpoints({
+  endpoints: (builder) => ({
+    executePython: builder.mutation<
+      ExecutePythonResponse,
+      ExecutePythonRequest
+    >({
+      query: ({ code, tables }) => ({
+        url: "/api/ee/transform/test-python",
+        method: "POST",
+        body: {
+          code,
+          tables,
+        },
+      }),
+      transformResponse: (response: any) => {
+        // Extract the nested result from the backend response
+        if (response?.result?.body) {
+          return {
+            output: response.result.body.output,
+            stdout: response.result.body.stdout,
+            stderr: response.result.body.stderr,
+            error: response.result.body.error,
+            exit_code: response.result.body.exit_code,
+            timeout: response.result.body.timeout,
+          };
+        }
+        return response;
+      },
+      transformErrorResponse: (response: any) => {
+        return {
+          status: response?.status || 500,
+          data: {
+            error: response?.data?.error || "Python execution failed",
+            stdout: response?.data?.stdout || "",
+            stderr: response?.data?.stderr || "",
+            exit_code: response?.data?.exit_code,
+          },
+        };
+      },
+    }),
+    cancelPython: builder.mutation<void, void>({
+      query: () => ({
+        url: "/api/ee/transform/test-python/cancel",
+        method: "POST",
+      }),
+    }),
+  }),
+  overrideExisting: true,
+});
+
+export const { useExecutePythonMutation, useCancelPythonMutation } =
+  pythonRunnerApi;

enterprise/frontend/src/metabase-enterprise/transforms/components/JobView/DependenciesSection/DependenciesSection.tsx

@@ -1,64 +1,36 @@
-import { push } from "react-router-redux";
 import { t } from "ttag";
 
-import { AdminContentTable } from "metabase/common/components/AdminContentTable";
-import EmptyState from "metabase/common/components/EmptyState";
-import { useDispatch } from "metabase/lib/redux";
-import { Box, Card, Center, Loader } from "metabase/ui";
+import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
 import { useListTransformJobTransformsQuery } from "metabase-enterprise/api";
-import type { Transform, TransformJobId } from "metabase-types/api";
+import type { TransformJobId } from "metabase-types/api";
 
 import { TitleSection } from "../../../components/TitleSection";
-import { getTransformUrl } from "../../../urls";
+import { TransformTable } from "../../../components/TransformTable";
+import { ListEmptyState } from "../../ListEmptyState";
 
-import S from "./DependenciesSection.module.css";
+type DependenciesSectionProps = {
+  jobId: TransformJobId;
+};
 
-export function DependenciesSection({ jobId }: { jobId: TransformJobId }) {
-  const { data: transforms, isLoading } =
-    useListTransformJobTransformsQuery(jobId);
-  const dispatch = useDispatch();
-
-  const handleRowClick = (transform: Transform) => {
-    dispatch(push(getTransformUrl(transform.id)));
-  };
+export function DependenciesSection({ jobId }: DependenciesSectionProps) {
+  const {
+    data: transforms = [],
+    error,
+    isLoading,
+  } = useListTransformJobTransformsQuery(jobId);
 
   return (
     <TitleSection
       label={t`Transforms`}
       description={t`Transforms will be run in this order.`}
     >
-      <Card p={0} shadow="none" withBorder>
-        {isLoading ? (
-          <Center>
-            <Loader m="xl" />
-          </Center>
-        ) : (
-          <AdminContentTable columnTitles={[t`Transform`, t`Target`]}>
-            {!transforms || transforms?.length === 0 ? (
-              <tr>
-                <td colSpan={2}>
-                  <Box p="md">
-                    <EmptyState
-                      message={t`There are no transforms for this job.`}
-                    />
-                  </Box>
-                </td>
-              </tr>
-            ) : (
-              transforms?.map((transform) => (
-                <tr
-                  key={transform.id}
-                  className={S.row}
-                  onClick={() => handleRowClick(transform)}
-                >
-                  <td>{transform.name}</td>
-                  <td>{transform.target.name}</td>
-                </tr>
-              ))
-            )}
-          </AdminContentTable>
-        )}
-      </Card>
+      {isLoading || error != null ? (
+        <LoadingAndErrorWrapper loading={isLoading} error={error} />
+      ) : transforms.length === 0 ? (
+        <ListEmptyState label={t`There are no transforms for this job.`} />
+      ) : (
+        <TransformTable transforms={transforms} />
+      )}
     </TitleSection>
   );
 }

enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/LogOutput.styled.tsx

@@ -0,0 +1,67 @@
+import { css } from "@emotion/react";
+import styled from "@emotion/styled";
+
+import { color } from "metabase/lib/colors";
+
+const LOG_PREFIX = ".react-ansi-style";
+
+const LOG_COLORS = {
+  black: color("text-dark"),
+  white: color("text-white"),
+  gray: color("text-medium"),
+  red: color("saturated-red"),
+  green: color("saturated-green"),
+  yellow: color("saturated-yellow"),
+  blue: color("saturated-blue"),
+  magenta: color("saturated-purple"),
+  cyan: "cyan",
+};
+
+export const LogOutputContainer = styled.div`
+  border: 1px solid var(--mb-color-border);
+  border-radius: 0.5rem;
+  background-color: var(--mb-color-bg-light);
+  font-family: Monaco, Menlo, "Ubuntu Mono", "Lucida Console", monospace;
+  font-size: 13px;
+  white-space: pre;
+  padding: 1rem;
+  overflow: auto;
+  line-height: 1.4;
+
+  ${LOG_PREFIX}-bold {
+    font-weight: bold;
+  }
+
+  ${LOG_PREFIX}-dim {
+    font-weight: lighter;
+  }
+
+  ${LOG_PREFIX}-italic {
+    font-style: italic;
+  }
+
+  ${LOG_PREFIX}-underline {
+    text-decoration: underline;
+  }
+
+  ${LOG_PREFIX}-inverse {
+    color: var(--mb-color-text-dark);
+    background-color: var(--mb-color-bg-white);
+  }
+
+  ${LOG_PREFIX}-hidden {
+    color: transparent;
+  }
+
+  ${LOG_PREFIX}-strikethrough {
+    text-decoration: line-through;
+  }
+
+  ${Object.entries(LOG_COLORS).map(
+    ([key, color]) => css`
+      ${LOG_PREFIX}-${key} {
+        color: ${color};
+      }
+    `,
+  )};
+`;

enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/LogOutput.tsx

@@ -0,0 +1,85 @@
+import * as React from "react";
+import { useEffect, useMemo, useRef, useState } from "react";
+import reactAnsiStyle from "react-ansi-style";
+
+import { LogOutputContainer } from "./LogOutput.styled";
+
+interface LogOutputProps {
+  content?: string;
+  showLineNumbers?: boolean;
+  maxLines?: number;
+}
+
+export function LogOutput({
+  content,
+  showLineNumbers = true,
+  maxLines = 20,
+}: LogOutputProps) {
+  const containerRef = useRef<HTMLDivElement>(null);
+  const [isUserScrolled, setIsUserScrolled] = useState(false);
+  const [shouldAutoScroll, setShouldAutoScroll] = useState(true);
+
+  const { displayContent, actualLines } = useMemo(() => {
+    if (!content || content.trim().length === 0) {
+      return { displayContent: "No output", actualLines: 1 };
+    }
+
+    const lines = content.split("\n");
+    let processedContent = content;
+
+    if (showLineNumbers) {
+      const maxLineLength = lines.length.toString().length;
+      processedContent = lines
+        .map(
+          (line, index) =>
+            `${(index + 1).toString().padStart(maxLineLength, " ")} | ${line}`,
+        )
+        .join("\n");
+    }
+
+    return {
+      displayContent: reactAnsiStyle(React, processedContent),
+      actualLines: lines.length,
+    };
+  }, [content, showLineNumbers]);
+
+  // Calculate the height based on actual lines (up to maxLines)
+  const linesToShow = Math.min(actualLines, maxLines);
+  const containerHeight = `${linesToShow * 1.4 + 2}em`; // 1.4 is line-height + padding
+  const maxContainerHeight = `${maxLines * 1.4 + 2}em`;
+
+  // Auto-scroll to bottom when content changes
+  useEffect(() => {
+    if (containerRef.current && shouldAutoScroll && !isUserScrolled) {
+      containerRef.current.scrollTop = containerRef.current.scrollHeight;
+    }
+  }, [displayContent, shouldAutoScroll, isUserScrolled]);
+
+  // Handle scroll events to detect user scrolling
+  const handleScroll = () => {
+    if (!containerRef.current) {
+      return;
+    }
+
+    const { scrollTop, scrollHeight, clientHeight } = containerRef.current;
+    const isAtBottom = Math.abs(scrollHeight - clientHeight - scrollTop) < 5;
+
+    if (isAtBottom) {
+      setIsUserScrolled(false);
+      setShouldAutoScroll(true);
+    } else {
+      setIsUserScrolled(true);
+      setShouldAutoScroll(false);
+    }
+  };
+
+  return (
+    <LogOutputContainer
+      ref={containerRef}
+      style={{ height: containerHeight, maxHeight: maxContainerHeight }}
+      onScroll={handleScroll}
+    >
+      {displayContent}
+    </LogOutputContainer>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/index.ts

@@ -0,0 +1 @@
+export { LogOutput } from "./LogOutput";

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonDataPicker/PythonDataPicker.module.css

@@ -0,0 +1,3 @@
+.dataPicker {
+  border-right: 1px solid var(--mb-color-border);
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonDataPicker/PythonDataPicker.tsx

@@ -0,0 +1,365 @@
+import { useState } from "react";
+import { t } from "ttag";
+
+import { useListDatabasesQuery, useListTablesQuery } from "metabase/api";
+import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
+import {
+  ActionIcon,
+  Box,
+  Button,
+  Group,
+  Icon,
+  Select,
+  Stack,
+  Text,
+  TextInput,
+} from "metabase/ui";
+import type { Database, Table } from "metabase-types/api";
+
+import S from "./PythonDataPicker.module.css";
+
+type TableSelection = {
+  id: string; // unique ID for this selection row
+  tableId: number | undefined;
+  alias: string;
+  aliasManuallySet?: boolean; // Track if user manually edited the alias
+};
+
+type PythonDataPickerProps = {
+  database?: number;
+  tables?: Record<string, number>; // alias -> table-id mapping
+  onChange: (
+    database: number,
+    tables: Record<string, { id: number; name: string }>,
+  ) => void;
+};
+
+// Helper function to slugify table names
+function slugify(text: string): string {
+  return text
+    .toLowerCase()
+    .replace(/[^\w\s-]/g, "") // Remove non-word chars
+    .replace(/[\s_-]+/g, "_") // Replace spaces, underscores, hyphens with single underscore
+    .replace(/^_+|_+$/g, ""); // Remove leading/trailing underscores
+}
+
+export function PythonDataPicker({
+  database,
+  tables,
+  onChange,
+}: PythonDataPickerProps) {
+  const [selectedDatabaseId, setSelectedDatabaseId] = useState<
+    number | undefined
+  >(database);
+
+  // Initialize table selections from props
+  const initializeTableSelections = (): TableSelection[] => {
+    if (tables && Object.keys(tables).length > 0) {
+      return Object.entries(tables).map(([alias, tableId], index) => ({
+        id: `table-${index}-${Date.now()}`,
+        tableId,
+        alias,
+        aliasManuallySet: true, // Existing aliases are considered manually set
+      }));
+    }
+    // Start with one empty row by default
+    return [
+      {
+        id: `table-0-${Date.now()}`,
+        tableId: undefined,
+        alias: "",
+        aliasManuallySet: false,
+      },
+    ];
+  };
+
+  const [tableSelections, setTableSelections] = useState<TableSelection[]>(
+    initializeTableSelections,
+  );
+
+  const {
+    data: databasesResponse,
+    isLoading: isLoadingDatabases,
+    error: databaseError,
+  } = useListDatabasesQuery();
+
+  const {
+    data: tablesData,
+    isLoading: isLoadingTables,
+    error: tablesError,
+  } = useListTablesQuery(
+    selectedDatabaseId
+      ? {
+          dbId: selectedDatabaseId,
+          include_hidden: false,
+          include_editable_data_model: true,
+        }
+      : undefined,
+    {
+      skip: !selectedDatabaseId,
+    },
+  );
+
+  const notifyParentOfChange = (
+    dbId: number | undefined,
+    selections: TableSelection[],
+    tableIdToName?: Map<number, string>,
+  ) => {
+    if (dbId) {
+      const tablesMap: Record<string, { id: number; name: string }> = {};
+
+      selections.forEach((selection) => {
+        if (selection.tableId && selection.alias) {
+          const tableName =
+            tableIdToName?.get(selection.tableId) ||
+            `Table ${selection.tableId}`;
+          tablesMap[selection.alias] = {
+            id: selection.tableId,
+            name: tableName,
+          };
+        }
+      });
+      onChange(dbId, tablesMap);
+    }
+  };
+
+  if (databaseError || tablesError) {
+    return <LoadingAndErrorWrapper error={databaseError || tablesError} />;
+  }
+
+  if (isLoadingDatabases) {
+    return <LoadingAndErrorWrapper loading />;
+  }
+
+  const databases = databasesResponse?.data || [];
+  const databaseOptions = databases.map((db: Database) => ({
+    value: db.id.toString(),
+    label: db.name,
+  }));
+
+  const availableTables = (tablesData || [])
+    .filter((tbl: Table) => tbl.db_id === selectedDatabaseId && tbl.active)
+    .map((tbl: Table) => ({
+      value: tbl.id.toString(),
+      label: tbl.display_name || tbl.name,
+      table: tbl,
+    }));
+
+  const tableIdToNameMap = new Map<number, string>(
+    availableTables.map((t) => [parseInt(t.value), t.table.name]),
+  );
+
+  const handleDatabaseChange = (value: string | null) => {
+    const dbId = value ? parseInt(value) : undefined;
+    setSelectedDatabaseId(dbId);
+    const newSelections = [
+      {
+        id: `table-0-${Date.now()}`,
+        tableId: undefined,
+        alias: "",
+        aliasManuallySet: false,
+      },
+    ];
+    setTableSelections(newSelections);
+    notifyParentOfChange(dbId, newSelections);
+  };
+
+  const handleTableChange = (selectionId: string, value: string | null) => {
+    const newSelections = tableSelections.map((selection) => {
+      if (selection.id === selectionId) {
+        const tableId = value ? parseInt(value) : undefined;
+        const table = availableTables.find((t) => t.value === value)?.table;
+
+        const newAlias = table ? slugify(table.name) : "";
+
+        return {
+          ...selection,
+          tableId,
+          alias: selection.aliasManuallySet ? selection.alias : newAlias,
+          aliasManuallySet: selection.aliasManuallySet || false,
+        };
+      }
+      return selection;
+    });
+    setTableSelections(newSelections);
+    notifyParentOfChange(selectedDatabaseId, newSelections, tableIdToNameMap);
+  };
+
+  const handleAliasChange = (selectionId: string, alias: string) => {
+    // Only allow valid Python identifier characters (letters, numbers, underscores)
+    // and ensure it doesn't start with a number
+    const validAlias = alias
+      .replace(/[^a-zA-Z0-9_]/g, "")
+      .replace(/^(\d)/, "_$1");
+
+    const newSelections = tableSelections.map((selection) =>
+      selection.id === selectionId
+        ? { ...selection, alias: validAlias, aliasManuallySet: true }
+        : selection,
+    );
+    setTableSelections(newSelections);
+    notifyParentOfChange(selectedDatabaseId, newSelections, tableIdToNameMap);
+  };
+
+  const handleResetAlias = (selectionId: string) => {
+    const newSelections = tableSelections.map((selection) => {
+      if (selection.id === selectionId && selection.tableId) {
+        const table = availableTables.find(
+          (t) => t.value === selection.tableId!.toString(),
+        )?.table;
+        if (table) {
+          return {
+            ...selection,
+            alias: slugify(table.name),
+            aliasManuallySet: false,
+          };
+        }
+      }
+      return selection;
+    });
+    setTableSelections(newSelections);
+    notifyParentOfChange(selectedDatabaseId, newSelections, tableIdToNameMap);
+  };
+
+  const handleAddTable = () => {
+    const newSelections = [
+      ...tableSelections,
+      {
+        id: `table-${tableSelections.length}-${Date.now()}`,
+        tableId: undefined,
+        alias: "",
+        aliasManuallySet: false,
+      },
+    ];
+    setTableSelections(newSelections);
+    // Don't notify parent for adding empty row
+  };
+
+  const handleRemoveTable = (selectionId: string) => {
+    const newSelections = tableSelections.filter(
+      (selection) => selection.id !== selectionId,
+    );
+    setTableSelections(newSelections);
+    notifyParentOfChange(selectedDatabaseId, newSelections, tableIdToNameMap);
+  };
+
+  const selectedTableIds = new Set(
+    tableSelections
+      .filter((s) => s.tableId !== undefined)
+      .map((s) => s.tableId?.toString()),
+  );
+
+  return (
+    <Stack p="md" gap="md" h="100%" className={S.dataPicker}>
+      <Box>
+        <Text size="sm" fw="bold" mb="xs">
+          {t`Source Database`}
+        </Text>
+        <Text size="xs" c="dimmed" mb="sm">
+          {t`Select the database that contains your source data`}
+        </Text>
+        <Select
+          data={databaseOptions}
+          value={selectedDatabaseId?.toString() || null}
+          onChange={handleDatabaseChange}
+          placeholder={t`Select a database`}
+          clearable
+        />
+      </Box>
+      {selectedDatabaseId && (
+        <Box>
+          <Text size="sm" fw="bold" mb="xs">
+            {t`Source Tables`}
+          </Text>
+          <Text size="xs" c="dimmed" mb="sm">
+            {t`Select tables to use as data sources and provide aliases for each`}
+          </Text>
+          <Stack gap="sm">
+            {tableSelections.map((selection, index) => {
+              // Filter available tables to exclude already selected ones (except current selection)
+              const filteredTables = availableTables.filter(
+                (table) =>
+                  !selectedTableIds.has(table.value) ||
+                  table.value === selection.tableId?.toString(),
+              );
+
+              const table = availableTables.find(
+                (t) => t.value === selection.tableId?.toString(),
+              )?.table;
+              const defaultAlias = table ? slugify(table.name) : "";
+              const showReset =
+                selection.aliasManuallySet &&
+                selection.alias !== defaultAlias &&
+                table;
+
+              return (
+                <Group key={selection.id} gap="sm" align="flex-end">
+                  <Select
+                    style={{ flex: 1 }}
+                    data={filteredTables}
+                    value={selection.tableId?.toString() || null}
+                    onChange={(value) => handleTableChange(selection.id, value)}
+                    placeholder={t`Select a table`}
+                    clearable
+                    disabled={isLoadingTables}
+                  />
+                  <TextInput
+                    style={{ flex: 0.7 }}
+                    value={selection.alias}
+                    onChange={(e) =>
+                      handleAliasChange(selection.id, e.target.value)
+                    }
+                    placeholder={t`Enter alias`}
+                    label={index === 0 ? t`Alias` : undefined}
+                    styles={{
+                      input: {
+                        color: selection.aliasManuallySet
+                          ? undefined
+                          : "var(--mb-color-text-light)",
+                      },
+                    }}
+                    rightSection={
+                      showReset ? (
+                        <ActionIcon
+                          onClick={() => handleResetAlias(selection.id)}
+                          aria-label={t`Reset alias to default`}
+                          color="gray"
+                          variant="subtle"
+                          size="xs"
+                        >
+                          <Icon name="refresh" size={12} />
+                        </ActionIcon>
+                      ) : null
+                    }
+                  />
+                  {tableSelections.length > 1 && (
+                    <ActionIcon
+                      onClick={() => handleRemoveTable(selection.id)}
+                      aria-label={t`Remove table`}
+                      color="gray"
+                      variant="subtle"
+                    >
+                      <Icon name="trash" />
+                    </ActionIcon>
+                  )}
+                </Group>
+              );
+            })}
+            <Button
+              leftSection={<Icon name="add" />}
+              variant="subtle"
+              onClick={handleAddTable}
+              disabled={
+                !tableSelections[tableSelections.length - 1]?.tableId ||
+                availableTables.length === 0 ||
+                selectedTableIds.size >= availableTables.length
+              }
+            >
+              {t`Add another table`}
+            </Button>
+          </Stack>
+        </Box>
+      )}
+    </Stack>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonDataPicker/index.ts

@@ -0,0 +1 @@
+export * from "./PythonDataPicker";

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonEditor/PythonEditor.module.css

@@ -0,0 +1,4 @@
+.editor {
+  width: 100%;
+  height: 100%;
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonEditor/PythonEditor.tsx

@@ -0,0 +1,28 @@
+import cx from "classnames";
+
+import { CodeEditor } from "metabase/common/components/CodeEditor";
+
+import S from "./PythonEditor.module.css";
+import { completion } from "./utils";
+
+export function PythonEditor({
+  value,
+  onChange,
+  withPandasCompletions,
+  className,
+}: {
+  value: string;
+  onChange: (value: string) => void;
+  withPandasCompletions?: boolean;
+  className?: string;
+}) {
+  return (
+    <CodeEditor
+      className={cx(S.editor, className)}
+      value={value}
+      onChange={onChange}
+      language="python"
+      extensions={withPandasCompletions ? completion : undefined}
+    />
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonEditor/index.ts

@@ -0,0 +1 @@
+export * from "./PythonEditor";

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonEditor/utils.ts

@@ -0,0 +1,675 @@
+import {
+  type Completion,
+  type CompletionContext,
+  autocompletion,
+} from "@codemirror/autocomplete";
+import { pythonLanguage } from "@codemirror/lang-python";
+
+// This is the closest you can get to "loading pandas definitions"
+
+const pandasDefinitions: Record<string, Completion[]> = {
+  pd: [
+    {
+      label: "DataFrame",
+      type: "class",
+      info: "Two-dimensional, size-mutable, potentially heterogeneous tabular data",
+    },
+    {
+      label: "Series",
+      type: "class",
+      info: "One-dimensional ndarray with axis labels",
+    },
+    {
+      label: "read_csv",
+      type: "function",
+      info: "Read a comma-separated values (csv) file into DataFrame",
+    },
+    {
+      label: "read_excel",
+      type: "function",
+      info: "Read an Excel file into a pandas DataFrame",
+    },
+    {
+      label: "read_json",
+      type: "function",
+      info: "Convert a JSON string to pandas object",
+    },
+    {
+      label: "read_sql",
+      type: "function",
+      info: "Read SQL query or database table into a DataFrame",
+    },
+    {
+      label: "read_parquet",
+      type: "function",
+      info: "Load a parquet object from file path",
+    },
+    {
+      label: "concat",
+      type: "function",
+      info: "Concatenate pandas objects along a particular axis",
+    },
+    {
+      label: "merge",
+      type: "function",
+      info: "Merge DataFrame or named Series objects with a database-style join",
+    },
+    {
+      label: "pivot_table",
+      type: "function",
+      info: "Create a spreadsheet-style pivot table as a DataFrame",
+    },
+    {
+      label: "crosstab",
+      type: "function",
+      info: "Compute a simple cross tabulation of two (or more) factors",
+    },
+    {
+      label: "get_dummies",
+      type: "function",
+      info: "Convert categorical variable(s) into dummy/indicator variables",
+    },
+    {
+      label: "date_range",
+      type: "function",
+      info: "Return a fixed frequency DatetimeIndex",
+    },
+    {
+      label: "to_datetime",
+      type: "function",
+      info: "Convert argument to datetime",
+    },
+    {
+      label: "to_numeric",
+      type: "function",
+      info: "Convert argument to a numeric type",
+    },
+    {
+      label: "cut",
+      type: "function",
+      info: "Bin values into discrete intervals",
+    },
+    {
+      label: "qcut",
+      type: "function",
+      info: "Quantile-based discretization function",
+    },
+    {
+      label: "isna",
+      type: "function",
+      info: "Detect missing values for an array-like object",
+    },
+    {
+      label: "isnull",
+      type: "function",
+      info: "Detect missing values for an array-like object",
+    },
+    {
+      label: "notna",
+      type: "function",
+      info: "Detect non-missing values for an array-like object",
+    },
+    {
+      label: "notnull",
+      type: "function",
+      info: "Detect non-missing values for an array-like object",
+    },
+  ],
+
+  // DataFrame methods and properties
+  DataFrame: [
+    // Viewing/Inspection
+    { label: "head", type: "method", info: "Return the first n rows" },
+    { label: "tail", type: "method", info: "Return the last n rows" },
+    {
+      label: "info",
+      type: "method",
+      info: "Print a concise summary of a DataFrame",
+    },
+    {
+      label: "describe",
+      type: "method",
+      info: "Generate descriptive statistics",
+    },
+    {
+      label: "shape",
+      type: "property",
+      info: "Return a tuple representing the dimensionality",
+    },
+    {
+      label: "size",
+      type: "property",
+      info: "Return an int representing the number of elements",
+    },
+    {
+      label: "columns",
+      type: "property",
+      info: "The column labels of the DataFrame",
+    },
+    {
+      label: "index",
+      type: "property",
+      info: "The index (row labels) of the DataFrame",
+    },
+    {
+      label: "dtypes",
+      type: "property",
+      info: "Return the dtypes in the DataFrame",
+    },
+    {
+      label: "values",
+      type: "property",
+      info: "Return a Numpy representation of the DataFrame",
+    },
+
+    // Selection/Indexing
+    {
+      label: "loc",
+      type: "property",
+      info: "Access a group of rows and columns by label(s)",
+    },
+    {
+      label: "iloc",
+      type: "property",
+      info: "Purely integer-location based indexing",
+    },
+    {
+      label: "at",
+      type: "property",
+      info: "Access a single value for a row/column label pair",
+    },
+    {
+      label: "iat",
+      type: "property",
+      info: "Access a single value for a row/column pair by integer position",
+    },
+    {
+      label: "query",
+      type: "method",
+      info: "Query the columns of a DataFrame with a boolean expression",
+    },
+    {
+      label: "filter",
+      type: "method",
+      info: "Subset the dataframe rows or columns according to labels",
+    },
+    {
+      label: "where",
+      type: "method",
+      info: "Replace values where the condition is False",
+    },
+    {
+      label: "mask",
+      type: "method",
+      info: "Replace values where the condition is True",
+    },
+
+    // Data Cleaning
+    {
+      label: "drop",
+      type: "method",
+      info: "Drop specified labels from rows or columns",
+    },
+    {
+      label: "drop_duplicates",
+      type: "method",
+      info: "Return DataFrame with duplicate rows removed",
+    },
+    { label: "dropna", type: "method", info: "Remove missing values" },
+    {
+      label: "fillna",
+      type: "method",
+      info: "Fill NA/NaN values using the specified method",
+    },
+    {
+      label: "replace",
+      type: "method",
+      info: "Replace values given in to_replace with value",
+    },
+    {
+      label: "interpolate",
+      type: "method",
+      info: "Fill NaN values using an interpolation method",
+    },
+    {
+      label: "duplicated",
+      type: "method",
+      info: "Return boolean Series denoting duplicate rows",
+    },
+    { label: "isnull", type: "method", info: "Detect missing values" },
+    { label: "isna", type: "method", info: "Detect missing values" },
+    {
+      label: "notnull",
+      type: "method",
+      info: "Detect existing (non-missing) values",
+    },
+    {
+      label: "notna",
+      type: "method",
+      info: "Detect existing (non-missing) values",
+    },
+
+    // Transformations
+    {
+      label: "sort_values",
+      type: "method",
+      info: "Sort by the values along either axis",
+    },
+    {
+      label: "sort_index",
+      type: "method",
+      info: "Sort object by labels (along an axis)",
+    },
+    {
+      label: "reset_index",
+      type: "method",
+      info: "Reset the index, or a level of it",
+    },
+    {
+      label: "set_index",
+      type: "method",
+      info: "Set the DataFrame index using existing columns",
+    },
+    {
+      label: "reindex",
+      type: "method",
+      info: "Conform Series/DataFrame to new index",
+    },
+    { label: "rename", type: "method", info: "Alter axes labels" },
+    {
+      label: "assign",
+      type: "method",
+      info: "Assign new columns to a DataFrame",
+    },
+    {
+      label: "pipe",
+      type: "method",
+      info: "Apply chainable functions that expect Series or DataFrames",
+    },
+
+    // Aggregation/GroupBy
+    {
+      label: "groupby",
+      type: "method",
+      info: "Group DataFrame using a mapper or by a Series of columns",
+    },
+    {
+      label: "agg",
+      type: "method",
+      info: "Aggregate using one or more operations",
+    },
+    {
+      label: "aggregate",
+      type: "method",
+      info: "Aggregate using one or more operations",
+    },
+    {
+      label: "apply",
+      type: "method",
+      info: "Apply a function along an axis of the DataFrame",
+    },
+    {
+      label: "applymap",
+      type: "method",
+      info: "Apply a function to a Dataframe elementwise",
+    },
+    {
+      label: "transform",
+      type: "method",
+      info: "Call func on self producing a DataFrame with transformed values",
+    },
+    {
+      label: "rolling",
+      type: "method",
+      info: "Provide rolling window calculations",
+    },
+    {
+      label: "expanding",
+      type: "method",
+      info: "Provide expanding transformations",
+    },
+    { label: "resample", type: "method", info: "Resample time-series data" },
+
+    // Statistical Methods
+    {
+      label: "sum",
+      type: "method",
+      info: "Return the sum of the values for the requested axis",
+    },
+    {
+      label: "mean",
+      type: "method",
+      info: "Return the mean of the values for the requested axis",
+    },
+    {
+      label: "median",
+      type: "method",
+      info: "Return the median of the values for the requested axis",
+    },
+    {
+      label: "mode",
+      type: "method",
+      info: "Get the mode(s) of each element along the selected axis",
+    },
+    {
+      label: "std",
+      type: "method",
+      info: "Return sample standard deviation over requested axis",
+    },
+    {
+      label: "var",
+      type: "method",
+      info: "Return unbiased variance over requested axis",
+    },
+    {
+      label: "min",
+      type: "method",
+      info: "Return the minimum of the values for the requested axis",
+    },
+    {
+      label: "max",
+      type: "method",
+      info: "Return the maximum of the values for the requested axis",
+    },
+    {
+      label: "count",
+      type: "method",
+      info: "Count non-NA cells for each column or row",
+    },
+    {
+      label: "nunique",
+      type: "method",
+      info: "Count number of distinct elements in specified axis",
+    },
+    {
+      label: "value_counts",
+      type: "method",
+      info: "Return a Series containing counts of unique values",
+    },
+    {
+      label: "corr",
+      type: "method",
+      info: "Compute pairwise correlation of columns",
+    },
+    {
+      label: "cov",
+      type: "method",
+      info: "Compute pairwise covariance of columns",
+    },
+    {
+      label: "rank",
+      type: "method",
+      info: "Compute numerical data ranks (1 through n) along axis",
+    },
+    {
+      label: "quantile",
+      type: "method",
+      info: "Return values at the given quantile over requested axis",
+    },
+    {
+      label: "clip",
+      type: "method",
+      info: "Trim values at input threshold(s)",
+    },
+
+    // Merging/Joining
+    {
+      label: "merge",
+      type: "method",
+      info: "Merge DataFrame or named Series objects",
+    },
+    {
+      label: "join",
+      type: "method",
+      info: "Join columns of another DataFrame",
+    },
+    {
+      label: "concat",
+      type: "method",
+      info: "Concatenate pandas objects along a particular axis",
+    },
+    {
+      label: "append",
+      type: "method",
+      info: "Append rows of other to the end of caller",
+    },
+    {
+      label: "update",
+      type: "method",
+      info: "Modify in place using non-NA values from another DataFrame",
+    },
+
+    // Reshaping
+    {
+      label: "pivot",
+      type: "method",
+      info: "Return reshaped DataFrame organized by given index / column values",
+    },
+    {
+      label: "pivot_table",
+      type: "method",
+      info: "Create a spreadsheet-style pivot table",
+    },
+    {
+      label: "melt",
+      type: "method",
+      info: "Unpivot a DataFrame from wide to long format",
+    },
+    {
+      label: "stack",
+      type: "method",
+      info: "Stack the prescribed level(s) from columns to index",
+    },
+    {
+      label: "unstack",
+      type: "method",
+      info: "Pivot a level of the (necessarily hierarchical) index labels",
+    },
+    { label: "transpose", type: "method", info: "Transpose index and columns" },
+    { label: "T", type: "property", info: "Transpose index and columns" },
+
+    // I/O
+    {
+      label: "to_csv",
+      type: "method",
+      info: "Write object to a comma-separated values (csv) file",
+    },
+    {
+      label: "to_excel",
+      type: "method",
+      info: "Write object to an Excel sheet",
+    },
+    {
+      label: "to_json",
+      type: "method",
+      info: "Convert the object to a JSON string",
+    },
+    {
+      label: "to_sql",
+      type: "method",
+      info: "Write records stored in a DataFrame to a SQL database",
+    },
+    {
+      label: "to_parquet",
+      type: "method",
+      info: "Write a DataFrame to the binary parquet format",
+    },
+    {
+      label: "to_dict",
+      type: "method",
+      info: "Convert the DataFrame to a dictionary",
+    },
+    {
+      label: "to_html",
+      type: "method",
+      info: "Render a DataFrame as an HTML table",
+    },
+    {
+      label: "to_string",
+      type: "method",
+      info: "Render a DataFrame to a console-friendly tabular output",
+    },
+    {
+      label: "to_numpy",
+      type: "method",
+      info: "Convert the DataFrame to a NumPy array",
+    },
+
+    // Time Series
+    {
+      label: "shift",
+      type: "method",
+      info: "Shift index by desired number of periods",
+    },
+    { label: "tshift", type: "method", info: "Shift the time index" },
+    {
+      label: "diff",
+      type: "method",
+      info: "First discrete difference of element",
+    },
+    {
+      label: "pct_change",
+      type: "method",
+      info: "Percentage change between the current and a prior element",
+    },
+
+    // Plotting (if matplotlib available)
+    {
+      label: "plot",
+      type: "method",
+      info: "Make plots of Series or DataFrame",
+    },
+    {
+      label: "hist",
+      type: "method",
+      info: "Make a histogram of the DataFrame's columns",
+    },
+    {
+      label: "boxplot",
+      type: "method",
+      info: "Make a box plot of the DataFrame columns",
+    },
+  ],
+
+  // Series methods (subset of DataFrame methods that make sense for Series)
+  Series: [
+    { label: "head", type: "method", info: "Return the first n values" },
+    { label: "tail", type: "method", info: "Return the last n values" },
+    {
+      label: "describe",
+      type: "method",
+      info: "Generate descriptive statistics",
+    },
+    {
+      label: "value_counts",
+      type: "method",
+      info: "Return a Series containing counts of unique values",
+    },
+    {
+      label: "unique",
+      type: "method",
+      info: "Return unique values of Series object",
+    },
+    {
+      label: "nunique",
+      type: "method",
+      info: "Return number of unique elements in the object",
+    },
+    { label: "sort_values", type: "method", info: "Sort by the values" },
+    {
+      label: "sort_index",
+      type: "method",
+      info: "Sort Series by index labels",
+    },
+    {
+      label: "reset_index",
+      type: "method",
+      info: "Generate a new DataFrame or Series with the index reset",
+    },
+    {
+      label: "dropna",
+      type: "method",
+      info: "Return a new Series with missing values removed",
+    },
+    {
+      label: "fillna",
+      type: "method",
+      info: "Fill NA/NaN values using the specified method",
+    },
+    {
+      label: "apply",
+      type: "method",
+      info: "Invoke function on values of Series",
+    },
+    {
+      label: "map",
+      type: "method",
+      info: "Map values of Series according to input correspondence",
+    },
+    {
+      label: "replace",
+      type: "method",
+      info: "Replace values given in to_replace with value",
+    },
+    {
+      label: "str",
+      type: "property",
+      info: "Vectorized string functions for Series and Index",
+    },
+    {
+      label: "dt",
+      type: "property",
+      info: "Accessor object for datetimelike properties",
+    },
+    {
+      label: "cat",
+      type: "property",
+      info: "Accessor object for categorical properties",
+    },
+  ],
+};
+
+function genericPandasCompletions(context: CompletionContext) {
+  const word = context.matchBefore(/[\w.]*/);
+  if (!word || (word.from === word.to && !context.explicit)) {
+    return null;
+  }
+
+  const text = word.text;
+
+  // Match patterns like "pd.", "df.", etc.
+  if (text.includes(".")) {
+    const parts = text.split(".");
+    const obj = parts.at(-2);
+    const partial = parts.at(-1) ?? "";
+
+    if (!obj || !pandasDefinitions[obj]) {
+      return null;
+    }
+
+    const options = pandasDefinitions[obj]
+      .filter(
+        (item) =>
+          !partial ||
+          item.label.toLowerCase().startsWith(partial.toLowerCase()),
+      )
+      .map((comp) => ({
+        ...comp,
+        info: undefined,
+        boost: comp.type === "method" ? 2 : comp.type === "function" ? 1 : 0,
+      }));
+
+    return {
+      filter: false,
+      from: word.from + text.length - partial.length,
+      options,
+    };
+  }
+}
+
+export const completion = [
+  pythonLanguage.data.of({
+    autocomplete: genericPandasCompletions,
+  }),
+  autocompletion(),
+];

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonResultsTable/PythonResultsTable.tsx

@@ -0,0 +1,181 @@
+import { useMemo } from "react";
+import { t } from "ttag";
+
+import { colors } from "metabase/lib/colors/colors";
+import { Alert, Box, Text } from "metabase/ui";
+
+interface PythonResultsTableProps {
+  csvData?: string;
+  error?: string;
+  stdout?: string;
+  stderr?: string;
+  isLoading?: boolean;
+}
+
+function parseCSV(csv: string): { headers: string[]; rows: string[][] } {
+  if (!csv || !csv.trim()) {
+    return { headers: [], rows: [] };
+  }
+
+  // Split by newlines to get rows
+  const lines = csv.trim().split("\n");
+  if (lines.length === 0) {
+    return { headers: [], rows: [] };
+  }
+
+  // Parse headers from first line
+  const headers = lines[0].split(",").map((h) => h.trim());
+
+  // Parse data rows
+  const rows = lines.slice(1).map((line) => {
+    // Simple CSV parsing - handles basic comma-separated values
+    // For more complex CSV (with quotes, escapes), we'd need a proper parser
+    return line.split(",").map((cell) => cell.trim());
+  });
+
+  return { headers, rows };
+}
+
+export function PythonResultsTable({
+  csvData,
+  error,
+  stdout,
+  stderr,
+  isLoading = false,
+}: PythonResultsTableProps) {
+  const { headers, rows } = useMemo(() => {
+    if (!csvData) {
+      return { headers: [], rows: [] };
+    }
+    const parsed = parseCSV(csvData);
+    return parsed;
+  }, [csvData]);
+
+  if (isLoading) {
+    return (
+      <Box p="lg">
+        <Text c="text-medium">{t`Running Python script...`}</Text>
+      </Box>
+    );
+  }
+
+  if (error) {
+    return (
+      <Box p="lg">
+        <Alert color="red" title={t`Execution Error`} mb="md">
+          {error}
+        </Alert>
+        {stdout && (
+          <Box mb="md">
+            <Text fw="bold" mb="xs">{t`Standard Output:`}</Text>
+            <Box
+              p="sm"
+              bg="bg-light"
+              style={{ fontFamily: "monospace", whiteSpace: "pre-wrap" }}
+            >
+              {stdout}
+            </Box>
+          </Box>
+        )}
+        {stderr && (
+          <Box>
+            <Text fw="bold" mb="xs">{t`Standard Error:`}</Text>
+            <Box
+              p="sm"
+              bg="bg-light"
+              style={{ fontFamily: "monospace", whiteSpace: "pre-wrap" }}
+            >
+              {stderr}
+            </Box>
+          </Box>
+        )}
+      </Box>
+    );
+  }
+
+  if (!csvData || headers.length === 0) {
+    return (
+      <Box p="lg">
+        <Text c="text-medium">{t`No results to display. Click "Run" to execute the script.`}</Text>
+        {stdout && (
+          <Box mt="md">
+            <Text fw="bold" mb="xs">{t`Standard Output:`}</Text>
+            <Box
+              p="sm"
+              bg="bg-light"
+              style={{ fontFamily: "monospace", whiteSpace: "pre-wrap" }}
+            >
+              {stdout}
+            </Box>
+          </Box>
+        )}
+      </Box>
+    );
+  }
+
+  return (
+    <Box
+      style={{
+        overflow: "auto",
+        padding: "1rem",
+        background: colors["bg-light"],
+        border: `1px solid ${colors["border"]}`,
+      }}
+    >
+      {stdout && (
+        <Box p="md" pb={0}>
+          <Text fw="bold" mb="xs">{t`Standard Output:`}</Text>
+          <Box
+            p="sm"
+            bg="bg-light"
+            style={{ fontFamily: "monospace", whiteSpace: "pre-wrap" }}
+            mb="md"
+          >
+            {stdout}
+          </Box>
+        </Box>
+      )}
+      <Box style={{ background: "white", padding: "1rem" }}>
+        <Text fw="bold" mb="md">{t`Results Table:`}</Text>
+        <table
+          style={{
+            width: "100%",
+            borderCollapse: "collapse",
+            border: `1px solid ${colors["border"]}`,
+          }}
+        >
+          <thead>
+            <tr style={{ backgroundColor: colors["bg-medium"] }}>
+              {headers.map((header, index) => (
+                <th
+                  key={index}
+                  style={{
+                    padding: "8px",
+                    textAlign: "left",
+                    borderBottom: `2px solid ${colors["border"]}`,
+                  }}
+                >
+                  {header}
+                </th>
+              ))}
+            </tr>
+          </thead>
+          <tbody>
+            {rows.map((row, rowIndex) => (
+              <tr
+                key={rowIndex}
+                style={{ borderBottom: `1px solid ${colors["bg-medium"]}` }}
+              >
+                {row.map((cell, cellIndex) => (
+                  <td key={cellIndex} style={{ padding: "8px" }}>
+                    {cell}
+                  </td>
+                ))}
+              </tr>
+            ))}
+          </tbody>
+        </table>
+      </Box>
+    </Box>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonResultsTable/index.ts

@@ -0,0 +1 @@
+export { PythonResultsTable } from "./PythonResultsTable";

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonTransformEditor/PythonTransformEditor.tsx

@@ -0,0 +1,235 @@
+import { useEffect, useState } from "react";
+
+import { Flex, Stack } from "metabase/ui";
+import type { PythonTransformSource } from "metabase-types/api";
+
+import { PythonDataPicker } from "../PythonDataPicker";
+import { EditorHeader } from "../QueryEditor/EditorHeader";
+import { PythonEditor } from "../QueryEditor/PythonEditor";
+
+type PythonTransformEditorProps = {
+  initialSource: PythonTransformSource;
+  isNew?: boolean;
+  isSaving?: boolean;
+  onSave: (newSource: PythonTransformSource) => void;
+  onCancel: () => void;
+  onSourceChange?: (newSource: PythonTransformSource) => void;
+};
+
+export function PythonTransformEditor({
+  initialSource,
+  isNew = true,
+  isSaving = false,
+  onSave,
+  onCancel,
+  onSourceChange,
+}: PythonTransformEditorProps) {
+  const [source, setSource] = useState(initialSource);
+  const [isSourceDirty, setIsSourceDirty] = useState(false);
+
+  useEffect(() => {
+    if (onSourceChange) {
+      onSourceChange(source);
+    }
+  }, [source, onSourceChange]);
+
+  const handleScriptChange = (body: string) => {
+    const newSource = {
+      ...source,
+      body,
+    };
+    setSource(newSource);
+    setIsSourceDirty(true);
+  };
+
+  const updateTransformSignature = (
+    script: string,
+    tables: Record<string, { id: number; name: string }>,
+  ): string => {
+    const tableAliases = Object.keys(tables);
+
+    const transformRegex = /^def\s+transform\s*\([^)]*\)\s*:\s*\n(\s*)/m;
+
+    const signatureOneLine = `def transform(${tableAliases.join(", ")}):`;
+    const newSignature =
+      signatureOneLine.length > 80 && tableAliases.length > 0
+        ? `def transform(\n${tableAliases.map((alias) => `    ${alias}`).join(",\n")},\n):`
+        : signatureOneLine;
+
+    if (transformRegex.test(script)) {
+      // Capture the existing indentation after the colon
+      const match = script.match(transformRegex);
+      const originalIndent = match ? match[1] : "    ";
+
+      let updatedScript = script.replace(
+        transformRegex,
+        newSignature + "\n" + originalIndent,
+      );
+
+      const argsRegex =
+        /(\s+"""[\s\S]*?)\s*Args:\s*\n([\s\S]*?)\s*(\n\s+Returns:|\n\s+""")/;
+      const argsMatch = updatedScript.match(argsRegex);
+
+      if (tableAliases.length > 0) {
+        const maxAliasLength = Math.max(...tableAliases.map((a) => a.length));
+        const newArgsSection = tableAliases
+          .map((alias) => {
+            const padding = " ".repeat(maxAliasLength - alias.length);
+            const tableName = tables[alias].name;
+            if (alias === tableName) {
+              return `        ${alias}:${padding} DataFrame containing the data from the corresponding table`;
+            }
+            return `        ${alias}:${padding} DataFrame containing the data from table "${tableName}"`;
+          })
+          .join("\n");
+
+        if (argsMatch) {
+          // Replace existing Args section
+          updatedScript = updatedScript.replace(
+            argsRegex,
+            `    $1
+
+    Args:
+${newArgsSection}
+$3`,
+          );
+        } else {
+          // No Args section exists, look for the pattern before Returns and insert Args
+          // Also remove the instructional text about selecting tables
+          const insertRegex =
+            /(\s+"""[\s\S]*?)\s*Select tables above to add them as function parameters\.\s*(\n\s+Returns:|\n\s+""")/;
+          const insertMatch = updatedScript.match(insertRegex);
+          if (insertMatch) {
+            updatedScript = updatedScript.replace(
+              insertRegex,
+              `$1
+
+    Args:
+${newArgsSection}
+$2`,
+            );
+          } else {
+            // Fallback: just look for Returns section without the specific text
+            // Remove any extra whitespace before Returns and ensure single blank line
+            const fallbackRegex = /(\s+"""[\s\S]*?)\s*(\n\s+Returns:|\n\s+""")/;
+            const fallbackMatch = updatedScript.match(fallbackRegex);
+            if (fallbackMatch) {
+              updatedScript = updatedScript.replace(
+                fallbackRegex,
+                `$1
+
+    Args:
+${newArgsSection}
+$2`,
+              );
+            }
+          }
+        }
+      } else if (argsMatch) {
+        // Remove existing Args section when no tables
+        updatedScript = updatedScript.replace(argsRegex, "$1$3");
+      }
+
+      return updatedScript;
+    }
+
+    const maxAliasLength =
+      tableAliases.length > 0
+        ? Math.max(...tableAliases.map((a) => a.length))
+        : 0;
+    const functionTemplate = `
+${newSignature}
+    """
+    Transform function that processes the input data.
+    ${
+      tableAliases.length > 0
+        ? `
+
+    Args:
+${tableAliases
+  .map((alias) => {
+    const padding = " ".repeat(maxAliasLength - alias.length);
+    const tableName = tables[alias].name;
+    if (alias === tableName) {
+      return `        ${alias}:${padding} DataFrame containing the data from the corresponding table`;
+    }
+    return `        ${alias}:${padding} DataFrame containing the data from table "${tableName}"`;
+  })
+  .join("\n")}`
+        : ""
+    }
+
+    Returns:
+        DataFrame: The transformed data
+    """
+    import pandas as pd
+    return pd.DataFrame()
+`;
+
+    return script + functionTemplate;
+  };
+
+  const handleDataChange = (
+    database: number,
+    tables: Record<string, { id: number; name: string }>,
+  ) => {
+    const updatedScript = updateTransformSignature(source.body, tables);
+
+    const sourceTables: Record<string, number> = {};
+    Object.entries(tables).forEach(([alias, tableInfo]) => {
+      sourceTables[alias] = tableInfo.id;
+    });
+
+    const newSource = {
+      ...source,
+      body: updatedScript,
+      "source-database": database,
+      "source-tables": sourceTables,
+    };
+    setSource(newSource);
+    setIsSourceDirty(true);
+  };
+
+  const handleSave = () => {
+    onSave(source);
+  };
+
+  const canSave = Boolean(
+    source.body.trim() &&
+      source["source-database"] &&
+      source["source-tables"] &&
+      Object.keys(source["source-tables"]).length > 0,
+  );
+
+  return (
+    <Stack
+      w="100%"
+      h="100%"
+      bg="bg-white"
+      data-testid="python-transform-editor"
+      gap={0}
+    >
+      <EditorHeader
+        isNew={isNew}
+        isSaving={isSaving}
+        canSave={canSave && (isNew || isSourceDirty)}
+        onSave={handleSave}
+        onCancel={onCancel}
+      />
+      <Flex h="100%" w="100%">
+        <PythonDataPicker
+          database={source["source-database"]}
+          tables={source["source-tables"]}
+          onChange={handleDataChange}
+        />
+
+        <PythonEditor
+          script={source.body}
+          isRunnable={true}
+          onChange={handleScriptChange}
+          tables={source["source-tables"]}
+        />
+      </Flex>
+    </Stack>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonTransformEditor/index.ts

@@ -0,0 +1 @@
+export { PythonTransformEditor } from "./PythonTransformEditor";

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/EditorBody/EditorBody.tsx

@@ -11,7 +11,13 @@ import { doesDatabaseSupportTransforms } from "metabase-enterprise/transforms/ut
 import type Question from "metabase-lib/v1/Question";
 import type Database from "metabase-lib/v1/metadata/Database";
 import type NativeQuery from "metabase-lib/v1/queries/NativeQuery";
-import type { Database as ApiDatabase, RecentItem } from "metabase-types/api";
+import type {
+  Database as ApiDatabase,
+  PythonTransformSource,
+  RecentItem,
+} from "metabase-types/api";
+
+import { PythonEditor } from "../PythonEditor";
 
 import S from "./EditorBody.module.css";
 import { ResizableBoxHandle } from "./ResizableBoxHandle";
@@ -21,10 +27,13 @@ const EDITOR_HEIGHT = 400;
 type EditorBodyProps = {
   question: Question;
   isNative: boolean;
+  isPython?: boolean;
+  pythonSource?: PythonTransformSource;
   isRunnable: boolean;
   isRunning: boolean;
   isResultDirty: boolean;
   onChange: (newQuestion: Question) => void;
+  onPythonChange?: (script: string) => void;
   onRunQuery: () => Promise<void>;
   onCancelQuery: () => void;
   databases: ApiDatabase[];
@@ -33,10 +42,13 @@ type EditorBodyProps = {
 export function EditorBody({
   question,
   isNative,
+  isPython = false,
+  pythonSource,
   isRunnable,
   isRunning,
   isResultDirty,
   onChange,
+  onPythonChange,
   onRunQuery,
   onCancelQuery,
   databases,
@@ -70,6 +82,24 @@ export function EditorBody({
 
   const dataPickerOptions = useDataPickerOptions({ databases });
 
+  const handlePythonChange = (script: string) => {
+    onPythonChange?.(script);
+  };
+
+  if (isPython) {
+    return (
+      <PythonEditor
+        script={pythonSource?.body || ""}
+        isRunnable={isRunnable}
+        isRunning={isRunning}
+        isResultDirty={isResultDirty}
+        onChange={handlePythonChange}
+        onRunScript={onRunQuery}
+        onCancelScript={onCancelQuery}
+      />
+    );
+  }
+
   return isNative ? (
     <NativeQueryEditor
       question={question}

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/EditorHeader/EditorHeader.tsx

@@ -23,6 +23,7 @@ export function EditorHeader({
       title={getTitle(isNew)}
       admin
       buttons={[
+        <Button key="cancel" small onClick={onCancel}>{t`Cancel`}</Button>,
         <Button
           key="save"
           onClick={onSave}
@@ -32,7 +33,6 @@ export function EditorHeader({
         >
           {getSaveButtonLabel(isNew, isSaving)}
         </Button>,
-        <Button key="cancel" small onClick={onCancel}>{t`Cancel`}</Button>,
       ]}
     />
   );

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/PythonEditor/ExecutionOutputTable.tsx

@@ -0,0 +1,25 @@
+import { useMemo } from "react";
+
+import { DataGrid, useDataGridInstance } from "metabase/data-grid";
+
+import { parseCSV } from "./utils";
+
+export function ExecutionOutputTable({ output }: { output?: string }) {
+  const { headers, rows } = useMemo(() => parseCSV(output || ""), [output]);
+
+  const tableProps = useDataGridInstance<string[], unknown>({
+    data: rows,
+    columnsOptions: headers.map((header, index) => ({
+      id: header,
+      name: header,
+      accessorFn: (row) => row[index],
+    })),
+    minGridWidth: 100,
+  });
+
+  if (!output || headers.length === 0) {
+    return null;
+  }
+
+  return <DataGrid {...tableProps} />;
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/PythonEditor/PythonEditor.module.css

@@ -0,0 +1,68 @@
+.root {
+  border-bottom: 1px solid var(--mb-color-border);
+}
+
+.editor {
+  width: 100%;
+  height: 100%;
+}
+
+.logs {
+  white-space: pre;
+  border: 1px solid var(--mb-color-border);
+  font-family: var(--mb-default-monospace-font-family);
+}
+
+.visualization {
+  flex: 1 0 auto;
+}
+
+.results {
+  width: 100%;
+  border-collapse: collapse;
+  border: 1px solid var(--mb-color-border);
+  border-radius: var(--mantine-radius-md);
+}
+
+.tableHeader {
+  background-color: var(--mb-color-bg-light);
+  padding: var(--mantine-spacing-sm);
+}
+
+.tableCell {
+  border-top: 1px solid var(--mb-color-border);
+  padding: var(--mantine-spacing-sm);
+}
+
+.error {
+  color: var(--mb-color-error);
+  background-color: var(--mb-color-bg-error);
+  border-bottom: 1px solid var(--mb-color-border);
+}
+
+.info {
+  color: var(--mb-color-text-dark);
+  border: 1px solid var(--mb-color-bg-medium);
+  background-color: var(--mb-color-bg-light);
+}
+
+.success {
+  color: var(--mb-color-success);
+  background-color: var(--mb-color-bg-success);
+  border-bottom: 1px solid var(--mb-color-border);
+}
+
+.loading {
+  position: absolute;
+  top: 0;
+  left: 0;
+  bottom: 0;
+  right: 0;
+  background-color: rgba(255, 255, 255, 0.5);
+}
+
+.libraryActions {
+  position: absolute;
+  top: 0;
+  right: 0;
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/PythonEditor/PythonEditor.tsx

@@ -0,0 +1,247 @@
+import type { ReactNode } from "react";
+import { ResizableBox } from "react-resizable";
+import { c, t } from "ttag";
+
+import EmptyCodeResult from "assets/img/empty-states/code.svg";
+import DebouncedFrame from "metabase/common/components/DebouncedFrame";
+import Link from "metabase/common/components/Link";
+import { LoadingSpinner } from "metabase/common/components/MetadataInfo/MetadataInfo.styled";
+import { isMac } from "metabase/lib/browser";
+import RunButtonWithTooltip from "metabase/query_builder/components/RunButtonWithTooltip";
+import { Box, Checkbox, Flex, Icon, Stack, Text, Title } from "metabase/ui";
+import { getPythonLibraryUrl } from "metabase-enterprise/transforms/urls";
+
+import { SHARED_LIB_IMPORT_NAME } from "../../../constants";
+import { PythonEditor as PythonCodeEditor } from "../../PythonEditor";
+import { ResizableBoxHandle } from "../EditorBody/ResizableBoxHandle";
+
+import { ExecutionOutputTable } from "./ExecutionOutputTable";
+import S from "./PythonEditor.module.css";
+import {
+  type ExecutionResult,
+  hasImport,
+  insertImport,
+  removeImport,
+  useTestPythonScript,
+} from "./utils";
+
+type PythonEditorProps = {
+  script: string;
+  isRunnable: boolean;
+  onChange: (script: string) => void;
+  onRunScript?: () => Promise<void>;
+  onCancelScript?: () => void;
+  tables?: Record<string, number>;
+};
+
+const EDITOR_HEIGHT = 400;
+
+export function PythonEditor({
+  script,
+  isRunnable,
+  onChange,
+  tables = {},
+}: PythonEditorProps) {
+  const { isRunning, isDirty, cancel, run, executionResult } =
+    useTestPythonScript(script, tables);
+
+  const hasSharedLib = hasImport(script, SHARED_LIB_IMPORT_NAME);
+  function handleToggleSharedLib() {
+    if (hasImport(script, SHARED_LIB_IMPORT_NAME)) {
+      onChange(removeImport(script, SHARED_LIB_IMPORT_NAME));
+    } else {
+      onChange(insertImport(script, SHARED_LIB_IMPORT_NAME));
+    }
+  }
+
+  return (
+    <Stack h="100%" w="100%" gap={0}>
+      <ResizableBox
+        className={S.root}
+        axis="y"
+        height={EDITOR_HEIGHT}
+        handle={<ResizableBoxHandle />}
+        resizeHandles={["s"]}
+      >
+        <Flex h="100%" align="end" bg="bg-light">
+          <PythonCodeEditor
+            value={script}
+            onChange={onChange}
+            withPandasCompletions
+          />
+
+          <Box p="md">
+            <RunButtonWithTooltip
+              disabled={!isRunnable}
+              isRunning={isRunning}
+              isDirty={isDirty}
+              onRun={run}
+              onCancel={cancel}
+              getTooltip={() => t`Run Python script`}
+            />
+          </Box>
+        </Flex>
+        <Stack className={S.libraryActions} p="md" gap="sm">
+          <Checkbox
+            label={t`Import shared library`}
+            checked={hasSharedLib}
+            onChange={handleToggleSharedLib}
+            size="sm"
+          />
+          <Flex
+            component={Link}
+            target="_blank"
+            to={getPythonLibraryUrl({ name: SHARED_LIB_IMPORT_NAME })}
+            gap="sm"
+          >
+            <Icon name="pencil" />
+            {t`Edit shared library`}
+          </Flex>
+        </Stack>
+      </ResizableBox>
+
+      <DebouncedFrame className={S.visualization}>
+        {executionResult ? (
+          <ExecutionResult executionResult={executionResult} />
+        ) : (
+          <EmptyState />
+        )}
+        {isRunning && <LoadingState />}
+      </DebouncedFrame>
+    </Stack>
+  );
+}
+
+function getRunQueryShortcut() {
+  return isMac() ? t`⌘ + return` : t`Ctrl + enter`;
+}
+
+function LoadingState() {
+  return (
+    <Flex p="md" className={S.loading}>
+      <LoadingSpinner />
+    </Flex>
+  );
+}
+
+function EmptyState() {
+  const keyboardShortcut = getRunQueryShortcut();
+
+  return (
+    <Flex h="100%" align="center" justify="center">
+      <Stack maw="25rem" gap={0} ta="center" align="center">
+        <Box maw="3rem" mb="0.75rem">
+          <img src={EmptyCodeResult} alt="Code prompt icon" />
+        </Box>
+        <Text c="text-medium">
+          {c("{0} refers to the keyboard shortcut")
+            .jt`To run your code, click on the Run button or type ${(
+            <b key="shortcut">({keyboardShortcut})</b>
+          )}`}
+        </Text>
+      </Stack>
+    </Flex>
+  );
+}
+
+function ExecutionResult({
+  executionResult,
+}: {
+  executionResult: ExecutionResult | null;
+}) {
+  if (!executionResult) {
+    return null;
+  }
+
+  const message = getMessageForExecutionResult(executionResult);
+
+  return (
+    <Stack gap={0}>
+      {message}
+
+      <ExecutionLogs
+        label={t`Standard output:`}
+        content={executionResult.stdout}
+      />
+      <ExecutionLogs
+        label={t`Standard error:`}
+        content={executionResult.stderr}
+      />
+
+      <ExecutionOutputTable output={executionResult.output} />
+    </Stack>
+  );
+}
+
+function getMessageForExecutionResult(executionResult: ExecutionResult) {
+  if (executionResult.error) {
+    return (
+      <Box className={S.error} p="sm">
+        <Flex gap="sm">
+          <Icon name="warning" />
+          <Stack gap={0}>
+            <Title order={5} mb="xs">
+              {t`An error occurred while executing your Python script`}
+            </Title>
+            {executionResult.error}
+          </Stack>
+        </Flex>
+      </Box>
+    );
+  }
+
+  if (!executionResult.output) {
+    return (
+      <Flex className={S.info} p="sm" bdrs="xs" gap="sm" align="center">
+        <Icon name="info" />
+        {t`No results to display.`}
+      </Flex>
+    );
+  }
+
+  return (
+    <Flex className={S.success} p="sm" gap="sm" align="center">
+      <Icon name="check" />
+      {t`Script executed successfully.`}
+    </Flex>
+  );
+}
+
+function ExecutionLogs({
+  label,
+  content,
+}: {
+  label: string;
+  content?: string | null;
+}) {
+  if (!content) {
+    return null;
+  }
+
+  return (
+    <Section title={label}>
+      <Box p="sm" bg="bg-light" mah="150px" bdrs="xs" className={S.logs}>
+        {content}
+      </Box>
+    </Section>
+  );
+}
+
+function Section({
+  title,
+  children,
+}: {
+  title: ReactNode;
+  children: ReactNode;
+}) {
+  return (
+    <Box>
+      {title && (
+        <Title order={5} mb="xs">
+          {title}
+        </Title>
+      )}
+      {children}
+    </Box>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/PythonEditor/index.ts

@@ -0,0 +1 @@
+export { PythonEditor } from "./PythonEditor";

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/PythonEditor/utils.ts

@@ -0,0 +1,131 @@
+import { useRef, useState } from "react";
+import { t } from "ttag";
+
+import {
+  type ExecutePythonResponse,
+  useExecutePythonMutation,
+} from "metabase-enterprise/transforms/api/python-runner";
+
+export function parseCSV(csv: string): { headers: string[]; rows: string[][] } {
+  if (!csv || !csv.trim()) {
+    return { headers: [], rows: [] };
+  }
+
+  const lines = csv.trim().split("\n");
+  if (lines.length === 0) {
+    return { headers: [], rows: [] };
+  }
+
+  const headers = lines[0].split(",").map((h) => h.trim());
+  const rows = lines.slice(1).map((line) => {
+    return line.split(",").map((cell) => cell.trim());
+  });
+
+  return { headers, rows };
+}
+
+export type ExecutionResult = {
+  output?: string;
+  stdout?: string;
+  stderr?: string;
+  error?: string;
+};
+
+type TestPythonScriptState = {
+  isRunning: boolean;
+  isDirty: boolean;
+  executionResult: ExecutionResult | null;
+  run: () => void;
+  cancel: () => void;
+};
+
+export function useTestPythonScript(
+  code: string,
+  tables: Record<string, number>,
+): TestPythonScriptState {
+  const [executePython, { isLoading: isRunning, originalArgs }] =
+    useExecutePythonMutation();
+  const abort = useRef<(() => void) | null>(null);
+  const [executionResult, setData] = useState<ExecutePythonResponse | null>(
+    null,
+  );
+
+  const isDirty = originalArgs?.code !== code;
+
+  const run = async () => {
+    const request = executePython({ code, tables });
+    abort.current = () => request.abort();
+
+    try {
+      const data = await request.unwrap();
+      setData(data);
+    } catch (error) {
+      if (typeof error === "object" && error !== null) {
+        if ("name" in error && error.name === "AbortError") {
+          setData({ error: t`Python script execution was canceled` });
+          return;
+        }
+
+        if ("message" in error && typeof error.message === "string") {
+          setData({ error: error.message });
+          return;
+        }
+
+        if (
+          "data" in error &&
+          typeof error.data === "object" &&
+          error.data !== null &&
+          "error" in error.data &&
+          typeof error.data.error === "string"
+        ) {
+          setData({ error: error?.data?.error });
+          return;
+        }
+      }
+
+      setData({ error: t`An unknown error occurred` });
+    }
+  };
+
+  const cancel = () => {
+    abort.current?.();
+  };
+
+  return {
+    isRunning,
+    isDirty,
+    cancel,
+    run,
+    executionResult,
+  };
+}
+
+export function insertImport(source: string, lib: string) {
+  const lines = source.split("\n");
+
+  // Find the first line that is not a comment
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    if (line.trim().startsWith("#")) {
+      continue;
+    }
+    lines.splice(i, 0, `import ${lib}`);
+    break;
+  }
+
+  return lines.join("\n");
+}
+
+function libImportRegex(lib: string) {
+  return new RegExp(`import ${lib}`, "g");
+}
+
+export function removeImport(source: string, lib: string) {
+  const lines = source.split("\n");
+  return lines.filter((line) => !libImportRegex(lib).test(line)).join("\n");
+}
+
+export function hasImport(source: string, lib: string) {
+  const regex = libImportRegex(lib);
+  return regex.test(source);
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/RunButton/RunButton.tsx

@@ -2,7 +2,7 @@ import { type ReactNode, type Ref, forwardRef, useState } from "react";
 import { useUpdateEffect } from "react-use";
 import { t } from "ttag";
 
-import { Button, Icon, Loader } from "metabase/ui";
+import { Button, Group, Icon, Loader } from "metabase/ui";
 import type { TransformRun } from "metabase-types/api";
 
 const RECENT_TIMEOUT = 5000;
@@ -12,6 +12,8 @@ type RunButtonProps = {
   isLoading: boolean;
   isDisabled?: boolean;
   onRun: () => void;
+  onCancel?: () => void;
+  isCanceling?: boolean;
 };
 
 export const RunButton = forwardRef(function RunButton(
@@ -20,6 +22,8 @@ export const RunButton = forwardRef(function RunButton(
     isLoading,
     isDisabled: isExternallyDisabled = false,
     onRun,
+    onCancel,
+    isCanceling = false,
   }: RunButtonProps,
   ref: Ref<HTMLButtonElement>,
 ) {
@@ -37,6 +41,35 @@ export const RunButton = forwardRef(function RunButton(
     return () => clearTimeout(timeoutId);
   }, [run]);
 
+  const showCancelButton = (run?.status === "started" || isLoading) && onCancel;
+
+  if (showCancelButton) {
+    return (
+      <Group gap="sm">
+        <Button
+          ref={ref}
+          variant="filled"
+          color={color}
+          leftSection={leftSection}
+          disabled={isDisabled}
+          data-testid="run-button"
+          onClick={onRun}
+        >
+          {label}
+        </Button>
+        <Button
+          variant="outline"
+          color="error"
+          disabled={isCanceling}
+          data-testid="cancel-button"
+          onClick={onCancel}
+        >
+          {isCanceling ? t`Canceling…` : t`Cancel`}
+        </Button>
+      </Group>
+    );
+  }
+
   return (
     <Button
       ref={ref}

enterprise/frontend/src/metabase-enterprise/transforms/components/TransformTable/TransformTable.module.css

@@ -0,0 +1,3 @@
+.row {
+  cursor: pointer;
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/TransformTable/TransformTable.tsx

@@ -0,0 +1,40 @@
+import { push } from "react-router-redux";
+import { t } from "ttag";
+
+import { AdminContentTable } from "metabase/common/components/AdminContentTable";
+import { useDispatch } from "metabase/lib/redux";
+import { Card } from "metabase/ui";
+import type { Transform } from "metabase-types/api";
+
+import { getTransformUrl } from "../../urls";
+
+import S from "./TransformTable.module.css";
+
+type TransformTableProps = {
+  transforms: Transform[];
+};
+
+export function TransformTable({ transforms }: TransformTableProps) {
+  const dispatch = useDispatch();
+
+  const handleRowClick = (transform: Transform) => {
+    dispatch(push(getTransformUrl(transform.id)));
+  };
+
+  return (
+    <Card p={0} shadow="none" withBorder>
+      <AdminContentTable columnTitles={[t`Transform`, t`Target`]}>
+        {transforms.map((transform) => (
+          <tr
+            key={transform.id}
+            className={S.row}
+            onClick={() => handleRowClick(transform)}
+          >
+            <td>{transform.name}</td>
+            <td>{transform.target.name}</td>
+          </tr>
+        ))}
+      </AdminContentTable>
+    </Card>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/TransformTable/index.ts

@@ -0,0 +1 @@
+export * from "./TransformTable";

enterprise/frontend/src/metabase-enterprise/transforms/constants.ts

@@ -6,3 +6,5 @@ export const POLLING_INTERVAL = isCypressActive ? 100 : 3000;
 
 export const FILTER_WIDGET_MIN_WIDTH = 300;
 export const FILTER_WIDGET_MAX_HEIGHT = 400;
+
+export const SHARED_LIB_IMPORT_NAME = "lib";

enterprise/frontend/src/metabase-enterprise/transforms/index.tsx

@@ -9,7 +9,9 @@ import { hasPremiumFeature } from "metabase-enterprise/settings";
 import { JobListPage } from "./pages/JobListPage";
 import { JobPage } from "./pages/JobPage";
 import { NewJobPage } from "./pages/NewJobPage";
+import { NewPythonTransformPage } from "./pages/NewPythonTransformPage";
 import { NewTransformQueryPage } from "./pages/NewTransformQueryPage";
+import { PythonLibraryEditorPage } from "./pages/PythonLibraryEditorPage";
 import { RunListPage } from "./pages/RunListPage";
 import { TransformListPage } from "./pages/TransformListPage";
 import { TransformPage } from "./pages/TransformPage";
@@ -32,7 +34,21 @@ if (hasPremiumFeature("transforms")) {
           <Route path="runs" component={RunListPage} />
           <Route path=":transformId" component={TransformPage} />
         </Route>
-        <Route path="new/:type" component={NewTransformQueryPage} />
+        <Route
+          component={(props) => <TransformPageLayout {...props} fullWidth />}
+        >
+          <Route path="library/:name" component={PythonLibraryEditorPage} />
+        </Route>
+        <Route
+          path="new/:type"
+          component={(props: any) =>
+            props.params.type === "python" ? (
+              <NewPythonTransformPage />
+            ) : (
+              <NewTransformQueryPage params={props.params} />
+            )
+          }
+        />
         <Route path="new/card/:cardId" component={NewTransformQueryPage} />
         <Route path=":transformId/query" component={TransformQueryPage} />
       </Route>

enterprise/frontend/src/metabase-enterprise/transforms/pages/NewPythonTransformPage/CreatePythonTransformModal/CreatePythonTransformModal.tsx

@@ -0,0 +1,154 @@
+import { Field, Form, Formik } from "formik";
+import { t } from "ttag";
+import * as Yup from "yup";
+
+import { Button, Group, Modal, Stack, TextInput } from "metabase/ui";
+import { useCreateTransformMutation } from "metabase-enterprise/api";
+import type { PythonTransformSource, Transform } from "metabase-types/api";
+
+type PythonTransformApiSource = {
+  type: "python";
+  body: string;
+  "source-database": number;
+  "source-tables": Record<string, number>;
+};
+
+type CreatePythonTransformModalProps = {
+  source: PythonTransformSource;
+  onCreate: (transform: Transform) => void;
+  onClose: () => void;
+};
+
+type FormValues = {
+  name: string;
+  description: string | null;
+  targetSchema: string | null;
+  targetName: string;
+};
+
+const getValidationSchema = () =>
+  Yup.object({
+    name: Yup.string().required(t`Name is required`),
+    description: Yup.string().nullable(),
+    targetSchema: Yup.string().nullable(),
+    targetName: Yup.string().required(t`Table name is required`),
+  });
+
+export function CreatePythonTransformModal({
+  source,
+  onCreate,
+  onClose,
+}: CreatePythonTransformModalProps) {
+  const [createTransform] = useCreateTransformMutation();
+
+  const initialValues: FormValues = {
+    name: "",
+    description: null,
+    targetSchema: "public",
+    targetName: "",
+  };
+
+  const handleSubmit = async (values: FormValues) => {
+    try {
+      const transformedSource: PythonTransformApiSource = {
+        type: "python",
+        body: source.body,
+        "source-database": source["source-database"],
+        "source-tables": source["source-tables"] || {},
+      };
+
+      const transform = await createTransform({
+        name: values.name,
+        description: values.description,
+        source: transformedSource as any,
+        target: {
+          type: "table",
+          schema: values.targetSchema?.trim() || null,
+          name: values.targetName,
+          database: source["source-database"], // for now the same as source
+        },
+      }).unwrap();
+
+      onCreate(transform);
+    } catch (error) {
+      console.error("Error creating Python transform:", error);
+      // TODO: Show error toast
+    }
+  };
+
+  return (
+    <Modal
+      opened
+      onClose={onClose}
+      title={t`Create Python Transform`}
+      size="md"
+    >
+      <Formik
+        initialValues={initialValues}
+        validationSchema={getValidationSchema()}
+        onSubmit={handleSubmit}
+      >
+        {({ isSubmitting, isValid }) => (
+          <Form>
+            <Stack gap="md">
+              <Field name="name">
+                {({ field, meta }: any) => (
+                  <TextInput
+                    {...field}
+                    label={t`Transform name`}
+                    placeholder={t`My Python Transform`}
+                    error={meta.touched && meta.error}
+                  />
+                )}
+              </Field>
+
+              <Field name="description">
+                {({ field }: any) => (
+                  <TextInput
+                    {...field}
+                    label={t`Description (optional)`}
+                    placeholder={t`What does this transform do?`}
+                  />
+                )}
+              </Field>
+
+              <Field name="targetName">
+                {({ field, meta }: any) => (
+                  <TextInput
+                    {...field}
+                    label={t`Output table name`}
+                    placeholder={t`my_python_transform`}
+                    error={meta.touched && meta.error}
+                  />
+                )}
+              </Field>
+
+              <Field name="targetSchema">
+                {({ field }: any) => (
+                  <TextInput
+                    {...field}
+                    label={t`Schema (optional)`}
+                    placeholder={t`public`}
+                  />
+                )}
+              </Field>
+
+              <Group justify="flex-end" mt="md">
+                <Button variant="subtle" onClick={onClose}>
+                  {t`Cancel`}
+                </Button>
+                <Button
+                  type="submit"
+                  loading={isSubmitting}
+                  disabled={!isValid}
+                >
+                  {t`Create transform`}
+                </Button>
+              </Group>
+            </Stack>
+          </Form>
+        )}
+      </Formik>
+    </Modal>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/pages/NewPythonTransformPage/CreatePythonTransformModal/index.ts

@@ -0,0 +1 @@
+export { CreatePythonTransformModal } from "./CreatePythonTransformModal";

enterprise/frontend/src/metabase-enterprise/transforms/pages/NewPythonTransformPage/NewPythonTransformPage.tsx

@@ -0,0 +1,71 @@
+import { useDisclosure } from "@mantine/hooks";
+import { useState } from "react";
+import { push } from "react-router-redux";
+
+import { useDispatch } from "metabase/lib/redux";
+import type { PythonTransformSource, Transform } from "metabase-types/api";
+
+import { PythonTransformEditor } from "../../components/PythonTransformEditor";
+import { getTransformListUrl, getTransformUrl } from "../../urls";
+
+import { CreatePythonTransformModal } from "./CreatePythonTransformModal";
+
+const DEFAULT_PYTHON_SOURCE: PythonTransformSource = {
+  type: "python",
+  "source-database": 1, // Default to first database, will be updated by user
+  "source-tables": {}, // Will be populated when user selects tables
+  body: `# Write your Python transformation script here
+import pandas as pd
+
+def transform():
+    """
+    Your transformation function.
+
+    Select tables above to add them as function parameters.
+
+    Returns:
+        DataFrame to write to the destination table
+    """
+    # Your transformation logic here
+    return pd.DataFrame([{"message": "Hello from Python transform!"}])`,
+};
+
+export function NewPythonTransformPage() {
+  const [source, setSource] = useState<PythonTransformSource>(
+    DEFAULT_PYTHON_SOURCE,
+  );
+  const [isModalOpened, { open: openModal, close: closeModal }] =
+    useDisclosure();
+  const dispatch = useDispatch();
+
+  const handleCreate = (transform: Transform) => {
+    dispatch(push(getTransformUrl(transform.id)));
+  };
+
+  const handleSaveClick = (newSource: PythonTransformSource) => {
+    setSource(newSource);
+    openModal();
+  };
+
+  const handleCancelClick = () => {
+    dispatch(push(getTransformListUrl()));
+  };
+
+  return (
+    <>
+      <PythonTransformEditor
+        initialSource={DEFAULT_PYTHON_SOURCE}
+        isNew
+        onSave={handleSaveClick}
+        onCancel={handleCancelClick}
+      />
+      {isModalOpened && (
+        <CreatePythonTransformModal
+          source={source}
+          onCreate={handleCreate}
+          onClose={closeModal}
+        />
+      )}
+    </>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/pages/NewPythonTransformPage/index.ts

@@ -0,0 +1 @@
+export { NewPythonTransformPage } from "./NewPythonTransformPage";

enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/CreateTransformModal/CreateTransformModal.tsx

@@ -107,7 +107,10 @@ function CreateTransformForm({
   }
 
   const handleSubmit = async (values: NewTransformValues) => {
-    const request = getCreateRequest(query, values);
+    if (!databaseId) {
+      throw new Error("Database ID is required");
+    }
+    const request = getCreateRequest(query, values, databaseId);
     const transform = await createTransform(request).unwrap();
 
     trackTransformCreated({ transformId: transform.id });
@@ -170,6 +173,7 @@ function getInitialValues(schemas: string[]): NewTransformValues {
 function getCreateRequest(
   query: DatasetQuery,
   { name, description, targetName, targetSchema }: NewTransformValues,
+  databaseId: number,
 ): CreateTransformRequest {
   return {
     name: name,
@@ -182,6 +186,7 @@ function getCreateRequest(
       type: "table",
       name: targetName,
       schema: targetSchema,
+      database: databaseId,
     },
   };
 }

enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/NewTransformQueryPage.tsx

@@ -90,7 +90,7 @@ function getParsedParams({
   cardId,
 }: NewTransformQueryPageParams): NewTransformQueryPageParsedParams {
   return {
-    type: type === "native" ? "native" : "query",
+    type: type === "native" ? "native" : type === "python" ? "query" : "query",
     cardId: cardId != null ? Urls.extractEntityId(cardId) : undefined,
   };
 }

enterprise/frontend/src/metabase-enterprise/transforms/pages/PythonLibraryEditorPage/PythonLibraryEditorPage.module.css

@@ -0,0 +1,5 @@
+.editor {
+  /* HACK: 133px is the height of the headers */
+  height: calc(100vh - 133px);
+  flex-grow: 1;
+}

enterprise/frontend/src/metabase-enterprise/transforms/pages/PythonLibraryEditorPage/PythonLibraryEditorPage.tsx

@@ -0,0 +1,119 @@
+import { useEffect, useState } from "react";
+import { t } from "ttag";
+
+import Button from "metabase/common/components/Button";
+import EditBar from "metabase/common/components/EditBar";
+import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
+import { useMetadataToasts } from "metabase/metadata/hooks";
+import { Box, Flex } from "metabase/ui";
+import {
+  useGetPythonLibraryQuery,
+  useUpdatePythonLibraryMutation,
+} from "metabase-enterprise/api/python-transform-library";
+import { PythonEditor } from "metabase-enterprise/transforms/components/PythonEditor";
+import type { PythonLibraryEditorPageParams } from "metabase-enterprise/transforms/types";
+
+import S from "./PythonLibraryEditorPage.module.css";
+
+type PythonLibraryEditorPageProps = {
+  params: PythonLibraryEditorPageParams;
+};
+
+export function PythonLibraryEditorPage({
+  params,
+}: PythonLibraryEditorPageProps) {
+  const { name } = params;
+  const [source, setSource] = useState("");
+
+  const {
+    data: library,
+    isLoading,
+    error,
+  } = useGetPythonLibraryQuery({ name });
+  const [updatePythonLibrary, { isLoading: isSaving }] =
+    useUpdatePythonLibraryMutation();
+  const { sendSuccessToast, sendErrorToast } = useMetadataToasts();
+
+  useEffect(() => {
+    setSource(library?.source || "");
+  }, [library?.source]);
+
+  const isDirty = source !== library?.source;
+
+  function handleRevert() {
+    setSource(library?.source || "");
+  }
+
+  async function handleSave() {
+    try {
+      await updatePythonLibrary({ name, source }).unwrap();
+      sendSuccessToast(t`Python library saved`);
+    } catch (error) {
+      sendErrorToast(t`Python library could not be saved`);
+    }
+  }
+
+  if (isLoading || error) {
+    return (
+      <Box p="md">
+        <LoadingAndErrorWrapper loading={isLoading} error={error} />
+      </Box>
+    );
+  }
+
+  return (
+    <Flex h="100%" w="100%" bg="bg-light" gap={0} direction="column">
+      <LibraryEditorHeader
+        onSave={handleSave}
+        onRevert={handleRevert}
+        isDirty={isDirty}
+        isSaving={isSaving}
+      />
+      <PythonEditor
+        value={source}
+        onChange={setSource}
+        withPandasCompletions
+        className={S.editor}
+      />
+    </Flex>
+  );
+}
+
+export function LibraryEditorHeader({
+  isDirty,
+  isSaving,
+  onSave,
+  onRevert,
+}: {
+  isDirty?: boolean;
+  isSaving?: boolean;
+  onSave: () => void;
+  onRevert: () => void;
+}) {
+  return (
+    <EditBar
+      title={t`You are editing the shared Python library`}
+      admin
+      buttons={[
+        <Button
+          key="save"
+          onClick={onRevert}
+          primary
+          small
+          disabled={!isDirty || isSaving}
+        >
+          {t`Revert`}
+        </Button>,
+        <Button
+          key="save"
+          onClick={onSave}
+          primary
+          small
+          disabled={!isDirty || isSaving}
+        >
+          {t`Save`}
+        </Button>,
+      ]}
+    />
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/pages/PythonLibraryEditorPage/index.ts

@@ -0,0 +1 @@
+export * from "./PythonLibraryEditorPage";

enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformListPage/CreateTransformMenu/CreateTransformMenu.tsx

@@ -85,6 +85,19 @@ export function CreateTransformMenu() {
               >
                 {t`SQL query`}
               </Menu.Item>
+              <Menu.Item
+                component={ForwardRefLink}
+                to={getNewTransformFromTypeUrl("python")}
+                leftSection={<Icon name="code_block" />}
+                onClick={() => {
+                  trackTransformCreate({
+                    triggeredFrom: "transform-page-create-menu",
+                    creationType: "python",
+                  });
+                }}
+              >
+                {t`Python script`}
+              </Menu.Item>
               <Menu.Item
                 leftSection={<Icon name="copy" />}
                 onClick={handleSavedQuestionClick}

enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/DependenciesSection/DependenciesSection.tsx

@@ -1,56 +1,33 @@
-import { push } from "react-router-redux";
 import { t } from "ttag";
 
-import { AdminContentTable } from "metabase/common/components/AdminContentTable";
-import { useDispatch } from "metabase/lib/redux";
-import { Card, Center, Loader } from "metabase/ui";
+import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
 import { useListTransformDependenciesQuery } from "metabase-enterprise/api";
 import type { Transform } from "metabase-types/api";
 
 import { TitleSection } from "../../../components/TitleSection";
-import { getTransformUrl } from "../../../urls";
-
-import S from "./DependenciesSection.module.css";
+import { TransformTable } from "../../../components/TransformTable";
 
 export function DependenciesSection({ transform }: { transform: Transform }) {
-  const { data: dependencies, isLoading } = useListTransformDependenciesQuery(
-    transform.id,
-  );
-  const dispatch = useDispatch();
+  const {
+    data: transforms = [],
+    error,
+    isLoading,
+  } = useListTransformDependenciesQuery(transform.id);
 
-  if (!isLoading && dependencies?.length === 0) {
+  if (isLoading || transforms.length === 0) {
     return null;
   }
 
-  const handleRowClick = (transform: Transform) => {
-    dispatch(push(getTransformUrl(transform.id)));
-  };
-
   return (
     <TitleSection
       label={t`Dependencies`}
       description={t`This transform depends on the output of the transforms below, so they need to be run first.`}
     >
-      <Card p={0} shadow="none" withBorder>
-        {isLoading ? (
-          <Center>
-            <Loader m="xl" />
-          </Center>
-        ) : (
-          <AdminContentTable columnTitles={[t`Transform`, t`Target`]}>
-            {dependencies?.map((transform) => (
-              <tr
-                key={transform.id}
-                className={S.row}
-                onClick={() => handleRowClick(transform)}
-              >
-                <th>{transform.name}</th>
-                <td>{transform.target.name}</td>
-              </tr>
-            ))}
-          </AdminContentTable>
-        )}
-      </Card>
+      {error != null ? (
+        <LoadingAndErrorWrapper error={error} />
+      ) : (
+        <TransformTable transforms={transforms} />
+      )}
     </TitleSection>
   );
 }

enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/ManageSection/ManageSection.tsx

@@ -19,9 +19,12 @@ type ManageSectionProps = {
 };
 
 export function ManageSection({ transform }: ManageSectionProps) {
+  const sectionLabel =
+    transform.source.type === "python" ? t`Python Script` : t`Query`;
+
   return (
     <TitleSection
-      label={t`Query`}
+      label={sectionLabel}
       rightSection={
         <Group>
           <EditQueryButton transform={transform} />
@@ -30,7 +33,14 @@ export function ManageSection({ transform }: ManageSectionProps) {
       }
     >
       <Card p={0} shadow="none" withBorder>
-        <QueryView query={transform.source.query} />
+        {transform.source.type === "query" && (
+          <QueryView query={transform.source.query} />
+        )}
+        {transform.source.type === "python" && (
+          <div style={{ padding: "1rem" }}>
+            <pre>{transform.source.body}</pre>
+          </div>
+        )}
       </Card>
     </TitleSection>
   );
@@ -41,13 +51,16 @@ type EditQueryButtonProps = {
 };
 
 function EditQueryButton({ transform }: EditQueryButtonProps) {
+  const buttonText =
+    transform.source.type === "python" ? t`Edit script` : t`Edit query`;
+
   return (
     <Button
       component={Link}
       to={getTransformQueryUrl(transform.id)}
       leftSection={<Icon name="pencil_lines" aria-hidden />}
     >
-      {t`Edit query`}
+      {buttonText}
     </Button>
   );
 }