Parametrise PythonLibraryEditor

- python-runner no longer simulates a single http api that does
everything (incl s3 interaction)
- Separates file lifetimes (shared storage) from execution
- Easier to see how to avoid reading into strings (e.g. for piping into
tmp files, follow up)

Earlier on we had a service that managed much of the I/O, metabase now
coordinates it so it makes little sense pretending there is still
an overall http api.

.clj-kondo/config/modules/config.edn

@@ -718,6 +718,7 @@
            metabase.native-query-snippets.core}
    :uses #{api
            collections
+           lib
            models
            permissions
            premium-features

.github/actions/await-port/action.yml

@@ -11,6 +11,18 @@ runs:
     - name: Wait for port ${{ inputs.port }} to be ready
       shell: bash
       run: |
+        echo "Waiting for port ${{ inputs.port }} to be ready..."
+        timeout=300  # 5 minutes timeout
+        elapsed=0
         while ! nc -z localhost ${{ inputs.port }}; do
-          sleep 0.1;
+          if [ $elapsed -ge $timeout ]; then
+            echo "❌ Timeout waiting for port ${{ inputs.port }} after ${timeout} seconds"
+            exit 1
+          fi
+          sleep 1
+          elapsed=$((elapsed + 1))
+          if [ $((elapsed % 10)) -eq 0 ]; then
+            echo "  Still waiting for port ${{ inputs.port }}... ($elapsed seconds)"
+          fi
         done
+        echo "✅ Port ${{ inputs.port }} is ready!"

.github/scripts/get-issue-data.js

@@ -0,0 +1,84 @@
+const githubSlackMap = require("../../release/github-slack-map.json");
+
+async function getIssueData({ context, github, manualIssueNumber }) {
+  let issueNumber;
+  let issueData;
+  let prReviews = [];
+  const triggerType = context.eventName;
+
+  if (context.eventName === "issues") {
+    issueNumber = context.payload.issue.number;
+    issueData = context.payload.issue;
+  } else if (context.eventName === "workflow_dispatch") {
+    issueNumber = parseInt(manualIssueNumber);
+    const { data } = await github.rest.issues.get({
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      issue_number: issueNumber,
+    });
+    issueData = data;
+  } else if (context.eventName === "pull_request_review") {
+    const prNumber = context.payload.pull_request.number;
+    const { data: prData } = await github.rest.pulls.get({
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      pull_number: prNumber,
+    });
+
+    // Extract issue number from branch name (e.g., "claude-fix/issue-12345-some-title")
+    const branchMatch = prData.head.ref.match(/claude-fix\/issue-(\d+)/);
+    if (!branchMatch) {
+      throw new Error(
+        `Could not extract issue number from branch name: ${prData.head.ref}`,
+      );
+    }
+
+    issueNumber = parseInt(branchMatch[1]);
+
+    const { data } = await github.rest.issues.get({
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      issue_number: issueNumber,
+    });
+    issueData = data;
+
+    const { data: reviews } = await github.rest.pulls.listReviews({
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      pull_number: prNumber,
+    });
+
+    prReviews = reviews.filter(
+      (review) =>
+        review.state === "CHANGES_REQUESTED" &&
+        isMetabaseMember(review.user.login),
+    );
+  }
+
+  const { data: comments } = await github.rest.issues.listComments({
+    owner: context.repo.owner,
+    repo: context.repo.repo,
+    issue_number: issueNumber,
+  });
+
+  const employeeComments = comments.filter((comment) =>
+    isMetabaseMember(comment.user.login),
+  );
+
+  return {
+    issueNumber,
+    issueData,
+    employeeComments,
+    prReviews,
+    triggerType,
+    isIssueAuthorEmployee: isMetabaseMember(issueData.user.login),
+  };
+}
+
+function isMetabaseMember(username) {
+  return username in githubSlackMap;
+}
+
+module.exports = {
+  getIssueData,
+};

.github/workflows/backend.yml

@@ -195,46 +195,6 @@ jobs:
       - name: Run cljfmt
         run: ./bin/mage cljfmt-all --force-check
 
-  be-python-runner-tests:
-    if: ${{ !inputs.skip }}
-    runs-on: ubuntu-22.04
-    timeout-minutes: 30
-    name: Python Runner Tests
-    env:
-      CI: 'true'
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-          token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
-      - name: Prepare back-end environment
-        uses: ./.github/actions/prepare-backend
-        with:
-          m2-cache-key: be-python-runner-tests
-      - name: Start python-runner using Makefile
-        run: |
-          echo "🚀 Starting python-runner using make run..."
-          cd python-runner
-
-          # Use the official make run command
-          make run
-
-          echo "✅ Python runner started successfully!"
-
-      - name: Run Python Runner tests
-        run: >-
-          clojure -X:dev:ee:ee-dev:driver:test:test/python-transform
-      - name: Upload Test Results
-        uses: ./.github/actions/upload-test-results
-        if: always()
-        with:
-          input-path: ./target/junit/
-          output-name: be-python-runner-tests
-          bucket: ${{ vars.AWS_S3_TEST_RESULTS_BUCKET }}
-          aws-access-key-id: ${{ secrets.AWS_TEST_RESULTS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}
-          aws-region: ${{ vars.AWS_REGION }}
-          trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
 
   be-tests-result:
     needs:
@@ -243,7 +203,6 @@ jobs:
       - be-cljfmt
       - be-check
       - be-tests
-      - be-python-runner-tests
     if: always() && !cancelled()
     runs-on: ubuntu-latest
     timeout-minutes: 5

.github/workflows/claude-code-bug-fix.yml

@@ -4,97 +4,22 @@ on:
   issues:
     types:
       - labeled
+  pull_request_review:
+    types:
+      - submitted
   workflow_dispatch:
     inputs:
       issue_number:
         description: "Issue number to fix"
         required: true
         type: string
+      override_employee_check:
+        description: "Override employee check for issue creator"
+        required: false
+        default: false
+        type: boolean
 
 jobs:
-  check-employee-and-issue:
-    runs-on: ubuntu-latest
-    timeout-minutes: 2
-    permissions:
-      contents: read
-      issues: read
-    if: |
-      (github.event_name == 'issues' && github.event.label.name == 'Claude Bug Fix' && github.event.issue.state == 'open') ||
-      (github.event_name == 'workflow_dispatch')
-    outputs:
-      is_employee: ${{ steps.check_employee.outputs.is_employee }}
-      issue_number: ${{ steps.get_issue.outputs.issue_number }}
-      issue_data: ${{ steps.get_issue.outputs.issue_data }}
-    steps:
-      - name: Generate GitHub App Token
-        uses: actions/create-github-app-token@v1
-        id: app-token
-        with:
-          app-id: ${{ secrets.METABASE_BOT_APP_ID }}
-          private-key: ${{ secrets.METABASE_BOT_APP_PRIVATE_KEY }}
-
-      - name: Get issue data
-        id: get_issue
-        uses: actions/github-script@v7
-        with:
-          script: |
-            let issueNumber, issueData;
-
-            if (context.eventName === 'issues') {
-              // Original issue label event
-              issueNumber = context.payload.issue.number;
-              issueData = context.payload.issue;
-            } else if (context.eventName === 'workflow_dispatch') {
-              // Manual trigger with issue number input
-              issueNumber = parseInt('${{ github.event.inputs.issue_number }}');
-              const { data } = await github.rest.issues.get({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                issue_number: issueNumber
-              });
-              issueData = data;
-            }
-
-            core.setOutput('issue_number', issueNumber);
-            core.setOutput('issue_data', JSON.stringify(issueData));
-
-            console.log(`Issue #${issueNumber}: ${issueData.title}`);
-            console.log(`Issue author: ${issueData.user.login}`);
-
-      - name: Check if issue creator is a Metabase employee
-        id: check_employee
-        uses: actions/github-script@v7
-        env:
-          ISSUE_DATA: ${{ steps.get_issue.outputs.issue_data }}
-        with:
-          github-token: ${{ steps.app-token.outputs.token }}
-          script: |
-            const issueData = JSON.parse(process.env.ISSUE_DATA);
-            const issueAuthor = issueData.user.login;
-
-            try {
-              // Check if user is a member of the metabase organization (including private members)
-              const { data: membership } = await github.rest.orgs.getMembershipForUser({
-                org: 'metabase',
-                username: issueAuthor
-              });
-
-              if (membership.state === 'active') {
-                console.log(`User ${issueAuthor} is an active member of the Metabase organization. Proceeding with bug fix automation.`);
-                core.setOutput('is_employee', 'true');
-              } else {
-                console.log(`User ${issueAuthor} is not an active member of the Metabase organization. Skipping bug fix automation.`);
-                core.setOutput('is_employee', 'false');
-              }
-            } catch (error) {
-              if (error.status === 404) {
-                console.log(`User ${issueAuthor} is not a member of the Metabase organization. Skipping bug fix automation.`);
-              } else {
-                console.error('Error checking organization membership:', error);
-              }
-              core.setOutput('is_employee', 'false');
-            }
-
   auto-fix-bug:
     runs-on: ubuntu-latest
     timeout-minutes: 30
@@ -103,8 +28,10 @@ jobs:
       issues: write
       pull-requests: write
       id-token: write
-    needs: check-employee-and-issue
-    if: needs.check-employee-and-issue.outputs.is_employee == 'true'
+    if: |
+      (github.event_name == 'issues' && github.event.label.name == 'Claude Bug Fix' && github.event.issue.state == 'open') ||
+      (github.event_name == 'workflow_dispatch') ||
+      (github.event_name == 'pull_request_review' && github.event.review.state == 'changes_requested' && startsWith(github.event.pull_request.head.ref, 'claude-fix/'))
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -112,67 +39,95 @@ jobs:
           fetch-depth: 0
           token: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Get issue data and check employee status
+        id: get_issue
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const { getIssueData } = require('${{ github.workspace }}/.github/scripts/get-issue-data.js');
+
+            try {
+              const result = await getIssueData({
+                context,
+                github,
+                manualIssueNumber: '${{ github.event.inputs.issue_number }}'
+              });
+
+              const { issueNumber, issueData, employeeComments, prReviews, triggerType, isIssueAuthorEmployee } = result;
+
+              // Check employee status and fail if not authorized
+              if (!result.isIssueAuthorEmployee && '${{ github.event.inputs.override_employee_check }}' !== 'true') {
+                  core.setFailed(`User ${result.issueData.user.login} is not a Metabase employee. Skipping bug fix automation.`);
+              }
+
+              core.setOutput('issue_number', issueNumber);
+              core.setOutput('issue_data', JSON.stringify(issueData));
+              core.setOutput('issue_comments', JSON.stringify(employeeComments));
+              core.setOutput('pr_reviews', JSON.stringify(prReviews));
+              core.setOutput('trigger_type', triggerType);
+            } catch (error) {
+              core.setFailed(error.message);
+            }
+
       - name: Set up git configuration
         run: |
           git config --global user.name "Claude Code[bot]"
           git config --global user.email "noreply@anthropic.com"
 
-      - name: Prepare dynamic issue info
-        id: prepare_issue_info
+      - name: Prepare issue context and bug fix prompt
+        id: prepare_prompt
         uses: actions/github-script@v7
         env:
-          ISSUE_DATA: ${{ needs.check-employee-and-issue.outputs.issue_data }}
+          ISSUE_DATA: ${{ steps.get_issue.outputs.issue_data }}
+          ISSUE_COMMENTS: ${{ steps.get_issue.outputs.issue_comments }}
+          PR_REVIEWS: ${{ steps.get_issue.outputs.pr_reviews }}
+          TRIGGER_TYPE: ${{ steps.get_issue.outputs.trigger_type }}
         with:
           script: |
             const issueData = JSON.parse(process.env.ISSUE_DATA);
-            const issueNumber = '${{ needs.check-employee-and-issue.outputs.issue_number }}';
+            const issueComments = JSON.parse(process.env.ISSUE_COMMENTS);
+            const prReviews = JSON.parse(process.env.PR_REVIEWS);
+            const triggerType = process.env.TRIGGER_TYPE;
+            const issueNumber = '${{ steps.get_issue.outputs.issue_number }}';
 
-            // Convert issue title to kebab case
+            // Convert issue title to kebab case and create branch name
             const issueTitle = issueData.title || 'untitled';
             const kebabTitle = issueTitle
               .toLowerCase()
               .replace(/[^a-z0-9]+/g, '-')
               .replace(/^-+|-+$/g, '')
               .replace(/-+/g, '-')
-              .substring(0, 50); // Limit length
+              .substring(0, 50);
+            const branchName = `claude-fix/issue-${issueNumber}-${kebabTitle}`;
 
-            // Create base branch name
-            const baseBranchName = `claude-fix/issue-${issueNumber}-${kebabTitle}`;
-            core.exportVariable('BASE_BRANCH_NAME', baseBranchName);
+            // Format data for bash environment
+            const formattedComments = issueComments.length > 0
+              ? issueComments.map(comment =>
+                  `**Comment by @${comment.user.login} on ${comment.created_at}:**\n${comment.body}`
+                ).join('\n\n---\n\n')
+              : '';
+            const formattedReviews = prReviews.length > 0
+              ? prReviews.map(review =>
+                  `**Review by @${review.user.login} on ${review.submitted_at} (Changes Requested):**\n${review.body || 'No review comment provided'}`
+                ).join('\n\n---\n\n')
+              : '';
 
-            // Export issue info for the prompt
+            core.exportVariable('BRANCH_NAME', branchName);
             core.exportVariable('ISSUE_NUMBER', issueNumber);
             core.exportVariable('ISSUE_TITLE', issueData.title);
             core.exportVariable('ISSUE_AUTHOR', issueData.user.login);
             core.exportVariable('ISSUE_CREATED', issueData.created_at);
             core.exportVariable('ISSUE_BODY', issueData.body || 'No description provided');
+            core.exportVariable('ISSUE_COMMENTS_FORMATTED', formattedComments);
+            core.exportVariable('PR_REVIEWS_FORMATTED', formattedReviews);
+            core.exportVariable('TRIGGER_TYPE', triggerType);
 
-      - name: Determine branch strategy
+      - name: Generate bug fix prompt
         run: |
-          # Use the base branch name (no counters)
-          BRANCH_NAME="${{ env.BASE_BRANCH_NAME }}"
-
-          # Check if this exact branch already exists
-          if git branch -r --list "origin/${BRANCH_NAME}" | grep -q .; then
-            echo "Branch already exists: $BRANCH_NAME"
-            echo "Will push to existing branch and skip PR creation"
-            echo "BRANCH_EXISTS=true" >> $GITHUB_ENV
-          else
-            echo "Creating new branch: $BRANCH_NAME"
-            echo "Will create new PR"
-            echo "BRANCH_EXISTS=false" >> $GITHUB_ENV
-          fi
-
-          # Export the branch name
-          echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
-
-      - name: Prepare bug fix prompt
-        id: prepare_prompt
-        run: |
-          # Prepare the prompt for Claude
           cat << 'EOF' >> $GITHUB_ENV
           BUG_FIX_PROMPT<<PROMPT_END
-          You are helping to automatically fix a bug reported in GitHub issue #${{ env.ISSUE_NUMBER }}.
+          You are helping to fix a bug reported in GitHub issue #${{ env.ISSUE_NUMBER }}.
 
           ## Bug Report Details
           **Title:** ${{ env.ISSUE_TITLE }}
@@ -182,8 +137,27 @@ jobs:
           **Description:**
           ${{ env.ISSUE_BODY }}
 
+          **Comments:**
+          ${{ env.ISSUE_COMMENTS_FORMATTED }}
+          EOF
+
+          # Add review feedback section only if this is a review trigger
+          if [ "${{ env.TRIGGER_TYPE }}" = "pull_request_review" ]; then
+            cat << 'EOF' >> $GITHUB_ENV
+
+          ## Previous Fix Attempt - Review Feedback (MUST ADDRESS)
+          Your previous automated fix attempt received the following feedback that needs to be addressed:
+
+          ${{ env.PR_REVIEWS_FORMATTED }}
+          EOF
+          fi
+
+          # Add the rest of the prompt (always the same)
+          cat << 'EOF' >> $GITHUB_ENV
+
           ## Instructions
-          1. **Analyze the bug report** - Understand what the issue is describing
+          1. **Analyze the issue description and comments** - Understand what the issue is describing
+             a. Pay particular attention to reproduction steps and if there's an agreed upon fix.
           2. **Search the codebase** - Use Grep and Glob tools to find relevant files
           3. **Implement a fix** - Follow existing code patterns and conventions
           4. **Test the fix** - Run appropriate tests using the commands in CLAUDE.md:
@@ -205,15 +179,26 @@ jobs:
           - Only modify actual production code in the main repository directories
           - The scratchpad will be cleaned up automatically - nothing from it will be committed
 
-          ## If You Cannot Fix The Bug
-          If you determine that you cannot create a definitive fix, create a file called `FAILED_FIX_ANALYSIS.md` with:
-          1. **Why No Fix** - Specific reason why you couldn't implement a fix
-          2. **Tool Limitations** - If you needed additional tools/access that weren't available
-
           Start by understanding the bug and then systematically work toward a solution.
           PROMPT_END
           EOF
 
+      - name: Determine branch strategy
+        run: |
+          BRANCH_NAME="${{ env.BRANCH_NAME }}"
+          if git branch -r --list "origin/${BRANCH_NAME}" | grep -q .; then
+            echo "Branch already exists: $BRANCH_NAME"
+            echo "Will push to existing branch and skip PR creation"
+            echo "BRANCH_EXISTS=true" >> $GITHUB_ENV
+          else
+            echo "Creating new branch: $BRANCH_NAME"
+            echo "Will create new PR"
+            echo "BRANCH_EXISTS=false" >> $GITHUB_ENV
+          fi
+
+          # Export the branch name
+          echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
+
       - name: Create scratchpad directory for Claude
         run: |
           # Create a scratchpad directory for Claude to use for temporary files
@@ -249,20 +234,6 @@ jobs:
       - name: Check for changes and analysis
         id: check_changes
         run: |
-          # Check if Claude created an analysis file first and store its content
-          if [ -f "FAILED_FIX_ANALYSIS.md" ]; then
-            echo "has_analysis=true" >> $GITHUB_OUTPUT
-            echo "Analysis file found"
-            # Store analysis content in a GitHub output (base64 encoded to handle multiline)
-            ANALYSIS_CONTENT=$(base64 -w 0 FAILED_FIX_ANALYSIS.md)
-            echo "analysis_content=$ANALYSIS_CONTENT" >> $GITHUB_OUTPUT
-            # Remove analysis file so it doesn't count as a code change
-            rm -f FAILED_FIX_ANALYSIS.md
-          else
-            echo "has_analysis=false" >> $GITHUB_OUTPUT
-            echo "No analysis file found"
-          fi
-
           # Now check for actual code changes (excluding analysis file)
           if [ -n "$(git status --porcelain)" ]; then
             echo "has_changes=true" >> $GITHUB_OUTPUT
@@ -278,25 +249,16 @@ jobs:
         id: prepare_reviewers
         uses: actions/github-script@v7
         env:
-          ISSUE_DATA: ${{ needs.check-employee-and-issue.outputs.issue_data }}
+          ISSUE_DATA: ${{ steps.get_issue.outputs.issue_data }}
         with:
           script: |
-            // Get existing team labels from the issue
             const issueData = JSON.parse(process.env.ISSUE_DATA);
             const issueLabels = issueData.labels.map(label => label.name);
             const teamLabels = issueLabels.filter(label => label.startsWith('.Team/'));
-
-            console.log('Issue labels:', issueLabels);
-            console.log('Team labels found:', teamLabels);
-
-            // Add issue creator as reviewer
             const issueCreator = issueData.user.login;
-
             core.setOutput('team_labels', teamLabels.join(','));
             core.setOutput('reviewers', issueCreator);
 
-            console.log('Reviewer:', issueCreator);
-
       - name: Create Pull Request
         if: steps.check_changes.outputs.has_changes == 'true'
         uses: peter-evans/create-pull-request@v6
@@ -363,25 +325,3 @@ jobs:
         with:
           script: |
             console.log('🤖 No code changes were made by Claude.');
-
-            // Check if Claude provided a detailed analysis
-            if ('${{ steps.check_changes.outputs.has_analysis }}' === 'true') {
-              try {
-                // Decode the base64-encoded analysis content
-                const analysisBase64 = '${{ steps.check_changes.outputs.analysis_content }}';
-                const analysisContent = Buffer.from(analysisBase64, 'base64').toString('utf8');
-                console.log('📋 Claude provided detailed analysis:');
-                console.log('='.repeat(50));
-                console.log(analysisContent);
-                console.log('='.repeat(50));
-              } catch (error) {
-                console.log('❌ Could not decode analysis content:', error);
-              }
-            } else {
-              console.log('ℹ️  No detailed analysis file was created by Claude.');
-              console.log('This could be because:');
-              console.log('- The issue requires more context or clarification');
-              console.log('- The fix is complex and requires human judgment');
-              console.log('- The issue is not a straightforward code bug');
-              console.log('- Claude couldn\'t locate the relevant code to modify');
-            }

.github/workflows/drivers.yml

@@ -338,6 +338,12 @@ jobs:
     name: H2
     steps:
     - uses: actions/checkout@v4
+      with:
+        submodules: true
+        token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+    - name: Start python-runner
+      working-directory: python-runner
+      run: make run-all
     - name: Test H2 driver
       uses: ./.github/actions/test-driver
       with:
@@ -442,6 +448,12 @@ jobs:
     name: "${{ matrix.version.name }} ${{ matrix.job.name }}"
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
+          token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      - name: Start python-runner
+        working-directory: python-runner
+        run: make run-all
       - name: Test ${{ matrix.version.name }}
         uses: ./.github/actions/test-driver
         with:
@@ -745,6 +757,12 @@ jobs:
           POSTGRES_HOST_AUTH_METHOD: trust
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
+          token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      - name: Start python-runner
+        working-directory: python-runner
+        run: make run-all
       - name: Test ${{ matrix.version.name }} (${{ matrix.job.name }})
         uses: ./.github/actions/test-driver
         with:

deps.edn

@@ -118,7 +118,7 @@
   net.sf.cssbox/cssbox                      {:mvn/version "5.0.2"               ; HTML / CSS rendering
                                              :exclusions  [org.slf4j/slf4j-api
                                                            junit/junit]}
-  net.thisptr/jackson-jq                    {:mvn/version "1.4.0"}              ; Java implementation of the JQ json query language
+  net.thisptr/jackson-jq                    {:mvn/version "1.6.0"}              ; Java implementation of the JQ json query language
   org.apache.commons/commons-compress       {:mvn/version "1.28.0"}             ; compression utils
   org.apache.commons/commons-lang3          {:mvn/version "3.18.0"}             ; helper methods for working with java.lang stuff
   org.apache.logging.log4j/log4j-1.2-api    {:mvn/version "2.25.1"}             ; apache logging framework
@@ -135,7 +135,7 @@
                                              :exclusions  [org.bouncycastle/bcpkix-jdk15on
                                                            org.bouncycastle/bcprov-jdk15on]}
   org.apache.poi/poi-ooxml-full             {:mvn/version "5.4.1"}
-  org.apache.sshd/sshd-core                 {:mvn/version "2.15.0"              ; ssh tunneling and test server
+  org.apache.sshd/sshd-core                 {:mvn/version "2.16.0"              ; ssh tunneling and test server
                                              :exclusions  [org.slf4j/slf4j-api
                                                            org.slf4j/jcl-over-slf4j]}
   org.apache.tika/tika-core                 {:mvn/version "3.2.2"}
@@ -147,7 +147,7 @@
   org.clj-commons/hickory                   {:mvn/version "0.7.7"               ; Parse HTML into Clojure data structures
                                              :exclusions [org.jsoup/jsoup]}
   org.clojars.pntblnk/clj-ldap              {:mvn/version "0.0.17"}             ; LDAP client
-  org.clojure/clojure                       {:mvn/version "1.12.1"}
+  org.clojure/clojure                       {:mvn/version "1.12.2"}
   org.clojure/core.async                    {:mvn/version "1.7.701"
                                              :exclusions  [org.clojure/tools.reader]}
   org.clojure/core.logic                    {:mvn/version "1.1.0"}              ; optimized pattern matching library for Clojure
@@ -255,7 +255,7 @@
     {:mvn/version "0.4.0"}                     ; Enables easy memory measurement
     com.gfredericks/test.chuck   {:mvn/version "0.2.15"}                    ; generating strings from regexes (useful with malli)
     criterium/criterium          {:mvn/version "0.4.6"}                     ; benchmarking library
-    djblue/portal                {:mvn/version "0.60.2"}                    ; ui for inspecting values
+    djblue/portal                {:mvn/version "0.61.0"}                    ; ui for inspecting values
     io.github.camsaul/humane-are {:mvn/version "1.0.2"}                     ; cleaner test output
     io.github.metabase/hawk      {:mvn/version "1.0.11"}                    ; test runner
     jonase/eastwood              {:mvn/version "1.4.3"                      ; inspects namespaces and reports possible problems using tools.analyzer
@@ -842,7 +842,6 @@
   ;;    clj -X:dev:ee:ee-dev:driver:test:test/python-transform
   :test/python-transform
   {:exec-args {:only [metabase-enterprise.transforms.api-test
-                      metabase.driver.table-creation-test
                       "enterprise/backend/test/metabase_enterprise/transforms"]}}
 
 ;; test the Query Processor

docs/ai/metabot.md

@@ -68,7 +68,9 @@ You can also ask Metabot to tell you about specific tables in your database.
 
 ## How Metabot uses the query builder
 
-When you ask Metabot to create a chart from natural language, it first looks for existing questions that might answer your request. If it finds a relevant question, it'll point you to that instead of creating something new. Otherwise, Metabot will use the [query builder](../questions/query-builder/editor.md) to create a new chart for you. Keep in mind that Metabot can only handle basic query builder operations. It's limited to single-level aggregation and grouping, so if you need more complex analysis, you can take over and refine the query yourself, or switch to the native editor.
+When you ask Metabot to create a chart from natural language, it first looks for existing questions that might answer your request. If it finds a relevant question, it'll point you to that instead of creating something new. Otherwise, Metabot will use the [query builder](../questions/query-builder/editor.md) to create a new chart for you. 
+
+Keep in mind that Metabot is still learning the query builder. Metabot can only handle basic query builder operations, and it lacks access to the library of [custom expressions](../questions/query-builder/expressions-list.md). Metabot is also limited to single-level aggregation and grouping, so if you need more complex analysis, you can take over and refine the query yourself, or switch to the SQL editor.
 
 ## Metabot in the native editor
 

docs/api.html

@@ -12771,7 +12771,7 @@ redirect_from:
                       },
                       "schema-filters-type" : {
                         "type" : "string",
-                        "enum" : [ "include", "exclude", "all" ]
+                        "enum" : [ "inclusion", "exclusion", "all" ]
                       }
                     },
                     "required" : [ "schema-filters-type", "schema-filters-patterns" ]
@@ -12849,7 +12849,7 @@ redirect_from:
                       },
                       "schema-filters-type" : {
                         "type" : "string",
-                        "enum" : [ "include", "exclude", "all" ]
+                        "enum" : [ "inclusion", "exclusion", "all" ]
                       }
                     },
                     "required" : [ "schema-filters-type", "schema-filters-patterns" ]
@@ -15864,7 +15864,35 @@ redirect_from:
       "get" : {
         "summary" : "GET /api/ee/transform-job/",
         "description" : "Get all transform jobs.",
-        "parameters" : [ ],
+        "parameters" : [ {
+          "in" : "query",
+          "name" : "last_run_start_time",
+          "required" : false,
+          "schema" : {
+            "type" : "string",
+            "minLength" : 1
+          }
+        }, {
+          "in" : "query",
+          "name" : "next_run_start_time",
+          "required" : false,
+          "schema" : {
+            "type" : "string",
+            "minLength" : 1
+          }
+        }, {
+          "in" : "query",
+          "name" : "transform_tag_ids",
+          "required" : false,
+          "schema" : {
+            "type" : "array",
+            "items" : {
+              "description" : "value must be an integer greater or equal to than zero.",
+              "type" : "integer",
+              "minimum" : 0
+            }
+          }
+        } ],
         "responses" : {
           "2XX" : {
             "description" : "Successful response"
@@ -16018,6 +16046,34 @@ redirect_from:
         "tags" : [ "/api/ee/transform-job" ]
       }
     },
+    "/api/ee/transform-job/{job-id}/transforms" : {
+      "get" : {
+        "summary" : "GET /api/ee/transform-job/{job-id}/transforms",
+        "description" : "Get the transforms of job specified by the job's ID.",
+        "parameters" : [ {
+          "in" : "path",
+          "name" : "job-id",
+          "required" : true,
+          "schema" : {
+            "type" : "integer",
+            "minimum" : 1
+          },
+          "description" : "value must be an integer greater than zero."
+        } ],
+        "responses" : {
+          "2XX" : {
+            "description" : "Successful response"
+          },
+          "4XX" : {
+            "description" : "Client error response"
+          },
+          "5XX" : {
+            "description" : "Server error response"
+          }
+        },
+        "tags" : [ "/api/ee/transform-job" ]
+      }
+    },
     "/api/ee/transform-tag/" : {
       "post" : {
         "summary" : "POST /api/ee/transform-tag/",
@@ -16448,6 +16504,34 @@ redirect_from:
         "tags" : [ "/api/ee/transform" ]
       }
     },
+    "/api/ee/transform/{id}/dependencies" : {
+      "get" : {
+        "summary" : "GET /api/ee/transform/{id}/dependencies",
+        "description" : "Get the dependencies of a specific transform.",
+        "parameters" : [ {
+          "in" : "path",
+          "name" : "id",
+          "required" : true,
+          "schema" : {
+            "type" : "integer",
+            "minimum" : 1
+          },
+          "description" : "value must be an integer greater than zero."
+        } ],
+        "responses" : {
+          "2XX" : {
+            "description" : "Successful response"
+          },
+          "4XX" : {
+            "description" : "Client error response"
+          },
+          "5XX" : {
+            "description" : "Server error response"
+          }
+        },
+        "tags" : [ "/api/ee/transform" ]
+      }
+    },
     "/api/ee/transform/{id}/run" : {
       "post" : {
         "summary" : "POST /api/ee/transform/{id}/run",

docs/configuring-metabase/config-template.md

@@ -117,6 +117,7 @@ config:
     db-connection-timeout-ms: 10000
     db-query-timeout-minutes: 20
     default-maps-enabled: true
+    disable-cors-on-localhost: false
     download-row-limit: null
     email-from-address: notifications@metabase.com
     email-from-address-override: notifications@metabase.com
@@ -134,7 +135,7 @@ config:
     email-smtp-username: null
     email-smtp-username-override: null
     embedding-app-origins-interactive: null
-    embedding-app-origins-sdk: ""
+    embedding-app-origins-sdk: ''
     embedding-homepage: hidden
     embedding-secret-key: null
     enable-embedding-interactive: false

docs/configuring-metabase/environment-variables.md

@@ -393,6 +393,15 @@ Timeout in minutes for databases query execution, both Metabase application data
 
 Whether or not the default GeoJSON maps are enabled.
 
+### `MB_DISABLE_CORS_ON_LOCALHOST`
+
+- Type: boolean
+- Default: `false`
+- [Exported as](../installation-and-operation/serialization.md): `disable-cors-on-localhost`.
+- [Configuration file name](./config-file.md): `disable-cors-on-localhost`
+
+Prevents the server from sending CORS headers for requests originating from localhost.
+
 ### `MB_DOWNLOAD_ROW_LIMIT`
 
 - Type: positive-integer
@@ -562,7 +571,7 @@ Allow these space delimited origins to embed Metabase interactive.
 > Only available on Metabase [Pro](https://www.metabase.com/product/pro) and [Enterprise](https://www.metabase.com/product/enterprise) plans.
 
 - Type: string
-- Default: `""`
+- Default: ``
 - [Configuration file name](./config-file.md): `embedding-app-origins-sdk`
 
 Allow Metabase SDK access to these space delimited origins.

docs/developers-guide/driver-changelog.md

@@ -14,10 +14,18 @@ title: Driver interface changelog
 
 - Added metabase.driver/compile-transform, metabase.driver/compile-drop-table, metabase.driver/execute-raw-queries!,
   metabase.driver/run-transform!, metabase.driver/drop-transform-target!, metabase.driver/native-query-deps,
-  metabase.driver/connection-details, metabase.driver/table-exists?, metabase.driver.sql/normalize-name,
-  metabase.driver.sql/default-schema, and metabase.driver.sql/find-table to implement sql transforms.
+  metabase.driver/connection-spec, metabase.driver/table-exists?, metabase.driver.sql/normalize-name,
+  and metabase.driver.sql/default-schema to implement sql transforms.
 
-- `driver/humanize-connection-error-message` now takes a list of all messages in the exception chain,
+- Added the driver multi-methods `metabase.driver/schema-exists?` and `metabase.driver/create-schema-if-needed!` which should be implemented by drivers that support `:schemas` and `:transforms/table`.
+
+- Added `metabase.driver/type->database-type` multimethod that returns the database type for a given Metabase type (from the type hierarchy) as a HoneySQL spec. Unlike `upload-type->database-type` which is specific to upload types, this method handles general Metabase base types.
+
+- Added `metabase.driver/rename-table!` multimethod to rename a table from one name to another. Both table names may be qualified by schema (e.g., `schema.table`). This is currently used for transform operations and uploads.
+
+- Added `metabase.driver/insert-from-source!` multimethod that abstracts data insertion from various sources into existing tables. This multimethod dispatches on both the driver and the data source type (`:rows` or `:csv-file`). It allows drivers to optimize based on the data source type and returns the number of rows inserted. Default implementations are provided for both `:rows` and `:csv-file` source types.
+
+- `metabase.driver/humanize-connection-error-message` now takes a list of all messages in the exception chain,
   instead of just the top-level message as a string.
 
 - Removed `driver/set-database-used`. Drivers should set default databases in their connection specs instead.

docs/installation-and-operation/development-instance.md

@@ -11,7 +11,7 @@ summary: Create development instances of Metabase for testing without paying per
 
 On Pro and Enterprise plans, you can purchase a development instance or development token as an add-on to your plan.
 
-- **Self-hosted**: you can purchase a dev token that you can use for one or more self-hosted development instances (meaning: you can reuse the same token for multiple dev instances at no additional cost). The development token will have access to the same set of paid features as your production token.
+- **Self-hosted**: you can purchase a dev token so you don't get billed for every test user acccount. You can use the same dev token for one or more self-hosted development instances (meaning: you can reuse the same token for multiple dev instances at no additional cost). The development token will have access to the same set of paid features as your production token.
 - **Cloud**: you can purchase a development instance hosted on [Metabase Cloud](https://www.metabase.com/cloud/). The development instance will have access to the same set of paid features as your production token.
 
 Dev instances are only available to Pro and Enterprise plans; they're not included in the base plans. Dev instances must be purchased as an add-on to your plan, after you've completed your trial and made your first payment.
@@ -22,6 +22,10 @@ That's the basic value proposition. Unlike production instances where you pay pe
 
 Once you're happy with the changes in your dev instance, you can [export your changes and import them](./serialization.md) into your production Metabase.
 
+## Even if you self-host, dev instance tokens can save you money
+
+While it's true that you can use your production token to test multiple Metabases when self-hosting, you'll still get charged for test user accounts in each of those Metabases. In that case, it might make sense to get a dev instance token so you're only charged a flat rate for as many dev instances as you like, with as many test users as you need.
+
 ## You can't use development instances in production
 
 The UI for development instances, including the UI for all its exports and notifications, is watermarked.

e2e/support/helpers/api/createSnippet.ts

@@ -1,12 +1,27 @@
-export function createSnippet({
-  name,
-  content,
-}: {
-  name: string;
+import type {
+  NativeQuerySnippet,
+  RegularCollectionId,
+} from "metabase-types/api";
+
+export type NativeQuerySnippetDetails = {
+  name?: string;
+  description?: string | null;
   content: string;
-}): Cypress.Chainable<Cypress.Response<NativeQuerySnippet>> {
-  cy.request("POST", "/api/native-query-snippet", {
+  collection_id?: RegularCollectionId | null;
+};
+
+export function createSnippet({
+  name = "Test snippet",
+  description = null,
+  content,
+  collection_id = null,
+}: NativeQuerySnippetDetails): Cypress.Chainable<
+  Cypress.Response<NativeQuerySnippet>
+> {
+  return cy.request("POST", "/api/native-query-snippet", {
     name,
+    description,
     content,
+    collection_id,
   });
 }

e2e/support/helpers/e2e-misc-helpers.js

@@ -281,6 +281,7 @@ export function interceptIfNotPreviouslyDefined({ method, url, alias } = {}) {
  * @param {boolean=} [options.addToDashboard]
  * @param {boolean=} [options.wrapId]
  * @param {string=} [options.idAlias]
+ * @param {Object=} [pickEntityOptions]
  */
 export function saveQuestion(
   name,

e2e/test/scenarios/admin-2/people.cy.spec.js

@@ -767,7 +767,7 @@ describe("scenarios > admin > people > group managers", () => {
       cy.url().should("match", /\/$/);
     });
 
-    it("can manage members from the people page", () => {
+    it("can manage members from the people page", { tags: "@flaky" }, () => {
       // Open membership select for a user
       // eslint-disable-next-line no-unscoped-text-selectors -- deprecated usage
       cy.findByText(noCollectionUserName)

e2e/test/scenarios/admin/transforms.cy.spec.ts

@@ -16,6 +16,7 @@ const TARGET_TABLE = "transform_table";
 const TARGET_TABLE_2 = "transform_table_2";
 const TARGET_SCHEMA = "Schema A";
 const TARGET_SCHEMA_2 = "Schema B";
+const CUSTOM_SCHEMA = "custom_schema";
 
 H.describeWithSnowplowEE("scenarios > admin > transforms", () => {
   beforeEach(() => {
@@ -33,6 +34,9 @@ H.describeWithSnowplowEE("scenarios > admin > transforms", () => {
     cy.intercept("DELETE", "/api/ee/transform/*/table").as(
       "deleteTransformTable",
     );
+    cy.intercept("GET", "/api/ee/transform/*/dependencies").as(
+      "transformDependencies",
+    );
     cy.intercept("POST", "/api/ee/transform-tag").as("createTag");
     cy.intercept("PUT", "/api/ee/transform-tag/*").as("updateTag");
     cy.intercept("DELETE", "/api/ee/transform-tag/*").as("deleteTag");
@@ -208,6 +212,90 @@ H.describeWithSnowplowEE("scenarios > admin > transforms", () => {
       });
     });
 
+    it("should be able to create a new schema when saving a transform", () => {
+      visitTransformListPage();
+      getTransformListPage().button("Create a transform").click();
+      H.popover().findByText("Query builder").click();
+
+      H.entityPickerModal().within(() => {
+        cy.findByText(DB_NAME).click();
+        cy.findByText(SOURCE_TABLE).click();
+      });
+      getQueryEditor().button("Save").click();
+      H.modal().within(() => {
+        cy.findByLabelText("Name").type("MBQL transform");
+        cy.findByLabelText("Table name").type(TARGET_TABLE);
+        cy.findByLabelText("Schema").clear().type(CUSTOM_SCHEMA);
+      });
+      H.popover().findByText("Create new schema").click();
+      H.modal().within(() => {
+        cy.button("Save").click();
+        cy.wait("@createTransform");
+        cy.wait("@transformDependencies");
+      });
+
+      getSchemaLink()
+        .should("have.text", CUSTOM_SCHEMA)
+        .should("have.attr", "aria-disabled", "true")
+        .realHover();
+
+      H.tooltip()
+        .should("be.visible")
+        .should(
+          "have.text",
+          "This schema will be created when the transform runs",
+        );
+
+      getTableLink()
+        .should("have.text", TARGET_TABLE)
+        .should("have.attr", "aria-disabled", "true");
+
+      cy.log("run the transform and verify the table");
+      runTransformAndWaitForSuccess();
+      getSchemaLink()
+        .should("have.text", CUSTOM_SCHEMA)
+        .should("have.attr", "aria-disabled", "false")
+        .realHover();
+
+      getTableLink().click();
+      H.queryBuilderHeader().findByText(CUSTOM_SCHEMA).should("be.visible");
+      H.assertQueryBuilderRowCount(3);
+    });
+
+    it("should be able to create a new table in an existing when saving a transform", () => {
+      visitTransformListPage();
+      getTransformListPage().button("Create a transform").click();
+      H.popover().findByText("Query builder").click();
+
+      H.entityPickerModal().within(() => {
+        cy.findByText(DB_NAME).click();
+        cy.findByText(SOURCE_TABLE).click();
+      });
+      getQueryEditor().button("Save").click();
+      H.modal().within(() => {
+        cy.findByLabelText("Name").type("MBQL transform");
+        cy.findByLabelText("Table name").type(TARGET_TABLE);
+        cy.button("Save").click();
+        cy.wait("@createTransform");
+      });
+
+      getSchemaLink()
+        .should("have.text", TARGET_SCHEMA)
+        .should("have.attr", "aria-disabled", "false");
+
+      getTableLink()
+        .should("have.text", TARGET_TABLE)
+        .should("have.attr", "aria-disabled", "true");
+
+      cy.log("run the transform and verify the table");
+      runTransformAndWaitForSuccess();
+
+      getSchemaLink().should("have.attr", "aria-disabled", "false");
+      getTableLink().should("have.attr", "aria-disabled", "false").click();
+
+      H.assertQueryBuilderRowCount(3);
+    });
+
     it("should not be possible to create an mbql transform from a table from an unsupported database", () => {
       visitTransformListPage();
       getTransformListPage().button("Create a transform").click();
@@ -458,16 +546,71 @@ H.describeWithSnowplowEE("scenarios > admin > transforms", () => {
         cy.button("Change target").click();
         cy.wait("@updateTransform");
       });
-      getTableLink().should("have.text", TARGET_TABLE_2);
-      getSchemaLink().should("have.text", TARGET_SCHEMA_2);
+
+      getSchemaLink()
+        .should("have.text", TARGET_SCHEMA_2)
+        .should("have.attr", "aria-disabled", "false");
+
+      getTableLink()
+        .should("have.text", TARGET_TABLE_2)
+        .should("have.attr", "aria-disabled", "true");
 
       cy.log("run the transform and verify the table");
       runTransformAndWaitForSuccess();
-      getTableLink().click();
+      getSchemaLink()
+        .should("have.text", TARGET_SCHEMA_2)
+        .should("have.attr", "aria-disabled", "false");
+
+      getTableLink()
+        .should("have.text", TARGET_TABLE_2)
+        .should("have.attr", "aria-disabled", "false")
+        .click();
+
       H.queryBuilderHeader().findByText(TARGET_SCHEMA_2).should("be.visible");
       H.assertQueryBuilderRowCount(3);
     });
 
+    it("should be possible to create a new schema", () => {
+      cy.log("create but do not run the transform");
+      createMbqlTransform({ visitTransform: true });
+
+      cy.log("modify the transform before running");
+      getTransformPage().button("Change target").click();
+      H.modal().within(() => {
+        cy.findByLabelText("Table name").should("have.value", TARGET_TABLE);
+        cy.findByLabelText("Schema").should("have.value", TARGET_SCHEMA);
+        cy.findByLabelText("Table name").clear().type(TARGET_TABLE_2);
+        cy.findByLabelText("Schema").clear().type(CUSTOM_SCHEMA);
+      });
+      H.popover().findByText("Create new schema").click();
+      H.modal().within(() => {
+        cy.button("Change target").click();
+        cy.wait("@updateTransform");
+      });
+
+      getSchemaLink()
+        .should("have.text", CUSTOM_SCHEMA)
+        .should("have.attr", "aria-disabled", "true");
+
+      getTableLink()
+        .should("have.text", TARGET_TABLE_2)
+        .should("have.attr", "aria-disabled", "true");
+
+      cy.log("run the transform and verify the table");
+      runTransformAndWaitForSuccess();
+      getSchemaLink()
+        .should("have.text", CUSTOM_SCHEMA)
+        .should("have.attr", "aria-disabled", "false");
+
+      getTableLink()
+        .should("have.text", TARGET_TABLE_2)
+        .should("have.attr", "aria-disabled", "false")
+        .click();
+
+      H.queryBuilderHeader().findByText(CUSTOM_SCHEMA).should("be.visible");
+      H.assertQueryBuilderRowCount(3);
+    });
+
     it("should be able to change the target after running a transform and keep the old target", () => {
       cy.log("create and run a transform");
       createMbqlTransform({ visitTransform: true });
@@ -483,11 +626,25 @@ H.describeWithSnowplowEE("scenarios > admin > transforms", () => {
         cy.button("Change target").click();
         cy.wait("@updateTransform");
       });
-      getTableLink().should("have.text", TARGET_TABLE_2);
+
+      getSchemaLink()
+        .should("have.text", TARGET_SCHEMA)
+        .should("have.attr", "aria-disabled", "false");
+
+      getTableLink()
+        .should("have.text", TARGET_TABLE_2)
+        .should("have.attr", "aria-disabled", "true");
 
       cy.log("run the transform and verify the new table");
       runTransformAndWaitForSuccess();
-      getTableLink().click();
+      getSchemaLink()
+        .should("have.text", TARGET_SCHEMA)
+        .should("have.attr", "aria-disabled", "false");
+
+      getTableLink()
+        .should("have.text", TARGET_TABLE_2)
+        .should("have.attr", "aria-disabled", "false")
+        .click();
       H.queryBuilderHeader()
         .findByText("Transform Table 2")
         .should("be.visible");
@@ -514,11 +671,26 @@ H.describeWithSnowplowEE("scenarios > admin > transforms", () => {
         cy.wait("@deleteTransformTable");
         cy.wait("@updateTransform");
       });
-      getTableLink().should("have.text", TARGET_TABLE_2);
+
+      getSchemaLink()
+        .should("have.text", TARGET_SCHEMA)
+        .should("have.attr", "aria-disabled", "false");
+
+      getTableLink()
+        .should("have.text", TARGET_TABLE_2)
+        .should("have.attr", "aria-disabled", "true");
 
       cy.log("run the transform and verify the new table");
       runTransformAndWaitForSuccess();
-      getTableLink().click();
+
+      getSchemaLink()
+        .should("have.text", TARGET_SCHEMA)
+        .should("have.attr", "aria-disabled", "false");
+
+      getTableLink()
+        .should("have.text", TARGET_TABLE_2)
+        .should("have.attr", "aria-disabled", "false")
+        .click();
       H.queryBuilderHeader()
         .findByText("Transform Table 2")
         .should("be.visible");
@@ -529,37 +701,43 @@ H.describeWithSnowplowEE("scenarios > admin > transforms", () => {
       assertTableDoesNotExistError();
     });
 
-    it("should be able to delete the target and restore the same target back", () => {
-      cy.log("create and run a transform");
-      createMbqlTransform({ visitTransform: true });
-      runTransformAndWaitForSuccess();
+    it(
+      "should be able to delete the target and restore the same target back",
+      { tags: "@flaky" },
+      () => {
+        cy.log("create and run a transform");
+        createMbqlTransform({ visitTransform: true });
+        runTransformAndWaitForSuccess();
 
-      cy.log("delete the old target without creating the new one");
-      getTransformPage().button("Change target").click();
-      H.modal().within(() => {
-        cy.findByLabelText("Table name").clear().type(TARGET_TABLE_2);
-        cy.findByLabelText("Delete transform_table").click();
-        cy.button("Change target and delete the old one").click();
-        cy.wait("@deleteTransformTable");
-        cy.wait("@updateTransform");
-      });
+        cy.log("delete the old target without creating the new one");
+        getTransformPage().button("Change target").click();
+        H.modal().within(() => {
+          cy.findByLabelText("Table name").clear().type(TARGET_TABLE_2);
+          cy.findByLabelText("Delete transform_table").click();
+          cy.button("Change target and delete the old one").click();
+          cy.wait("@deleteTransformTable");
+          cy.wait("@updateTransform");
+        });
 
-      cy.log("change the target back to the original one");
-      getTransformPage().button("Change target").click();
-      H.modal().within(() => {
-        cy.findByLabelText("Table name").clear().type(TARGET_TABLE);
-        cy.button("Change target").click();
-        cy.wait("@updateTransform");
-      });
+        cy.log("change the target back to the original one");
+        getTransformPage().button("Change target").click();
+        H.modal().within(() => {
+          cy.findByLabelText("Table name").clear().type(TARGET_TABLE);
+          cy.button("Change target").click();
+          cy.wait("@updateTransform");
+        });
 
-      cy.log("run the transform to re-create the original target");
-      runTransformAndWaitForSuccess();
+        cy.log("run the transform to re-create the original target");
+        runTransformAndWaitForSuccess();
 
-      cy.log("verify the target is available");
-      getTableLink().click();
-      H.queryBuilderHeader().findByText("Transform Table").should("be.visible");
-      H.assertQueryBuilderRowCount(3);
-    });
+        cy.log("verify the target is available");
+        getTableLink().click();
+        H.queryBuilderHeader()
+          .findByText("Transform Table")
+          .should("be.visible");
+        H.assertQueryBuilderRowCount(3);
+      },
+    );
 
     it("should not allow to overwrite an existing table when changing the target", () => {
       createMbqlTransform({ visitTransform: true });
@@ -969,6 +1147,55 @@ H.describeWithSnowplowEE("scenarios > admin > transforms", () => {
   });
 });
 
+describe("scenarios > admin > transforms > databases without :schemas", () => {
+  const DB_NAME = "QA MySQL8";
+
+  beforeEach(() => {
+    H.restore("mysql-8");
+    cy.signInAsAdmin();
+    H.activateToken("bleeding-edge");
+    H.resyncDatabase({ dbId: WRITABLE_DB_ID });
+
+    cy.intercept("PUT", "/api/field/*").as("updateField");
+    cy.intercept("POST", "/api/ee/transform").as("createTransform");
+    cy.intercept("PUT", "/api/ee/transform/*").as("updateTransform");
+    cy.intercept("DELETE", "/api/ee/transform/*").as("deleteTransform");
+    cy.intercept("DELETE", "/api/ee/transform/*/table").as(
+      "deleteTransformTable",
+    );
+    cy.intercept("POST", "/api/ee/transform-tag").as("createTag");
+    cy.intercept("PUT", "/api/ee/transform-tag/*").as("updateTag");
+    cy.intercept("DELETE", "/api/ee/transform-tag/*").as("deleteTag");
+  });
+
+  it("should be not be possible to create a new schema when updating a transform target", () => {
+    createMbqlTransform({
+      databaseId: WRITABLE_DB_ID,
+      sourceTable: "ORDERS",
+      visitTransform: true,
+      targetSchema: null,
+    });
+
+    getTransformPage().button("Change target").click();
+
+    H.modal().findByLabelText("Schema").should("not.exist");
+  });
+
+  it("should be not be possible to create a new schema when the database does not support schemas", () => {
+    cy.log("create a new transform");
+    visitTransformListPage();
+    getTransformListPage().button("Create a transform").click();
+    H.popover().findByText("Query builder").click();
+
+    H.entityPickerModal().within(() => {
+      cy.findByText(DB_NAME).click();
+      cy.findByText("Orders").click();
+    });
+    getQueryEditor().button("Save").click();
+    H.modal().findByLabelText("Schema").should("not.exist");
+  });
+});
+
 describe("scenarios > admin > transforms > jobs", () => {
   beforeEach(() => {
     H.restore("postgres-writable");
@@ -995,7 +1222,9 @@ describe("scenarios > admin > transforms > jobs", () => {
         cy.findByPlaceholderText("Name").should("have.value", "New job");
         cy.findByPlaceholderText("No description yet").should("have.value", "");
         getCronInput().should("have.value", "0 0 * * ?");
-        cy.findByText("This job will run at 12:00 AM").should("be.visible");
+        cy.findByText("This job will run at 12:00 AM, UTC-07:00").should(
+          "be.visible",
+        );
       });
     });
 
@@ -1023,7 +1252,9 @@ describe("scenarios > admin > transforms > jobs", () => {
           "Description",
         );
         getCronInput().should("have.value", "0 * * * ?");
-        cy.findByText("This job will run every hour").should("be.visible");
+        cy.findByText("This job will run every hour, UTC-07:00").should(
+          "be.visible",
+        );
         cy.findByText("daily").should("be.visible");
       });
     });
@@ -1084,7 +1315,9 @@ describe("scenarios > admin > transforms > jobs", () => {
       H.createTransformJob({ name: "New job" }, { visitTransformJob: true });
       getJobPage().within(() => {
         getCronInput().clear().type("0 * * * ?").blur();
-        cy.findByText("This job will run every hour").should("be.visible");
+        cy.findByText("This job will run every hour, UTC-07:00").should(
+          "be.visible",
+        );
       });
       H.undoToast().findByText("Job schedule updated").should("be.visible");
       getJobPage().within(() => {
@@ -1194,56 +1427,213 @@ describe("scenarios > admin > transforms > jobs", () => {
     });
   });
 
-  describe("dependencies", () => {
-    it("should render a table of transforms that will be run by a job", () => {
-      H.createTransformTag({ name: "Foo" }).then(({ body: tag }) => {
-        createMbqlTransform({
-          name: "Transform A",
-          tagIds: [],
-          targetTable: "table_a",
-          visitTransform: true,
-        }).then(runTransformAndWaitForSuccess);
+  describe("filtering", () => {
+    it("should be able to filter jobs ", () => {
+      cy.log("run hourly job so know that was recently run");
+      visitJobListPage();
 
-        createMbqlTransform({
-          name: "Transform B",
-          tagIds: [tag.id],
-          sourceTable: "table_a",
-          targetTable: "table_b",
-          visitTransform: true,
-        }).then(runTransformAndWaitForSuccess);
+      getContentTable().findByText("Hourly job").click();
+      runJobAndWaitForSuccess();
 
-        createMbqlTransform({
-          name: "Transform C",
-          tagIds: [tag.id],
-          sourceTable: "table_b",
-          targetTable: "table_c",
-          visitTransform: true,
-        });
+      visitJobListPage();
 
-        H.createTransformJob(
-          { name: "Foo job", tag_ids: [tag.id] },
-          { visitTransformJob: true },
-        );
-
-        H.main().findByText("Transforms").should("be.visible");
+      function testLastRunFilter() {
+        cy.log("no filters");
         getContentTable().within(() => {
-          // 3 transforms plus the header row
-          cy.findAllByRole("row").should("have.length", 4);
-
-          // Check the existence and also their order
-          cy.findAllByRole("row").eq(1).should("contain", "Transform A");
-          cy.findAllByRole("row").eq(2).should("contain", "Transform B");
-          cy.findAllByRole("row").eq(3).should("contain", "Transform C");
+          cy.findByText("Hourly job").should("be.visible");
+          cy.findByText("Daily job").should("be.visible");
+          cy.findByText("Weekly job").should("be.visible");
+          cy.findByText("Monthly job").should("be.visible");
         });
+
+        cy.log("last run at - add a filter");
+        getLastRunAtFilterWidget().click();
+        H.popover().findByText("Today").click();
+
+        getLastRunAtFilterWidget().should("contain", "Today");
+        getContentTable().within(() => {
+          cy.findByText("Hourly job").should("be.visible");
+          cy.findByText("Daily job").should("not.exist");
+          cy.findByText("Weekly job").should("not.exist");
+          cy.findByText("Monthly job").should("not.exist");
+        });
+
+        cy.log("last run at filter - update a filter");
+        getLastRunAtFilterWidget().click();
+        H.popover().findByText("Week").click();
+        getLastRunAtFilterWidget().should("contain", "This week");
+        getContentTable().within(() => {
+          cy.findByText("Hourly job").should("be.visible");
+          cy.findByText("Daily job").should("not.exist");
+          cy.findByText("Weekly job").should("not.exist");
+          cy.findByText("Monthly job").should("not.exist");
+        });
+
+        cy.log("last run at filter - remove filter");
+        getLastRunAtFilterWidget().button("Remove filter").click();
+        getContentTable().within(() => {
+          cy.findByText("Hourly job").should("be.visible");
+          cy.findByText("Daily job").should("be.visible");
+          cy.findByText("Weekly job").should("be.visible");
+          cy.findByText("Monthly job").should("be.visible");
+        });
+      }
+
+      function testNextRunFilter() {
+        cy.log("no filters");
+        getContentTable().within(() => {
+          cy.findByText("Hourly job").should("be.visible");
+          cy.findByText("Daily job").should("be.visible");
+          cy.findByText("Weekly job").should("be.visible");
+          cy.findByText("Monthly job").should("be.visible");
+        });
+
+        cy.log("next run - add a filter");
+        getNextRunFilterWidget().click();
+        H.popover().within(() => {
+          cy.findByText("Relative date range…").click();
+          cy.findByText("Current").click();
+          cy.findByText("Week").click();
+        });
+
+        getNextRunFilterWidget().should("contain", "This week");
+        getContentTable().within(() => {
+          cy.findByText("Hourly job").should("be.visible");
+          cy.findByText("Daily job").should("be.visible");
+          cy.findByText("Weekly job").should("not.exist");
+          cy.findByText("Monthly job").should("not.exist");
+        });
+
+        cy.log("next run filter - update a filter");
+        getNextRunFilterWidget().click();
+        H.popover().within(() => {
+          cy.findByText("Next").click();
+          cy.findByDisplayValue(30).clear().type("2");
+          cy.button("Apply").click();
+        });
+        getNextRunFilterWidget().should("contain", "Next 2 weeks");
+        getContentTable().within(() => {
+          cy.findByText("Hourly job").should("not.exist");
+          cy.findByText("Daily job").should("not.exist");
+          cy.findByText("Weekly job").should("be.visible");
+          cy.findByText("Monthly job").should("not.exist");
+        });
+
+        cy.log("next run filter - remove filter");
+        getNextRunFilterWidget().button("Remove filter").click();
+        getContentTable().within(() => {
+          cy.findByText("Hourly job").should("be.visible");
+          cy.findByText("Daily job").should("be.visible");
+          cy.findByText("Weekly job").should("be.visible");
+          cy.findByText("Monthly job").should("be.visible");
+        });
+      }
+
+      function testTagFilter() {
+        cy.log("no filters");
+        getContentTable().within(() => {
+          cy.findByText("Hourly job").should("be.visible");
+          cy.findByText("Daily job").should("be.visible");
+          cy.findByText("Weekly job").should("be.visible");
+          cy.findByText("Monthly job").should("be.visible");
+        });
+
+        cy.log("tag filter - add a filter");
+        getTagFilterWidget().click();
+        H.popover().within(() => {
+          cy.findByText("hourly").click();
+          cy.button("Add filter").click();
+        });
+        getTagFilterWidget().findByText("hourly").should("be.visible");
+        getContentTable().within(() => {
+          cy.findByText("Hourly job").should("be.visible");
+          cy.findByText("Daily job").should("not.exist");
+          cy.findByText("Weekly job").should("not.exist");
+          cy.findByText("Monthly job").should("not.exist");
+        });
+
+        cy.log("tag filter - update a filter");
+        getTagFilterWidget().click();
+        H.popover().within(() => {
+          cy.findByText("hourly").click();
+          cy.findByText("weekly").click();
+          cy.button("Update filter").click();
+        });
+
+        getTagFilterWidget().findByText("weekly").should("be.visible");
+        getContentTable().within(() => {
+          cy.findByText("Hourly job").should("not.exist");
+          cy.findByText("Daily job").should("not.exist");
+          cy.findByText("Weekly job").should("be.visible");
+          cy.findByText("Monthly job").should("not.exist");
+        });
+
+        cy.log("tag filter - multiple options");
+        getTagFilterWidget().click();
+        H.popover().within(() => {
+          cy.findByText("monthly").click();
+          cy.button("Update filter").click();
+        });
+        getTagFilterWidget().findByText("2 tags").should("be.visible");
+        getContentTable().within(() => {
+          cy.findByText("Hourly job").should("not.exist");
+          cy.findByText("Daily job").should("not.exist");
+          cy.findByText("Weekly job").should("be.visible");
+          cy.findByText("Monthly job").should("be.visible");
+        });
+
+        cy.log("tag filter - remove filter");
+        getTagFilterWidget().button("Remove filter").click();
+        getTagFilterWidget().findByText("2 tags").should("not.exist");
+        getContentTable().within(() => {
+          cy.findByText("Hourly job").should("be.visible");
+          cy.findByText("Daily job").should("be.visible");
+          cy.findByText("Weekly job").should("be.visible");
+          cy.findByText("Monthly job").should("be.visible");
+        });
+      }
+
+      testLastRunFilter();
+      testNextRunFilter();
+      testTagFilter();
+    });
+  });
+
+  describe("dependencies", () => {
+    it("should render a table of dependencies", () => {
+      createMbqlTransform({
+        name: "Transform A",
+        targetTable: "table_a",
+        visitTransform: true,
+      }).then(runTransformAndWaitForSuccess);
+
+      createMbqlTransform({
+        name: "Transform B",
+        sourceTable: "table_a",
+        targetTable: "table_b",
+        visitTransform: true,
+      }).then(runTransformAndWaitForSuccess);
+
+      createMbqlTransform({
+        name: "Transform C",
+        sourceTable: "table_b",
+        targetTable: "table_c",
+        visitTransform: true,
+      });
+
+      H.main().findByText("Dependencies").scrollIntoView().should("be.visible");
+      getContentTable().within(() => {
+        // 1 transform plus the header row
+        cy.findAllByRole("row").should("have.length", 2);
+
+        // Check the existence and also their order
+        cy.findAllByRole("row").eq(1).should("contain", "Transform B");
       });
     });
 
-    it("should render an empty table of transforms if there are no transforms for this job", () => {
-      H.createTransformJob({ name: "Foo job" }, { visitTransformJob: true });
-      getContentTable()
-        .findByText("There are no transforms for this job.")
-        .scrollIntoView()
-        .should("be.visible");
+    it("should no dependencies table if the transform has no dependencies", () => {
+      createMbqlTransform({ name: "Transform A", visitTransform: true });
+      H.main().findByText("Dependencies").should("not.exist");
     });
   });
 });
@@ -1665,6 +2055,14 @@ function getTransformFilterWidget() {
   return cy.findByRole("group", { name: "Transform" });
 }
 
+function getLastRunAtFilterWidget() {
+  return cy.findByRole("group", { name: "Last run at" });
+}
+
+function getNextRunFilterWidget() {
+  return cy.findByRole("group", { name: "Next run" });
+}
+
 function getStatusFilterWidget() {
   return cy.findByRole("group", { name: "Status" });
 }
@@ -1718,17 +2116,19 @@ function createMbqlTransform({
   targetTable = TARGET_TABLE,
   targetSchema = TARGET_SCHEMA,
   tagIds,
+  databaseId,
   name = "MBQL transform",
   visitTransform,
 }: {
   sourceTable?: string;
   targetTable?: string;
-  targetSchema?: string;
+  targetSchema?: string | null;
   tagIds?: TransformTagId[];
   name?: string;
+  databaseId?: number;
   visitTransform?: boolean;
 } = {}) {
-  return H.getTableId({ name: sourceTable }).then((tableId) => {
+  return H.getTableId({ databaseId, name: sourceTable }).then((tableId) => {
     return H.createTransform(
       {
         name,
@@ -1845,8 +2245,8 @@ function assertOptionSelected(name: string) {
 
 function assertOptionNotSelected(name: string) {
   cy.findByRole("option", { name }).should(
-    "not.have.attr",
+    "have.attr",
     "aria-selected",
-    "true",
+    "false",
   );
 }

e2e/test/scenarios/collections/uploads.cy.spec.js

@@ -15,80 +15,84 @@ H.describeWithSnowplow(
       cy.intercept("POST", "/api/table/*/replace-csv").as("replaceCSV");
     });
 
-    it("Can upload a CSV file to an empty postgres schema", () => {
-      const testFile = VALID_CSV_FILES[0];
-      const EMPTY_SCHEMA_NAME = "empty_uploads";
+    it(
+      "Can upload a CSV file to an empty postgres schema",
+      { tags: "@flaky" },
+      () => {
+        const testFile = VALID_CSV_FILES[0];
+        const EMPTY_SCHEMA_NAME = "empty_uploads";
 
-      cy.intercept("PUT", "/api/setting/*").as("saveSettings");
-      cy.intercept("GET", "/api/database").as("databaseList");
+        cy.intercept("PUT", "/api/setting/*").as("saveSettings");
+        cy.intercept("GET", "/api/database").as("databaseList");
 
-      H.restore("postgres-writable");
-      cy.signInAsAdmin();
+        H.restore("postgres-writable");
+        cy.signInAsAdmin();
 
-      H.queryWritableDB(
-        "DROP SCHEMA IF EXISTS empty_uploads CASCADE;",
-        "postgres",
-      );
-      H.queryWritableDB(
-        "CREATE SCHEMA IF NOT EXISTS empty_uploads;",
-        "postgres",
-      );
-
-      cy.request("POST", "/api/collection", {
-        name: "Uploads Collection",
-        parent_id: null,
-      }).then(({ body: { id: collectionId } }) => {
-        cy.wrap(collectionId).as("collectionId");
-      });
-      cy.visit("/admin/settings/uploads");
-
-      cy.findByLabelText("Upload Settings Form")
-        .findByPlaceholderText("Select a database")
-        .click();
-      H.popover().findByText("Writable Postgres12").click();
-      cy.findByLabelText("Upload Settings Form")
-        .findByPlaceholderText("Select a schema")
-        .click();
-
-      H.popover().findByText(EMPTY_SCHEMA_NAME).click();
-
-      cy.findByLabelText("Upload Settings Form")
-        .button("Enable uploads")
-        .click();
-
-      cy.wait(["@saveSettings", "@databaseList"]);
-
-      uploadFileToCollection(testFile);
-
-      const tableQuery = `SELECT * FROM information_schema.tables WHERE table_name LIKE '%${testFile.tableName}_%' ORDER BY table_name DESC LIMIT 1;`;
-
-      H.queryWritableDB(tableQuery, "postgres").then((result) => {
-        expect(result.rows.length).to.equal(1);
-        const tableName = result.rows[0].table_name;
         H.queryWritableDB(
-          `SELECT count(*) FROM ${EMPTY_SCHEMA_NAME}.${tableName};`,
+          "DROP SCHEMA IF EXISTS empty_uploads CASCADE;",
           "postgres",
-        ).then((result) => {
-          expect(Number(result.rows[0].count)).to.equal(testFile.rowCount);
+        );
+        H.queryWritableDB(
+          "CREATE SCHEMA IF NOT EXISTS empty_uploads;",
+          "postgres",
+        );
+
+        cy.request("POST", "/api/collection", {
+          name: "Uploads Collection",
+          parent_id: null,
+        }).then(({ body: { id: collectionId } }) => {
+          cy.wrap(collectionId).as("collectionId");
         });
-      });
+        cy.visit("/admin/settings/uploads");
 
-      cy.log(
-        "Ensure that table is visible in admin without refreshing (metabase#38041)",
-      );
+        cy.findByLabelText("Upload Settings Form")
+          .findByPlaceholderText("Select a database")
+          .click();
+        H.popover().findByText("Writable Postgres12").click();
+        cy.findByLabelText("Upload Settings Form")
+          .findByPlaceholderText("Select a schema")
+          .click();
 
-      cy.findByTestId("app-bar")
-        .findByRole("button", { name: "Settings" })
-        .click();
-      H.popover().findByText("Admin settings").click();
+        H.popover().findByText(EMPTY_SCHEMA_NAME).click();
 
-      cy.findByRole("link", { name: "Table Metadata" }).click();
+        cy.findByLabelText("Upload Settings Form")
+          .button("Enable uploads")
+          .click();
 
-      H.DataModel.TablePicker.getDatabase("Writable Postgres12").click();
-      H.DataModel.TablePicker.getSchema(EMPTY_SCHEMA_NAME).click();
-      H.DataModel.TablePicker.getTables().should("have.length", 1);
-      H.DataModel.TablePicker.getTable("Dog Breeds").should("be.visible");
-    });
+        cy.wait(["@saveSettings", "@databaseList"]);
+
+        uploadFileToCollection(testFile);
+
+        const tableQuery = `SELECT * FROM information_schema.tables WHERE table_name LIKE '%${testFile.tableName}_%' ORDER BY table_name DESC LIMIT 1;`;
+
+        H.queryWritableDB(tableQuery, "postgres").then((result) => {
+          expect(result.rows.length).to.equal(1);
+          const tableName = result.rows[0].table_name;
+          H.queryWritableDB(
+            `SELECT count(*) FROM ${EMPTY_SCHEMA_NAME}.${tableName};`,
+            "postgres",
+          ).then((result) => {
+            expect(Number(result.rows[0].count)).to.equal(testFile.rowCount);
+          });
+        });
+
+        cy.log(
+          "Ensure that table is visible in admin without refreshing (metabase#38041)",
+        );
+
+        cy.findByTestId("app-bar")
+          .findByRole("button", { name: "Settings" })
+          .click();
+        H.popover().findByText("Admin settings").click();
+
+        cy.findByRole("link", { name: "Table Metadata" }).click();
+
+        H.DataModel.TablePicker.getDatabase("Writable Postgres12").click();
+        H.DataModel.TablePicker.getSchema(EMPTY_SCHEMA_NAME).click();
+        H.DataModel.TablePicker.getTables().should("have.length", 1);
+        H.DataModel.TablePicker.getTable("Dog Breeds").should("be.visible");
+      },
+    );
 
     ["postgres", "mysql"].forEach((dialect) => {
       describe(`CSV Uploading (${dialect})`, () => {

e2e/test/scenarios/embedding/embedding-reproductions.cy.spec.js

@@ -957,20 +957,24 @@ describe("issue 40660", () => {
     });
   });
 
-  it("static dashboard content shouldn't overflow its container (metabase#40660)", () => {
-    H.openStaticEmbeddingModal({
-      activeTab: "parameters",
-      previewMode: "preview",
-    });
+  it(
+    "static dashboard content shouldn't overflow its container (metabase#40660)",
+    { tags: "@flaky" },
+    () => {
+      H.openStaticEmbeddingModal({
+        activeTab: "parameters",
+        previewMode: "preview",
+      });
 
-    H.getIframeBody().within(() => {
-      cy.findByTestId("embed-frame").scrollTo("bottom");
+      H.getIframeBody().within(() => {
+        cy.findByTestId("embed-frame").scrollTo("bottom");
 
-      cy.findByRole("link", { name: "Powered by Metabase" }).should(
-        "be.visible",
-      );
-    });
-  });
+        cy.findByRole("link", { name: "Powered by Metabase" }).should(
+          "be.visible",
+        );
+      });
+    },
+  );
 });
 
 // Skipped since it does not make sense when CSP is disabled

e2e/test/scenarios/models/models-list-view.cy.spec.js

@@ -123,6 +123,17 @@ describe("scenarios > models list view", () => {
         cy.findByText("2.07").should("not.exist");
       });
 
+      cy.log("Verify that empty column preview displays placeholder value");
+      cy.get("@rightColumns").within(() => {
+        cy.get("input").type("DISC");
+      });
+      H.popover().within(() => {
+        cy.findByText("DISCOUNT").click();
+      });
+      cy.get("@listPreview").within(() => {
+        cy.findByText("$123.46").should("be.visible");
+      });
+
       cy.log("Update list item icon");
       cy.get("@listPreview").within(() => {
         cy.findByRole("img").should("have.attr", "aria-label", "document icon");

e2e/test/scenarios/models/reproductions.cy.spec.ts

@@ -143,31 +143,35 @@ describe("issue 35711", () => {
     cy.signInAsAdmin();
   });
 
-  it("can edit metadata of a model with a custom column (metabase#35711)", () => {
-    H.createQuestion(
-      {
-        type: "model",
-        query: {
-          "source-table": ORDERS_ID,
-          expressions: {
-            "Custom column": ["-", DISCOUNT_FIELD_REF, 1],
+  it(
+    "can edit metadata of a model with a custom column (metabase#35711)",
+    { tags: "@flaky" },
+    () => {
+      H.createQuestion(
+        {
+          type: "model",
+          query: {
+            "source-table": ORDERS_ID,
+            expressions: {
+              "Custom column": ["-", DISCOUNT_FIELD_REF, 1],
+            },
+            limit: 5, // optimization
           },
-          limit: 5, // optimization
         },
-      },
-      { visitQuestion: true },
-    );
+        { visitQuestion: true },
+      );
 
-    H.openQuestionActions();
-    H.popover().findByText("Edit metadata").click();
-    H.waitForLoaderToBeRemoved();
+      H.openQuestionActions();
+      H.popover().findByText("Edit metadata").click();
+      H.waitForLoaderToBeRemoved();
 
-    reorderTaxAndTotalColumns();
-    assertNoError();
+      reorderTaxAndTotalColumns();
+      assertNoError();
 
-    cy.findByTestId("editor-tabs-query-name").click();
-    assertNoError();
-  });
+      cy.findByTestId("editor-tabs-query-name").click();
+      assertNoError();
+    },
+  );
 });
 
 describe("issues 25884 and 34349", () => {
@@ -977,20 +981,26 @@ describe("issue 39993", () => {
     cy.intercept("PUT", "/api/card/*").as("updateModel");
   });
 
-  it("should preserve viz settings for models with custom expressions (metabase#39993)", () => {
-    H.createQuestion(modelDetails).then(({ body: card }) =>
-      H.visitModel(card.id),
-    );
-    H.openQuestionActions();
-    H.popover().findByText("Edit metadata").click();
-    H.waitForLoaderToBeRemoved();
-    cy.log("drag & drop the custom column 100 px to the left");
-    H.moveDnDKitElement(H.tableHeaderColumn(columnName), { horizontal: -100 });
-    cy.button("Save changes").click();
-    cy.wait("@updateModel");
-    cy.findAllByTestId("header-cell").eq(0).should("have.text", "Exp");
-    cy.findAllByTestId("header-cell").eq(1).should("have.text", "ID");
-  });
+  it(
+    "should preserve viz settings for models with custom expressions (metabase#39993)",
+    { tags: "@flaky" },
+    () => {
+      H.createQuestion(modelDetails).then(({ body: card }) =>
+        H.visitModel(card.id),
+      );
+      H.openQuestionActions();
+      H.popover().findByText("Edit metadata").click();
+      H.waitForLoaderToBeRemoved();
+      cy.log("drag & drop the custom column 100 px to the left");
+      H.moveDnDKitElement(H.tableHeaderColumn(columnName), {
+        horizontal: -100,
+      });
+      cy.button("Save changes").click();
+      cy.wait("@updateModel");
+      cy.findAllByTestId("header-cell").eq(0).should("have.text", "Exp");
+      cy.findAllByTestId("header-cell").eq(1).should("have.text", "ID");
+    },
+  );
 });
 
 describe("issue 34574", () => {
@@ -1154,36 +1164,40 @@ describe("issue 36161", () => {
     cy.intercept("POST", "/api/dataset").as("dataset");
   });
 
-  it("should allow to override metadata for custom columns (metabase#36161)", () => {
-    H.visitModel(ORDERS_MODEL_ID);
-    cy.wait("@dataset");
+  it(
+    "should allow to override metadata for custom columns (metabase#36161)",
+    { tags: "@flaky" },
+    () => {
+      H.visitModel(ORDERS_MODEL_ID);
+      cy.wait("@dataset");
 
-    H.openQuestionActions("Edit query definition");
-    H.getNotebookStep("data").button("Pick columns").click();
-    H.popover().findByText("Select all").click();
-    H.getNotebookStep("data").button("Custom column").click();
-    H.enterCustomColumnDetails({ formula: "[ID]", name: "ID2" });
-    H.popover().button("Done").click();
-    H.getNotebookStep("expression").icon("add").click();
-    H.enterCustomColumnDetails({ formula: "[ID]", name: "ID3" });
-    H.popover().button("Done").click();
-    H.runButtonOverlay().click();
-    cy.wait("@dataset");
-    cy.findByTestId("editor-tabs-columns-name").click();
-    H.openColumnOptions("ID2");
-    H.renameColumn("ID2", "ID2 custom");
-    H.openColumnOptions("ID3");
-    H.renameColumn("ID3", "ID3 custom");
-    H.saveMetadataChanges();
+      H.openQuestionActions("Edit query definition");
+      H.getNotebookStep("data").button("Pick columns").click();
+      H.popover().findByText("Select all").click();
+      H.getNotebookStep("data").button("Custom column").click();
+      H.enterCustomColumnDetails({ formula: "[ID]", name: "ID2" });
+      H.popover().button("Done").click();
+      H.getNotebookStep("expression").icon("add").click();
+      H.enterCustomColumnDetails({ formula: "[ID]", name: "ID3" });
+      H.popover().button("Done").click();
+      H.runButtonOverlay().click();
+      cy.wait("@dataset");
+      cy.findByTestId("editor-tabs-columns-name").click();
+      H.openColumnOptions("ID2");
+      H.renameColumn("ID2", "ID2 custom");
+      H.openColumnOptions("ID3");
+      H.renameColumn("ID3", "ID3 custom");
+      H.saveMetadataChanges();
 
-    H.openNotebook();
-    H.getNotebookStep("data").button("Filter").click();
-    H.popover().within(() => {
-      cy.findByText("ID").should("be.visible");
-      cy.findByText("ID2 custom").should("be.visible");
-      cy.findByText("ID3 custom").should("be.visible");
-    });
-  });
+      H.openNotebook();
+      H.getNotebookStep("data").button("Filter").click();
+      H.popover().within(() => {
+        cy.findByText("ID").should("be.visible");
+        cy.findByText("ID2 custom").should("be.visible");
+        cy.findByText("ID3 custom").should("be.visible");
+      });
+    },
+  );
 });
 
 describe("issue 34514", () => {

e2e/test/scenarios/native/snippet-tags.cy.spec.ts

@@ -0,0 +1,404 @@
+const { H } = cy;
+
+import { ORDERS_QUESTION_ID } from "e2e/support/cypress_sample_instance_data";
+
+describe("scenarios > native > snippet tags", () => {
+  beforeEach(() => {
+    H.restore();
+    cy.signInAsAdmin();
+  });
+
+  it("should be able to create a snippet with variable tags", () => {
+    H.startNewNativeQuestion();
+    H.NativeEditor.type("select id from products where ");
+
+    cy.log("create a snippet");
+    getEditorTopBar().icon("snippet").click();
+    getEditorSidebar().findByText("Create snippet").click();
+    H.modal().within(() => {
+      getSnippetContentInput().type("category = {{category}}", {
+        parseSpecialCharSequences: false,
+      });
+      getSnippetNameInput().type("variable-snippet");
+      cy.button("Save").click();
+    });
+
+    cy.log("verify that the snippet was inserted");
+    H.NativeEditor.get().should(
+      "contain",
+      "select id from products where {{snippet: variable-snippet}}",
+    );
+
+    cy.log("verify that the query can be run");
+    getEditorTopBar().findByPlaceholderText("Category").type("Widget");
+    H.runNativeQuery();
+    H.assertQueryBuilderRowCount(54);
+  });
+
+  it("should be able to create a snippet with card tags", () => {
+    H.startNewNativeQuestion();
+    H.NativeEditor.type("select * from ");
+
+    cy.log("create a snippet");
+    getEditorTopBar().icon("snippet").click();
+    getEditorSidebar().findByText("Create snippet").click();
+    H.modal().within(() => {
+      getSnippetContentInput().type(`{{#${ORDERS_QUESTION_ID}}}`, {
+        parseSpecialCharSequences: false,
+      });
+      getSnippetNameInput().type("card-snippet");
+      cy.button("Save").click();
+    });
+
+    cy.log("verify that the snippet was inserted");
+    H.NativeEditor.get().should(
+      "contain",
+      "select * from {{snippet: card-snippet}}",
+    );
+
+    cy.log("verify that the query can be run");
+    H.runNativeQuery();
+    H.tableInteractive().should("be.visible");
+  });
+
+  it("should be able to create a snippet with snippet tags", () => {
+    H.startNewNativeQuestion();
+    H.NativeEditor.type("select * from ");
+
+    cy.log("create a snippet");
+    getEditorTopBar().icon("snippet").click();
+    getEditorSidebar().findByText("Create snippet").click();
+    H.modal().within(() => {
+      getSnippetContentInput().type("category = {{category}}", {
+        parseSpecialCharSequences: false,
+      });
+      getSnippetNameInput().type("snippet1");
+      cy.button("Save").click();
+    });
+
+    cy.log("create a snippet that uses the previous snippet");
+    getEditorTopBar().icon("snippet").click();
+    getEditorSidebar().icon("add").click();
+    H.popover().findByText("New snippet").click();
+    H.modal().within(() => {
+      getSnippetContentInput().type("{{snippet: snippet1}}", {
+        parseSpecialCharSequences: false,
+      });
+      getSnippetNameInput().type("snippet2");
+      cy.button("Save").click();
+    });
+
+    cy.log("verify that the snippet can used");
+    H.NativeEditor.clear();
+    H.NativeEditor.type("select id from products where {{snippet: snippet2}}");
+    getEditorTopBar().findByLabelText("Category").type("Gizmo");
+    H.runNativeQuery();
+    H.tableInteractive().should("be.visible");
+  });
+
+  it("should be able to create a tag with the same name as the inner snippet tag", () => {
+    createQuestionAndSnippet().then(({ card }) => {
+      H.visitQuestion(card.id);
+    });
+
+    cy.log("add a local tag with the same name");
+    getEditorVisibilityToggler().click();
+    H.NativeEditor.type(" and category = {{filter}}");
+
+    cy.log("verify that the query can be run");
+    getEditorTopBar().findByPlaceholderText("Filter").type("Widget");
+    H.runNativeQuery();
+    H.assertQueryBuilderRowCount(54);
+  });
+
+  it("should be able to use snippets where tags overlap", () => {
+    cy.log("create snippets with overlapping tags");
+    H.createSnippet({
+      name: "snippet1",
+      content: "category = {{category1}} or category = {{category2}}",
+    });
+    H.createSnippet({
+      name: "snippet2",
+      content: "category = {{category2}}",
+    });
+
+    cy.log("use both snippets in a query");
+    H.startNewNativeQuestion();
+    H.NativeEditor.type(
+      "select id from products where {{snippet: snippet1}} or {{snippet: snippet2}}",
+    );
+
+    cy.log("verify that the query can be run");
+    getEditorTopBar().findByPlaceholderText("Category1").type("Widget");
+    getEditorTopBar().findByPlaceholderText("Category2").type("Gadget");
+    H.runNativeQuery();
+    H.assertQueryBuilderRowCount(107);
+  });
+
+  it("should be able to update a snippet and change tags", () => {
+    createQuestionAndSnippet().then(({ card }) => {
+      H.visitQuestion(card.id);
+    });
+
+    cy.log("update the snippet");
+    getEditorVisibilityToggler().click();
+    getEditorTopBar().icon("snippet").click();
+    getEditorSidebar().within(() => {
+      cy.icon("chevrondown").click({ force: true });
+      cy.button(/Edit/).click();
+    });
+    H.modal().within(() => {
+      getSnippetContentInput()
+        .clear()
+        .type("ean = {{ean}} or vendor = {{vendor}}", {
+          parseSpecialCharSequences: false,
+        });
+      cy.button("Save").click();
+    });
+
+    cy.log("verify that the tags in the query were updated");
+    getEditorTopBar().within(() => {
+      cy.findByPlaceholderText("Filter").should("not.exist");
+      cy.findByPlaceholderText("Ean").type("1018947080336");
+      cy.findByPlaceholderText("Vendor").type("Balistreri-Ankunding");
+    });
+    H.runNativeQuery();
+    H.assertTableRowsCount(2);
+  });
+
+  it("should be able to change the inner tag type to Number", () => {
+    createQuestionAndSnippet({ snippetContent: "ID = {{filter}}" }).then(
+      ({ card }) => {
+        H.visitQuestion(card.id);
+      },
+    );
+
+    cy.log("change the type");
+    getEditorVisibilityToggler().click();
+    getEditorTopBar().icon("variable").click();
+    getVariableTypeSelect().click();
+    H.popover().findByText("Number").click();
+
+    cy.log("verify that the parameter can be used");
+    getEditorTopBar().findByPlaceholderText("Filter").type("10");
+    H.runNativeQuery();
+    H.assertQueryBuilderRowCount(1);
+  });
+
+  it("should be able to change the inner tag type to Field Filter", () => {
+    createQuestionAndSnippet({ snippetContent: "{{filter}}" }).then(
+      ({ card }) => {
+        H.visitQuestion(card.id);
+      },
+    );
+
+    cy.log("change the type");
+    getEditorVisibilityToggler().click();
+    getEditorTopBar().icon("variable").click();
+    getVariableTypeSelect().click();
+    H.popover().findByText("Field Filter").click();
+    H.popover().findByText("Products").click();
+    H.popover().findByText("Category").click();
+
+    cy.log("verify that the parameter can be used");
+    H.filterWidget().click();
+    H.popover().within(() => {
+      cy.findByText("Gadget").click();
+      cy.button("Add filter").click();
+    });
+    H.runNativeQuery();
+    H.assertQueryBuilderRowCount(53);
+  });
+
+  it("should be able to open a question with an old snippet name", () => {
+    createQuestionAndSnippet().then(({ card, snippet }) => {
+      cy.request("PUT", `/api/native-query-snippet/${snippet.id}`, {
+        name: "category-snippet",
+      });
+      cy.visit(`/question/${card.id}`);
+    });
+
+    cy.log("verify that the parameter can be used");
+    getEditorTopBar().findByPlaceholderText("Filter").type("Widget");
+    H.queryBuilderHeader().icon("play").click();
+    H.assertQueryBuilderRowCount(54);
+  });
+
+  it("should not be able to create card cycles via snippets", () => {
+    cy.log("create a card and a snippet referencing this card");
+    H.createNativeQuestion({
+      native: {
+        query: "select 1",
+      },
+    }).then(({ body: card }) => {
+      H.createSnippet({
+        name: "cycle-snippet",
+        content: `{{#${card.id}}}`,
+      });
+      H.visitQuestion(card.id);
+    });
+
+    cy.log("create a cycle and try to save the card");
+    getEditorVisibilityToggler().click();
+    H.NativeEditor.clear();
+    H.NativeEditor.type("select * from {{snippet: cycle-snippet}}");
+    H.queryBuilderHeader().button("Save").click();
+    H.modal().within(() => {
+      cy.button("Save").click();
+      cy.findByText("Cannot save card with cycles.").should("be.visible");
+    });
+  });
+
+  it("should not be able to create snippet cycles via snippets", () => {
+    cy.log("create two snippets, one using another");
+    H.createSnippet({
+      name: "snippet1",
+      content: "1",
+    });
+    H.createSnippet({
+      name: "snippet2",
+      content: "{{snippet: snippet1}}",
+    });
+
+    cy.log("open the modal for the first snippet");
+    H.startNewNativeQuestion();
+    getEditorTopBar().icon("snippet").click();
+    getEditorSidebar().within(() => {
+      cy.icon("chevrondown")
+        .should("have.length", 2)
+        .first()
+        .click({ force: true });
+      cy.button(/Edit/).click();
+    });
+
+    cy.log("create a cycle and save the first snippet");
+    H.modal().within(() => {
+      getSnippetContentInput()
+        .clear()
+        .type("{{snippet: snippet2}}", { parseSpecialCharSequences: false })
+        .click();
+      cy.button("Save").click();
+    });
+
+    cy.log("run the query and verify that we detect a cycle");
+    H.NativeEditor.type("select {{snippet: snippet1}}");
+    H.runNativeQuery();
+    H.queryBuilderMain()
+      .findByText(
+        'This query has circular referencing sub-queries. The snippet "snippet2" and the snippet "snippet1" seem to be part of the problem.',
+      )
+      .should("be.visible");
+
+    cy.log(
+      "try to save a question with a snippet cycle and verify we detect it",
+    );
+    H.queryBuilderHeader().findByText("Save").click();
+    H.modal().within(() => {
+      cy.findByLabelText("Name").type("SQL");
+      cy.button("Save").click();
+      cy.findByText("Cannot save card with cycles.").should("be.visible");
+      cy.button("Cancel").click();
+    });
+
+    cy.log(
+      "remove a cycle, save the card, add a cycle again and verify that we detect it on update",
+    );
+    H.NativeEditor.clear().type("select 1");
+    H.saveQuestion("SQL", undefined, {
+      tab: "Browse",
+      path: ["Our analytics"],
+    });
+    H.NativeEditor.clear().type("select {{snippet: snippet1}}");
+    H.queryBuilderHeader().button("Save").click();
+    H.modal().within(() => {
+      cy.button("Save").click();
+      cy.findByText("Cannot save card with cycles.").should("be.visible");
+    });
+  });
+
+  it("should work for a public or embedded question", () => {
+    createQuestionAndSnippet().then(({ card }) => {
+      cy.log("public question - no parameter value");
+      H.visitPublicQuestion(card.id);
+      H.assertTableRowsCount(200);
+
+      cy.log("public question - with parameter value");
+      H.filterWidget().findByLabelText("Filter").type("Gadget{enter}");
+      H.assertTableRowsCount(53);
+
+      cy.log("embedded question - no parameter value");
+      H.visitEmbeddedPage({
+        resource: { question: card.id },
+        params: {},
+      });
+      H.assertTableRowsCount(200);
+
+      cy.log("embedded question - with parameter value");
+      H.filterWidget().findByLabelText("Filter").type("Gadget{enter}");
+      H.assertTableRowsCount(53);
+    });
+  });
+});
+
+function getEditorSidebar() {
+  return cy.findByTestId("sidebar-right");
+}
+
+function getEditorTopBar() {
+  return cy.findByTestId("native-query-top-bar");
+}
+
+function getEditorVisibilityToggler() {
+  return cy.findByTestId("visibility-toggler");
+}
+
+function getVariableTypeSelect() {
+  return cy.findByTestId("variable-type-select");
+}
+
+function getSnippetNameInput() {
+  return cy.findByLabelText("Give your snippet a name");
+}
+
+function getSnippetContentInput() {
+  return cy.findByLabelText("Enter some SQL here so you can reuse it later");
+}
+
+function createQuestionAndSnippet({
+  snippetContent = "category = {{filter}}",
+}: {
+  snippetContent?: string;
+} = {}) {
+  return H.createSnippet({
+    name: "filter-snippet",
+    content: snippetContent,
+  }).then(({ body: snippet }) => {
+    return H.createNativeQuestion({
+      native: {
+        query: "select id from products [[where {{snippet: filter-snippet}}]]",
+        "template-tags": {
+          "snippet: filter-snippet": {
+            id: "4b77cc1f-ea70-4ef6-84db-58432fce6928",
+            name: "snippet: filter-snippet",
+            "display-name": "snippet: filter-snippet",
+            type: "snippet",
+            "snippet-id": snippet.id,
+            "snippet-name": snippet.name,
+          },
+          filter: {
+            id: "4b77cc1f-ea70-4ef6-84db-58432fce6929",
+            name: "filter",
+            "display-name": "Filter",
+            type: "text",
+          },
+        },
+      },
+      enable_embedding: true,
+      embedding_params: {
+        filter: "enabled",
+      },
+    }).then(({ body: card }) => {
+      return { card, snippet };
+    });
+  });
+}

e2e/test/scenarios/onboarding/setup/setup-embedding.cy.spec.ts

@@ -75,195 +75,201 @@ H.describeWithSnowplowEE("scenarios > setup embedding (EMB-477)", () => {
       .should("be.visible");
   });
 
-  it("[cloud-hosted] should allow users to go through the embedding setup and onboarding flow", () => {
-    // mock `is-hosted`, we can't set the token before the user is created, we'll set it right after that
-    H.activateToken("pro-cloud");
+  it(
+    "[cloud-hosted] should allow users to go through the embedding setup and onboarding flow",
+    { tags: "@flaky" },
+    () => {
+      // mock `is-hosted`, we can't set the token before the user is created, we'll set it right after that
+      H.activateToken("pro-cloud");
 
-    mockCloudHosted();
+      mockCloudHosted();
 
-    cy.visit(
-      "/setup/embedding?first_name=Firstname&last_name=Lastname&email=testy@metabase.test&site_name=Epic Team",
-    );
-
-    cy.log("0: Welcome step");
-    assertEmbeddingOnboardingPageLoaded();
-
-    H.expectUnstructuredSnowplowEvent({
-      event: "embedding_setup_step_seen",
-      event_detail: "welcome",
-    });
-
-    step().within(() => {
-      cy.findByText(
-        /Let's get you up and running with a starting setup for embedded analytics/,
-      ).should("be.visible");
-      cy.button("Start").should("be.visible").click();
-    });
-
-    cy.log("1: User creation step");
-    sidebar().within(() => {
-      cy.findByRole("listitem", { current: "step" }).should(
-        "have.text",
-        "Set up your account",
-      );
-    });
-    step().within(() => {
-      cy.findByRole("heading", { name: "What should we call you?" }).should(
-        "be.visible",
+      cy.visit(
+        "/setup/embedding?first_name=Firstname&last_name=Lastname&email=testy@metabase.test&site_name=Epic Team",
       );
 
+      cy.log("0: Welcome step");
+      assertEmbeddingOnboardingPageLoaded();
+
       H.expectUnstructuredSnowplowEvent({
         event: "embedding_setup_step_seen",
-        event_detail: "user-creation",
+        event_detail: "welcome",
       });
 
-      cy.findByLabelText("First name").should("have.value", "Firstname");
-      cy.findByLabelText("Last name").should("have.value", "Lastname");
-      cy.findByLabelText("Email").should("have.value", "testy@metabase.test");
-      cy.findByLabelText("Company or team name").should(
-        "have.value",
-        "Epic Team",
-      );
+      step().within(() => {
+        cy.findByText(
+          /Let's get you up and running with a starting setup for embedded analytics/,
+        ).should("be.visible");
+        cy.button("Start").should("be.visible").click();
+      });
 
-      fillOutUserForm();
+      cy.log("1: User creation step");
+      sidebar().within(() => {
+        cy.findByRole("listitem", { current: "step" }).should(
+          "have.text",
+          "Set up your account",
+        );
+      });
+      step().within(() => {
+        cy.findByRole("heading", { name: "What should we call you?" }).should(
+          "be.visible",
+        );
 
-      cy.intercept("POST", "/api/setup").as("setup");
-      cy.button("Next").should("be.enabled").click();
-    });
-
-    cy.log(
-      "Now we have a user we simulate being on cloud by setting a Metatabase Token",
-    );
-    cy.wait("@setup");
-
-    // set a real token so we have real token-features for the rest of the flow
-    H.activateToken("pro-cloud");
-
-    cy.log("2: Data connection step");
-    sidebar().within(() => {
-      cy.findByRole("listitem", { current: "step" }).should(
-        "have.text",
-        "Connect to your data",
-      );
-    });
-    step().within(() => {
-      cy.findByRole("heading", { name: "Connect to your data" }).should(
-        "be.visible",
-      );
-
-      cy.findByRole("option", { name: "PostgreSQL" }).click();
-      fillOutDatabaseForm();
-
-      cy.log("simulate database that takes some time to fully sync");
-      const NO_OF_MIN_RETRY = 4;
-      let currentRetry = 0;
-      cy.intercept("GET", "api/database", (req) => {
-        req.on("response", (res) => {
-          currentRetry++;
-          const [postgresDb] = res.body.data;
-          if (
-            postgresDb.initial_sync_status !== "complete" ||
-            currentRetry < NO_OF_MIN_RETRY
-          ) {
-            // Simulate syncing not complete until we reach a certain number of retries
-            postgresDb.initial_sync_status = "incomplete";
-          }
+        H.expectUnstructuredSnowplowEvent({
+          event: "embedding_setup_step_seen",
+          event_detail: "user-creation",
         });
+
+        cy.findByLabelText("First name").should("have.value", "Firstname");
+        cy.findByLabelText("Last name").should("have.value", "Lastname");
+        cy.findByLabelText("Email").should("have.value", "testy@metabase.test");
+        cy.findByLabelText("Company or team name").should(
+          "have.value",
+          "Epic Team",
+        );
+
+        fillOutUserForm();
+
+        cy.intercept("POST", "/api/setup").as("setup");
+        cy.button("Next").should("be.enabled").click();
       });
-      cy.button("Connect database").should("be.enabled").click();
-    });
 
-    cy.log("3: Content generation step");
-    sidebar().within(() => {
-      cy.findByRole("listitem", { current: "step" }).should(
-        "have.text",
-        "Generate starter content",
+      cy.log(
+        "Now we have a user we simulate being on cloud by setting a Metatabase Token",
       );
-    });
+      cy.wait("@setup");
 
-    const SECOND = 1000;
-    // Waiting for the database sync to complete
-    step()
-      .findByRole("heading", {
-        name: "Select Tables to Embed",
-        timeout: 10 * SECOND,
-      })
-      .should("be.visible");
+      // set a real token so we have real token-features for the rest of the flow
+      H.activateToken("pro-cloud");
 
-    expectNoDatabaseStatus();
-
-    step().within(() => {
-      cy.findByRole("checkbox", { name: "feedback" })
-        .should("be.enabled")
-        .check();
-      cy.findByRole("checkbox", { name: "orders" })
-        .should("be.enabled")
-        .check();
-      cy.findByRole("checkbox", { name: "products" })
-        .should("be.enabled")
-        .check();
-
-      cy.button("Continue").should("be.enabled").click();
-    });
-
-    cy.log("4: Processing step");
-
-    step().within(() => {
-      cy.findByRole("heading", { name: "Setting Up Your Analytics" }).should(
-        "be.visible",
-      );
-      /**
-       * Since this loader will go away automatically it's crucial to wait for it to go away
-       * before start asserting the next steps.
-       */
-      cy.findByText("Creating dashboards...").should("not.exist");
-    });
-
-    cy.log("5: Embed in your app step");
-    sidebar().within(() => {
-      cy.findByRole("listitem", { current: "step" }).should(
-        "have.text",
-        "Add to your app",
-      );
-    });
-
-    step().within(() => {
-      cy.findByRole("heading", { name: "Add to your app" }).should(
-        "be.visible",
-      );
-      cy.findByRole("tab", { name: 'A look at "Feedback"' }).click();
-      H.getIframeBody().within(() => {
-        cy.findByText('A look at "Feedback"').should("be.visible");
+      cy.log("2: Data connection step");
+      sidebar().within(() => {
+        cy.findByRole("listitem", { current: "step" }).should(
+          "have.text",
+          "Connect to your data",
+        );
       });
-      cy.findByRole("code").contains(
-        new RegExp(
-          `<iframe src="${Cypress.config("baseUrl")}/dashboard/\\d+" width="800px" height="500px" />`,
-        ),
+      step().within(() => {
+        cy.findByRole("heading", { name: "Connect to your data" }).should(
+          "be.visible",
+        );
+
+        cy.findByRole("option", { name: "PostgreSQL" }).click();
+        fillOutDatabaseForm();
+
+        cy.log("simulate database that takes some time to fully sync");
+        const NO_OF_MIN_RETRY = 4;
+        let currentRetry = 0;
+        cy.intercept("GET", "api/database", (req) => {
+          req.on("response", (res) => {
+            currentRetry++;
+            const [postgresDb] = res.body.data;
+            if (
+              postgresDb.initial_sync_status !== "complete" ||
+              currentRetry < NO_OF_MIN_RETRY
+            ) {
+              // Simulate syncing not complete until we reach a certain number of retries
+              postgresDb.initial_sync_status = "incomplete";
+            }
+          });
+        });
+        cy.button("Connect database").should("be.enabled").click();
+      });
+
+      cy.log("3: Content generation step");
+      sidebar().within(() => {
+        cy.findByRole("listitem", { current: "step" }).should(
+          "have.text",
+          "Generate starter content",
+        );
+      });
+
+      const SECOND = 1000;
+      // Waiting for the database sync to complete
+      step()
+        .findByRole("heading", {
+          name: "Select Tables to Embed",
+          timeout: 10 * SECOND,
+        })
+        .should("be.visible");
+
+      expectNoDatabaseStatus();
+
+      step().within(() => {
+        cy.findByRole("checkbox", { name: "feedback" })
+          .should("be.enabled")
+          .check();
+        cy.findByRole("checkbox", { name: "orders" })
+          .should("be.enabled")
+          .check();
+        cy.findByRole("checkbox", { name: "products" })
+          .should("be.enabled")
+          .check();
+
+        cy.button("Continue").should("be.enabled").click();
+      });
+
+      cy.log("4: Processing step");
+
+      step().within(() => {
+        cy.findByRole("heading", { name: "Setting Up Your Analytics" }).should(
+          "be.visible",
+        );
+        /**
+         * Since this loader will go away automatically it's crucial to wait for it to go away
+         * before start asserting the next steps.
+         */
+        cy.findByText("Creating dashboards...").should("not.exist");
+      });
+
+      cy.log("5: Embed in your app step");
+      sidebar().within(() => {
+        cy.findByRole("listitem", { current: "step" }).should(
+          "have.text",
+          "Add to your app",
+        );
+      });
+
+      step().within(() => {
+        cy.findByRole("heading", { name: "Add to your app" }).should(
+          "be.visible",
+        );
+        cy.findByRole("tab", { name: 'A look at "Feedback"' }).click();
+        H.getIframeBody().within(() => {
+          cy.findByText('A look at "Feedback"').should("be.visible");
+        });
+        cy.findByRole("code").contains(
+          new RegExp(
+            `<iframe src="${Cypress.config("baseUrl")}/dashboard/\\d+" width="800px" height="500px" />`,
+          ),
+        );
+
+        cy.button("I see Metabase").click();
+
+        cy.findByRole("heading", { name: "You're on your way!" }).should(
+          "be.visible",
+        );
+        cy.findByRole("link", { name: "Take me to Metabase" }).click();
+      });
+
+      cy.log("5: Metabase");
+      cy.button("New").should("be.visible");
+
+      cy.log(
+        "Only one 'Automatically Generated Dashboards' collection should be created",
       );
-
-      cy.button("I see Metabase").click();
-
-      cy.findByRole("heading", { name: "You're on your way!" }).should(
-        "be.visible",
-      );
-      cy.findByRole("link", { name: "Take me to Metabase" }).click();
-    });
-
-    cy.log("5: Metabase");
-    cy.button("New").should("be.visible");
-
-    cy.log(
-      "Only one 'Automatically Generated Dashboards' collection should be created",
-    );
-    sidebar().within(() => {
-      cy.findByText("Automatically Generated Dashboards").should("be.visible");
-      cy.findByText("Automatically Generated Dashboards").should(
-        "have.length",
-        1,
-      );
-    });
-    expectNoDatabaseStatus();
-  });
+      sidebar().within(() => {
+        cy.findByText("Automatically Generated Dashboards").should(
+          "be.visible",
+        );
+        cy.findByText("Automatically Generated Dashboards").should(
+          "have.length",
+          1,
+        );
+      });
+      expectNoDatabaseStatus();
+    },
+  );
 });
 
 /**

e2e/test/scenarios/permissions/admin-permissions.cy.spec.js

@@ -310,7 +310,7 @@ describe("scenarios > admin > permissions", { tags: "@OSS" }, () => {
     });
 
     context("group focused view", () => {
-      it("shows filterable list of groups", () => {
+      it("shows filterable list of groups", { tags: "@flaky" }, () => {
         cy.visit("/admin/permissions");
 
         // no groups selected initially and it shows an empty state

e2e/test/scenarios/permissions/sandboxing/sandboxing-via-ui.cy.spec.ts

@@ -78,12 +78,16 @@ describe(
       H.restore("sandboxing-snapshot" as any);
     });
 
-    it("shows all data before sandboxing policy is applied", () => {
-      signInAs(gizmoViewer);
-      assertNoResultsOrValuesAreSandboxed(dashboard, sandboxableQuestions);
-      signInAs(widgetViewer);
-      assertNoResultsOrValuesAreSandboxed(dashboard, sandboxableQuestions);
-    });
+    it(
+      "shows all data before sandboxing policy is applied",
+      { tags: "@flaky" },
+      () => {
+        signInAs(gizmoViewer);
+        assertNoResultsOrValuesAreSandboxed(dashboard, sandboxableQuestions);
+        signInAs(widgetViewer);
+        assertNoResultsOrValuesAreSandboxed(dashboard, sandboxableQuestions);
+      },
+    );
 
     describe("we can apply a sandbox policy", () => {
       beforeEach(() => {

e2e/test/scenarios/question-reproductions/reproductions-3.cy.spec.js

@@ -1955,7 +1955,7 @@ describe("issue 45063", { tags: "@flaky" }, () => {
       });
     });
 
-    it("should work with models", () => {
+    it("should work with models", { tags: "@flaky" }, () => {
       createGuiModel({ sourceTableId: ORDERS_ID });
       verifyRemappedFilter({
         visitCard: () => cy.get("@modelId").then(H.visitModel),

e2e/test/scenarios/question-reproductions/reproductions.cy.spec.ts

@@ -1202,3 +1202,91 @@ describe("issue 52872", () => {
       });
   });
 });
+
+describe("issue 35561", () => {
+  beforeEach(() => {
+    H.restore();
+    cy.signInAsAdmin();
+  });
+
+  it("should display a functional fk drill for both null and non-null values", () => {
+    H.createNativeQuestion(
+      {
+        name: "Model",
+        native: {
+          query: `SELECT 1 AS ID, CAST(NULL AS INT) AS USER_ID
+UNION ALL
+SELECT 2 AS ID, 3 AS USER_ID
+`,
+        },
+        type: "model",
+      },
+      {},
+    ).then(async ({ body: model }) => {
+      await cy.request("POST", `/api/card/${model.id}/query`);
+      await H.setModelMetadata(model.id, (field) => {
+        if (field.name === "USER_ID") {
+          return {
+            ...field,
+            display_name: "User ID",
+            semantic_type: "type/FK",
+            fk_target_field_id: ORDERS.ID,
+          };
+        }
+        return field;
+      });
+      H.createQuestion(
+        {
+          name: "Question",
+          query: {
+            "source-table": `card__${model.id}`,
+          },
+          display: "table",
+        },
+        { wrapId: true, visitQuestion: true },
+      );
+    });
+
+    cy.findByTestId("table-root")
+      .findAllByRole("row")
+      .first()
+      .within(() => {
+        cy.get("[data-column-id='USER_ID']").click();
+      });
+    H.popover().within(() => {
+      cy.findByText("View Models with no User").should("be.visible").click();
+    });
+    cy.findByTestId("table-body").within(() => {
+      cy.findAllByRole("row").should("have.length", 1);
+      cy.findAllByRole("row")
+        .first()
+        .within(() => {
+          cy.get("[data-column-id='USER_ID']").within(() => {
+            cy.get("[data-testid=cell-data]").should("not.exist");
+          });
+        });
+    });
+
+    H.visitQuestion("@questionId");
+
+    cy.findByTestId("table-root")
+      .findAllByRole("row")
+      .eq(1)
+      .within(() => {
+        cy.get("[data-column-id='USER_ID']").click();
+      });
+    H.popover().within(() => {
+      cy.findByText("View this User's Models").should("be.visible").click();
+    });
+    cy.findByTestId("table-body").within(() => {
+      cy.findAllByRole("row").should("have.length", 1);
+      cy.findAllByRole("row")
+        .first()
+        .within(() => {
+          cy.get("[data-column-id='USER_ID']").within(() => {
+            cy.get("[data-testid=cell-data]").should("have.text", "3");
+          });
+        });
+    });
+  });
+});

e2e/test/scenarios/table-editing/table-editing.cy.spec.ts

@@ -304,15 +304,20 @@ describe("scenarios > table-editing", () => {
           cy.findAllByRole("button").last().click();
         });
 
-        cy.wait("@updateTableData").then(({ response }) => {
+        cy.wait("@updateTableData").then(({ response, request }) => {
           const targetDate = dayjs(
             new Date(2024, 4, day, hour, minute, 0),
           ).format("YYYY-MM-DDTHH:mm:ss");
-          const savedDate = response?.body.outputs[0].row.CREATED_AT;
 
-          expect(response?.body.outputs[0].op).to.equal("updated");
-          // Omit timezone offset
-          expect(savedDate.startsWith(targetDate)).to.be.true;
+          const requestDate = request.body.params.CREATED_AT;
+          const responseDate = response?.body.outputs[0].row.CREATED_AT;
+
+          // Check if request date matches the response date.
+          // In theory FE should preserve the initial input date timezone offset (might be based on the CI environment)
+          expect(requestDate).to.equal(responseDate);
+
+          // Check if date param matches the input date (without timezone offset at offset 19)
+          expect(requestDate.slice(0, 19)).to.equal(targetDate);
         });
 
         H.undoToast().findByText("Successfully updated").should("be.visible");

enterprise/backend/src/metabase_enterprise/harbormaster/client.clj

@@ -2,6 +2,7 @@
   "API client for interfacing with Harbormaster on store-api-url."
   (:require
    [clj-http.client :as http]
+   [clojure.core.memoize :as memoize]
    [clojure.string :as str]
    [martian.clj-http :as martian-http]
    [martian.core :as martian]
@@ -135,7 +136,8 @@
    {:headers {"Authorization" (str "Bearer " api-key)}}))
 
 (def ^:private create-client-memo
-  (memoize create-client))
+  (memoize/ttl create-client
+               :ttl/threshold (m.util/minutes->ms 5)))
 
 (defn- client []
   (let [store-api-url (store-api/store-api-url)

enterprise/backend/src/metabase_enterprise/semantic_search/repair.clj

@@ -13,7 +13,7 @@
    [medley.core :as m]
    [metabase.util :as u]
    [metabase.util.log :as log]
-   [nano-id.core :as nano-id]
+   [metabase.util.string :as u.str]
    [next.jdbc :as jdbc]
    [next.jdbc.result-set :as jdbc.rs]))
 
@@ -93,7 +93,7 @@
   Format: repair_<millis-since-epoch>_<short-id>"
   []
   (let [millis-since-epoch (t/to-millis-from-epoch (t/instant))
-        short-id           (u/lower-case-en (subs (nano-id/nano-id) 0 6))]
+        short-id           (u/lower-case-en (u.str/random-string 6))]
     (format "repair_%d_%s" millis-since-epoch short-id)))
 
 (defn with-repair-table!

enterprise/backend/src/metabase_enterprise/transforms/api.clj

@@ -1,6 +1,7 @@
 (ns metabase-enterprise.transforms.api
   (:require
    [clojure.core.async :as a]
+   [clojure.string :as str]
    [metabase-enterprise.transforms.api.transform-job]
    [metabase-enterprise.transforms.api.transform-tag]
    [metabase-enterprise.transforms.canceling :as trasnforms.canceling]
@@ -45,7 +46,7 @@
 
 (mr/def ::transform-target
   [:map
-   [:database :int]
+   [:database {:optional true} :int]
    [:type [:enum "table"]]
    [:schema {:optional true} [:or ms/NonBlankString :nil]]
    [:name :string]])
@@ -85,12 +86,6 @@
       (api/check-400 (driver.u/supports? (:engine database) feature database)
                      (deferred-tru "The database does not support the requested transform target type.")))))
 
-(defn- localize-timestamps
-  [run]
-  (-> run
-      (u/update-some :start_time transforms.util/local-timestamp-string)
-      (u/update-some :end_time   transforms.util/local-timestamp-string)))
-
 (api.macros/defendpoint :get "/"
   "Get a list of transforms."
   [_route-params
@@ -98,7 +93,7 @@
   (api/check-superuser)
   (-> (t2/select :model/Transform)
       (t2/hydrate :last_run :transform_tag_ids)
-      (->> (map #(update % :last_run localize-timestamps)))))
+      (->> (map #(update % :last_run transforms.util/localize-run-timestamps)))))
 
 (api.macros/defendpoint :post "/"
   "Create a new transform."
@@ -137,7 +132,7 @@
         target-table (transforms.util/target-table database-id target :active true)]
     (-> transform
         (t2/hydrate :last_run :transform_tag_ids)
-        (u/update-some :last_run localize-timestamps)
+        (u/update-some :last_run transforms.util/localize-run-timestamps)
         (assoc :table target-table))))
 
 (api.macros/defendpoint :get "/:id/dependencies"
@@ -170,7 +165,7 @@
   (-> (transform-run/paged-runs (assoc query-params
                                        :offset (request/offset)
                                        :limit  (request/limit)))
-      (update :data #(map localize-timestamps %))))
+      (update :data #(map transforms.util/localize-run-timestamps %))))
 
 (api.macros/defendpoint :put "/:id"
   "Update a transform."
@@ -281,20 +276,21 @@
         cancel-chan (a/promise-chan)]
     (trasnforms.canceling/chan-start-run! run-id cancel-chan)
     (try
-      (let [{:keys [body status]} (transforms.execute/call-python-runner-api! (:code body) (:tables body) run-id cancel-chan)]
+      (let [{:keys [response output events]} (transforms.execute/test-python-transform! (:code body) (:tables body) run-id cancel-chan)
+            {:keys [body status]} response]
         (if (= status 200)
           (do
             (log/info "Python test execution succeeded")
             (-> (response/response {:message (deferred-tru "Python code executed successfully")
-                                    :result  {:body body}})
+                                    :result  {:body (assoc body :output output)}})
                 (assoc :status 200)))
           (do
             (log/error "Error executing Python test code")
             (-> (response/response {:message   (deferred-tru "Python code execution failed")
-                                    :error     (:error body)
-                                    :stdout    (:stdout body)
-                                    :stderr    (:stderr body)
-                                    :exit_code (:exit-code body)})
+                                    :error     body
+                                    :stdout    (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
+                                    :stderr    (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))
+                                    :exit_code (:exit_code (:exit_code body))})
                 (assoc :status status)))))
       (finally
         (trasnforms.canceling/chan-end-run! run-id)))))

enterprise/backend/src/metabase_enterprise/transforms/api/transform_job.clj

@@ -4,9 +4,12 @@
    [metabase-enterprise.transforms.jobs :as transforms.jobs]
    [metabase-enterprise.transforms.models.transform-job :as transform-job]
    [metabase-enterprise.transforms.schedule :as transforms.schedule]
+   [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.api.common :as api]
    [metabase.api.macros :as api.macros]
    [metabase.api.routes.common :refer [+auth]]
+   [metabase.driver.common.parameters.dates :as params.dates]
+   [metabase.util.date-2 :as u.date]
    [metabase.util.i18n :refer [deferred-tru]]
    [metabase.util.jvm :as u.jvm]
    [metabase.util.log :as log]
@@ -124,6 +127,29 @@
   (let [job (api/check-404 (t2/select-one :model/TransformJob :id job-id))]
     (t2/hydrate job :tag_ids :last_run)))
 
+(defn- add-next-run
+  [{id :id :as job}]
+  (if-let [start-time (-> id transforms.schedule/existing-trigger :next-fire-time)]
+    (assoc job :next_run {:start_time (str (transforms.util/->instant start-time))})
+    job))
+
+(defn- matching-timestamp?
+  [job field-path {:keys [start end]}]
+  (when-let [field-instant (transforms.util/->instant (get-in job field-path))]
+    (let [start-instant (some-> start u.date/parse transforms.util/->instant)
+          end-instant (some-> end u.date/parse transforms.util/->instant)]
+      (and (or (nil? start)
+               (not (.isBefore field-instant start-instant)))
+           (or (nil? end)
+               (.isAfter end-instant field-instant))))))
+
+(defn- ->date-field-filter-xf
+  [field-path filter-value]
+  (let [range (some-> filter-value (params.dates/date-string->range {:inclusive-end? false}))]
+    (if range
+      (filter #(matching-timestamp? % field-path range))
+      identity)))
+
 (api.macros/defendpoint :get "/:job-id/transforms"
   "Get the transforms of job specified by the job's ID."
   [{:keys [job-id]} :- [:map
@@ -136,11 +162,25 @@
 (api.macros/defendpoint :get "/"
   "Get all transform jobs."
   [_route-params
-   _query-params]
+   {:keys [last_run_start_time next_run_start_time transform_tag_ids]} :-
+   [:map
+    [:last_run_start_time {:optional true} [:maybe ms/NonBlankString]]
+    [:next_run_start_time {:optional true} [:maybe ms/NonBlankString]]
+    [:transform_tag_ids   {:optional true} [:maybe (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]]]]
   (log/info "Getting all transform jobs")
   (api/check-superuser)
-  (let [jobs (t2/select :model/TransformJob {:order-by [[:created_at :desc]]})]
-    (t2/hydrate jobs :tag_ids :last_run)))
+  (let [jobs (t2/select :model/TransformJob {:order-by [[:created_at :desc]]})
+        transform-tag-ids (-> transform_tag_ids set not-empty)]
+    (into []
+          (comp (map add-next-run)
+                (->date-field-filter-xf [:last_run :start_time] last_run_start_time)
+                (->date-field-filter-xf [:next_run :start_time] next_run_start_time)
+                (if transform-tag-ids
+                  (filter #(some transform-tag-ids (:tag_ids %)))
+                  identity)
+                (map #(update % :last_run transforms.util/localize-run-timestamps))
+                (map #(update % :next_run transforms.util/localize-run-timestamps)))
+          (t2/hydrate jobs :tag_ids :last_run))))
 
 (def ^{:arglists '([request respond raise])} routes
   "`/api/ee/transform-job` routes."

enterprise/backend/src/metabase_enterprise/transforms/execute.clj

@@ -14,7 +14,6 @@
    [metabase.lib.schema.common :as schema.common]
    [metabase.query-processor.pipeline :as qp.pipeline]
    [metabase.util :as u]
-   [metabase.util.json :as json]
    [metabase.util.jvm :as u.jvm]
    [metabase.util.log :as log]
    [metabase.util.malli.registry :as mr]
@@ -29,7 +28,7 @@
 (mr/def ::transform-details
   [:map
    [:transform-type [:enum {:decode/normalize schema.common/normalize-keyword} :table]]
-   [:connection-details :any]
+   [:conn-spec :any]
    [:query :string]
    [:output-table [:keyword {:decode/normalize schema.common/normalize-keyword}]]])
 
@@ -37,24 +36,6 @@
   [:map
    [:overwrite? :boolean]])
 
-(defmacro with-transform-lifecycle
-  "Macro to manage transform run lifecycle. Starts a run, executes body,
-   and handles success/failure appropriately.
-
-   Usage:
-   (with-transform-lifecycle [run-id transform-id opts]
-     ;; body that returns result on success or throws on failure
-     )"
-  [[run-id-sym [transform-id opts]] & body]
-  `(let [{~run-id-sym :id} (transform-run/start-run! ~transform-id ~opts)]
-     (try
-       (let [result# (do ~@body)]
-         (transform-run/succeed-started-run! ~run-id-sym)
-         result#)
-       (catch Throwable t#
-         (transform-run/fail-started-run! ~run-id-sym {:message (.getMessage t#)})
-         (throw t#)))))
-
 (defn- sync-target!
   ([transform-id run-id]
    (let [{:keys [source target]} (t2/select-one :model/Transform transform-id)
@@ -68,9 +49,12 @@
 
 (defn run-cancelable-transform!
   "Run a compiled transform"
-  [run-id run-transform!]
+  [run-id driver {:keys [db conn-spec output-schema]} run-transform!]
   ;; local run is responsible for status, using canceling lifecycle
   (try
+    (when (driver.u/supports? driver :schemas db)
+      (when-not (driver/schema-exists? driver (:id db) output-schema)
+        (driver/create-schema-if-needed! driver conn-spec output-schema)))
     (canceling/chan-start-timeout-vthread! run-id (transforms.settings/transform-timeout))
     (let [cancel-chan (a/promise-chan)
           ret (binding [qp.pipeline/*canceled-chan* cancel-chan]
@@ -107,9 +91,11 @@
      (let [db (get-in source [:query :database])
            {driver :engine :as database} (t2/select-one :model/Database db)
            feature (transforms.util/required-database-feature transform)
-           transform-details {:transform-type (keyword (:type target))
-                              :connection-details (driver/connection-details driver database)
+           transform-details {:db db
+                              :transform-type (keyword (:type target))
+                              :conn-spec (driver/connection-spec driver database)
                               :query (transforms.util/compile-source source)
+                              :output-schema (:schema target)
                               :output-table (transforms.util/qualified-table-name driver target)}
            opts {:overwrite? true}]
        (when-not (driver.u/supports? driver feature database)
@@ -120,7 +106,7 @@
          (when start-promise
            (deliver start-promise [:started run-id]))
          (log/info "Executing transform" id "with target" (pr-str target))
-         (run-cancelable-transform! run-id (fn [_cancel-chan] (driver/run-transform! driver transform-details opts)))
+         (run-cancelable-transform! run-id driver transform-details (fn [_cancel-chan] (driver/run-transform! driver transform-details opts)))
          (transforms.instrumentation/with-stage-timing [run-id :sync :table-sync]
            (sync-target! target database run-id))))
      (catch Throwable t
@@ -201,15 +187,6 @@
       Closeable
       (close [_] (cleanup)))))
 
-(defn call-python-runner-api!
-  "Call the Python runner API endpoint to execute Python code.
-   Returns the result map or throws on error."
-  [code table-name->id run-id cancel-chan]
-  ;; TODO probably don't need this hack anymore, double check
-  (transforms.instrumentation/with-python-api-timing [run-id]
-    (update (python-runner/execute-python-code run-id code table-name->id cancel-chan)
-            :body #(if (string? %) json/decode+kw %))))
-
 (defn- debug-info-str [{:keys [exit-code events]}]
   (str/join "\n"
             ;; todo this was temporary before the log stuff, needs a rethink at some point
@@ -222,31 +199,78 @@
              "======"
              (format "exit code %d" exit-code)]))
 
-(defn- transfer-file-to-db [driver db {:keys [target] :as transform} metadata temp-file]
-  (let [table-name (transforms.util/qualified-table-name driver target)
-        table-schema {:name table-name
-                      :columns (mapv (fn [{:keys [name dtype]}]
+(defn- create-table-and-insert-data!
+  "Create a table from metadata and insert data from source."
+  [driver db-id table-name metadata data-source]
+  (let [table-schema {:name (if (keyword? table-name) table-name (keyword table-name))
+                      :columns (mapv (fn [{:keys [name dtype base_type database_type]}]
                                        {:name name
-                                        :type (transforms.util/dtype->base-type dtype)
+                                        :type (or (some->> base_type (keyword "type"))
+                                                  (transforms.util/dtype->base-type dtype))
+                                        :database-type database_type
                                         :nullable? true})
                                      (:fields metadata))}
+        data-source (assoc data-source :table-schema table-schema)]
+    (transforms.util/create-table-from-schema! driver db-id table-schema)
+    (driver/insert-from-source! driver db-id table-name (mapv :name (:columns table-schema)) data-source)))
+
+(defn- transfer-file-to-db [driver db {:keys [target] :as transform} metadata temp-file]
+  (let [table-name (transforms.util/qualified-table-name driver target)
+        table-exists? (transforms.util/target-table-exists? transform)
         data-source {:type :csv-file
                      :file temp-file}]
-    ;; TODO: should be transactional, perharps go through driver/run-transform!
-    (transforms.util/delete-target-table! transform)
-    (transforms.util/create-table-from-schema! driver (:id db) table-schema)
-    (driver/insert-from-source! driver (:id db) table-name (mapv :name (:columns table-schema)) data-source)))
+    (if table-exists?
+      ;; Table exists - use temp table + atomic swap pattern
+      (let [temp-table-name (keyword (str (u/qualified-name table-name) "_temp_" (System/currentTimeMillis)))]
+        (log/info "Existing table deletected, Create then swap")
+        (try
+          (create-table-and-insert-data! driver (:id db) temp-table-name metadata data-source)
+          ;; TODO: These operations should be within a transaction for atomicity
+          (transforms.util/drop-table! driver (:id db) table-name)
+          (transforms.util/rename-table! driver (:id db) temp-table-name table-name)
+          (catch Exception e
+            (log/error e "Failed to transfer data to table")
+            (try
+              (transforms.util/drop-table! driver (:id db) temp-table-name)
+              (catch Exception _))
+            (throw e))))
+      ;; Table doesn't exist - create directly with target name
+      (do
+        (log/info "New table")
+        (create-table-and-insert-data! driver (:id db) table-name metadata data-source)))))
+
+(defn test-python-transform!
+  "Execute a transform in test mode (does not write result into a table)."
+  [code tables->id run-id cancel-chan]
+  (with-open [shared-storage-ref (python-runner/open-s3-shared-storage! tables->id)]
+    (let [server-url              (transforms.settings/python-execution-server-url)
+          _                       (python-runner/copy-tables-to-s3! {:run-id         run-id
+                                                                     :shared-storage @shared-storage-ref
+                                                                     :table-name->id tables->id
+                                                                     :cancel-chan    cancel-chan})
+          _                       (python-runner/open-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan
+          response
+          (python-runner/execute-python-code-http-call!
+           {:server-url     server-url
+            :code           code
+            :run-id         run-id
+            :table-name->id tables->id
+            :shared-storage @shared-storage-ref})
+          {:keys [output events]} (python-runner/read-output-objects @shared-storage-ref)]
+      {:response response
+       :events   events
+       :output   output})))
 
 (defn- run-python-transform! [{:keys [source] :as transform} db run-id cancel-chan message-log]
   (with-open [log-thread-ref     (open-log-thread! run-id message-log)
               shared-storage-ref (python-runner/open-s3-shared-storage! (:source-tables source))]
-    (let [driver                        (:engine db)
-          server-url                    (transforms.settings/python-execution-server-url)
-          _                             (python-runner/copy-tables-to-s3! {:run-id                  run-id
-                                                                           :shared-storage @shared-storage-ref
-                                                                           :table-name->id (:source-tables source)
-                                                                           :cancel-chan    cancel-chan})
-          _                             (python-runner/open-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan
+    (let [driver     (:engine db)
+          server-url (transforms.settings/python-execution-server-url)
+          _          (python-runner/copy-tables-to-s3! {:run-id                  run-id
+                                                        :shared-storage @shared-storage-ref
+                                                        :table-name->id (:source-tables source)
+                                                        :cancel-chan    cancel-chan})
+          _          (python-runner/open-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan
           {:keys [status body] :as response}
           (python-runner/execute-python-code-http-call!
            {:server-url     server-url
@@ -302,13 +326,19 @@
                                                :python      nil
                                                :post-python []}))
             {:keys [target] transform-id :id} transform
-            db (t2/select-one :model/Database (:database target))
+            {driver :engine :as db} (t2/select-one :model/Database (:database target))
             {run-id :id} (try-start-unless-already-running transform-id run-method)]
         (some-> start-promise (deliver [:started run-id]))
         (log! message-log "Executing Python transform")
         (log/info "Executing Python transform" transform-id "with target" (pr-str target))
         (let [start-ms (u/start-timer)
-              result   (run-cancelable-transform! run-id (fn [cancel-chan] (run-python-transform! transform db run-id cancel-chan message-log)))]
+              transform-details
+              {:db db
+               :transform-type (keyword (:type target))
+               :conn-spec (driver/connection-spec driver db)
+               :output-schema (:schema target)
+               :output-table (transforms.util/qualified-table-name driver target)}
+              result   (run-cancelable-transform! run-id driver transform-details (fn [cancel-chan] (run-python-transform! transform db run-id cancel-chan message-log)))]
           (transforms.instrumentation/with-stage-timing [run-id :sync :table-sync]
             (sync-target! target db run-id))
           (log! message-log (format "Python execution finished in %s" (Duration/ofMillis (u/since-ms start-ms))))

enterprise/backend/src/metabase_enterprise/transforms/ordering.clj

@@ -16,23 +16,29 @@
 (set! *warn-on-reflection* true)
 
 (defn- transform-deps [transform]
-  (let [query (-> (get-in transform [:source :query])
-                  transforms.util/massage-sql-query
-                  qp.preprocess/preprocess
-                  ;; legacy usage -- don't do things like this going forward
-                  #_{:clj-kondo/ignore [:discouraged-var]}
-                  lib/->legacy-MBQL)]
-    (case (:type query)
-      :native (driver/native-query-deps (-> (qp.store/metadata-provider)
-                                            lib.metadata/database
-                                            :engine)
-                                        (get-in query [:native :query]))
-      :query (into #{}
-                   (keep #(clojure.core.match/match %
-                            [:source-table source] (when (int? source)
-                                                     source)
-                            _ nil))
-                   (tree-seq coll? seq query)))))
+  (if (transforms.util/python-transform? transform)
+    ;; Python transforms have dependencies via source-tables mapping
+    (into #{}
+          (map #(hash-map :table %))
+          (vals (get-in transform [:source :source-tables])))
+    ;; Query transforms have dependencies via SQL query analysis
+    (let [query (-> (get-in transform [:source :query])
+                    transforms.util/massage-sql-query
+                    qp.preprocess/preprocess
+                    ;; legacy usage -- don't do things like this going forward
+                    #_{:clj-kondo/ignore [:discouraged-var]}
+                    lib/->legacy-MBQL)]
+      (case (:type query)
+        :native (driver/native-query-deps (-> (qp.store/metadata-provider)
+                                              lib.metadata/database
+                                              :engine)
+                                          (get-in query [:native :query]))
+        :query (into #{}
+                     (keep #(clojure.core.match/match %
+                              [:source-table source] (when (int? source)
+                                                       {:table source})
+                              _ nil))
+                     (tree-seq coll? seq query))))))
 
 (defn- dependency-map [transforms]
   (into {}
@@ -56,28 +62,32 @@
 
   The result is a map of transform id -> #{transform ids the transform depends on}. Dependencies are limited to just
   the transforms in the original list -- if a transform depends on some transform not in the list, the 'extra'
-  dependency is ignored. Python transforms have no dependencies and run sequentially."
+  dependency is ignored. Both query and Python transforms can have dependencies on tables produced by other transforms."
   [transforms]
-  (let [{python-transforms true
-         sql-transforms false} (group-by transforms.util/python-transform? transforms)
 
-        ;; TODO: figure out python transforms deps
-        python-deps (into {} (map (fn [transform] [(:id transform) #{}])) python-transforms)
+  (let [;; Group all transforms by their database
+        transforms-by-db (->> transforms
+                              (map (fn [transform]
+                                     (let [db-id (if (transforms.util/python-transform? transform)
+                                                   (get-in transform [:target :database])
+                                                   (get-in transform [:source :query :database]))]
+                                       {db-id [transform]})))
+                              (apply merge-with into))
 
-        sql-transforms-by-db (->> sql-transforms
-                                  (map (fn [transform]
-                                         {(get-in transform [:source :query :database]) [transform]}))
-                                  (apply merge-with into))
-
-        {:keys [output-tables dependencies]} (->> sql-transforms-by-db
+        transform-ids (into #{} (map :id) transforms)
+        {:keys [output-tables dependencies]} (->> transforms-by-db
                                                   (map (fn [[db-id db-transforms]]
                                                          (qp.store/with-metadata-provider db-id
                                                            {:output-tables (output-table-map db-transforms)
                                                             :dependencies (dependency-map db-transforms)})))
                                                   (apply merge-with merge))
 
-        sql-deps (update-vals dependencies #(into #{} (keep output-tables) %))]
-    (merge python-deps sql-deps)))
+        all-deps (update-vals dependencies #(into #{}
+                                                  (keep (fn [{:keys [table transform]}]
+                                                          (or (output-tables table)
+                                                              (transform-ids transform))))
+                                                  %))]
+    all-deps))
 
 (defn find-cycle
   "Finds a path containing a cycle in the directed graph `node->children`.
@@ -123,9 +133,12 @@
                                transforms)
         db-id (get-in to-check [:source :query :database])]
     (qp.store/with-metadata-provider db-id
-      (let [output-tables (output-table-map (filter #(= (get-in % [:source :query :database]) db-id)
-                                                    transforms))
-            node->children #(->> % transforms-by-id transform-deps (keep output-tables))
+      (let [db-transforms (filter #(= (get-in % [:source :query :database]) db-id) transforms)
+            output-tables (output-table-map db-transforms)
+            transform-ids (into #{} (map :id) db-transforms)
+            node->children #(->> % transforms-by-id transform-deps (keep (fn [{:keys [table transform]}]
+                                                                           (or (output-tables table)
+                                                                               (transform-ids transform)))))
             id->name (comp :name transforms-by-id)
             cycle (find-cycle node->children [transform-id])]
         (when cycle

enterprise/backend/src/metabase_enterprise/transforms/python_runner.clj

@@ -4,10 +4,12 @@
    [clojure.core.async :as a]
    [clojure.java.io :as io]
    [clojure.string :as str]
+   [medley.core :as m]
    [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
    [metabase-enterprise.transforms.settings :as transforms.settings]
    [metabase.driver :as driver]
    [metabase.query-processor.compile :as qp.compile]
+   [metabase.query-processor.pipeline :as qp.pipeline]
    ^{:clj-kondo/ignore [:discouraged-namespace]}
    [metabase.query-processor.store :as qp.store]
    [metabase.util.json :as json]
@@ -19,7 +21,6 @@
    (java.net URI)
    (java.nio.charset StandardCharsets)
    (java.time Duration)
-   (java.util.concurrent CancellationException)
    (software.amazon.awssdk.auth.credentials AwsBasicCredentials StaticCredentialsProvider)
    (software.amazon.awssdk.auth.credentials DefaultCredentialsProvider)
    (software.amazon.awssdk.core.sync RequestBody)
@@ -60,7 +61,8 @@
                   :type     :native
                   :native   native}]
       (qp.store/with-metadata-provider db-id
-        (driver/execute-reducible-query driver query {:canceled-chan cancel-chan} respond)))))
+        (binding [qp.pipeline/*canceled-chan* cancel-chan]
+          (driver/execute-reducible-query driver query {:canceled-chan cancel-chan} respond))))))
 
 (defn- root-type
   [base-type]
@@ -71,6 +73,8 @@
            :type/Boolean
            :type/DateTimeWithTZ
            :type/DateTime
+           :type/Dictionary
+           :type/JSON
            :type/Date
            :type/Text])))
 
@@ -100,23 +104,42 @@
                             :semantic_type  (some-> (effective-semantic-type-i-think col-meta) name)
                             :effective_type (some-> (or (:effective_type col-meta) (:database_type col-meta)) name)
                             ;; TODO get this passed through
-                            #_#_:field_id   (:id col-meta)})
+                            :field_id       (:id col-meta)})
                          cols-meta)
    :table_metadata {:table_id table-id}})
 
+(defn- maybe-fixup-v [col v]
+  ;; clickhouse returns bigdecimals for int64 values
+  (if (and (isa? (:base_type col) :type/Integer)
+           (or (instance? BigDecimal v)
+               (float? v)))
+    (bigint v)
+    v))
+
 (defn- write-table-data-to-file! [id temp-file cancel-chan]
-  (let [db-id         (t2/select-one-fn :db_id (t2/table-name :model/Table) :id id)
-        driver        (t2/select-one-fn :engine :model/Database db-id)
-        ;; TODO: limit
-        query         {:source-table id}
-        manifest-atom (atom nil)]
+  (let [db-id           (t2/select-one-fn :db_id (t2/table-name :model/Table) :id id)
+        driver          (t2/select-one-fn :engine :model/Database db-id)
+        all-fields-meta (t2/select [:model/Field :id :name :base_type :effective_type :semantic_type :database_type :database_position :nfc_path :parent_id]
+                                   :table_id id
+                                   :active true
+                                   {:order-by [[:database_position :asc]]})
+        fields-meta (filter #(and (nil? (:parent_id %)) (nil? (:nfc_path %))) all-fields-meta)
+        query {:source-table id}
+        manifest (generate-manifest id fields-meta)]
     (execute-mbql-query driver db-id query
                         (fn [{cols-meta :cols} reducible-rows]
-                          (reset! manifest-atom (generate-manifest id cols-meta))
                           (with-open [os (io/output-stream temp-file)]
-                            (write-to-stream! os (mapv :name cols-meta) reducible-rows)))
+                            (let [filtered-col-meta (m/index-by :name fields-meta)
+                                  filtered-cols (filter #(contains? filtered-col-meta (:name %)) cols-meta)
+                                  col-name->index (into {} (map-indexed (fn [i col] [(:name col) i]) cols-meta))
+                                  filtered-indices (mapv (fn [col] [(col-name->index (:name col)) col]) filtered-cols)
+                                  filtered-rows (eduction (map (fn [row] (mapv (fn [[idx col]]
+                                                                                 (let [v (nth row idx)]
+                                                                                   (maybe-fixup-v col v))) filtered-indices)))
+                                                          reducible-rows)]
+                              (write-to-stream! os (mapv :name filtered-cols) filtered-rows))))
                         cancel-chan)
-    @manifest-atom))
+    manifest))
 
 (defn- maybe-with-endpoint-s3-client [^S3ClientBuilder builder endpoint]
   (let [region (transforms.settings/python-storage-s-3-region)]
@@ -256,8 +279,10 @@
   (try
     (let [^DeleteObjectRequest request (build-delete-object-request bucket-name key)]
       (.deleteObject s3-client request))
-    (catch Exception _
-      ;; Ignore deletion errors - object might not exist or we might not have permissions
+    (catch Exception e
+      (log/debugf e "Error deleting s3 object %s" key)
+      ;; Ignore deletion errors - object might not exist, or we might not have permissions
+      ;; NOTE: we plan to put general retention on the bucket so that objects will eventually be deleted
       nil)))
 
 (defn- cleanup-s3-objects [^S3Client s3-client bucket-name s3-keys]
@@ -293,6 +318,9 @@
         bucket-name         (transforms.settings/python-storage-s-3-bucket)
         loc
         (fn [method relative-path]
+          ;; save a tracking root to a well known location
+          ;; later can ls these to collect those objects whose ttl has elapsed
+          ;; (OR setup a retention policy in s3?)
           (let [path (str work-dir-name "/" relative-path)]
             {:path   path
              :method method
@@ -301,16 +329,17 @@
     {:s3-client   (create-s3-client)                        ;; do not like mixing interactive things with descriptions, but its damn convenient to have it here for now
      :bucket-name bucket-name
      :objects
-     (-> (into
-          {:output          (loc :put "output.csv")
-           :output-manifest (loc :put "output-manifest.json")
-           :events          (loc :put "events.jsonl")}
-          (for [[table-name id] table-name->id]
-            {[:table id :manifest] (loc :get (str "-table-" (name table-name) "-" id ".manifest.json"))
-             [:table id :data]     (loc :get (str "-table-" (name table-name) "-" id ".jsonl"))})))}))
+     (into
+      {:output          (loc :put "output.csv")
+       :output-manifest (loc :put "output-manifest.json")
+       :events          (loc :put "events.jsonl")}
+      (for [[table-name id] table-name->id]
+        {[:table id :manifest] (loc :get (str "-table-" (name table-name) "-" id ".manifest.json"))
+         [:table id :data]     (loc :get (str "-table-" (name table-name) "-" id ".jsonl"))}))}))
 
 (defn open-s3-shared-storage!
-  "Returns a deref'able shared storage value, (.close) will cleanup any s3 objects named in storage (data files for tables and so on)."
+  "Returns a deref'able shared storage value, (.close) will optimistically delete any s3 objects named in storage (data files for tables, metadata files etc).
+  The intention is the bucket specifies a generic object retention policy to ensure objects are eventually deleted (e.g. because the process dies during writing and .close never gets called)"
   ^Closeable [table-name->id]
   (let [shared-storage (s3-shared-storage table-name->id)]
     (reify IDeref
@@ -318,10 +347,13 @@
       Closeable
       (close [_] (cleanup-s3-objects (:s3-client shared-storage) (:bucket-name shared-storage) (map :path (vals (:objects shared-storage))))))))
 
-(defn copy-tables-to-s3! [{:keys [run-id
-                                  shared-storage
-                                  table-name->id
-                                  cancel-chan]}]
+(defn copy-tables-to-s3!
+  "Writes table content to their corresponding objects named in shared-storage, see (open-shared-storage!).
+  Blocks until all tables are fully written and committed to shared storage."
+  [{:keys [run-id
+           shared-storage
+           table-name->id
+           cancel-chan]}]
   (doseq [id (vals table-name->id)
           :let [{:keys [s3-client bucket-name objects]} shared-storage
                 {data-path :path}     (get objects [:table id :data])
@@ -350,7 +382,10 @@
           (safe-delete temp-file)
           (safe-delete manifest-file))))))
 
-(defn execute-python-code-http-call! [{:keys [server-url code run-id table-name->id shared-storage]}]
+(defn execute-python-code-http-call!
+  "Calls the /execute endpoint of the python runner. Blocks until the run either succeeds or fails and returns
+  the response from the server."
+  [{:keys [server-url code run-id table-name->id shared-storage]}]
   (let [{:keys [objects]} shared-storage
         {:keys [output output-manifest events]} objects
 
@@ -366,12 +401,13 @@
                                   :table_mapping       table-name->url
                                   :manifest_mapping    table-name->manifest-url}
 
-        response                 (http/post (str server-url "/execute")
-                                            {:content-type     :json
-                                             :accept           :json
-                                             :body             (json/encode payload)
-                                             :throw-exceptions false
-                                             :as               :json})]
+        response                 (transforms.instrumentation/with-python-api-timing [run-id]
+                                   (http/post (str server-url "/execute")
+                                              {:content-type     :json
+                                               :accept           :json
+                                               :body             (json/encode payload)
+                                               :throw-exceptions false
+                                               :as               :json}))]
     ;; when a 500 is returned we observe a string in the body (despite the python returning json)
     ;; always try to parse the returned string as json before yielding (could tighten this up at some point)
     (update response :body (fn [string-if-error]
@@ -393,7 +429,7 @@
                       :async?       true}
                      identity identity))))
 
-;; temporary (just keep the files alive longer and stream them directly (i.e into tmp file for write)
+;; temporary, we should not need to realize data/events files into memory longer term
 (defn read-output-objects
   "Temporary function that strings/jsons stuff in S3 and returns it for compatibility."
   [{:keys [s3-client bucket-name objects]}]

enterprise/backend/src/metabase_enterprise/transforms/schedule.clj

@@ -61,12 +61,17 @@
     (log/info "Creating trigger for transform job" job-id "with schedule" schedule)
     (task/add-trigger! (build-trigger job-id schedule))))
 
+(defn existing-trigger
+  "Return the existing trigger for `job-id-or-trigger`, or nil, if the trigger doesn't exist."
+  [job-id-or-trigger]
+  (first (task/existing-triggers (job-key job-id-or-trigger)
+                                 (trigger-key job-id-or-trigger))))
+
 (defn- delete-trigger!
   "Delete the trigger for a transform job."
   ([job-id-or-trigger]
    (if (number? job-id-or-trigger)
-     (if-let [trigger (first (task/existing-triggers (job-key job-id-or-trigger)
-                                                     (trigger-key job-id-or-trigger)))]
+     (if-let [trigger (existing-trigger job-id-or-trigger)]
        (delete-trigger! trigger)
        (log/info "No trigger for this transform job exists"))
      (do

enterprise/backend/src/metabase_enterprise/transforms/settings.clj

@@ -6,7 +6,7 @@
 
 (set! *warn-on-reflection* true)
 
-(def ^:privatg is-not-prod? (or config/is-dev? config/is-test?))
+(def ^:private is-not-prod? (or config/is-dev? config/is-test?))
 
 (setting/defsetting transform-timeout
   (deferred-tru "The timeout for a transform job, in minutes.")

enterprise/backend/src/metabase_enterprise/transforms/util.clj

@@ -1,17 +1,19 @@
 (ns metabase-enterprise.transforms.util
   (:require
    [clojure.string :as str]
+   [java-time.api :as t]
    [metabase.driver :as driver]
    [metabase.lib.schema.common :as lib.schema.common]
    [metabase.query-processor.compile :as qp.compile]
    [metabase.sync.core :as sync]
+   [metabase.util :as u]
    [metabase.util.date-2 :as u.date]
    [metabase.util.log :as log]
    [metabase.util.malli :as mu]
    [metabase.util.malli.registry :as mr]
    [toucan2.core :as t2])
   (:import
-   (java.time Instant LocalDateTime OffsetDateTime ZonedDateTime ZoneId)
+   (java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)
    (java.util Date)))
 
 (set! *warn-on-reflection* true)
@@ -35,11 +37,12 @@
 
 (defn target-table-exists?
   "Test if the target table of a transform already exists."
-  [{:keys [source target] :as transform}]
-  (when (query-transform? transform)
-    (let [db-id (-> source :query :database)
-          {driver :engine :as database} (t2/select-one :model/Database db-id)]
-      (driver/table-exists? driver database target))))
+  [{:keys [source target] :as _transform}]
+  (let [db-id (or (-> source :query :database)
+                  ;; python transform target
+                  (-> target :database))
+        {driver :engine :as database} (t2/select-one :model/Database db-id)]
+    (driver/table-exists? driver database target)))
 
 (defn target-table
   "Load the `target` table of a transform from the database specified by `database-id`."
@@ -108,23 +111,35 @@
   [transform]
   (case (-> transform :target :type)
     "table"             :transforms/table))
-(defn local-timestamp
-  "Convert the timestamp t to a ZonedDateTime instance in the system timezone."
-  [t]
-  (condp instance? t
-    String         (recur (u.date/parse t))
-    Date           (recur (.toInstant ^Date t))
-    Instant        (ZonedDateTime/ofInstant t (ZoneId/systemDefault))
-    LocalDateTime  (ZonedDateTime/of t (ZoneId/systemDefault))
-    OffsetDateTime (.atZoneSameInstant ^OffsetDateTime t (ZoneId/systemDefault))
-    ZonedDateTime  (.withZoneSameInstant ^ZonedDateTime t (ZoneId/systemDefault))
-    (throw (ex-info (str "Cannot convert timestamp " t " of type " (type t) " to a ZonedDateTime")
-                    {:timestamp t}))))
 
-(defn local-timestamp-string
+(defn ->instant
+  "Convert a temporal value `t` to an Instant in the system timezone."
+  ^Instant [t]
+  (when t
+    (condp instance? t
+      Instant        t
+      Date           (.toInstant ^Date t)
+      OffsetDateTime (.toInstant ^OffsetDateTime t)
+      ZonedDateTime  (.toInstant ^ZonedDateTime t)
+      LocalDateTime  (recur (.atZone ^LocalDateTime t (t/zone-id)))
+      String         (recur (u.date/parse t))
+      LocalTime      (recur (.atDate ^LocalTime t (t/local-date)))
+      OffsetTime     (recur (.atDate ^OffsetTime t (t/local-date)))
+      LocalDate      (recur (.atStartOfDay ^LocalDate t))
+      (throw (ex-info (str "Cannot convert temporal " t " of type " (type t) " to an Instant")
+                      {:temporal t})))))
+
+(defn utc-timestamp-string
   "Convert the timestamp t to a string encoding the it in the system timezone."
   [t]
-  (-> t local-timestamp str))
+  (-> t ->instant str))
+
+(defn localize-run-timestamps
+  "Convert the timestamps of a `run` to ISO strings in UTC."
+  [run]
+  (-> run
+      (u/update-some :start_time utc-timestamp-string)
+      (u/update-some :end_time   utc-timestamp-string)))
 
 (mr/def ::column-definition
   [:map
@@ -158,8 +173,27 @@
    database-id :- pos-int?
    table-schema :- ::table-definition]
   (let [{:keys [columns] table-name :name} table-schema
-        column-definitions (into {} (map (fn [{:keys [name type]}] [name (driver/type->database-type driver type)]))
+        column-definitions (into {} (map (fn [{:keys [name type database-type]}]
+                                           (let [database-type (when-not (and (= driver :bigquery-cloud-sdk)
+                                                                              (#{"ARRAY" "RECORD"} database-type))
+                                                                 database-type)
+                                                 db-type (if database-type
+                                                           [[:raw database-type]]
+                                                           (driver/type->database-type driver type))]
+                                             [name db-type])))
                                  columns)
         primary-key-opts (select-keys table-schema [:primary-key])]
     (log/infof "Creating table %s with %d columns" table-name (count columns))
     (driver/create-table! driver database-id table-name column-definitions primary-key-opts)))
+
+(defn rename-table!
+  "kename a table in the database."
+  [driver database-id old-table-name new-table-name]
+  (log/infof "Renaming table %s to %s" old-table-name new-table-name)
+  (driver/rename-table! driver database-id old-table-name new-table-name))
+
+(defn drop-table!
+  "Drop a table in the database."
+  [driver database-id table-name]
+  (log/infof "Dropping table %s" table-name)
+  (driver/drop-table! driver database-id table-name))

enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj

@@ -60,6 +60,7 @@
       (testing "Migration of simultaneous init attempt is blocked"
         (let [original-write-fn @#'semantic.db.migration/write-successful-migration!
               original-migrate-fn @#'semantic.db.connection/do-with-migrate-tx
+              original-maybe-migrate @#'semantic.db.migration/maybe-migrate!
               results (atom {:executed-migrations 0
                              :log []})]
           (with-redefs-fn {;; TODO: why in ci init index won't succeed -- http on embedding
@@ -72,6 +73,12 @@
                                (apply original-migrate-fn args)
                                (swap! results update :log conj [tid :ended (t/local-date-time)]))
                              nil)
+                           ;; add delay into migration function to ensure that thread performing migration
+                           ;; finishes earlier
+                           #'semantic.db.migration/maybe-migrate!
+                           (fn [& args]
+                             (Thread/sleep 200)
+                             (apply original-maybe-migrate args))
                            #'semantic.db.migration/write-successful-migration!
                            (fn [& args]
                              (Thread/sleep 2000)

enterprise/backend/test/metabase_enterprise/semantic_search/query_test.clj

@@ -21,13 +21,16 @@
     (mt/with-premium-features #{:semantic-search}
       (mt/as-admin
         (semantic.tu/with-index!
-          (testing "Dog-related query finds dog content"
-            (let [results (semantic.tu/query-index {:search-string "puppy"})]
-              (is (= "Dog Training Guide" (-> results first :name)))))
+          (semantic.tu/with-only-semantic-weights
+            (testing "Dog-related query finds dog content"
+              (let [results (-> (semantic.tu/query-index {:search-string "puppy"})
+                                semantic.tu/filter-for-mock-embeddings)]
+                (is (= "Dog Training Guide" (-> results first :name)))))
 
-          (testing "Bird-related query finds bird content"
-            (let [results (semantic.tu/query-index {:search-string "avian"})]
-              (is (= "Bird Watching Tips" (-> results first :name))))))))))
+            (testing "Bird-related query finds bird content"
+              (let [results (-> (semantic.tu/query-index {:search-string "avian"})
+                                semantic.tu/filter-for-mock-embeddings)]
+                (is (= "Bird Watching Tips" (-> results first :name)))))))))))
 
 (defn- index-of-name
   "Return the index of the item with :name `name` in `coll`"
@@ -42,68 +45,69 @@
     (mt/with-premium-features #{:semantic-search}
       (mt/as-admin
         (semantic.tu/with-index!
-          (let [not-top-10? #(or (nil? %) (< 10 %))]
-            (doseq [{:keys [name desc search-string search-native-query? expected-index?]}
-                    [{:name "Dog Training Guide"
-                      :desc "contains"
-                      :search-string "AVG(tricks)"
-                      :search-native-query? false
-                      :expected-index? not-top-10?}
-                     {:name "Dog Training Guide"
-                      :desc "contains"
-                      :search-string "AVG(tricks)"
-                      :search-native-query? true
-                      :expected-index? zero?}
-                     {:name "Dog Training Guide"
-                      :desc "contains"
-                      :search-string "GROUP BY breed"
-                      :search-native-query? false
-                      :expected-index? not-top-10?}
-                     {:name "Dog Training Guide"
-                      :desc "contains"
-                      :search-string "GROUP BY breed"
-                      :search-native-query? true
-                      :expected-index? zero?}
-                     {:name "Dog Training Guide"
-                      :desc "does not contain"
-                      :search-string "SUM(tricks)"
-                      :search-native-query? false
-                      :expected-index? not-top-10?}
-                     {:name "Dog Training Guide"
-                      :desc "does not contain"
-                      :search-string "SUM(tricks)"
-                      :search-native-query? true
-                      :expected-index? not-top-10?}
-                     {:name "Bird Watching Tips"
-                      :desc "does not contain (no native query)"
-                      :search-string "AVG(tricks)"
-                      :search-native-query? true
-                      :expected-index? not-top-10?}
-                     {:name "Dog Training Guide"
-                      :desc "semantic search"
-                      :search-string "puppy"
-                      :search-native-query? true
-                      :expected-index? zero?}
-                     {:name "Bird Watching Tips"
-                      :desc "semantic search"
-                      :search-string "avian"
-                      :search-native-query? true
-                      :expected-index? zero?}
-                     {:name "Dog Training Guide"
-                      :desc "non-native keyword query"
-                      :search-string "Training"
-                      :search-native-query? true
-                      :expected-index? zero?}
-                     {:name "Bird Watching Tips"
-                      :desc "non-native keyword query"
-                      :search-string "Watching"
-                      :search-native-query? true
-                      :expected-index? zero?}]]
-              (testing (format "\n%s %s %s %s" name desc search-string search-native-query?)
-                (is (-> (semantic.tu/query-index {:search-string search-string
-                                                  :search-native-query search-native-query?})
-                        (index-of-name name)
-                        expected-index?))))))))))
+          (semantic.tu/with-only-semantic-weights
+            (let [not-top-10? #(or (nil? %) (< 10 %))]
+              (doseq [{:keys [name desc search-string search-native-query? expected-index?]}
+                      [{:name "Dog Training Guide"
+                        :desc "contains"
+                        :search-string "AVG(tricks)"
+                        :search-native-query? false
+                        :expected-index? not-top-10?}
+                       {:name "Dog Training Guide"
+                        :desc "contains"
+                        :search-string "AVG(tricks)"
+                        :search-native-query? true
+                        :expected-index? zero?}
+                       {:name "Dog Training Guide"
+                        :desc "contains"
+                        :search-string "GROUP BY breed"
+                        :search-native-query? false
+                        :expected-index? not-top-10?}
+                       {:name "Dog Training Guide"
+                        :desc "contains"
+                        :search-string "GROUP BY breed"
+                        :search-native-query? true
+                        :expected-index? zero?}
+                       {:name "Dog Training Guide"
+                        :desc "does not contain"
+                        :search-string "SUM(tricks)"
+                        :search-native-query? false
+                        :expected-index? not-top-10?}
+                       {:name "Dog Training Guide"
+                        :desc "does not contain"
+                        :search-string "SUM(tricks)"
+                        :search-native-query? true
+                        :expected-index? not-top-10?}
+                       {:name "Bird Watching Tips"
+                        :desc "does not contain (no native query)"
+                        :search-string "AVG(tricks)"
+                        :search-native-query? true
+                        :expected-index? not-top-10?}
+                       {:name "Dog Training Guide"
+                        :desc "semantic search"
+                        :search-string "puppy"
+                        :search-native-query? true
+                        :expected-index? zero?}
+                       {:name "Bird Watching Tips"
+                        :desc "semantic search"
+                        :search-string "avian"
+                        :search-native-query? true
+                        :expected-index? zero?}
+                       {:name "Dog Training Guide"
+                        :desc "non-native keyword query"
+                        :search-string "Training"
+                        :search-native-query? true
+                        :expected-index? zero?}
+                       {:name "Bird Watching Tips"
+                        :desc "non-native keyword query"
+                        :search-string "Watching"
+                        :search-native-query? true
+                        :expected-index? zero?}]]
+                (testing (format "\n%s %s %s %s" name desc search-string search-native-query?)
+                  (is (-> (semantic.tu/query-index {:search-string search-string
+                                                    :search-native-query search-native-query?})
+                          (index-of-name name)
+                          expected-index?)))))))))))
 
 (deftest model-filtering-test
   (testing "Filter results by model type"
@@ -256,10 +260,13 @@
       (mt/as-admin
         (binding [semantic.index/*batch-size* 1]
           (semantic.tu/with-index!
-            (testing "Horse-related query finds horse content"
-              (let [results (semantic.tu/query-index {:search-string "marine mammal"})]
-                (is (= "Whale Communication" (-> results first :name)))))
+            (semantic.tu/with-only-semantic-weights
+              (testing "Horse-related query finds horse content"
+                (let [results (-> (semantic.tu/query-index {:search-string "marine mammal"})
+                                  semantic.tu/filter-for-mock-embeddings)]
+                  (is (= "Whale Communication" (-> results first :name)))))
 
-            (testing "Tiger-related query finds tiger content"
-              (let [results (semantic.tu/query-index {:search-string "endangered species"})]
-                (is (= "Tiger Conservation" (-> results first :name)))))))))))
+              (testing "Tiger-related query finds tiger content"
+                (let [results (-> (semantic.tu/query-index {:search-string "endangered species"})
+                                  semantic.tu/filter-for-mock-embeddings)]
+                  (is (= "Tiger Conservation" (-> results first :name))))))))))))

enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.clj

@@ -47,6 +47,13 @@
   [gate-contents id]
   (some #(when (= (:id %) id) %) gate-contents))
 
+(defn- clear-gate-table!
+  "Clear all entries from the gate table to ensure clean test state"
+  [pgvector gate-table-name]
+  (jdbc/execute! pgvector
+                 (-> {:delete-from [(keyword gate-table-name)]}
+                     (sql/format :quoted true))))
+
 (deftest repair-index-integration-test
   (testing "repair-index! properly handles document additions and deletions via gate table"
     (mt/with-premium-features #{:semantic-search}
@@ -54,6 +61,7 @@
         (let [pgvector       semantic.tu/db
               index-metadata semantic.tu/mock-index-metadata
               gate-table     (:gate-table-name index-metadata)
+              _              (clear-gate-table! pgvector gate-table)
               initial-docs   [(create-test-document "card" 1 "Dog Training Guide")
                               (create-test-document "card" 2 "Cat Behavior Study")
                               (create-test-document "dashboard" 3 "Animal Stats")]]
@@ -87,17 +95,17 @@
         (let [pgvector       semantic.tu/db
               index-metadata semantic.tu/mock-index-metadata
               gate-table     (:gate-table-name index-metadata)
-              initial-docs   [(create-test-document "card" 1 "Dog Training Guide")]]
+              initial-docs   [(create-test-document "card" 6 "Dog Training Guide")]]
           (semantic.core/update-index! initial-docs)
           (semantic.tu/index-all!)
 
           (testing "repair table is cleaned up after successful repair"
             (let [test-repair-table-name "repair_table_cleanup_test"]
               (with-redefs [semantic.repair/repair-table-name (constantly test-repair-table-name)]
-                (semantic.core/repair-index! [(create-test-document "card" 2 "New Test Card")])
+                (semantic.core/repair-index! [(create-test-document "card" 7 "New Test Card")])
 
                 (let [gate-contents (gate-table-contents pgvector gate-table)]
-                  (is (tombstone? (gate-entry-by-id gate-contents "card_1")))
-                  (is (some? (gate-entry-by-id gate-contents "card_2")))
+                  (is (tombstone? (gate-entry-by-id gate-contents "card_6")))
+                  (is (some? (gate-entry-by-id gate-contents "card_7")))
                   (is (not (semantic.util/table-exists? pgvector test-repair-table-name))
                       "Repair table should be cleaned up after repair-index! completes"))))))))))

enterprise/backend/test/metabase_enterprise/semantic_search/scoring_test.clj

@@ -6,7 +6,6 @@
    [metabase-enterprise.semantic-search.scoring :as semantic.scoring]
    [metabase-enterprise.semantic-search.test-util :as semantic.tu]
    [metabase.app-db.core :as mdb]
-   [metabase.search.appdb.scoring-test :refer [with-weights]]
    [metabase.search.config :as search.config]
    [metabase.test :as mt])
   (:import
@@ -56,13 +55,17 @@
 (defn search-results
   "Like search-results* but with a sanity check that search without weights returns a different result."
   [ranker-key search-string & {:as raw-ctx}]
-  (let [result   (with-weights {ranker-key 1} (search-results* search-string raw-ctx))
-        inverted (with-weights {ranker-key -1} (search-results* search-string raw-ctx))]
+  (let [result   (semantic.tu/with-weights {ranker-key  1} (search-results* search-string raw-ctx))
+        inverted (semantic.tu/with-weights {ranker-key -1} (search-results* search-string raw-ctx))]
     ;; note that this may not be a strict reversal, due to ties.
     (is (not= inverted result)
         "sanity check: search-no-weights should be different")
     result))
 
+;;
+;; index-based scorers
+;;
+
 (deftest rrf-test
   (mt/with-premium-features #{:semantic-search}
     (with-index-contents!
@@ -143,7 +146,7 @@
         (is (= [["dataset" 1 "card ancient"]
                 ["metric"  3 "card old"]
                 ["card"    2 "card recent"]]
-               (with-weights {:model 1.0 :model/dataset 1.0}
+               (semantic.tu/with-weights {:model 1.0 :model/dataset 1.0}
                  (search-results* "card"))))))))
 
 (deftest recency-test
@@ -181,9 +184,9 @@
          {:model "dashboard" :id 2 :name "view dashboard" :view_count 0}
          {:model "dataset"   :id 3 :name "view dataset"   :view_count 0}]
         ;; fix some test flakes where dataset 3 exists and has some sort of recent views
-        (with-weights (assoc (search.config/weights :default)
-                             :user-recency 0
-                             :rrf 0)
+        (semantic.tu/with-weights (assoc (search.config/weights :default)
+                                         :user-recency 0
+                                         :rrf 0)
           (is (=? [{:model "dashboard", :id 2, :name "view dashboard"}
                    {:model "card",      :id 1, :name "view card"}
                    {:model "dataset",   :id 3, :name "view dataset"}]
@@ -202,8 +205,8 @@
 (defn indifferent?
   "Check that the results and their order do not depend on the given ranker."
   [ranker-key search-string & {:as raw-ctx}]
-  (= (with-weights {ranker-key 1} (search-results* search-string raw-ctx))
-     (with-weights {ranker-key -1} (search-results* search-string raw-ctx))))
+  (= (semantic.tu/with-weights {ranker-key  1} (search-results* search-string raw-ctx))
+     (semantic.tu/with-weights {ranker-key -1} (search-results* search-string raw-ctx))))
 
 (deftest dashboard-count-test-2
   (mt/with-premium-features #{:semantic-search}
@@ -213,6 +216,44 @@
          {:model "card" :id 2 :name "card" :dashboardcard_count 201}]
         (is (indifferent? :dashboard "card"))))))
 
+;;;
+;;; "premium" index-based scorers
+;;;
+
+(deftest official-collection-test
+  (with-index-contents!
+    [{:model "collection" :id 1 :name "collection normal" :official_collection false}
+     {:model "collection" :id 2 :name "collection official" :official_collection true}]
+    (testing "official collections has higher rank"
+      (mt/with-premium-features #{:semantic-search :official-collections}
+        (is (= [["collection" 2 "collection official"]
+                ["collection" 1 "collection normal"]]
+               (search-results :official-collection "collection")))))
+    (testing "only if feature is enabled"
+      (mt/with-premium-features #{:semantic-search}
+        (is (= [["collection" 1 "collection normal"]
+                ["collection" 2 "collection official"]]
+               (search-results* "collection")))))))
+
+(deftest verified-test
+  (with-index-contents!
+    [{:model "card" :id 1 :name "card normal" :verified false}
+     {:model "card" :id 2 :name "card verified" :verified true}]
+    (testing "verified items have higher rank"
+      (mt/with-premium-features #{:semantic-search :content-verification}
+        (is (= [["card" 2 "card verified"]
+                ["card" 1 "card normal"]]
+               (search-results :verified "card")))))
+    (testing "only if feature is enabled"
+      (mt/with-premium-features #{:semantic-search}
+        (is (= [["card" 1 "card normal"]
+                ["card" 2 "card verified"]]
+               (search-results* "card")))))))
+
+;;
+;; appdb-based scorers
+;;
+
 (deftest bookmark-test
   (mt/with-premium-features #{:semantic-search}
     ;; TODO (BOT-360) enable for :mysql

enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj

@@ -15,6 +15,7 @@
    [metabase-enterprise.semantic-search.indexer :as semantic.indexer]
    [metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]
    [metabase-enterprise.semantic-search.util :as semantic.util]
+   [metabase.search.config :as search.config]
    [metabase.search.ingestion :as search.ingestion]
    [metabase.test :as mt]
    [metabase.util :as u]
@@ -82,6 +83,18 @@
   [db-name & body]
   `(do-with-test-db! ~db-name (fn [] ~@body)))
 
+(defmacro with-weights
+  "Execute `body` overriding search weights with `weight-map`."
+  [weight-map & body]
+  `(mt/with-dynamic-fn-redefs [search.config/weights (constantly ~weight-map)]
+     ~@body))
+
+(defmacro with-only-semantic-weights
+  "Execute `body` with only the semantic search hybrid scorer weights active."
+  [& body]
+  `(with-weights {:rrf 1}
+     ~@body))
+
 (def ^:private init-delay
   (delay
     (when-not @semantic.db.datasource/data-source

enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj

@@ -963,6 +963,64 @@
                    :content "11 = 11"}
                   (t2/select-one :model/NativeQuerySnippet :entity_id (:entity_id snippet)))))))))
 
+(deftest snippet-template-tags-import-test
+  (testing "Template tags import preserves nil, empty, and populated values"
+
+    (testing "Missing template_tags field -> {} when selected"
+      (mt/with-empty-h2-app-db!
+        (let [snippet-data {:serdes/meta [{:model "NativeQuerySnippet"
+                                           :id    "test-entity-1"
+                                           :label "test_snippet_1"}]
+                            :name        "no-tags-field"
+                            :content     "WHERE id = {{id}}"
+                            :creator_id  "test@example.com"
+                            :entity_id   "test-entity-1"}
+              ingestion    (ingestion-in-memory [snippet-data])]
+          (serdes.load/load-metabase! ingestion)
+          (let [template-tags (t2/select-one-fn :template_tags :model/NativeQuerySnippet :entity_id "test-entity-1")]
+            ;; Toucan hooks compute the template-tags:
+            (is (=? {"id" {:type         :text
+                           :display-name "ID"
+                           :name         "id"}}
+                    template-tags))))))
+
+    (testing "Empty map template_tags -> preserved as empty map"
+      (mt/with-empty-h2-app-db!
+        (let [snippet-data {:serdes/meta   [{:model "NativeQuerySnippet"
+                                             :id    "test-entity-2"
+                                             :label "test_snippet_2"}]
+                            :name          "empty-tags"
+                            :content       "SELECT 1"
+                            :template_tags {}
+                            :creator_id    "test@example.com"
+                            :entity_id     "test-entity-2"}
+              ingestion    (ingestion-in-memory [snippet-data])]
+          (serdes.load/load-metabase! ingestion)
+          (let [template-tags (t2/select-one-fn :template_tags :model/NativeQuerySnippet :entity_id "test-entity-2")]
+            (is (= {} template-tags))))))
+
+    (testing "Snippet template tags get preserved rather than recalculated"
+      (mt/with-empty-h2-app-db!
+        (let [snippet-data {:serdes/meta   [{:model "NativeQuerySnippet"
+                                             :id    "test-entity-3"
+                                             :label "test_snippet_3"}]
+                            :name          "with-tags"
+                            :content       "WHERE id = {{snippet: id}}"
+                            :template_tags {"id" {:type         :snippet
+                                                  :name         "snippet: id"
+                                                  :snippet-name "id"
+                                                  ;; Definitely not calculated that way:
+                                                  :display-name "Snippet: WOOP"}}
+                            :creator_id    "test@example.com"
+                            :entity_id     "test-entity-3"}
+              ingestion    (ingestion-in-memory [snippet-data])]
+          (serdes.load/load-metabase! ingestion)
+          (let [template-tags (t2/select-one-fn :template_tags :model/NativeQuerySnippet :entity_id "test-entity-3")]
+            (is (=? {"snippet: id" {:type         :snippet
+                                    :name         "snippet: id"
+                                    :display-name "Snippet: WOOP"}}
+                    template-tags))))))))
+
 (deftest load-action-test
   (let [serialized (atom nil)
         eid (u/generate-nano-id)]

enterprise/backend/test/metabase_enterprise/transforms/api/transform_job_test.clj

@@ -3,9 +3,13 @@
    [clojure.test :refer :all]
    [metabase-enterprise.transforms.models.transform-job]
    [metabase-enterprise.transforms.models.transform-tag]
+   [metabase-enterprise.transforms.schedule :as transforms.schedule]
+   [metabase-enterprise.transforms.test-util :refer [parse-instant utc-timestamp]]
    [metabase.test :as mt]
    [toucan2.core :as t2]))
 
+(set! *warn-on-reflection* true)
+
 (deftest create-job-test
   (testing "POST /api/ee/transform-job"
     (mt/with-premium-features #{:transforms}
@@ -77,16 +81,58 @@
 (deftest list-jobs-test
   (testing "GET /api/ee/transform-job"
     (mt/with-premium-features #{:transforms}
-      (mt/with-temp [:model/TransformJob job1 {:name "Job 1" :schedule "0 0 0 * * ?"}
-                     :model/TransformJob job2 {:name "Job 2" :schedule "0 0 */4 * * ?"}]
-        (let [response (mt/user-http-request :crowberto :get 200 "ee/transform-job")
-              job-ids  (set (map :id response))]
-          (is (contains? job-ids (:id job1)))
-          (is (contains? job-ids (:id job2)))
-          (is (every? #(or
-                        (and (not= (:id %) (:id job1))
-                             (not= (:id %) (:id job2)))
-                        (nil? (:last_run %))) response)))))))
+      (let [at-5-second-schedule "5 * * * * ?"]
+        (mt/with-temp [:model/TransformTag {t1-id :id} {:name "test-tag"}
+                       :model/TransformTag {t2-id :id} {:name "second test-tag"}
+                       :model/TransformJob {j1-id :id} {:name "Job 1" :schedule at-5-second-schedule}
+                       :model/TransformJob {j2-id :id} {:name "Job 2" :schedule "0 0 0 * * ?"}
+                       :model/TransformJob {j3-id :id} {:name "Job 3" :schedule "0 0 0 * * ?"}
+                       :model/TransformJobTransformTag _ {:job_id j1-id :tag_id t1-id :position 0}
+                       :model/TransformJobTransformTag _ {:job_id j2-id :tag_id t1-id :position 1}
+                       :model/TransformJobTransformTag _ {:job_id j3-id :tag_id t2-id :position 0}
+                       :model/TransformJobRun _ {:job_id j1-id :status "timeout" :run_method "cron"
+                                                 :start_time (parse-instant "2025-08-25T10:12:11")
+                                                 :end_time (parse-instant "2025-08-26T10:52:17")}
+                       :model/TransformJobRun _ {:job_id j2-id :status "started" :run_method "manual"
+                                                 :start_time (parse-instant "2025-08-26T10:12:11")
+                                                 :end_time nil
+                                                 :is_active true}]
+          (let [our-job-ids #{j1-id j2-id j3-id}
+                returned-jobs (fn [response] (into [] (filter (comp our-job-ids :id)) response))
+                returned-job-ids (fn [response] (into #{} (keep (comp our-job-ids :id)) response))]
+            (testing "listing without filtering"
+              (let [response (mt/user-http-request :crowberto :get 200 "ee/transform-job")]
+                (is (= our-job-ids (returned-job-ids response)))))
+            (testing "filtering by transform_tag_ids"
+              (let [response (mt/user-http-request :crowberto :get 200 "ee/transform-job" :transform_tag_ids [t2-id])]
+                (is (=? [{:schedule "0 0 0 * * ?"
+                          :tag_ids [t2-id]
+                          :name "Job 3"
+                          :id j3-id}]
+                        (returned-jobs response)))))
+            (testing "filtering by last_run_start_time"
+              (let [response (mt/user-http-request :crowberto :get 200 "ee/transform-job" :last_run_start_time "2025-08-26~")]
+                (is (= #{j2-id} (returned-job-ids response)))))
+            (testing "filtering by last_run_end_time without scheduled job"
+              (let [response (mt/user-http-request :crowberto :get 200 "ee/transform-job" :next_run_start_time "2025-08-26~")]
+                (is (= #{} (returned-job-ids response)))))
+            (testing "filtering by last_run_end_time with scheduled job"
+              (mt/with-temp-scheduler!
+                (transforms.schedule/initialize-job! {:id j1-id, :schedule at-5-second-schedule})
+                (try
+                  (let [response (mt/user-http-request :crowberto :get 200 "ee/transform-job" :next_run_start_time "2025-08-27~")]
+                    (is (=? [{:id j1-id
+                              :last_run {:job_id j1-id
+                                         :run_method "cron"
+                                         :start_time (utc-timestamp "2025-08-25T10:12:11")
+                                         :end_time (utc-timestamp "2025-08-26T10:52:17")}
+                              :name "Job 1"
+                              :next_run {:start_time #"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ"}
+                              :schedule at-5-second-schedule
+                              :tag_ids [t1-id]}]
+                            (returned-jobs response))))
+                  (finally
+                    (transforms.schedule/delete-job! j1-id)))))))))))
 
 (deftest update-job-test
   (testing "PUT /api/ee/transform-job/:id"

enterprise/backend/test/metabase_enterprise/transforms/api_test.clj

@@ -8,7 +8,7 @@
    [metabase-enterprise.transforms.models.transform-transform-tag]
    [metabase-enterprise.transforms.query-test-util :as query-test-util]
    [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
-   [metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]
+   [metabase-enterprise.transforms.test-util :as transforms.tu :refer [parse-instant with-transform-cleanup! utc-timestamp]]
    [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.driver :as driver]
    [metabase.legacy-mbql.normalize :as mbql.normalize]
@@ -17,11 +17,8 @@
    [metabase.lib.metadata :as lib.metadata]
    [metabase.query-processor :as qp]
    [metabase.test :as mt]
-   [metabase.test.util :as tu]
    [metabase.util :as u]
-   [toucan2.core :as t2])
-  (:import
-   (java.time LocalDateTime ZonedDateTime ZoneId)))
+   [toucan2.core :as t2]))
 
 (set! *warn-on-reflection* true)
 
@@ -363,8 +360,8 @@
                                         :target  (assoc target :database (mt/id))}
                     {transform-id :id} (mt/user-http-request :crowberto :post 200 "ee/transform"
                                                              original)]
-                (do (test-run transform-id)
-                    (wait-for-table table-name 5000))
+                (test-run transform-id)
+                (wait-for-table table-name 5000)
                 (is (true? (driver/table-exists? driver/*driver* (mt/db) target)))
                 (is (= [["Alice" 25] ["Bob" 30]]
                        (transforms.tu/table-rows table-name)))))))))))
@@ -432,20 +429,6 @@
   (let [response (apply mt/user-http-request :crowberto :get 200 "ee/transform/run" filters)]
     (filter our-pred (:data response))))
 
-(defn- parse-timestamp
-  ^ZonedDateTime [timestamp-string]
-  (-> timestamp-string
-      LocalDateTime/parse
-      (.atZone (ZoneId/systemDefault))))
-
-(defn- parse-instant
-  [timestamp-string]
-  (-> timestamp-string parse-timestamp .toInstant))
-
-(defn- zoned-timestamp
-  [timestamp-string]
-  (-> timestamp-string parse-timestamp str))
-
 (deftest get-runs-filter-by-multiple-statuses-test
   (testing "GET /api/ee/transform/run - filter by multiple statuses"
     (mt/with-premium-features #{:transforms}
@@ -457,9 +440,9 @@
                      :model/TransformRun {r1-id  :id} {:transform_id t0-id :status "succeeded" :run_method "manual"
                                                        :start_time (parse-instant "2025-08-26T10:12:11")
                                                        :end_time (parse-instant "2025-08-27T10:52:17")}
-                     :model/TransformRun {_r2-id :id} {:transform_id t1-id :status "succeeded" :run_method "cron"
-                                                       :start_time (parse-instant "2025-08-22T10:12:11")
-                                                       :end_time (parse-instant "2025-08-22T10:12:17")}
+                     :model/TransformRun {r2-id :id} {:transform_id t1-id :status "succeeded" :run_method "cron"
+                                                      :start_time (parse-instant "2025-08-22T10:12:11")
+                                                      :end_time (parse-instant "2025-08-22T10:12:17")}
                      :model/TransformRun {r3-id :id} {:transform_id t1-id :status "succeeded" :run_method "manual"
                                                       :start_time (parse-instant "2025-08-22T23:57:34")
                                                       :end_time (parse-instant "2025-08-23T00:17:41")}
@@ -474,15 +457,15 @@
                                                        :end_time nil :is_active true}]
         (let [our-run-pred (comp #{t0-id t1-id} :transform_id)
               t0-runs [{:id r1-id
-                        :start_time (zoned-timestamp "2025-08-26T10:12:11")
-                        :end_time (zoned-timestamp "2025-08-27T10:52:17")
+                        :start_time (utc-timestamp "2025-08-26T10:12:11")
+                        :end_time (utc-timestamp "2025-08-27T10:52:17")
                         :run_method "manual"
                         :status "succeeded"
                         :transform {:id t0-id}
                         :transform_id t0-id}
                        {:id r0-id
-                        :start_time (zoned-timestamp "2025-08-25T10:12:11")
-                        :end_time (zoned-timestamp "2025-08-26T10:52:17")
+                        :start_time (utc-timestamp "2025-08-25T10:12:11")
+                        :end_time (utc-timestamp "2025-08-26T10:52:17")
                         :run_method "cron"
                         :status "timeout"
                         :transform {:id t0-id}
@@ -494,21 +477,33 @@
               (is (every? #(contains? statuses (:status %)) our-runs))))
           (testing "Filter by 'start_time'"
             (is (=? [{:id r1-id
-                      :start_time (zoned-timestamp "2025-08-26T10:12:11")
-                      :end_time (zoned-timestamp "2025-08-27T10:52:17")
+                      :start_time (utc-timestamp "2025-08-26T10:12:11")
+                      :end_time (utc-timestamp "2025-08-27T10:52:17")
                       :run_method "manual"
                       :status "succeeded"
                       :transform {:id t0-id}
                       :transform_id t0-id}]
                     (transform-runs our-run-pred :start_time "2025-08-26~")))
             (let [our-runs (transform-runs our-run-pred :start_time "~2025-08-25")]
-              (is (= 6 (count our-runs)))
-              (is (every? (comp neg-int? #(compare % "2025-08-26") :start_time) our-runs)))
+              (is (= 6 (count our-runs))))
             (let [our-runs (transform-runs our-run-pred :start_time "2025-08-22~2025-08-23")]
-              (is (= 2 (count our-runs)))
-              (is (every? (every-pred (comp nat-int? #(compare % "2025-08-22") :start_time)
-                                      (comp neg-int? #(compare % "2025-08-24") :start_time))
-                          our-runs))))
+              (is (=? [{:transform {:id t1-id}
+                        :run_method "manual"
+                        :is_active nil
+                        :start_time (utc-timestamp "2025-08-22T23:57:34")
+                        :end_time (utc-timestamp "2025-08-23T00:17:41")
+                        :transform_id t1-id
+                        :status "succeeded"
+                        :id r3-id}
+                       {:transform {:id t1-id}
+                        :run_method "cron"
+                        :is_active nil
+                        :start_time (utc-timestamp "2025-08-22T10:12:11")
+                        :end_time (utc-timestamp "2025-08-22T10:12:17")
+                        :transform_id t1-id
+                        :status "succeeded"
+                        :id r2-id}]
+                      our-runs))))
           (testing "Filter by 'end_time'"
             (is (=? t0-runs
                     (transform-runs our-run-pred :end_time "2025-08-26~")))
@@ -526,8 +521,8 @@
             (is (=? [{:id r3-id
                       :status "succeeded"
                       :run_method "manual"
-                      :start_time (zoned-timestamp "2025-08-22T23:57:34")
-                      :end_time (zoned-timestamp "2025-08-23T00:17:41")
+                      :start_time (utc-timestamp "2025-08-22T23:57:34")
+                      :end_time (utc-timestamp "2025-08-23T00:17:41")
                       :transform {:id t1-id}
                       :transform_id t1-id}]
                     (transform-runs our-run-pred :run_methods ["manual"] :start_time "~2025-08-25" :end_time "~2025-08-23")))))))))

enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj

@@ -5,15 +5,18 @@
    [metabase-enterprise.transforms.execute :as transforms.execute]
    [metabase-enterprise.transforms.query-test-util :as query-test-util]
    [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
-   [metabase-enterprise.transforms.test-util :refer [with-transform-cleanup!]]
+   [metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]
    [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.driver :as driver]
+   [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
    [metabase.lib.core :as lib]
    [metabase.lib.metadata :as lib.metadata]
    [metabase.query-processor :as qp]
+   [metabase.sync.core :as sync]
    [metabase.test :as mt]
    [metabase.test.data.sql :as sql.tx]
-   [toucan2.core :as t2]))
+   [toucan2.core :as t2]
+   [toucan2.util :as u]))
 
 (set! *warn-on-reflection* true)
 
@@ -146,3 +149,193 @@
           (let [table-exists? (driver/table-exists? driver db-id {:schema schema-name :name table-name})]
             (is (some? table-exists?) "Table should exist in the database schema")
             (driver/drop-table! driver db-id (:name table-schema))))))))
+
+(deftest transform-schema-created-if-needed-test
+  (mt/test-drivers (mt/normal-driver-select {:+features [:transforms/table :schemas]})
+    (mt/dataset transforms-dataset/transforms-test
+      (with-transform-cleanup! [target-table {:type   "table"
+                                              :schema (str "transform_schema_" (mt/random-name))
+                                              :name   "widget_products"}]
+        (let [mp (mt/metadata-provider)
+              transforms-products (lib.metadata/table mp (mt/id :transforms_products))
+              products-category (lib.metadata/field mp (mt/id :transforms_products :category))
+              products-id (lib.metadata/field mp (mt/id :transforms_products :id))
+              query (-> (lib/query mp transforms-products)
+                        (lib/filter (lib/= products-category "Widget"))
+                        (lib/order-by products-id :asc))]
+          (mt/with-temp [:model/Transform transform {:name   "transform"
+                                                     :source {:type  :query
+                                                              :query query}
+                                                     :target target-table}]
+            (transforms.execute/run-mbql-transform! transform {:run-method :manual})
+            (let [table-result      (wait-for-table (:name target-table) 10000)
+                  query-result (->> (lib/query mp table-result)
+                                    (qp/process-query)
+                                    (mt/formatted-rows [int str str 2.0 str])
+                                    (sort-by first <))]
+              (is (= [[1 "Widget A" "Widget" 19.99 "2024-01-01T10:00:00Z"]
+                      [7 "Widget B" "Widget" 24.99 "2024-01-07T10:00:00Z"]
+                      [9 "Widget C" "Widget" 14.99 "2024-01-09T10:00:00Z"]
+                      [10 "Widget D" "Widget" 34.99 "2024-01-10T10:00:00Z"]
+                      [15 "Widget E" "Widget" 44.99 "2024-01-15T10:00:00Z"]]
+                     query-result)))))))))
+
+;; TODO(rileythomp, 2025-08-28): Make this test driver agnostic
+(deftest no-create-schema-permissions-test
+  (mt/test-driver :postgres
+    (mt/dataset transforms-dataset/transforms-test
+      (let [details (:details (mt/db))
+            password (:password details)
+            no-schema-user (u/lower-case-en (mt/random-name))
+            spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))]
+        (try
+          (driver/execute-raw-queries! driver/*driver* spec
+                                       [[(format "CREATE ROLE %s WITH LOGIN PASSWORD '%s';" no-schema-user password)]
+                                        [(format "REVOKE CREATE ON DATABASE \"%s\" FROM %s;" (:db details) no-schema-user)]
+                                        [(format "GRANT USAGE ON SCHEMA public TO %s;" no-schema-user)]
+                                        [(format "GRANT SELECT ON ALL TABLES IN SCHEMA public TO %s;" no-schema-user)]
+                                        [(format "GRANT CREATE ON SCHEMA public TO %s;" no-schema-user)]])
+          (mt/with-temp [:model/Database db {:engine :postgres
+                                             :details (assoc details
+                                                             :user no-schema-user)}]
+            (mt/with-db db
+              (sync/sync-database! db {:scan :schema})
+              (let [mp (mt/metadata-provider)
+                    transforms-products (lib.metadata/table mp (mt/id :transforms_products))
+                    products-category (lib.metadata/field mp (mt/id :transforms_products :category))
+                    products-id (lib.metadata/field mp (mt/id :transforms_products :id))
+                    query (-> (lib/query mp transforms-products)
+                              (lib/filter (lib/= products-category "Widget"))
+                              (lib/order-by products-id :asc))]
+                (testing "user without create schema permissions should be able to create tables in existing schema"
+                  (with-transform-cleanup! [target-table {:type   "table"
+                                                          :schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))
+                                                          :name   "widget_products"}]
+                    (mt/with-temp [:model/Transform transform {:name   "transform"
+                                                               :source {:type  :query
+                                                                        :query query}
+                                                               :target target-table}]
+                      (transforms.execute/run-mbql-transform! transform {:run-method :manual})
+                      (let [table-result      (wait-for-table (:name target-table) 10000)
+                            query-result (->> (lib/query mp table-result)
+                                              (qp/process-query)
+                                              (mt/formatted-rows [int str str 2.0 str])
+                                              (sort-by first <))]
+                        (is (= [[1 "Widget A" "Widget" 19.99 "2024-01-01T10:00:00Z"]
+                                [7 "Widget B" "Widget" 24.99 "2024-01-07T10:00:00Z"]
+                                [9 "Widget C" "Widget" 14.99 "2024-01-09T10:00:00Z"]
+                                [10 "Widget D" "Widget" 34.99 "2024-01-10T10:00:00Z"]
+                                [15 "Widget E" "Widget" 44.99 "2024-01-15T10:00:00Z"]]
+                               query-result))))))
+                (testing "user without create schema permissions should not be able to create a new schema"
+                  (with-transform-cleanup! [target-table {:type   "table"
+                                                          :schema (str "transform_schema_" (mt/random-name))
+                                                          :name   "widget_products"}]
+                    (mt/with-temp [:model/Transform transform {:name   "transform"
+                                                               :source {:type  :query
+                                                                        :query query}
+                                                               :target target-table}]
+
+                      (is (thrown-with-msg?
+                           clojure.lang.ExceptionInfo
+                           #"ERROR: permission denied for database transforms-test"
+                           (transforms.execute/run-mbql-transform! transform {:run-method :manual})))))))))
+          (finally
+            (driver/execute-raw-queries! driver/*driver* spec
+                                         [[(format "DROP OWNED BY %s;" no-schema-user)]
+                                          [(format "DROP ROLE IF EXISTS %s;" no-schema-user)]])))))))
+
+(deftest atomic-python-transform-swap-test
+  (testing "Python transform execution with atomic table swap"
+    (mt/test-drivers #{:h2 :postgres}
+      (mt/with-premium-features #{:transforms}
+        (mt/dataset transforms-dataset/transforms-test
+          (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
+            (with-transform-cleanup! [{table-name :name :as target} {:type   "table"
+                                                                     :schema schema
+                                                                     :name   "atomic_python_swap"}]
+              (let [initial-transform {:name   "Python Transform Initial"
+                                       :source {:type  "python"
+                                                :source-tables {}
+                                                :body  (str "import pandas as pd\n"
+                                                            "\n"
+                                                            "def transform():\n"
+                                                            "    return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}
+                                       :target (assoc target :database (mt/id))}]
+                (mt/with-temp [:model/Transform transform initial-transform]
+                  (transforms.execute/execute-python-transform! transform {:run-method :manual})
+                  (wait-for-table table-name 10000)
+
+                  (let [initial-rows (transforms.tu/table-rows table-name)]
+                    (is (= [["Alice" 25] ["Bob" 30]] initial-rows) "Initial data should be Alice and Bob")
+
+                    (t2/update! :model/Transform (:id transform)
+                                {:source {:type "python"
+                                          :source-tables {}
+                                          :body (str "import pandas as pd\n"
+                                                     "\n"
+                                                     "def transform():\n"
+                                                     "    return pd.DataFrame({'name': ['Charlie', 'Diana', 'Eve'], 'age': [35, 40, 45]})")}}))
+
+                  ;; Test atomic swap by intercepting drop operation
+                  (let [drop-latch (java.util.concurrent.CountDownLatch. 1)
+                        original-drop-table! transforms.util/drop-table!]
+                    (with-redefs [transforms.util/drop-table! (fn [driver db-id table-name]
+                                                                ;; Wait for latch before proceeding with drop
+                                                                (.await drop-latch)
+                                                                (original-drop-table! driver db-id table-name))]
+                      ;; Start transform in background
+                      (let [transform-future (future
+                                               (transforms.execute/execute-python-transform!
+                                                (t2/select-one :model/Transform (:id transform))
+                                                {:run-method :manual}))]
+                        ;; Give transform time to create temp table and reach the drop operation
+                        (Thread/sleep 1000)
+                        ;; Verify old table is still accessible during transform
+                        (is (= [["Alice" 25] ["Bob" 30]]
+                               (transforms.tu/table-rows table-name))
+                            "Original data should still be accessible during transform")
+                        ;; Release the latch to allow drop and complete transform
+                        (.countDown drop-latch)
+                        ;; Wait for transform to complete
+                        @transform-future
+                        ;; Verify final state
+                        (is (= [["Charlie" 35] ["Diana" 40] ["Eve" 45]]
+                               (transforms.tu/table-rows table-name))
+                            "Table should now contain Charlie, Diana, and Eve")))))))))))))
+
+(deftest python-transform-temp-table-cleanup-test
+  (testing "Python transform cleans up temp tables on success"
+    (mt/test-drivers #{:h2 :postgres}
+      (mt/with-premium-features #{:transforms}
+        (mt/dataset transforms-dataset/transforms-test
+          (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
+            (with-transform-cleanup! [{table-name :name :as target} {:type   "table"
+                                                                     :schema schema
+                                                                     :name   "python_cleanup_test"}]
+              (let [transform-def {:name   "Python Transform Cleanup"
+                                   :source {:type  "python"
+                                            :source-tables {}
+                                            :body  (str "import pandas as pd\n"
+                                                        "\n"
+                                                        "def transform():\n"
+                                                        "    return pd.DataFrame({'col1': [1, 2, 3], 'col2': ['a', 'b', 'c']})")}
+                                   :target (assoc target :database (mt/id))}]
+                (mt/with-temp [:model/Transform transform transform-def]
+                  ;; Run initial transform
+                  (transforms.execute/execute-python-transform! transform {:run-method :manual})
+                  (wait-for-table table-name 10000)
+
+                  ;; Run again to trigger temp table creation and swap
+                  (transforms.execute/execute-python-transform! transform {:run-method :manual})
+
+                  ;; Check that no temp tables remain
+                  (let [db-id (mt/id)
+                        tables (t2/select :model/Table :db_id db-id :active true)
+                        temp-table-pattern (re-pattern (str ".*" table-name "_temp_.*"))]
+                    (is (not-any? #(re-matches temp-table-pattern (:name %)) tables)
+                        "No temp tables should remain after successful Python transform")
+
+                    ;; Verify the final data is correct
+                    (is (= [[1 "a"] [2 "b"] [3 "c"]] (transforms.tu/table-rows table-name))
+                        "Table should contain the expected data after swap")))))))))))

enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj

@@ -1,25 +1,46 @@
 (ns ^:mb/driver-tests metabase-enterprise.transforms.ordering-test
   (:require
    [clojure.test :refer :all]
+   [metabase-enterprise.transforms.execute :as transforms.execute]
    [metabase-enterprise.transforms.ordering :as ordering]
+   [metabase.driver :as driver]
+   [metabase.driver.sql :as driver.sql]
    [metabase.query-processor.compile :as qp.compile]
    [metabase.test :as mt]
+   [metabase.test.data.sql :as sql.tx]
    [toucan2.core :as t2]))
 
-(defn- make-transform [{:keys [query name]}]
-  {:source {:type "query",
-            :query query}
-   :name "transform1"
-   :target {:schema "public"
-            :name (or name "orders_2")
-            :type "table"}})
+(defn- make-transform [query & [name schema]]
+  (let [name (or name (mt/random-name))
+        default-schema (some-> driver/*driver* driver.sql/default-schema)
+        schema (or schema default-schema "public")]
+    {:source {:type :query
+              :query query}
+     :name (str "transform_" name)
+     :target {:schema schema
+              :name name
+              :type :table}}))
+
+(defn- make-python-transform [source-tables & [name schema target-db]]
+  (let [name (or name (mt/random-name))
+        default-schema (some-> driver/*driver* driver.sql/default-schema)
+        schema (or schema default-schema "public")
+        target-db (or target-db (mt/id))]
+    {:source {:type "python"
+              :source-tables source-tables
+              :body "# Python code here\nresult = df"}
+     :name (str "python_transform_" name)
+     :target {:database target-db
+              :schema schema
+              :name name
+              :type "table"}}))
 
 (deftest basic-ordering-test
   (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
     (mt/with-temp [:model/Transform {t1 :id} (make-transform
-                                              {:query {:database (mt/id),
-                                                       :type "query",
-                                                       :query {:source-table (mt/id :orders)}}})]
+                                              {:database (mt/id),
+                                               :type "query",
+                                               :query {:source-table (mt/id :orders)}})]
       (is (= {t1 #{}}
              (ordering/transform-ordering (t2/select :model/Transform :id t1)))))))
 
@@ -30,14 +51,15 @@
                    :model/Field _ {:table_id table
                                    :name "foo"}
                    :model/Transform {parent :id} (make-transform
-                                                  {:query {:database (mt/id),
-                                                           :type "query",
-                                                           :query {:source-table (mt/id :orders)}}})
+                                                  {:database (mt/id),
+                                                   :type "query",
+                                                   :query {:source-table (mt/id :orders)}}
+                                                  "orders_2")
                    :model/Transform {child :id} (make-transform
-                                                 {:query {:database (mt/id)
-                                                          :type "query"
-                                                          :query {:source-table table}}
-                                                  :name "orders_3"})]
+                                                 {:database (mt/id)
+                                                  :type "query"
+                                                  :query {:source-table table}}
+                                                 "orders_3")]
       (is (= {parent #{}
               child #{parent}}
              (ordering/transform-ordering (t2/select :model/Transform :id [:in [parent child]])))))))
@@ -46,36 +68,134 @@
   (mt/with-metadata-provider (mt/id)
     (#'ordering/transform-deps transform)))
 
+(deftest not-run-transform-dependency-ordering-test
+  (mt/test-driver :postgres
+    (testing "dependencies are correctly identified when no transforms have been run yet"
+      (mt/with-temp [:model/Transform {t1 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type :native,
+                                                 :native {:query "SELECT * FROM orders LIMIT 100"}}
+                                                "orders_transform")
+                     :model/Transform {t2 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type :native,
+                                                 :native {:query "SELECT * FROM products LIMIT 100"}}
+                                                "products_transform")
+                     :model/Transform {t3 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type :native,
+                                                 :native {:query "SELECT * FROM orders_transform ot
+                                                                  LEFT JOIN products p
+                                                                  ON ot.product_id = p.id
+                                                                  LIMIT 100"}}
+                                                "orders_transform_products")
+                     :model/Transform {t4 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type :native,
+                                                 :native {:query "SELECT * FROM orders_transform ot
+                                                                  LEFT JOIN products_transform pt
+                                                                  ON ot.product_id = pt.id
+                                                                  LIMIT 100"}}
+                                                "orders_transform_products_transform")]
+        (is (= #{{:transform t1} {:transform t2}}
+               (transform-deps-for-db (t2/select-one :model/Transform  t4))))
+        (is (= {t1 #{}
+                t2 #{}
+                t3 #{t1}
+                t4 #{t1 t2}}
+               (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3 t4]]))))))
+    (testing "dependencies are correctly identified when some transform have been run and some haven't"
+      (mt/with-temp [:model/Transform {t1 :id :as transform1} (make-transform
+                                                               {:database (mt/id),
+                                                                :type :native,
+                                                                :native {:query "SELECT * FROM checkins LIMIT 100"}}
+                                                               "checkins_transform")
+                     :model/Transform {t2 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type :native,
+                                                 :native {:query "SELECT * FROM venues LIMIT 100"}}
+                                                "venues_transform")
+                     :model/Transform {t3 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type :native,
+                                                 :native {:query "SELECT * FROM venues_transform vt
+                                                                  LEFT JOIN checkins_transform ct
+                                                                  ON vt.id = ct.venue_id
+                                                                  LIMIT 100"}}
+                                                "venues_transform_2")]
+        (try
+          (transforms.execute/run-mbql-transform! transform1 {:run-method :manual})
+          (let [table1 (t2/select-one-pk :model/Table :name "checkins_transform")]
+            (is (= #{{:table table1} {:transform t2}}
+                   (transform-deps-for-db (t2/select-one :model/Transform  t3)))))
+          (is (= {t1 #{}
+                  t2 #{}
+                  t3 #{t1 t2}}
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]]))))
+          (finally
+            (t2/delete! :model/Table :name "checkins_transform")))))))
+
+(deftest bigquery-native-transform-dependency-ordering-test
+  (mt/test-driver :bigquery-cloud-sdk
+    (let [[dataset] (sql.tx/qualified-name-components :bigquery-cloud-sdk nil)]
+      (testing "dependencies are correctly identified when some transform have been run and some haven't"
+        (mt/with-temp [:model/Transform {t1 :id} (make-transform
+                                                  {:database (mt/id),
+                                                   :type :native,
+                                                   :native {:query (format "SELECT * FROM `%s.checkins` LIMIT 100" dataset)}}
+                                                  "checkins_transform" dataset)
+                       :model/Table {table1 :id} {:schema dataset
+                                                  :name "checkins_transform"}
+                       :model/Transform {t2 :id} (make-transform
+                                                  {:database (mt/id),
+                                                   :type :native,
+                                                   :native {:query (format "SELECT * FROM `%s.venues` LIMIT 100" dataset)}}
+                                                  "venues_transform" dataset)
+                       :model/Transform {t3 :id} (make-transform
+                                                  {:database (mt/id),
+                                                   :type :native,
+                                                   :native {:query (format "SELECT * FROM `%s.venues_transform` vt
+                                                                            LEFT JOIN `%s.checkins_transform` ct
+                                                                            ON vt.id = ct.venue_id
+                                                                            LIMIT 100" dataset dataset)}}
+                                                  "venues_transform_2" dataset)]
+          (is (= #{{:table table1} {:transform t2}}
+                 (transform-deps-for-db (t2/select-one :model/Transform  t3))))
+          (is (= {t1 #{}
+                  t2 #{}
+                  t3 #{t1 t2}}
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]])))))))))
+
 (deftest basic-dependencies-test
   (mt/with-temp [:model/Transform {t1 :id} (make-transform
-                                            {:query {:database (mt/id),
-                                                     :type "query",
-                                                     :query {:source-table (mt/id :orders)}}})]
-    (is (= #{(mt/id :orders)}
+                                            {:database (mt/id),
+                                             :type "query",
+                                             :query {:source-table (mt/id :orders)}})]
+    (is (= #{{:table (mt/id :orders)}}
            (transform-deps-for-db (t2/select-one :model/Transform :id t1))))))
 
 (deftest joined-dependencies-test
   (mt/with-temp [:model/Transform {t1 :id} (make-transform
-                                            {:query {:database (mt/id),
-                                                     :type "query",
-                                                     :query {:source-table (mt/id :orders),
-                                                             :joins
-                                                             [{:fields "all",
-                                                               :strategy "left-join",
-                                                               :alias "Products",
-                                                               :condition
-                                                               [:=
-                                                                [:field
-                                                                 (mt/id :orders :product_id)
-                                                                 {:base-type "type/Integer"}]
-                                                                [:field
-                                                                 (mt/id :products :id)
-                                                                 {:base-type "type/Integer",
-                                                                  :join-alias "Products"}]],
-                                                               :source-table (mt/id :products)}]},
-                                                     :parameters []}})]
-    (is (= #{(mt/id :orders)
-             (mt/id :products)}
+                                            {:database (mt/id),
+                                             :type "query",
+                                             :query {:source-table (mt/id :orders),
+                                                     :joins
+                                                     [{:fields "all",
+                                                       :strategy "left-join",
+                                                       :alias "Products",
+                                                       :condition
+                                                       [:=
+                                                        [:field
+                                                         (mt/id :orders :product_id)
+                                                         {:base-type "type/Integer"}]
+                                                        [:field
+                                                         (mt/id :products :id)
+                                                         {:base-type "type/Integer",
+                                                          :join-alias "Products"}]],
+                                                       :source-table (mt/id :products)}]},
+                                             :parameters []})]
+    (is (= #{{:table (mt/id :orders)}
+             {:table (mt/id :products)}}
            (transform-deps-for-db (t2/select-one :model/Transform :id t1))))))
 
 (deftest card-dependencies-test
@@ -99,27 +219,27 @@
                                                                      :source-table (mt/id :products)}]},
                                                            :parameters []}}
                    :model/Transform {t1 :id} (make-transform
-                                              {:query {:database (mt/id),
-                                                       :type "query",
-                                                       :query
-                                                       {:aggregation [["count"]],
-                                                        :breakout [["field" "Products__category" {:base-type "type/Text"}]],
-                                                        :source-table (str "card__" card)},
-                                                       :parameters []}})]
-      (is (= #{(mt/id :orders)
-               (mt/id :products)}
+                                              {:database (mt/id),
+                                               :type "query",
+                                               :query
+                                               {:aggregation [["count"]],
+                                                :breakout [["field" "Products__category" {:base-type "type/Text"}]],
+                                                :source-table (str "card__" card)},
+                                               :parameters []})]
+      (is (= #{{:table (mt/id :orders)}
+               {:table (mt/id :products)}}
              (transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))
 
 (deftest native-dependencies-test
   (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
     (mt/with-temp [:model/Transform {t1 :id} (make-transform
-                                              {:query {:database (mt/id),
-                                                       :type :native
-                                                       :native (qp.compile/compile
-                                                                {:database (mt/id),
-                                                                 :type "query",
-                                                                 :query {:source-table (mt/id :orders)}})}})]
-      (is (= #{(mt/id :orders)}
+                                              {:database (mt/id),
+                                               :type :native
+                                               :native (qp.compile/compile
+                                                        {:database (mt/id),
+                                                         :type "query",
+                                                         :query {:source-table (mt/id :orders)}})})]
+      (is (= #{{:table (mt/id :orders)}}
              (transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))
 
 (deftest native-card-dependencies-test
@@ -145,21 +265,20 @@
                                                                      :source-table (mt/id :products)}]},
                                                            :parameters []}}
                    :model/Transform {t1 :id} (make-transform
-                                              {:query {:database (mt/id),
-                                                       :type :native,
-                                                       :native {:query
-                                                                (mt/native-query-with-card-template-tag
-                                                                 (:engine (mt/db))
-                                                                 (str "#" card)),
-
-                                                                :template-tags
-                                                                {(keyword (str "#" card))
-                                                                 {:type "card"
-                                                                  :name "card"
-                                                                  :display-name "card"
-                                                                  :card-id card}}}}})]
-      (is (= #{(mt/id :orders)
-               (mt/id :products)}
+                                              {:database (mt/id),
+                                               :type :native,
+                                               :native {:query
+                                                        (mt/native-query-with-card-template-tag
+                                                         (:engine (mt/db))
+                                                         (str "#" card)),
+                                                        :template-tags
+                                                        {(keyword (str "#" card))
+                                                         {:type "card"
+                                                          :name "card"
+                                                          :display-name "card"
+                                                          :card-id card}}}})]
+      (is (= #{{:table (mt/id :orders)}
+               {:table (mt/id :products)}}
              (transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))
 
 (defn- rotations
@@ -181,3 +300,173 @@
         cycles (rotations [3 4 6 7])]
     (is (contains? cycles
                    (ordering/find-cycle test-graph)))))
+
+(deftest get-transform-cycle-test
+  (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+    (testing "cycle is detected in transform referencing itself"
+      (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                :name "table_1"}
+                     :model/Field _ {:table_id table1
+                                     :name "foo"}
+                     :model/Transform {t1 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type "query",
+                                                 :query {:source-table table1}}
+                                                "table_1")]
+        (is (= {:cycle-str "transform_table_1"
+                :cycle [t1]}
+               (ordering/get-transform-cycle (t2/select-one :model/Transform :id t1))))))
+    (testing "cycle is caught in 2 transforms referencing each other"
+      (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                :name "table_1"}
+                     :model/Field _ {:table_id table1
+                                     :name "foo"}
+                     :model/Table {table2 :id} {:schema "public"
+                                                :name "table_2"}
+                     :model/Field _ {:table_id table2
+                                     :name "foo"}
+                     :model/Transform {t1 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type "query",
+                                                 :query {:source-table table1}}
+                                                "table_2")
+                     :model/Transform {t2 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type "query",
+                                                 :query {:source-table table2}}
+                                                "table_1")]
+        (is (= {:cycle-str "transform_table_2 -> transform_table_1",
+                :cycle [t1 t2]}
+               (ordering/get-transform-cycle (t2/select-one :model/Transform :id t1))))))
+    (testing "cycle is detected in 3 transforms referencing each other"
+      (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                :name "table_1"}
+                     :model/Field _ {:table_id table1
+                                     :name "foo"}
+                     :model/Table {table2 :id} {:schema "public"
+                                                :name "table_2"}
+                     :model/Field _ {:table_id table2
+                                     :name "foo"}
+                     :model/Table {table3 :id} {:schema "public"
+                                                :name "table_3"}
+                     :model/Field _ {:table_id table3
+                                     :name "foo"}
+                     :model/Transform {t1 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type "query",
+                                                 :query {:source-table table1}}
+                                                "table_2")
+                     :model/Transform {t2 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type "query",
+                                                 :query {:source-table table2}}
+                                                "table_3")
+                     :model/Transform {t3 :id} (make-transform
+                                                {:database (mt/id),
+                                                 :type "query",
+                                                 :query {:source-table table3}}
+                                                "table_1")]
+        (is (= {:cycle-str "transform_table_2 -> transform_table_1 -> transform_table_3",
+                :cycle [t1 t3 t2]}
+               (ordering/get-transform-cycle (t2/select-one :model/Transform :id t1)))))))
+
+  (deftest python-transform-basic-dependencies-test
+    (testing "Python transforms with source-tables dependencies are extracted correctly"
+      (mt/with-temp [:model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)
+                                                                       "products" (mt/id :products)})]
+        (is (= #{{:table (mt/id :orders)}
+                 {:table (mt/id :products)}}
+               (transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))
+
+  (deftest python-transform-ordering-test
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+      (testing "Python transforms are ordered correctly based on source-tables dependencies"
+        (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                  :name "output_1"}
+                       :model/Field _ {:table_id table1
+                                       :name "foo"}
+                       :model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)} "output_1")
+                       :model/Transform {t2 :id} (make-python-transform {"output_1" table1} "output_2")]
+          (is (= {t1 #{}
+                  t2 #{t1}}
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2]]))))))))
+
+  (deftest python-transform-multiple-dependencies-test
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+      (testing "Python transforms with multiple table dependencies"
+        (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                  :name "output_1"}
+                       :model/Field _ {:table_id table1
+                                       :name "foo"}
+                       :model/Table {table2 :id} {:schema "public"
+                                                  :name "output_2"}
+                       :model/Field _ {:table_id table2
+                                       :name "bar"}
+                       :model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)} "output_1")
+                       :model/Transform {t2 :id} (make-python-transform {"products" (mt/id :products)} "output_2")
+                       :model/Transform {t3 :id} (make-python-transform {"output_1" table1
+                                                                         "output_2" table2} "final_output")]
+          (is (= {t1 #{}
+                  t2 #{}
+                  t3 #{t1 t2}}
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]]))))))))
+
+  (deftest mixed-transform-ordering-test
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+      (testing "Python and query transforms are ordered together correctly"
+        (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                  :name "sql_output"}
+                       :model/Field _ {:table_id table1
+                                       :name "foo"}
+                       :model/Table {table2 :id} {:schema "public"
+                                                  :name "python_output"}
+                       :model/Field _ {:table_id table2
+                                       :name "bar"}
+                       ;; SQL transform that depends on orders table
+                       :model/Transform {t1 :id} (make-transform
+                                                  {:database (mt/id),
+                                                   :type "query",
+                                                   :query {:source-table (mt/id :orders)}}
+                                                  "sql_output")
+                       ;; Python transform that depends on the SQL transform's output
+                       :model/Transform {t2 :id} (make-python-transform {"sql_output" table1} "python_output")
+                       ;; Another SQL transform that depends on the Python transform's output
+                       :model/Transform {t3 :id} (make-transform
+                                                  {:database (mt/id)
+                                                   :type "query"
+                                                   :query {:source-table table2}}
+                                                  "final_output")]
+          (is (= {t1 #{}
+                  t2 #{t1}
+                  t3 #{t2}}
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]])))))))))
+
+#_(deftest mixed-database-transform-ordering-test
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+      (testing "Python and query transforms are ordered together correctly, across multiple databases"
+      ;; orders (db-1) -> sql_output (db-1) -> python_output (db-2) -> final_output (db-2)
+        (mt/with-temp [:model/Database db-2    {}
+                       :model/Table    table-1 {:db_id (mt/id),    :schema "public", :name "sql_output"}
+                       :model/Table    table-2 {:db_id (:id db-2), :schema "public", :name "python_output"}
+                       :model/Field    _       {:table_id (:id table-1), :name "foo"}
+                       :model/Field    _       {:table_id (:id table-2), :name "bar"}
+
+                     ;; SQL transform that depends on orders table
+                       :model/Transform {t1 :id} (make-transform
+                                                  {:database (mt/id)
+                                                   :type     "query"
+                                                   :query    {:source-table (mt/id :orders)}}
+                                                  (:name table-1))
+                     ;; Python transform that depends on the SQL transform's output
+                       :model/Transform {t2 :id} (make-python-transform {"sql_output" (:id table-1)} (:name table-2))
+                     ;; Another SQL transform that depends on the Python transform's output
+                       :model/Transform {t3 :id} (make-transform
+                                                  {:database (:id db-2)
+                                                   :type     "query"
+                                                   :query    {:source-table (:id table-2)}}
+                                                  "final_output")]
+          (is (= {t1 #{}
+                  t2 #{t1}
+                  t3 #{t2}}
+
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]]))))))))

enterprise/backend/test/metabase_enterprise/transforms/python_runner_test.clj

@@ -8,15 +8,16 @@
    [java-time.api :as t]
    [metabase-enterprise.transforms.python-runner :as python-runner]
    [metabase-enterprise.transforms.schedule :as transforms.schedule]
+   [metabase-enterprise.transforms.settings :as transforms.settings]
    [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
    [metabase-enterprise.transforms.test-util :as transforms.tu]
    [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.driver :as driver]
-   [metabase-enterprise.transforms.settings :as transforms.settings]
    [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
    [metabase.sync.core :as sync]
    [metabase.task.core :as task]
    [metabase.test :as mt]
+   [metabase.test.data.interface :as tx]
    [metabase.test.data.sql :as sql.tx]
    [metabase.util :as u]
    [metabase.util.json :as json]
@@ -65,12 +66,13 @@
                                                                         :run-id         test-id
                                                                         :table-name->id table-name->id
                                                                         :shared-storage @shared-storage-ref})
-          {:keys [output events]} (python-runner/read-output-objects @shared-storage-ref)]
+          {:keys [output output-manifest events]} (python-runner/read-output-objects @shared-storage-ref)]
       ;; not sure about munging this all together but its what tests expect for now
       (merge (:body response)
-             {:output output
-              :stdout (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
-              :stderr (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))}))))
+             {:output          output
+              :output-manifest output-manifest
+              :stdout          (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
+              :stderr          (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))}))))
 
 (deftest ^:parallel transform-function-basic-test
   (testing "executes transform function and returns CSV output"
@@ -80,7 +82,7 @@
                               "    return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")
           result         (execute {:code transform-code})]
       (is (=? {:output "name,age\nAlice,25\nBob,30\n"
-               :stdout "Successfully saved 2 rows to CSV\nSuccessfully saved output manifest with 2 fields"
+               :stdout "Successfully saved 2 rows to S3\nSuccessfully saved output manifest with 2 fields"
                :stderr ""}
               result)))))
 
@@ -109,7 +111,7 @@
                                                  "    raise ValueError('Something went wrong')")})
           expected-template (str "ERROR: Transform function failed: Something went wrong\n"
                                  "Traceback (most recent call last):\n"
-                                 "  File \"/app/transform_runner.py\", line ___LINE___, in main\n"
+                                 "  File \"/app/external/transform_runner.py\", line ___LINE___, in main\n"
                                  "    result = script.transform()\n"
                                  "             ^^^^^^^^^^^^^^^^^^\n"
                                  "  File \"___PATH___/script.py\", line 2, in transform\n"
@@ -131,7 +133,7 @@
                               "    return pd.DataFrame(data)")
           result         (execute {:code transform-code})]
       (is (=? {:output "x,y,z\n1,10,a\n2,20,b\n3,30,c\n"
-               :stdout "Successfully saved 3 rows to CSV\nSuccessfully saved output manifest with 3 fields"
+               :stdout "Successfully saved 3 rows to S3\nSuccessfully saved output manifest with 3 fields"
                :stderr ""}
               result)))))
 
@@ -145,7 +147,7 @@
                                 "    return pd.DataFrame(data)")
             result         (execute {:code transform-code})]
         (is (=? {:output "name,score\nCharlie,85\nDana,92\n"
-                 :stdout "Successfully saved 2 rows to CSV\nSuccessfully saved output manifest with 2 fields"
+                 :stdout "Successfully saved 2 rows to S3\nSuccessfully saved output manifest with 2 fields"
                  :stderr ""}
                 result))))))
 
@@ -168,53 +170,52 @@
                                        :tables {"students" (mt/id :students)}})]
 
           (is (=? {:output   "id,name,score\n1,Alice,85\n2,Bob,92\n3,Charlie,88\n4,Dana,90\n"
-                   :metadata #(= (json/decode+kw %)
-                                 {:version         "0.1.0"
-                                  :fields          [{:base_type      "Integer",
-                                                     :dtype          "int64",
-                                                     :database_type  "int4",
-                                                     :effective_type "int4",
-                                                     :name           "id",
-                                                     :semantic_type  nil,
-                                                     :root_type      "Integer"}
-                                                    {:base_type      "Text",
-                                                     :dtype          "object",
-                                                     :database_type  "varchar",
-                                                     :effective_type "varchar",
-                                                     :name           "name",
-                                                     :semantic_type  nil,
-                                                     :root_type      "Text"}
-                                                    {:base_type      "Integer",
-                                                     :dtype          "int64",
-                                                     :database_type  "int4",
-                                                     :effective_type "int4",
-                                                     :name           "score",
-                                                     :semantic_type  nil,
-                                                     :root_type      "Integer"}],
-                                  :source_metadata {:fields         [{:base_type      "Integer",
-                                                                      :dtype          "int64",
-                                                                      :database_type  "int4",
-                                                                      :effective_type "int4",
-                                                                      :name           "id",
-                                                                      :semantic_type  nil,
-                                                                      :root_type      "Integer"}
-                                                                     {:base_type      "Text",
-                                                                      :dtype          "object",
-                                                                      :database_type  "varchar",
-                                                                      :effective_type "varchar",
-                                                                      :name           "name",
-                                                                      :semantic_type  nil,
-                                                                      :root_type      "Text"}
-                                                                     {:base_type      "Integer",
-                                                                      :dtype          "int64",
-                                                                      :database_type  "int4",
-                                                                      :effective_type "int4",
-                                                                      :name           "score",
-                                                                      :semantic_type  nil,
-                                                                      :root_type      "Integer"}],
-                                                    :table_metadata {:table_id (mt/id :students)},
-                                                    :version        "0.1.0"}})
-                   :stdout   (str "Successfully saved 4 rows to CSV\n"
+                   :output-manifest {:fields [{:base_type "Integer",
+                                               :database_type "int4",
+                                               :dtype "int64",
+                                               :effective_type "Integer",
+                                               :name "id",
+                                               :root_type "Integer",
+                                               :semantic_type "PK"}
+                                              {:base_type "Text",
+                                               :database_type "varchar",
+                                               :dtype "object",
+                                               :effective_type "Text",
+                                               :name "name",
+                                               :root_type "Text",
+                                               :semantic_type "Name"}
+                                              {:base_type "Integer",
+                                               :database_type "int4",
+                                               :dtype "int64",
+                                               :effective_type "Integer",
+                                               :name "score",
+                                               :root_type "Integer",
+                                               :semantic_type "Score"}],
+                                     :source_metadata {:fields [{:base_type "Integer",
+                                                                 :database_type "int4",
+                                                                 :dtype "int64",
+                                                                 :effective_type "Integer",
+                                                                 :name "id",
+                                                                 :root_type "Integer",
+                                                                 :semantic_type "PK"}
+                                                                {:base_type "Text",
+                                                                 :database_type "varchar",
+                                                                 :dtype "object",
+                                                                 :effective_type "Text",
+                                                                 :name "name",
+                                                                 :root_type "Text",
+                                                                 :semantic_type "Name"}
+                                                                {:base_type "Integer",
+                                                                 :database_type "int4",
+                                                                 :dtype "int64",
+                                                                 :effective_type "Integer",
+                                                                 :name "score",
+                                                                 :root_type "Integer",
+                                                                 :semantic_type "Score"}],
+                                                       :table_metadata {:table_id (mt/malli=? int?)},
+                                                       :version "0.1.0"},
+                                     :version "0.1.0"}
+                   :stdout   (str "Successfully saved 4 rows to S3\n"
                                   "Successfully saved output manifest with 3 fields")
                    :stderr   ""}
                   result)))))))
@@ -244,11 +245,10 @@
                                        :tables {"students" (mt/id :students)}})]
 
           (is (=? {:output   "student_count,average_score\n4,88.75\n"
-                   :metadata #(= (json/decode+kw %)
-                                 {:version        "0.1.0",
-                                  :fields         [{:name "student_count", :dtype "int64"}
-                                                   {:name "average_score", :dtype "float64"}]})
-                   :stdout   (str "Successfully saved 1 rows to CSV\n"
+                   :output-manifest {:version "0.1.0",
+                                     :fields  [{:name "student_count", :dtype "int64"}
+                                               {:name "average_score", :dtype "float64"}]}
+                   :stdout   (str "Successfully saved 1 rows to S3\n"
                                   "Successfully saved output manifest with 2 fields")
                    :stderr   ""}
                   result)))))))
@@ -299,7 +299,7 @@
             [row1 row2 row3] rows
             header-to-index (zipmap headers (range))
             get-col (fn [row col-name] (nth row (header-to-index col-name)))
-            metadata (some-> (:metadata result) json/decode+kw)]
+            metadata (:output-manifest result)]
 
         (is (= (set ["id" "name" "description" "count" "price" "is_active" "created_date" "updated_at" "scheduled_for"])
                (set headers)))
@@ -361,11 +361,13 @@
                 (is (true? (t2/exists? :model/Table :name (:name target))))))))))))
 
 (deftest transform-type-roundtrip-test
-  (mt/test-drivers #{:postgres :h2 :mysql :bigquery-cloud-sdk :redshift :snowflake :sqlserver}
+  (mt/test-drivers #{:postgres :h2 :mysql :bigquery-cloud-sdk :redshift :snowflake :sqlserver :clickhouse}
     (mt/with-empty-db
       (let [driver       driver/*driver*
             db-id        (mt/id)
-            table-name   (mt/random-name)
+            table-name   (if (= driver :redshift)
+                           (tx/db-qualified-table-name (get-in (mt/db) [:settings :database-source-dataset-name]) (mt/random-name))
+                           (mt/random-name))
             schema-name  (sql.tx/session-schema driver)
             qualified-table-name (if schema-name
                                    (keyword schema-name table-name)
@@ -392,14 +394,16 @@
                                           (mapv :name (:columns table-schema))
                                           {:type :rows :data row-values})
 
-            _ (sync/sync-database! (mt/db))
+            _ (sync/sync-database! (mt/db) {:scan :schema})
+
             transform-code (str "import pandas as pd\n"
                                 "\n"
                                 "def transform(" table-name "):\n"
                                 "    return " table-name)
             result (execute {:code transform-code
                              :tables {table-name (mt/id qualified-table-name)}})
-            metadata (some-> (:metadata result) json/decode+kw)]
+
+            metadata (:output-manifest result)]
 
         (testing "All expected columns are present"
           (is (= #{"id" "price" "active" "created_tz" "created_at" "created_date" "description"}
@@ -409,7 +413,7 @@
           (is (= {"id"           :type/Integer
                   "price"        :type/Float
                   "active"       :type/Boolean
-                  "created_tz"   (if (= driver/*driver* :mysql) :type/DateTime :type/DateTimeWithTZ)
+                  "created_tz"   :type/DateTimeWithTZ
                   "created_at"   :type/DateTime
                   "created_date" :type/Date
                   "description"  :type/Text}
@@ -417,4 +421,4 @@
                    [name (transforms.util/dtype->base-type dtype)]))))
 
         ;; cleanup
-        (driver/drop-table! driver db-id (:name table-schema))))))
+        (driver/drop-table! driver db-id qualified-table-name)))))

enterprise/backend/test/metabase_enterprise/transforms/test_util.clj

@@ -5,7 +5,11 @@
    [metabase.driver :as driver]
    [metabase.test :as mt]
    [metabase.util :as u]
-   [toucan2.core :as t2]))
+   [toucan2.core :as t2])
+  (:import
+   (java.time Instant LocalDateTime ZonedDateTime ZoneId)))
+
+(set! *warn-on-reflection* true)
 
 (defn drop-target!
   "Drop transform target `target` and clean up its metadata.
@@ -53,3 +57,20 @@
   (mt/rows (mt/process-query {:database (mt/id)
                               :query    {:source-table (t2/select-one-pk :model/Table :name table-name)}
                               :type     :query})))
+
+(defn parse-timestamp
+  "Parse a local datetime and convert it to a ZonedDateTime in the default timezone."
+  ^ZonedDateTime [timestamp-string]
+  (-> timestamp-string
+      LocalDateTime/parse
+      (.atZone (ZoneId/systemDefault))))
+
+(defn parse-instant
+  "Parse a local datetime and convert it to an Instant in the default timezone."
+  ^Instant [timestamp-string]
+  (-> timestamp-string parse-timestamp .toInstant))
+
+(defn utc-timestamp
+  "Parse a local datetime and convert it to a string encoding a ZonedDateTime in the default timezone."
+  ^String [timestamp-string]
+  (-> timestamp-string parse-instant str))

enterprise/frontend/src/embedding-sdk-package/package.template.json

@@ -1,6 +1,6 @@
 {
   "name": "@metabase/embedding-sdk-react",
-  "version": "0.57.0-alpha.1",
+  "version": "0.57.0-alpha.2",
   "description": "Metabase Embedding SDK for React",
   "bin": "./dist/cli.js",
   "repository": {

enterprise/frontend/src/metabase-enterprise/api/python-transform-library.ts

@@ -0,0 +1,35 @@
+import { Api } from "metabase/api";
+
+export type PythonLibrary = {
+  source: string;
+};
+
+export type GetPythonLibraryRequest = {
+  name: string;
+};
+
+export type UpdatePythonLibraryRequest = {
+  name: string;
+  source: string;
+};
+
+export const pythonLibraryApi = Api.injectEndpoints({
+  endpoints: (builder) => ({
+    getPythonLibrary: builder.query<PythonLibrary, GetPythonLibraryRequest>({
+      query: ({ name }) => ({
+        url: `/api/ee/python-transform/user-modules/${name}`,
+        method: "GET",
+      }),
+    }),
+    updatePythonLibrary: builder.mutation<void, UpdatePythonLibraryRequest>({
+      query: ({ name, source }) => ({
+        url: `/api/ee/python-transform/user-modules/${name}`,
+        method: "PUT",
+        body: { source },
+      }),
+    }),
+  }),
+});
+
+export const { useGetPythonLibraryQuery, useUpdatePythonLibraryMutation } =
+  pythonLibraryApi;

enterprise/frontend/src/metabase-enterprise/api/transform-job.ts

@@ -1,5 +1,6 @@
 import type {
   CreateTransformJobRequest,
+  ListTransformJobsRequest,
   Transform,
   TransformJob,
   TransformJobId,
@@ -19,7 +20,7 @@ import {
 
 export const transformJobApi = EnterpriseApi.injectEndpoints({
   endpoints: (builder) => ({
-    listTransformJobs: builder.query<TransformJob[], void>({
+    listTransformJobs: builder.query<TransformJob[], ListTransformJobsRequest>({
       query: (params) => ({
         method: "GET",
         url: "/api/ee/transform-job",

enterprise/frontend/src/metabase-enterprise/content_translation/tests/utils.unit.spec.ts

@@ -101,13 +101,13 @@ describe("content translation utils", () => {
       ).toBe("Whitespace Translation");
     });
 
-    it("should handle case sensitivity", () => {
+    it("should be case insensitive for msgid (metabase#61795)", () => {
       expect(translateContentString(mockDictionary, "es", "hello")).toBe(
-        "hello",
-      ); // no match
-      expect(translateContentString(mockDictionary, "ES", "Hello")).toBe(
-        "Hello",
-      ); // no match
+        "Hola",
+      );
+      expect(translateContentString(mockDictionary, "es", "WORLD")).toBe(
+        "Mundo",
+      );
     });
   });
 

enterprise/frontend/src/metabase-enterprise/content_translation/utils.ts

@@ -43,8 +43,11 @@ export const translateContentString: TranslateContentStringFunction = (
     return msgid;
   }
 
+  const lowerCaseMsgId = msgid.toLowerCase();
+
   const msgstr = dictionary?.find(
-    (row) => row.locale === locale && row.msgid === msgid,
+    (row) =>
+      row.locale === locale && row.msgid.toLowerCase() === lowerCaseMsgId,
   )?.msgstr;
 
   if (!msgstr || !msgstr.trim()) {

enterprise/frontend/src/metabase-enterprise/transforms/api/python-runner.ts

@@ -1,11 +1,11 @@
 import { Api } from "metabase/api";
 
-interface ExecutePythonRequest {
+export interface ExecutePythonRequest {
   code: string;
   tables: Record<string, number>;
 }
 
-interface ExecutePythonResponse {
+export interface ExecutePythonResponse {
   output?: string;
   stdout?: string;
   stderr?: string;

enterprise/frontend/src/metabase-enterprise/transforms/components/JobView/DependenciesSection/DependenciesSection.tsx

@@ -1,64 +1,36 @@
-import { push } from "react-router-redux";
 import { t } from "ttag";
 
-import { AdminContentTable } from "metabase/common/components/AdminContentTable";
-import EmptyState from "metabase/common/components/EmptyState";
-import { useDispatch } from "metabase/lib/redux";
-import { Box, Card, Center, Loader } from "metabase/ui";
+import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
 import { useListTransformJobTransformsQuery } from "metabase-enterprise/api";
-import type { Transform, TransformJobId } from "metabase-types/api";
+import type { TransformJobId } from "metabase-types/api";
 
 import { TitleSection } from "../../../components/TitleSection";
-import { getTransformUrl } from "../../../urls";
+import { TransformTable } from "../../../components/TransformTable";
+import { ListEmptyState } from "../../ListEmptyState";
 
-import S from "./DependenciesSection.module.css";
+type DependenciesSectionProps = {
+  jobId: TransformJobId;
+};
 
-export function DependenciesSection({ jobId }: { jobId: TransformJobId }) {
-  const { data: transforms, isLoading } =
-    useListTransformJobTransformsQuery(jobId);
-  const dispatch = useDispatch();
-
-  const handleRowClick = (transform: Transform) => {
-    dispatch(push(getTransformUrl(transform.id)));
-  };
+export function DependenciesSection({ jobId }: DependenciesSectionProps) {
+  const {
+    data: transforms = [],
+    error,
+    isLoading,
+  } = useListTransformJobTransformsQuery(jobId);
 
   return (
     <TitleSection
       label={t`Transforms`}
       description={t`Transforms will be run in this order.`}
     >
-      <Card p={0} shadow="none" withBorder>
-        {isLoading ? (
-          <Center>
-            <Loader m="xl" />
-          </Center>
-        ) : (
-          <AdminContentTable columnTitles={[t`Transform`, t`Target`]}>
-            {!transforms || transforms?.length === 0 ? (
-              <tr>
-                <td colSpan={2}>
-                  <Box p="md">
-                    <EmptyState
-                      message={t`There are no transforms for this job.`}
-                    />
-                  </Box>
-                </td>
-              </tr>
-            ) : (
-              transforms?.map((transform) => (
-                <tr
-                  key={transform.id}
-                  className={S.row}
-                  onClick={() => handleRowClick(transform)}
-                >
-                  <td>{transform.name}</td>
-                  <td>{transform.target.name}</td>
-                </tr>
-              ))
-            )}
-          </AdminContentTable>
-        )}
-      </Card>
+      {isLoading || error != null ? (
+        <LoadingAndErrorWrapper loading={isLoading} error={error} />
+      ) : transforms.length === 0 ? (
+        <ListEmptyState label={t`There are no transforms for this job.`} />
+      ) : (
+        <TransformTable transforms={transforms} />
+      )}
     </TitleSection>
   );
 }

enterprise/frontend/src/metabase-enterprise/transforms/components/JobView/ScheduleSection/ScheduleSection.tsx

@@ -2,10 +2,12 @@ import { useState } from "react";
 import { t } from "ttag";
 
 import { CronExpressionInput } from "metabase/common/components/CronExpressioInput";
+import { useSetting } from "metabase/common/hooks";
 import {
   formatCronExpressionForUI,
   getScheduleExplanation,
 } from "metabase/lib/cron";
+import { timezoneToUTCOffset } from "metabase/lib/time-dayjs";
 import { useMetadataToasts } from "metabase/metadata/hooks";
 import { Box, Divider, Group, Icon, Tooltip } from "metabase/ui";
 import {
@@ -31,6 +33,10 @@ export function ScheduleSection({
     formatCronExpressionForUI(job.schedule),
   );
   const scheduleExplanation = getScheduleExplanation(schedule);
+  const systemTimezone = useSetting("system-timezone") ?? "UTC";
+  const timezoneOffset = timezoneToUTCOffset(systemTimezone);
+  const timezoneExplanation =
+    timezoneOffset === "+00:00" ? "UTC" : `UTC${timezoneOffset}`;
 
   return (
     <SplitSection
@@ -53,7 +59,7 @@ export function ScheduleSection({
         {scheduleExplanation != null && (
           <Group gap="sm" c="text-secondary">
             <Icon name="calendar" />
-            {t`This job will run ${scheduleExplanation}`}
+            {t`This job will run ${scheduleExplanation}, ${timezoneExplanation}`}
           </Group>
         )}
         <RunButtonSection job={job} />

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonDataPicker/PythonDataPicker.module.css

@@ -0,0 +1,3 @@
+.dataPicker {
+  border-right: 1px solid var(--mb-color-border);
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonDataPicker/PythonDataPicker.tsx

@@ -16,6 +16,8 @@ import {
 } from "metabase/ui";
 import type { Database, Table } from "metabase-types/api";
 
+import S from "./PythonDataPicker.module.css";
+
 type TableSelection = {
   id: string; // unique ID for this selection row
   tableId: number | undefined;
@@ -248,7 +250,7 @@ export function PythonDataPicker({
   );
 
   return (
-    <Stack gap="md">
+    <Stack p="md" gap="md" h="100%" className={S.dataPicker}>
       <Box>
         <Text size="sm" fw="bold" mb="xs">
           {t`Source Database`}

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonEditor/PythonEditor.module.css

@@ -0,0 +1,4 @@
+.editor {
+  width: 100%;
+  height: 100%;
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonEditor/PythonEditor.tsx

@@ -0,0 +1,28 @@
+import cx from "classnames";
+
+import { CodeEditor } from "metabase/common/components/CodeEditor";
+
+import S from "./PythonEditor.module.css";
+import { completion } from "./utils";
+
+export function PythonEditor({
+  value,
+  onChange,
+  withPandasCompletions,
+  className,
+}: {
+  value: string;
+  onChange: (value: string) => void;
+  withPandasCompletions?: boolean;
+  className?: string;
+}) {
+  return (
+    <CodeEditor
+      className={cx(S.editor, className)}
+      value={value}
+      onChange={onChange}
+      language="python"
+      extensions={withPandasCompletions ? completion : undefined}
+    />
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonEditor/index.ts

@@ -0,0 +1 @@
+export * from "./PythonEditor";

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonEditor/utils.ts

@@ -0,0 +1,675 @@
+import {
+  type Completion,
+  type CompletionContext,
+  autocompletion,
+} from "@codemirror/autocomplete";
+import { pythonLanguage } from "@codemirror/lang-python";
+
+// This is the closest you can get to "loading pandas definitions"
+
+const pandasDefinitions: Record<string, Completion[]> = {
+  pd: [
+    {
+      label: "DataFrame",
+      type: "class",
+      info: "Two-dimensional, size-mutable, potentially heterogeneous tabular data",
+    },
+    {
+      label: "Series",
+      type: "class",
+      info: "One-dimensional ndarray with axis labels",
+    },
+    {
+      label: "read_csv",
+      type: "function",
+      info: "Read a comma-separated values (csv) file into DataFrame",
+    },
+    {
+      label: "read_excel",
+      type: "function",
+      info: "Read an Excel file into a pandas DataFrame",
+    },
+    {
+      label: "read_json",
+      type: "function",
+      info: "Convert a JSON string to pandas object",
+    },
+    {
+      label: "read_sql",
+      type: "function",
+      info: "Read SQL query or database table into a DataFrame",
+    },
+    {
+      label: "read_parquet",
+      type: "function",
+      info: "Load a parquet object from file path",
+    },
+    {
+      label: "concat",
+      type: "function",
+      info: "Concatenate pandas objects along a particular axis",
+    },
+    {
+      label: "merge",
+      type: "function",
+      info: "Merge DataFrame or named Series objects with a database-style join",
+    },
+    {
+      label: "pivot_table",
+      type: "function",
+      info: "Create a spreadsheet-style pivot table as a DataFrame",
+    },
+    {
+      label: "crosstab",
+      type: "function",
+      info: "Compute a simple cross tabulation of two (or more) factors",
+    },
+    {
+      label: "get_dummies",
+      type: "function",
+      info: "Convert categorical variable(s) into dummy/indicator variables",
+    },
+    {
+      label: "date_range",
+      type: "function",
+      info: "Return a fixed frequency DatetimeIndex",
+    },
+    {
+      label: "to_datetime",
+      type: "function",
+      info: "Convert argument to datetime",
+    },
+    {
+      label: "to_numeric",
+      type: "function",
+      info: "Convert argument to a numeric type",
+    },
+    {
+      label: "cut",
+      type: "function",
+      info: "Bin values into discrete intervals",
+    },
+    {
+      label: "qcut",
+      type: "function",
+      info: "Quantile-based discretization function",
+    },
+    {
+      label: "isna",
+      type: "function",
+      info: "Detect missing values for an array-like object",
+    },
+    {
+      label: "isnull",
+      type: "function",
+      info: "Detect missing values for an array-like object",
+    },
+    {
+      label: "notna",
+      type: "function",
+      info: "Detect non-missing values for an array-like object",
+    },
+    {
+      label: "notnull",
+      type: "function",
+      info: "Detect non-missing values for an array-like object",
+    },
+  ],
+
+  // DataFrame methods and properties
+  DataFrame: [
+    // Viewing/Inspection
+    { label: "head", type: "method", info: "Return the first n rows" },
+    { label: "tail", type: "method", info: "Return the last n rows" },
+    {
+      label: "info",
+      type: "method",
+      info: "Print a concise summary of a DataFrame",
+    },
+    {
+      label: "describe",
+      type: "method",
+      info: "Generate descriptive statistics",
+    },
+    {
+      label: "shape",
+      type: "property",
+      info: "Return a tuple representing the dimensionality",
+    },
+    {
+      label: "size",
+      type: "property",
+      info: "Return an int representing the number of elements",
+    },
+    {
+      label: "columns",
+      type: "property",
+      info: "The column labels of the DataFrame",
+    },
+    {
+      label: "index",
+      type: "property",
+      info: "The index (row labels) of the DataFrame",
+    },
+    {
+      label: "dtypes",
+      type: "property",
+      info: "Return the dtypes in the DataFrame",
+    },
+    {
+      label: "values",
+      type: "property",
+      info: "Return a Numpy representation of the DataFrame",
+    },
+
+    // Selection/Indexing
+    {
+      label: "loc",
+      type: "property",
+      info: "Access a group of rows and columns by label(s)",
+    },
+    {
+      label: "iloc",
+      type: "property",
+      info: "Purely integer-location based indexing",
+    },
+    {
+      label: "at",
+      type: "property",
+      info: "Access a single value for a row/column label pair",
+    },
+    {
+      label: "iat",
+      type: "property",
+      info: "Access a single value for a row/column pair by integer position",
+    },
+    {
+      label: "query",
+      type: "method",
+      info: "Query the columns of a DataFrame with a boolean expression",
+    },
+    {
+      label: "filter",
+      type: "method",
+      info: "Subset the dataframe rows or columns according to labels",
+    },
+    {
+      label: "where",
+      type: "method",
+      info: "Replace values where the condition is False",
+    },
+    {
+      label: "mask",
+      type: "method",
+      info: "Replace values where the condition is True",
+    },
+
+    // Data Cleaning
+    {
+      label: "drop",
+      type: "method",
+      info: "Drop specified labels from rows or columns",
+    },
+    {
+      label: "drop_duplicates",
+      type: "method",
+      info: "Return DataFrame with duplicate rows removed",
+    },
+    { label: "dropna", type: "method", info: "Remove missing values" },
+    {
+      label: "fillna",
+      type: "method",
+      info: "Fill NA/NaN values using the specified method",
+    },
+    {
+      label: "replace",
+      type: "method",
+      info: "Replace values given in to_replace with value",
+    },
+    {
+      label: "interpolate",
+      type: "method",
+      info: "Fill NaN values using an interpolation method",
+    },
+    {
+      label: "duplicated",
+      type: "method",
+      info: "Return boolean Series denoting duplicate rows",
+    },
+    { label: "isnull", type: "method", info: "Detect missing values" },
+    { label: "isna", type: "method", info: "Detect missing values" },
+    {
+      label: "notnull",
+      type: "method",
+      info: "Detect existing (non-missing) values",
+    },
+    {
+      label: "notna",
+      type: "method",
+      info: "Detect existing (non-missing) values",
+    },
+
+    // Transformations
+    {
+      label: "sort_values",
+      type: "method",
+      info: "Sort by the values along either axis",
+    },
+    {
+      label: "sort_index",
+      type: "method",
+      info: "Sort object by labels (along an axis)",
+    },
+    {
+      label: "reset_index",
+      type: "method",
+      info: "Reset the index, or a level of it",
+    },
+    {
+      label: "set_index",
+      type: "method",
+      info: "Set the DataFrame index using existing columns",
+    },
+    {
+      label: "reindex",
+      type: "method",
+      info: "Conform Series/DataFrame to new index",
+    },
+    { label: "rename", type: "method", info: "Alter axes labels" },
+    {
+      label: "assign",
+      type: "method",
+      info: "Assign new columns to a DataFrame",
+    },
+    {
+      label: "pipe",
+      type: "method",
+      info: "Apply chainable functions that expect Series or DataFrames",
+    },
+
+    // Aggregation/GroupBy
+    {
+      label: "groupby",
+      type: "method",
+      info: "Group DataFrame using a mapper or by a Series of columns",
+    },
+    {
+      label: "agg",
+      type: "method",
+      info: "Aggregate using one or more operations",
+    },
+    {
+      label: "aggregate",
+      type: "method",
+      info: "Aggregate using one or more operations",
+    },
+    {
+      label: "apply",
+      type: "method",
+      info: "Apply a function along an axis of the DataFrame",
+    },
+    {
+      label: "applymap",
+      type: "method",
+      info: "Apply a function to a Dataframe elementwise",
+    },
+    {
+      label: "transform",
+      type: "method",
+      info: "Call func on self producing a DataFrame with transformed values",
+    },
+    {
+      label: "rolling",
+      type: "method",
+      info: "Provide rolling window calculations",
+    },
+    {
+      label: "expanding",
+      type: "method",
+      info: "Provide expanding transformations",
+    },
+    { label: "resample", type: "method", info: "Resample time-series data" },
+
+    // Statistical Methods
+    {
+      label: "sum",
+      type: "method",
+      info: "Return the sum of the values for the requested axis",
+    },
+    {
+      label: "mean",
+      type: "method",
+      info: "Return the mean of the values for the requested axis",
+    },
+    {
+      label: "median",
+      type: "method",
+      info: "Return the median of the values for the requested axis",
+    },
+    {
+      label: "mode",
+      type: "method",
+      info: "Get the mode(s) of each element along the selected axis",
+    },
+    {
+      label: "std",
+      type: "method",
+      info: "Return sample standard deviation over requested axis",
+    },
+    {
+      label: "var",
+      type: "method",
+      info: "Return unbiased variance over requested axis",
+    },
+    {
+      label: "min",
+      type: "method",
+      info: "Return the minimum of the values for the requested axis",
+    },
+    {
+      label: "max",
+      type: "method",
+      info: "Return the maximum of the values for the requested axis",
+    },
+    {
+      label: "count",
+      type: "method",
+      info: "Count non-NA cells for each column or row",
+    },
+    {
+      label: "nunique",
+      type: "method",
+      info: "Count number of distinct elements in specified axis",
+    },
+    {
+      label: "value_counts",
+      type: "method",
+      info: "Return a Series containing counts of unique values",
+    },
+    {
+      label: "corr",
+      type: "method",
+      info: "Compute pairwise correlation of columns",
+    },
+    {
+      label: "cov",
+      type: "method",
+      info: "Compute pairwise covariance of columns",
+    },
+    {
+      label: "rank",
+      type: "method",
+      info: "Compute numerical data ranks (1 through n) along axis",
+    },
+    {
+      label: "quantile",
+      type: "method",
+      info: "Return values at the given quantile over requested axis",
+    },
+    {
+      label: "clip",
+      type: "method",
+      info: "Trim values at input threshold(s)",
+    },
+
+    // Merging/Joining
+    {
+      label: "merge",
+      type: "method",
+      info: "Merge DataFrame or named Series objects",
+    },
+    {
+      label: "join",
+      type: "method",
+      info: "Join columns of another DataFrame",
+    },
+    {
+      label: "concat",
+      type: "method",
+      info: "Concatenate pandas objects along a particular axis",
+    },
+    {
+      label: "append",
+      type: "method",
+      info: "Append rows of other to the end of caller",
+    },
+    {
+      label: "update",
+      type: "method",
+      info: "Modify in place using non-NA values from another DataFrame",
+    },
+
+    // Reshaping
+    {
+      label: "pivot",
+      type: "method",
+      info: "Return reshaped DataFrame organized by given index / column values",
+    },
+    {
+      label: "pivot_table",
+      type: "method",
+      info: "Create a spreadsheet-style pivot table",
+    },
+    {
+      label: "melt",
+      type: "method",
+      info: "Unpivot a DataFrame from wide to long format",
+    },
+    {
+      label: "stack",
+      type: "method",
+      info: "Stack the prescribed level(s) from columns to index",
+    },
+    {
+      label: "unstack",
+      type: "method",
+      info: "Pivot a level of the (necessarily hierarchical) index labels",
+    },
+    { label: "transpose", type: "method", info: "Transpose index and columns" },
+    { label: "T", type: "property", info: "Transpose index and columns" },
+
+    // I/O
+    {
+      label: "to_csv",
+      type: "method",
+      info: "Write object to a comma-separated values (csv) file",
+    },
+    {
+      label: "to_excel",
+      type: "method",
+      info: "Write object to an Excel sheet",
+    },
+    {
+      label: "to_json",
+      type: "method",
+      info: "Convert the object to a JSON string",
+    },
+    {
+      label: "to_sql",
+      type: "method",
+      info: "Write records stored in a DataFrame to a SQL database",
+    },
+    {
+      label: "to_parquet",
+      type: "method",
+      info: "Write a DataFrame to the binary parquet format",
+    },
+    {
+      label: "to_dict",
+      type: "method",
+      info: "Convert the DataFrame to a dictionary",
+    },
+    {
+      label: "to_html",
+      type: "method",
+      info: "Render a DataFrame as an HTML table",
+    },
+    {
+      label: "to_string",
+      type: "method",
+      info: "Render a DataFrame to a console-friendly tabular output",
+    },
+    {
+      label: "to_numpy",
+      type: "method",
+      info: "Convert the DataFrame to a NumPy array",
+    },
+
+    // Time Series
+    {
+      label: "shift",
+      type: "method",
+      info: "Shift index by desired number of periods",
+    },
+    { label: "tshift", type: "method", info: "Shift the time index" },
+    {
+      label: "diff",
+      type: "method",
+      info: "First discrete difference of element",
+    },
+    {
+      label: "pct_change",
+      type: "method",
+      info: "Percentage change between the current and a prior element",
+    },
+
+    // Plotting (if matplotlib available)
+    {
+      label: "plot",
+      type: "method",
+      info: "Make plots of Series or DataFrame",
+    },
+    {
+      label: "hist",
+      type: "method",
+      info: "Make a histogram of the DataFrame's columns",
+    },
+    {
+      label: "boxplot",
+      type: "method",
+      info: "Make a box plot of the DataFrame columns",
+    },
+  ],
+
+  // Series methods (subset of DataFrame methods that make sense for Series)
+  Series: [
+    { label: "head", type: "method", info: "Return the first n values" },
+    { label: "tail", type: "method", info: "Return the last n values" },
+    {
+      label: "describe",
+      type: "method",
+      info: "Generate descriptive statistics",
+    },
+    {
+      label: "value_counts",
+      type: "method",
+      info: "Return a Series containing counts of unique values",
+    },
+    {
+      label: "unique",
+      type: "method",
+      info: "Return unique values of Series object",
+    },
+    {
+      label: "nunique",
+      type: "method",
+      info: "Return number of unique elements in the object",
+    },
+    { label: "sort_values", type: "method", info: "Sort by the values" },
+    {
+      label: "sort_index",
+      type: "method",
+      info: "Sort Series by index labels",
+    },
+    {
+      label: "reset_index",
+      type: "method",
+      info: "Generate a new DataFrame or Series with the index reset",
+    },
+    {
+      label: "dropna",
+      type: "method",
+      info: "Return a new Series with missing values removed",
+    },
+    {
+      label: "fillna",
+      type: "method",
+      info: "Fill NA/NaN values using the specified method",
+    },
+    {
+      label: "apply",
+      type: "method",
+      info: "Invoke function on values of Series",
+    },
+    {
+      label: "map",
+      type: "method",
+      info: "Map values of Series according to input correspondence",
+    },
+    {
+      label: "replace",
+      type: "method",
+      info: "Replace values given in to_replace with value",
+    },
+    {
+      label: "str",
+      type: "property",
+      info: "Vectorized string functions for Series and Index",
+    },
+    {
+      label: "dt",
+      type: "property",
+      info: "Accessor object for datetimelike properties",
+    },
+    {
+      label: "cat",
+      type: "property",
+      info: "Accessor object for categorical properties",
+    },
+  ],
+};
+
+function genericPandasCompletions(context: CompletionContext) {
+  const word = context.matchBefore(/[\w.]*/);
+  if (!word || (word.from === word.to && !context.explicit)) {
+    return null;
+  }
+
+  const text = word.text;
+
+  // Match patterns like "pd.", "df.", etc.
+  if (text.includes(".")) {
+    const parts = text.split(".");
+    const obj = parts.at(-2);
+    const partial = parts.at(-1) ?? "";
+
+    if (!obj || !pandasDefinitions[obj]) {
+      return null;
+    }
+
+    const options = pandasDefinitions[obj]
+      .filter(
+        (item) =>
+          !partial ||
+          item.label.toLowerCase().startsWith(partial.toLowerCase()),
+      )
+      .map((comp) => ({
+        ...comp,
+        info: undefined,
+        boost: comp.type === "method" ? 2 : comp.type === "function" ? 1 : 0,
+      }));
+
+    return {
+      filter: false,
+      from: word.from + text.length - partial.length,
+      options,
+    };
+  }
+}
+
+export const completion = [
+  pythonLanguage.data.of({
+    autocomplete: genericPandasCompletions,
+  }),
+  autocompletion(),
+];

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonTransformEditor/PythonTransformEditor.tsx

@@ -1,19 +1,19 @@
 import { useEffect, useState } from "react";
 
-import { Flex } from "metabase/ui";
-import type { TransformSource } from "metabase-types/api";
+import { Flex, Stack } from "metabase/ui";
+import type { PythonTransformSource } from "metabase-types/api";
 
 import { PythonDataPicker } from "../PythonDataPicker";
 import { EditorHeader } from "../QueryEditor/EditorHeader";
 import { PythonEditor } from "../QueryEditor/PythonEditor";
 
 type PythonTransformEditorProps = {
-  initialSource: TransformSource & { type: "python" };
+  initialSource: PythonTransformSource;
   isNew?: boolean;
   isSaving?: boolean;
-  onSave: (newSource: TransformSource & { type: "python" }) => void;
+  onSave: (newSource: PythonTransformSource) => void;
   onCancel: () => void;
-  onSourceChange?: (newSource: TransformSource & { type: "python" }) => void;
+  onSourceChange?: (newSource: PythonTransformSource) => void;
 };
 
 export function PythonTransformEditor({
@@ -202,12 +202,12 @@ ${tableAliases
   );
 
   return (
-    <Flex
+    <Stack
       w="100%"
       h="100%"
-      direction="column"
       bg="bg-white"
       data-testid="python-transform-editor"
+      gap={0}
     >
       <EditorHeader
         isNew={isNew}
@@ -216,22 +216,20 @@ ${tableAliases
         onSave={handleSave}
         onCancel={onCancel}
       />
-      <Flex direction="column" style={{ flex: 1 }} p="md">
+      <Flex h="100%" w="100%">
         <PythonDataPicker
           database={source["source-database"]}
           tables={source["source-tables"]}
           onChange={handleDataChange}
         />
 
-        <Flex direction="column" style={{ flex: 1, marginTop: "1rem" }}>
-          <PythonEditor
-            script={source.body}
-            isRunnable={true}
-            onChange={handleScriptChange}
-            tables={source["source-tables"]}
-          />
-        </Flex>
+        <PythonEditor
+          script={source.body}
+          isRunnable={true}
+          onChange={handleScriptChange}
+          tables={source["source-tables"]}
+        />
       </Flex>
-    </Flex>
+    </Stack>
   );
 }

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/EditorBody/EditorBody.tsx

@@ -13,8 +13,8 @@ import type Database from "metabase-lib/v1/metadata/Database";
 import type NativeQuery from "metabase-lib/v1/queries/NativeQuery";
 import type {
   Database as ApiDatabase,
+  PythonTransformSource,
   RecentItem,
-  TransformSource,
 } from "metabase-types/api";
 
 import { PythonEditor } from "../PythonEditor";
@@ -28,7 +28,7 @@ type EditorBodyProps = {
   question: Question;
   isNative: boolean;
   isPython?: boolean;
-  pythonSource?: TransformSource & { type: "python" };
+  pythonSource?: PythonTransformSource;
   isRunnable: boolean;
   isRunning: boolean;
   isResultDirty: boolean;

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/EditorHeader/EditorHeader.tsx

@@ -23,6 +23,7 @@ export function EditorHeader({
       title={getTitle(isNew)}
       admin
       buttons={[
+        <Button key="cancel" small onClick={onCancel}>{t`Cancel`}</Button>,
         <Button
           key="save"
           onClick={onSave}
@@ -32,7 +33,6 @@ export function EditorHeader({
         >
           {getSaveButtonLabel(isNew, isSaving)}
         </Button>,
-        <Button key="cancel" small onClick={onCancel}>{t`Cancel`}</Button>,
       ]}
     />
   );

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/PythonEditor/ExecutionOutputTable.tsx

@@ -0,0 +1,25 @@
+import { useMemo } from "react";
+
+import { DataGrid, useDataGridInstance } from "metabase/data-grid";
+
+import { parseCSV } from "./utils";
+
+export function ExecutionOutputTable({ output }: { output?: string }) {
+  const { headers, rows } = useMemo(() => parseCSV(output || ""), [output]);
+
+  const tableProps = useDataGridInstance<string[], unknown>({
+    data: rows,
+    columnsOptions: headers.map((header, index) => ({
+      id: header,
+      name: header,
+      accessorFn: (row) => row[index],
+    })),
+    minGridWidth: 100,
+  });
+
+  if (!output || headers.length === 0) {
+    return null;
+  }
+
+  return <DataGrid {...tableProps} />;
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/PythonEditor/PythonEditor.module.css

@@ -0,0 +1,68 @@
+.root {
+  border-bottom: 1px solid var(--mb-color-border);
+}
+
+.editor {
+  width: 100%;
+  height: 100%;
+}
+
+.logs {
+  white-space: pre;
+  border: 1px solid var(--mb-color-border);
+  font-family: var(--mb-default-monospace-font-family);
+}
+
+.visualization {
+  flex: 1 0 auto;
+}
+
+.results {
+  width: 100%;
+  border-collapse: collapse;
+  border: 1px solid var(--mb-color-border);
+  border-radius: var(--mantine-radius-md);
+}
+
+.tableHeader {
+  background-color: var(--mb-color-bg-light);
+  padding: var(--mantine-spacing-sm);
+}
+
+.tableCell {
+  border-top: 1px solid var(--mb-color-border);
+  padding: var(--mantine-spacing-sm);
+}
+
+.error {
+  color: var(--mb-color-error);
+  background-color: var(--mb-color-bg-error);
+  border-bottom: 1px solid var(--mb-color-border);
+}
+
+.info {
+  color: var(--mb-color-text-dark);
+  border: 1px solid var(--mb-color-bg-medium);
+  background-color: var(--mb-color-bg-light);
+}
+
+.success {
+  color: var(--mb-color-success);
+  background-color: var(--mb-color-bg-success);
+  border-bottom: 1px solid var(--mb-color-border);
+}
+
+.loading {
+  position: absolute;
+  top: 0;
+  left: 0;
+  bottom: 0;
+  right: 0;
+  background-color: rgba(255, 255, 255, 0.5);
+}
+
+.libraryActions {
+  position: absolute;
+  top: 0;
+  right: 0;
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/PythonEditor/PythonEditor.tsx

@@ -1,327 +1,247 @@
-import { useState } from "react";
-import { t } from "ttag";
+import type { ReactNode } from "react";
+import { ResizableBox } from "react-resizable";
+import { c, t } from "ttag";
 
-import { CodeEditor } from "metabase/common/components/CodeEditor";
-import { color } from "metabase/lib/colors";
-import { Alert, Box, Button, Group, Loader, Text } from "metabase/ui";
+import EmptyCodeResult from "assets/img/empty-states/code.svg";
+import DebouncedFrame from "metabase/common/components/DebouncedFrame";
+import Link from "metabase/common/components/Link";
+import { LoadingSpinner } from "metabase/common/components/MetadataInfo/MetadataInfo.styled";
+import { isMac } from "metabase/lib/browser";
+import RunButtonWithTooltip from "metabase/query_builder/components/RunButtonWithTooltip";
+import { Box, Checkbox, Flex, Icon, Stack, Text, Title } from "metabase/ui";
+import { getPythonLibraryUrl } from "metabase-enterprise/transforms/urls";
 
+import { SHARED_LIB_IMPORT_NAME } from "../../../constants";
+import { PythonEditor as PythonCodeEditor } from "../../PythonEditor";
+import { ResizableBoxHandle } from "../EditorBody/ResizableBoxHandle";
+
+import { ExecutionOutputTable } from "./ExecutionOutputTable";
+import S from "./PythonEditor.module.css";
 import {
-  useCancelPythonMutation,
-  useExecutePythonMutation,
-} from "../../../api/python-runner";
+  type ExecutionResult,
+  hasImport,
+  insertImport,
+  removeImport,
+  useTestPythonScript,
+} from "./utils";
 
 type PythonEditorProps = {
   script: string;
   isRunnable: boolean;
-  isRunning?: boolean;
-  isResultDirty?: boolean;
   onChange: (script: string) => void;
   onRunScript?: () => Promise<void>;
   onCancelScript?: () => void;
   tables?: Record<string, number>;
 };
 
-interface ExecutionResult {
-  output?: string;
-  stdout?: string;
-  stderr?: string;
-  error?: string;
-}
-
-function parseCSV(csv: string): { headers: string[]; rows: string[][] } {
-  if (!csv || !csv.trim()) {
-    return { headers: [], rows: [] };
-  }
-
-  const lines = csv.trim().split("\n");
-  if (lines.length === 0) {
-    return { headers: [], rows: [] };
-  }
-
-  const headers = lines[0].split(",").map((h) => h.trim());
-  const rows = lines.slice(1).map((line) => {
-    return line.split(",").map((cell) => cell.trim());
-  });
-
-  return { headers, rows };
-}
+const EDITOR_HEIGHT = 400;
 
 export function PythonEditor({
   script,
   isRunnable,
-  isRunning: isRunningProp,
-  isResultDirty: _isResultDirty,
   onChange,
-  onRunScript,
-  onCancelScript,
   tables = {},
 }: PythonEditorProps) {
-  const [localIsRunning, setLocalIsRunning] = useState(false);
-  const isRunning =
-    isRunningProp !== undefined ? isRunningProp : localIsRunning;
-  const [executionResult, setExecutionResult] =
-    useState<ExecutionResult | null>(null);
-  const [executePython] = useExecutePythonMutation();
-  const [cancelPython] = useCancelPythonMutation();
-  const handleScriptChange = (newScript: string) => {
-    onChange(newScript);
-    // Don't clear results on every keystroke - keep them visible for reference
-  };
+  const { isRunning, isDirty, cancel, run, executionResult } =
+    useTestPythonScript(script, tables);
 
-  const handleRunScript = async () => {
-    if (onRunScript) {
-      await onRunScript();
-      return;
+  const hasSharedLib = hasImport(script, SHARED_LIB_IMPORT_NAME);
+  function handleToggleSharedLib() {
+    if (hasImport(script, SHARED_LIB_IMPORT_NAME)) {
+      onChange(removeImport(script, SHARED_LIB_IMPORT_NAME));
+    } else {
+      onChange(insertImport(script, SHARED_LIB_IMPORT_NAME));
     }
-    setLocalIsRunning(true);
-    setExecutionResult(null);
-
-    try {
-      const result = await executePython({
-        code: script,
-        tables: tables,
-      }).unwrap();
-
-      setExecutionResult({
-        output: result.output,
-        stdout: result.stdout,
-        stderr: result.stderr,
-      });
-    } catch (error: any) {
-      // The API returns error data directly in error.data
-      const errorData = error?.data || {};
-      const errorMessage =
-        errorData.error || error?.message || "Failed to execute Python script";
-      const stdout = errorData.stdout || "";
-      const stderr = errorData.stderr || "";
-
-      setExecutionResult({
-        error: errorMessage,
-        stdout: stdout,
-        stderr: stderr,
-      });
-    } finally {
-      setLocalIsRunning(false);
-    }
-  };
-
-  const handleCancelScript = async () => {
-    if (onCancelScript) {
-      onCancelScript();
-      return;
-    }
-    try {
-      await cancelPython().unwrap();
-      setLocalIsRunning(false);
-      setExecutionResult({
-        error: t`Python script execution was canceled`,
-      });
-    } catch (error) {
-      console.error("Failed to cancel Python script:", error);
-      // Still set running to false since the cancel might have worked on the server
-      setLocalIsRunning(false);
-      setExecutionResult({
-        error: t`Python script execution was canceled (cancel request may have failed)`,
-      });
-    }
-  };
-
-  const { headers, rows } = parseCSV(executionResult?.output || "");
+  }
 
   return (
-    <Box h="100%" style={{ display: "flex", flexDirection: "column" }}>
-      {/* Python Script Editor */}
-      <Box
-        style={{ flex: "1 1 300px", minHeight: "300px", overflow: "hidden" }}
+    <Stack h="100%" w="100%" gap={0}>
+      <ResizableBox
+        className={S.root}
+        axis="y"
+        height={EDITOR_HEIGHT}
+        handle={<ResizableBoxHandle />}
+        resizeHandles={["s"]}
       >
-        <CodeEditor
-          value={script}
-          onChange={handleScriptChange}
-          language="python"
-        />
-      </Box>
+        <Flex h="100%" align="end" bg="bg-light">
+          <PythonCodeEditor
+            value={script}
+            onChange={onChange}
+            withPandasCompletions
+          />
 
-      {/* Results Section - positioned between editor and buttons */}
-      {(executionResult || isRunning) && (
-        <Box
-          mt="md"
-          style={{ flex: "0 0 auto", overflow: "auto", maxHeight: "40%" }}
-        >
-          {isRunning ? (
-            <Box p="md">
-              <Text c="text-medium">{t`Running Python script...`}</Text>
-            </Box>
-          ) : executionResult?.error ? (
-            <>
-              <Box p="md">
-                <Alert color="red" title={t`Execution Error`}>
-                  {executionResult.error}
-                </Alert>
-              </Box>
-              {executionResult.stdout && (
-                <Box
-                  mt="md"
-                  p="md"
-                  bg="bg-light"
-                  style={{ maxHeight: "150px", overflow: "auto" }}
-                >
-                  <Text fw="bold" mb="xs">{t`Standard Output:`}</Text>
-                  <Box
-                    style={{ fontFamily: "monospace", whiteSpace: "pre-wrap" }}
-                  >
-                    {executionResult.stdout}
-                  </Box>
-                </Box>
-              )}
-              {executionResult.stderr && (
-                <Box
-                  mt="md"
-                  p="md"
-                  bg="bg-light"
-                  style={{ maxHeight: "150px", overflow: "auto" }}
-                >
-                  <Text fw="bold" mb="xs">{t`Standard Error:`}</Text>
-                  <Box
-                    style={{ fontFamily: "monospace", whiteSpace: "pre-wrap" }}
-                  >
-                    {executionResult.stderr}
-                  </Box>
-                </Box>
-              )}
-            </>
-          ) : !executionResult?.output || headers.length === 0 ? (
-            <>
-              <Box p="md">
-                <Text c="text-medium">{t`No results to display.`}</Text>
-              </Box>
-              {executionResult?.stdout && (
-                <Box
-                  mt="md"
-                  p="md"
-                  bg="bg-light"
-                  style={{ maxHeight: "150px", overflow: "auto" }}
-                >
-                  <Text fw="bold" mb="xs">{t`Standard Output:`}</Text>
-                  <Box
-                    style={{ fontFamily: "monospace", whiteSpace: "pre-wrap" }}
-                  >
-                    {executionResult.stdout}
-                  </Box>
-                </Box>
-              )}
-              {executionResult?.stderr && (
-                <Box
-                  mt="md"
-                  p="md"
-                  bg="bg-light"
-                  style={{ maxHeight: "150px", overflow: "auto" }}
-                >
-                  <Text fw="bold" mb="xs">{t`Standard Error:`}</Text>
-                  <Box
-                    style={{ fontFamily: "monospace", whiteSpace: "pre-wrap" }}
-                  >
-                    {executionResult.stderr}
-                  </Box>
-                </Box>
-              )}
-            </>
-          ) : (
-            <>
-              {executionResult.stdout && (
-                <Box
-                  mt="md"
-                  p="md"
-                  bg="bg-light"
-                  style={{ maxHeight: "150px", overflow: "auto" }}
-                >
-                  <Text fw="bold" mb="xs">{t`Standard Output:`}</Text>
-                  <Box
-                    style={{ fontFamily: "monospace", whiteSpace: "pre-wrap" }}
-                  >
-                    {executionResult.stdout}
-                  </Box>
-                </Box>
-              )}
-              {executionResult.stderr && (
-                <Box
-                  mt="md"
-                  p="md"
-                  bg="bg-light"
-                  style={{ maxHeight: "150px", overflow: "auto" }}
-                >
-                  <Text fw="bold" mb="xs">{t`Standard Error:`}</Text>
-                  <Box
-                    style={{ fontFamily: "monospace", whiteSpace: "pre-wrap" }}
-                  >
-                    {executionResult.stderr}
-                  </Box>
-                </Box>
-              )}
-              <Box mt="md" style={{ maxHeight: "300px", overflow: "auto" }}>
-                <Text fw="bold" mb="md">{t`Results Table:`}</Text>
-                <table
-                  style={{
-                    width: "100%",
-                    borderCollapse: "collapse",
-                    border: `1px solid ${color("border")}`,
-                  }}
-                >
-                  <thead>
-                    <tr style={{ backgroundColor: color("bg-medium") }}>
-                      {headers.map((header, index) => (
-                        <th
-                          key={index}
-                          style={{
-                            padding: "8px",
-                            textAlign: "left",
-                            borderBottom: `2px solid ${color("border")}`,
-                          }}
-                        >
-                          {header}
-                        </th>
-                      ))}
-                    </tr>
-                  </thead>
-                  <tbody>
-                    {rows.map((row, rowIndex) => (
-                      <tr
-                        key={rowIndex}
-                        style={{ borderBottom: `1px solid ${color("border")}` }}
-                      >
-                        {row.map((cell, cellIndex) => (
-                          <td key={cellIndex} style={{ padding: "8px" }}>
-                            {cell}
-                          </td>
-                        ))}
-                      </tr>
-                    ))}
-                  </tbody>
-                </table>
-              </Box>
-            </>
-          )}
-        </Box>
-      )}
-
-      {/* Action Buttons */}
-      <Box mt="md" style={{ flex: "0 0 auto" }}>
-        <Group gap="sm">
-          <Button
-            variant="filled"
-            style={
-              isRunning
-                ? { backgroundColor: color("error"), color: color("white") }
-                : undefined
-            }
-            leftSection={isRunning ? undefined : <span>▶</span>}
-            onClick={isRunning ? handleCancelScript : handleRunScript}
-            disabled={!isRunnable && !isRunning}
+          <Box p="md">
+            <RunButtonWithTooltip
+              disabled={!isRunnable}
+              isRunning={isRunning}
+              isDirty={isDirty}
+              onRun={run}
+              onCancel={cancel}
+              getTooltip={() => t`Run Python script`}
+            />
+          </Box>
+        </Flex>
+        <Stack className={S.libraryActions} p="md" gap="sm">
+          <Checkbox
+            label={t`Import shared library`}
+            checked={hasSharedLib}
+            onChange={handleToggleSharedLib}
+            size="sm"
+          />
+          <Flex
+            component={Link}
+            target="_blank"
+            to={getPythonLibraryUrl({ name: SHARED_LIB_IMPORT_NAME })}
+            gap="sm"
           >
-            {isRunning ? t`Cancel` : t`Run Python Script`}
-          </Button>
-          {isRunning && <Loader size="sm" />}
-        </Group>
+            <Icon name="pencil" />
+            {t`Edit shared library`}
+          </Flex>
+        </Stack>
+      </ResizableBox>
+
+      <DebouncedFrame className={S.visualization}>
+        {executionResult ? (
+          <ExecutionResult executionResult={executionResult} />
+        ) : (
+          <EmptyState />
+        )}
+        {isRunning && <LoadingState />}
+      </DebouncedFrame>
+    </Stack>
+  );
+}
+
+function getRunQueryShortcut() {
+  return isMac() ? t`⌘ + return` : t`Ctrl + enter`;
+}
+
+function LoadingState() {
+  return (
+    <Flex p="md" className={S.loading}>
+      <LoadingSpinner />
+    </Flex>
+  );
+}
+
+function EmptyState() {
+  const keyboardShortcut = getRunQueryShortcut();
+
+  return (
+    <Flex h="100%" align="center" justify="center">
+      <Stack maw="25rem" gap={0} ta="center" align="center">
+        <Box maw="3rem" mb="0.75rem">
+          <img src={EmptyCodeResult} alt="Code prompt icon" />
+        </Box>
+        <Text c="text-medium">
+          {c("{0} refers to the keyboard shortcut")
+            .jt`To run your code, click on the Run button or type ${(
+            <b key="shortcut">({keyboardShortcut})</b>
+          )}`}
+        </Text>
+      </Stack>
+    </Flex>
+  );
+}
+
+function ExecutionResult({
+  executionResult,
+}: {
+  executionResult: ExecutionResult | null;
+}) {
+  if (!executionResult) {
+    return null;
+  }
+
+  const message = getMessageForExecutionResult(executionResult);
+
+  return (
+    <Stack gap={0}>
+      {message}
+
+      <ExecutionLogs
+        label={t`Standard output:`}
+        content={executionResult.stdout}
+      />
+      <ExecutionLogs
+        label={t`Standard error:`}
+        content={executionResult.stderr}
+      />
+
+      <ExecutionOutputTable output={executionResult.output} />
+    </Stack>
+  );
+}
+
+function getMessageForExecutionResult(executionResult: ExecutionResult) {
+  if (executionResult.error) {
+    return (
+      <Box className={S.error} p="sm">
+        <Flex gap="sm">
+          <Icon name="warning" />
+          <Stack gap={0}>
+            <Title order={5} mb="xs">
+              {t`An error occurred while executing your Python script`}
+            </Title>
+            {executionResult.error}
+          </Stack>
+        </Flex>
       </Box>
+    );
+  }
+
+  if (!executionResult.output) {
+    return (
+      <Flex className={S.info} p="sm" bdrs="xs" gap="sm" align="center">
+        <Icon name="info" />
+        {t`No results to display.`}
+      </Flex>
+    );
+  }
+
+  return (
+    <Flex className={S.success} p="sm" gap="sm" align="center">
+      <Icon name="check" />
+      {t`Script executed successfully.`}
+    </Flex>
+  );
+}
+
+function ExecutionLogs({
+  label,
+  content,
+}: {
+  label: string;
+  content?: string | null;
+}) {
+  if (!content) {
+    return null;
+  }
+
+  return (
+    <Section title={label}>
+      <Box p="sm" bg="bg-light" mah="150px" bdrs="xs" className={S.logs}>
+        {content}
+      </Box>
+    </Section>
+  );
+}
+
+function Section({
+  title,
+  children,
+}: {
+  title: ReactNode;
+  children: ReactNode;
+}) {
+  return (
+    <Box>
+      {title && (
+        <Title order={5} mb="xs">
+          {title}
+        </Title>
+      )}
+      {children}
     </Box>
   );
 }

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/PythonEditor/utils.ts

@@ -0,0 +1,131 @@
+import { useRef, useState } from "react";
+import { t } from "ttag";
+
+import {
+  type ExecutePythonResponse,
+  useExecutePythonMutation,
+} from "metabase-enterprise/transforms/api/python-runner";
+
+export function parseCSV(csv: string): { headers: string[]; rows: string[][] } {
+  if (!csv || !csv.trim()) {
+    return { headers: [], rows: [] };
+  }
+
+  const lines = csv.trim().split("\n");
+  if (lines.length === 0) {
+    return { headers: [], rows: [] };
+  }
+
+  const headers = lines[0].split(",").map((h) => h.trim());
+  const rows = lines.slice(1).map((line) => {
+    return line.split(",").map((cell) => cell.trim());
+  });
+
+  return { headers, rows };
+}
+
+export type ExecutionResult = {
+  output?: string;
+  stdout?: string;
+  stderr?: string;
+  error?: string;
+};
+
+type TestPythonScriptState = {
+  isRunning: boolean;
+  isDirty: boolean;
+  executionResult: ExecutionResult | null;
+  run: () => void;
+  cancel: () => void;
+};
+
+export function useTestPythonScript(
+  code: string,
+  tables: Record<string, number>,
+): TestPythonScriptState {
+  const [executePython, { isLoading: isRunning, originalArgs }] =
+    useExecutePythonMutation();
+  const abort = useRef<(() => void) | null>(null);
+  const [executionResult, setData] = useState<ExecutePythonResponse | null>(
+    null,
+  );
+
+  const isDirty = originalArgs?.code !== code;
+
+  const run = async () => {
+    const request = executePython({ code, tables });
+    abort.current = () => request.abort();
+
+    try {
+      const data = await request.unwrap();
+      setData(data);
+    } catch (error) {
+      if (typeof error === "object" && error !== null) {
+        if ("name" in error && error.name === "AbortError") {
+          setData({ error: t`Python script execution was canceled` });
+          return;
+        }
+
+        if ("message" in error && typeof error.message === "string") {
+          setData({ error: error.message });
+          return;
+        }
+
+        if (
+          "data" in error &&
+          typeof error.data === "object" &&
+          error.data !== null &&
+          "error" in error.data &&
+          typeof error.data.error === "string"
+        ) {
+          setData({ error: error?.data?.error });
+          return;
+        }
+      }
+
+      setData({ error: t`An unknown error occurred` });
+    }
+  };
+
+  const cancel = () => {
+    abort.current?.();
+  };
+
+  return {
+    isRunning,
+    isDirty,
+    cancel,
+    run,
+    executionResult,
+  };
+}
+
+export function insertImport(source: string, lib: string) {
+  const lines = source.split("\n");
+
+  // Find the first line that is not a comment
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    if (line.trim().startsWith("#")) {
+      continue;
+    }
+    lines.splice(i, 0, `import ${lib}`);
+    break;
+  }
+
+  return lines.join("\n");
+}
+
+function libImportRegex(lib: string) {
+  return new RegExp(`import ${lib}`, "g");
+}
+
+export function removeImport(source: string, lib: string) {
+  const lines = source.split("\n");
+  return lines.filter((line) => !libImportRegex(lib).test(line)).join("\n");
+}
+
+export function hasImport(source: string, lib: string) {
+  const regex = libImportRegex(lib);
+  return regex.test(source);
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/SchemaFormSelect/SchemaFormSelect.tsx

@@ -0,0 +1,59 @@
+import { useField } from "formik";
+import { useState } from "react";
+import { jt, t } from "ttag";
+import _ from "underscore";
+
+import { FormField, FormSelect, type FormSelectProps } from "metabase/forms";
+import { SelectItem, Text } from "metabase/ui";
+
+export function SchemaFormSelect(props: FormSelectProps & { data: string[] }) {
+  const { data, name, ...rest } = props;
+  const [{ value }] = useField(name);
+
+  const [searchValue, setSearchValue] = useState(value);
+  const dataWithNewItem = _.uniq([
+    ...data,
+    ...(searchValue !== "" ? [searchValue] : []),
+  ]);
+
+  const isNewValue = value !== "" && !data.includes(value);
+
+  return (
+    <FormField>
+      <FormSelect
+        {...rest}
+        name={name}
+        data={dataWithNewItem}
+        searchable
+        searchValue={searchValue}
+        onSearchChange={setSearchValue}
+        renderOption={({ option }) => {
+          const { value } = option;
+          if (data.includes(value)) {
+            return <ExistingSelectItem value={value} />;
+          }
+          return <NewSelectItem value={value} />;
+        }}
+      />
+      {isNewValue && (
+        <Text size="sm" c="text-secondary" mt="sm">
+          {t`This schema will be created the first time the transform runs.`}
+        </Text>
+      )}
+    </FormField>
+  );
+}
+
+function ExistingSelectItem({ value }: { value: string }) {
+  return <SelectItem>{value}</SelectItem>;
+}
+
+function NewSelectItem({ value }: { value: string }) {
+  return (
+    <SelectItem>
+      <Text c="inherit" lh="inherit">
+        {jt`Create new schema ${(<strong key="value">{value}</strong>)}`}
+      </Text>
+    </SelectItem>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/SchemaFormSelect/SchemaFormSelect.unit.spec.tsx

@@ -0,0 +1,106 @@
+import userEvent from "@testing-library/user-event";
+
+import { act, renderWithProviders, screen } from "__support__/ui";
+import { FormProvider } from "metabase/forms";
+
+import { SchemaFormSelect } from "./SchemaFormSelect";
+
+function setup({
+  initialValue = "",
+  name = "schema",
+  schemas = [],
+  onSubmit = jest.fn(),
+}: {
+  initialValue?: string;
+  name?: string;
+  schemas?: string[];
+  onSubmit?: (values: Record<string, string>) => void;
+} = {}) {
+  const initialValues = { [name]: initialValue };
+
+  renderWithProviders(
+    <FormProvider initialValues={initialValues} onSubmit={onSubmit}>
+      <SchemaFormSelect name={name} data={schemas} />
+    </FormProvider>,
+  );
+}
+
+describe("SchemaFormSelect", () => {
+  it("should render existing schemas as options and filter them accordingly", async () => {
+    setup({
+      schemas: ["foo", "bar"],
+    });
+
+    const input = screen.getByRole("textbox");
+    act(() => input.focus());
+
+    expect(screen.getByText("foo")).toBeInTheDocument();
+    expect(screen.getByText("bar")).toBeInTheDocument();
+    expect(screen.queryByText(/Create new schema/)).not.toBeInTheDocument();
+
+    await userEvent.type(input, "foo");
+    expect(screen.getByText("foo")).toBeInTheDocument();
+    expect(screen.queryByText("bar")).not.toBeInTheDocument();
+
+    await userEvent.click(screen.getByText("foo"));
+    expect(input).toHaveValue("foo");
+
+    expect(
+      screen.queryByText(
+        "This schema will be created the first time the transform runs.",
+      ),
+    ).not.toBeInTheDocument();
+  });
+
+  it("should be possible to create new schemas", async () => {
+    setup({
+      schemas: ["foo", "bar"],
+    });
+
+    const input = screen.getByRole("textbox");
+    act(() => input.focus());
+
+    await userEvent.type(input, "fo");
+    expect(screen.getByText("foo")).toBeInTheDocument();
+    expect(screen.getByText(/Create new schema/)).toBeInTheDocument();
+
+    await userEvent.click(screen.getByText(/Create new schema/));
+    expect(input).toHaveValue("fo");
+
+    expect(
+      screen.getByText(
+        "This schema will be created the first time the transform runs.",
+      ),
+    ).toBeInTheDocument();
+  });
+
+  it("should render initial value when it is an existing schema", async () => {
+    setup({
+      initialValue: "foo",
+      schemas: ["foo", "bar"],
+    });
+
+    const input = screen.getByRole("textbox");
+    expect(input).toHaveValue("foo");
+    expect(
+      screen.queryByText(
+        "This schema will be created the first time the transform runs.",
+      ),
+    ).not.toBeInTheDocument();
+  });
+
+  it("should render initial value when it is a new schema", async () => {
+    setup({
+      initialValue: "quu",
+      schemas: ["foo", "bar"],
+    });
+
+    const input = screen.getByRole("textbox");
+    expect(input).toHaveValue("quu");
+    expect(
+      screen.getByText(
+        "This schema will be created the first time the transform runs.",
+      ),
+    ).toBeInTheDocument();
+  });
+});

enterprise/frontend/src/metabase-enterprise/transforms/components/SchemaFormSelect/index.ts

@@ -0,0 +1 @@
+export * from "./SchemaFormSelect";

enterprise/frontend/src/metabase-enterprise/transforms/components/TagFilterWidget/TagFilterWidget.tsx

@@ -13,8 +13,11 @@ import {
 } from "metabase/ui";
 import type { TransformTag, TransformTagId } from "metabase-types/api";
 
+import {
+  FILTER_WIDGET_MAX_HEIGHT,
+  FILTER_WIDGET_MIN_WIDTH,
+} from "../../constants";
 import { FilterFieldSet } from "../FilterFieldSet";
-import { MAX_HEIGHT, MIN_WIDTH } from "../constants";
 
 type TagFilterWidgetProps = {
   tagIds: TransformTagId[];
@@ -94,9 +97,9 @@ function TagFilterForm({
   };
 
   return (
-    <Box component="form" miw={MIN_WIDTH} onSubmit={handleSubmit}>
+    <Box component="form" miw={FILTER_WIDGET_MIN_WIDTH} onSubmit={handleSubmit}>
       {tags.length > 0 ? (
-        <Box mah={MAX_HEIGHT} style={{ overflow: "auto" }}>
+        <Box mah={FILTER_WIDGET_MAX_HEIGHT} style={{ overflow: "auto" }}>
           <Checkbox.Group
             value={tagIds.map(getTagIdValue)}
             onChange={handleChange}

enterprise/frontend/src/metabase-enterprise/transforms/components/TimeFilterWidget/TimeFilterWidget.tsx

@@ -1,5 +1,6 @@
 import { useDisclosure } from "@mantine/hooks";
 
+import type { DatePickerShortcut } from "metabase/querying/filters/types";
 import { SearchFilterDateDisplay } from "metabase/search/components/SearchFilterDateDisplay";
 import { SearchFilterDatePicker } from "metabase/search/components/SearchFilterDatePicker";
 import { Popover } from "metabase/ui";
@@ -10,10 +11,12 @@ export function TimeFilterWidget({
   label,
   value,
   onChange,
+  availableShortcuts,
 }: {
   label: string;
   value: string | undefined;
   onChange: (value: string | undefined) => void;
+  availableShortcuts?: DatePickerShortcut[];
 }) {
   const [isOpened, { toggle, close }] = useDisclosure();
 
@@ -42,7 +45,11 @@ export function TimeFilterWidget({
         />
       </Popover.Target>
       <Popover.Dropdown>
-        <SearchFilterDatePicker value={value ?? null} onChange={handleChange} />
+        <SearchFilterDatePicker
+          value={value ?? null}
+          onChange={handleChange}
+          availableShortcuts={availableShortcuts}
+        />
       </Popover.Dropdown>
     </Popover>
   );

enterprise/frontend/src/metabase-enterprise/transforms/components/TimezoneIndicator/TimezoneIndicator.tsx

@@ -0,0 +1,17 @@
+import { t } from "ttag";
+
+import { useSetting } from "metabase/common/hooks";
+import { Icon } from "metabase/ui";
+
+export function TimezoneIndicator() {
+  const systemTimezone = useSetting("system-timezone");
+
+  return (
+    <Icon
+      name="info_outline"
+      size={16}
+      tooltip={t`Timezone is ${systemTimezone}`}
+      tooltipPosition="bottom"
+    />
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/TimezoneIndicator/index.ts

@@ -0,0 +1 @@
+export * from "./TimezoneIndicator";

enterprise/frontend/src/metabase-enterprise/transforms/components/TransformTable/TransformTable.module.css

@@ -0,0 +1,3 @@
+.row {
+  cursor: pointer;
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/TransformTable/TransformTable.tsx

@@ -0,0 +1,40 @@
+import { push } from "react-router-redux";
+import { t } from "ttag";
+
+import { AdminContentTable } from "metabase/common/components/AdminContentTable";
+import { useDispatch } from "metabase/lib/redux";
+import { Card } from "metabase/ui";
+import type { Transform } from "metabase-types/api";
+
+import { getTransformUrl } from "../../urls";
+
+import S from "./TransformTable.module.css";
+
+type TransformTableProps = {
+  transforms: Transform[];
+};
+
+export function TransformTable({ transforms }: TransformTableProps) {
+  const dispatch = useDispatch();
+
+  const handleRowClick = (transform: Transform) => {
+    dispatch(push(getTransformUrl(transform.id)));
+  };
+
+  return (
+    <Card p={0} shadow="none" withBorder>
+      <AdminContentTable columnTitles={[t`Transform`, t`Target`]}>
+        {transforms.map((transform) => (
+          <tr
+            key={transform.id}
+            className={S.row}
+            onClick={() => handleRowClick(transform)}
+          >
+            <td>{transform.name}</td>
+            <td>{transform.target.name}</td>
+          </tr>
+        ))}
+      </AdminContentTable>
+    </Card>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/TransformTable/index.ts

@@ -0,0 +1 @@
+export * from "./TransformTable";

enterprise/frontend/src/metabase-enterprise/transforms/constants.ts

@@ -3,3 +3,8 @@ import { isCypressActive } from "metabase/env";
 export const NAME_MAX_LENGTH = 254;
 
 export const POLLING_INTERVAL = isCypressActive ? 100 : 3000;
+
+export const FILTER_WIDGET_MIN_WIDTH = 300;
+export const FILTER_WIDGET_MAX_HEIGHT = 400;
+
+export const SHARED_LIB_IMPORT_NAME = "lib";

enterprise/frontend/src/metabase-enterprise/transforms/index.tsx

@@ -11,6 +11,7 @@ import { JobPage } from "./pages/JobPage";
 import { NewJobPage } from "./pages/NewJobPage";
 import { NewPythonTransformPage } from "./pages/NewPythonTransformPage";
 import { NewTransformQueryPage } from "./pages/NewTransformQueryPage";
+import { PythonLibraryEditorPage } from "./pages/PythonLibraryEditorPage";
 import { RunListPage } from "./pages/RunListPage";
 import { TransformListPage } from "./pages/TransformListPage";
 import { TransformPage } from "./pages/TransformPage";
@@ -33,6 +34,11 @@ if (hasPremiumFeature("transforms")) {
           <Route path="runs" component={RunListPage} />
           <Route path=":transformId" component={TransformPage} />
         </Route>
+        <Route
+          component={(props) => <TransformPageLayout {...props} fullWidth />}
+        >
+          <Route path="library/:name" component={PythonLibraryEditorPage} />
+        </Route>
         <Route
           path="new/:type"
           component={(props: any) =>

enterprise/frontend/src/metabase-enterprise/transforms/pages/JobListPage/JobFilterList/JobFilterList.tsx

@@ -0,0 +1,56 @@
+import { replace } from "react-router-redux";
+import { t } from "ttag";
+
+import { useDispatch } from "metabase/lib/redux";
+import type { DatePickerShortcut } from "metabase/querying/filters/types";
+import { Group } from "metabase/ui";
+import { getJobListUrl } from "metabase-enterprise/transforms/urls";
+import type { TransformTag, TransformTagId } from "metabase-types/api";
+
+import { TagFilterWidget } from "../../../components/TagFilterWidget";
+import { TimeFilterWidget } from "../../../components/TimeFilterWidget";
+import type { JobListParams } from "../../../types";
+
+type FilterListProps = {
+  tags: TransformTag[];
+  params: JobListParams;
+};
+
+const NO_SHORTCUTS: DatePickerShortcut[] = [];
+
+export function JobFilterList({ tags, params }: FilterListProps) {
+  const dispatch = useDispatch();
+
+  const handleLastRunStartTimeChange = (lastRunStartTime?: string) => {
+    dispatch(replace(getJobListUrl({ ...params, lastRunStartTime })));
+  };
+
+  const handleNextRunStartTimeChange = (nextRunStartTime?: string) => {
+    dispatch(replace(getJobListUrl({ ...params, nextRunStartTime })));
+  };
+
+  const handleTagsChange = (tagIds: TransformTagId[]) => {
+    dispatch(replace(getJobListUrl({ ...params, transformTagIds: tagIds })));
+  };
+
+  return (
+    <Group>
+      <TimeFilterWidget
+        label={t`Last run at`}
+        value={params.lastRunStartTime}
+        onChange={handleLastRunStartTimeChange}
+      />
+      <TimeFilterWidget
+        label={t`Next run`}
+        value={params.nextRunStartTime}
+        onChange={handleNextRunStartTimeChange}
+        availableShortcuts={NO_SHORTCUTS}
+      />
+      <TagFilterWidget
+        tagIds={params.transformTagIds ?? []}
+        tags={tags}
+        onChange={handleTagsChange}
+      />
+    </Group>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/pages/JobListPage/JobFilterList/index.ts

@@ -0,0 +1 @@
+export * from "./JobFilterList";

enterprise/frontend/src/metabase-enterprise/transforms/pages/JobListPage/JobList/JobList.tsx

@@ -3,28 +3,35 @@ import { t } from "ttag";
 
 import { AdminContentTable } from "metabase/common/components/AdminContentTable";
 import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
+import { useSetting } from "metabase/common/hooks";
 import { useDispatch } from "metabase/lib/redux";
-import { parseTimestamp } from "metabase/lib/time-dayjs";
-import { Card } from "metabase/ui";
+import { Card, Flex } from "metabase/ui";
 import {
   useListTransformJobsQuery,
   useListTransformTagsQuery,
 } from "metabase-enterprise/api";
+import { TimezoneIndicator } from "metabase-enterprise/transforms/components/TimezoneIndicator";
+import type { JobListParams } from "metabase-enterprise/transforms/types";
 import type { TransformJob } from "metabase-types/api";
 
 import { ListEmptyState } from "../../../components/ListEmptyState";
-import { RunStatusInfo } from "../../../components/RunStatusInfo";
 import { TagList } from "../../../components/TagList";
 import { getJobUrl } from "../../../urls";
+import { parseTimestampWithTimezone } from "../../../utils";
 
 import S from "./JobList.module.css";
 
-export function JobList() {
+export function JobList({ params }: { params: JobListParams }) {
+  const systemTimezone = useSetting("system-timezone");
   const {
     data: jobs = [],
     isLoading: isLoadingJobs,
     error: jobsError,
-  } = useListTransformJobsQuery();
+  } = useListTransformJobsQuery({
+    last_run_start_time: params.lastRunStartTime,
+    next_run_start_time: params.nextRunStartTime,
+    transform_tag_ids: params.transformTagIds,
+  });
   const {
     data: tags = [],
     isLoading: isLoadingTags,
@@ -49,7 +56,17 @@ export function JobList() {
   return (
     <Card p={0} shadow="none" withBorder>
       <AdminContentTable
-        columnTitles={[t`Job`, t`Last run at`, t`Last run status`, t`Tags`]}
+        columnTitles={[
+          t`Job`,
+          <Flex align="center" gap="xs" key="last-run-at">
+            {t`Last run at`} <TimezoneIndicator />
+          </Flex>,
+          <Flex align="center" gap="xs" key="next-run">
+            {t`Next run`} <TimezoneIndicator />
+          </Flex>,
+          t`Transforms`,
+          t`Tags`,
+        ]}
       >
         {jobs.map((job) => (
           <tr
@@ -60,22 +77,21 @@ export function JobList() {
             <td>{job.name}</td>
             <td className={S.nowrap}>
               {job.last_run?.start_time
-                ? parseTimestamp(job.last_run?.start_time).format("lll")
+                ? parseTimestampWithTimezone(
+                    job.last_run?.start_time,
+                    systemTimezone,
+                  ).format("lll")
                 : null}
             </td>
             <td className={S.nowrap}>
-              {job.last_run != null ? (
-                <RunStatusInfo
-                  status={job.last_run.status}
-                  message={job.last_run.message}
-                  endTime={
-                    job.last_run.end_time != null
-                      ? parseTimestamp(job.last_run.end_time).toDate()
-                      : null
-                  }
-                />
-              ) : null}
+              {job.next_run?.start_time
+                ? parseTimestampWithTimezone(
+                    job.next_run?.start_time,
+                    systemTimezone,
+                  ).format("lll")
+                : null}
             </td>
+            <td className={S.nowrap}></td>
             <td>
               <TagList tags={tags} tagIds={job.tag_ids ?? []} />
             </td>

enterprise/frontend/src/metabase-enterprise/transforms/pages/JobListPage/JobListPage.tsx

@@ -1,13 +1,30 @@
+import type { Location } from "history";
 import { Link } from "react-router";
 import { t } from "ttag";
 
+import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
 import { Box, Button, Group, Icon, Stack, Title } from "metabase/ui";
+import { useListTransformTagsQuery } from "metabase-enterprise/api";
 
 import { getNewJobUrl } from "../../urls";
 
+import { JobFilterList } from "./JobFilterList";
 import { JobList } from "./JobList";
+import { getParsedParams } from "./utils";
+
+type JobListPageProps = {
+  location: Location;
+};
+
+export function JobListPage({ location }: JobListPageProps) {
+  const params = getParsedParams(location);
+
+  const { data: tags = [], isLoading, error } = useListTransformTagsQuery();
+
+  if (!tags || isLoading || error != null) {
+    return <LoadingAndErrorWrapper loading={isLoading} error={error} />;
+  }
 
-export function JobListPage() {
   return (
     <Stack gap="xl" data-testid="job-list-page">
       <Group justify="space-between">
@@ -24,7 +41,9 @@ export function JobListPage() {
           {t`Create a job`}
         </Button>
       </Group>
-      <JobList />
+
+      <JobFilterList params={params} tags={tags} />
+      <JobList params={params} />
     </Stack>
   );
 }

enterprise/frontend/src/metabase-enterprise/transforms/pages/JobListPage/utils.ts

@@ -0,0 +1,39 @@
+import type { Location } from "history";
+
+import { isNotNull } from "metabase/lib/types";
+import type { JobListParams } from "metabase-enterprise/transforms/types";
+
+export function getParsedParams(location: Location): JobListParams {
+  const { transformTagIds, lastRunStartTime, nextRunStartTime } =
+    location.query;
+
+  return {
+    lastRunStartTime: parseTime(lastRunStartTime),
+    nextRunStartTime: parseTime(nextRunStartTime),
+    transformTagIds: parseList(transformTagIds, parseNumber),
+  };
+}
+
+function parseTime(value: unknown): string | undefined {
+  if (typeof value === "string") {
+    return value;
+  }
+  return undefined;
+}
+
+function parseList<T>(
+  value: unknown,
+  parseItem: (value: unknown) => T | undefined,
+): T[] | undefined {
+  if (typeof value === "string") {
+    const item = parseItem(value);
+    return item != null ? [item] : [];
+  }
+  if (Array.isArray(value)) {
+    return value.map(parseItem).filter(isNotNull);
+  }
+}
+
+function parseNumber(value: unknown) {
+  return typeof value === "string" ? parseInt(value, 10) : undefined;
+}

enterprise/frontend/src/metabase-enterprise/transforms/pages/NewPythonTransformPage/CreatePythonTransformModal/CreatePythonTransformModal.tsx

@@ -4,7 +4,7 @@ import * as Yup from "yup";
 
 import { Button, Group, Modal, Stack, TextInput } from "metabase/ui";
 import { useCreateTransformMutation } from "metabase-enterprise/api";
-import type { Transform, TransformSource } from "metabase-types/api";
+import type { PythonTransformSource, Transform } from "metabase-types/api";
 
 type PythonTransformApiSource = {
   type: "python";
@@ -14,7 +14,7 @@ type PythonTransformApiSource = {
 };
 
 type CreatePythonTransformModalProps = {
-  source: TransformSource & { type: "python" };
+  source: PythonTransformSource;
   onCreate: (transform: Transform) => void;
   onClose: () => void;
 };

enterprise/frontend/src/metabase-enterprise/transforms/pages/NewPythonTransformPage/NewPythonTransformPage.tsx

@@ -3,14 +3,14 @@ import { useState } from "react";
 import { push } from "react-router-redux";
 
 import { useDispatch } from "metabase/lib/redux";
-import type { Transform, TransformSource } from "metabase-types/api";
+import type { PythonTransformSource, Transform } from "metabase-types/api";
 
 import { PythonTransformEditor } from "../../components/PythonTransformEditor";
 import { getTransformListUrl, getTransformUrl } from "../../urls";
 
 import { CreatePythonTransformModal } from "./CreatePythonTransformModal";
 
-const DEFAULT_PYTHON_SOURCE: TransformSource & { type: "python" } = {
+const DEFAULT_PYTHON_SOURCE: PythonTransformSource = {
   type: "python",
   "source-database": 1, // Default to first database, will be updated by user
   "source-tables": {}, // Will be populated when user selects tables
@@ -20,7 +20,7 @@ import pandas as pd
 def transform():
     """
     Your transformation function.
-    
+
     Select tables above to add them as function parameters.
 
     Returns:
@@ -31,7 +31,7 @@ def transform():
 };
 
 export function NewPythonTransformPage() {
-  const [source, setSource] = useState<TransformSource & { type: "python" }>(
+  const [source, setSource] = useState<PythonTransformSource>(
     DEFAULT_PYTHON_SOURCE,
   );
   const [isModalOpened, { open: openModal, close: closeModal }] =
@@ -42,7 +42,7 @@ export function NewPythonTransformPage() {
     dispatch(push(getTransformUrl(transform.id)));
   };
 
-  const handleSaveClick = (newSource: TransformSource & { type: "python" }) => {
+  const handleSaveClick = (newSource: PythonTransformSource) => {
     setSource(newSource);
     openModal();
   };

enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/CreateTransformModal/CreateTransformModal.tsx

@@ -2,13 +2,17 @@ import { useMemo } from "react";
 import { t } from "ttag";
 import * as Yup from "yup";
 
-import { skipToken, useListDatabaseSchemasQuery } from "metabase/api";
+import { hasFeature } from "metabase/admin/databases/utils";
+import {
+  skipToken,
+  useGetDatabaseQuery,
+  useListDatabaseSchemasQuery,
+} from "metabase/api";
 import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
 import {
   Form,
   FormErrorMessage,
   FormProvider,
-  FormSelect,
   FormSubmitButton,
   FormTextInput,
 } from "metabase/forms";
@@ -16,6 +20,7 @@ import * as Errors from "metabase/lib/errors";
 import { Box, Button, FocusTrap, Group, Modal, Stack } from "metabase/ui";
 import { useCreateTransformMutation } from "metabase-enterprise/api";
 import { trackTransformCreated } from "metabase-enterprise/transforms/analytics";
+import { SchemaFormSelect } from "metabase-enterprise/transforms/components/SchemaFormSelect";
 import type {
   CreateTransformRequest,
   DatasetQuery,
@@ -71,14 +76,26 @@ function CreateTransformForm({
   onClose,
 }: CreateTransformFormProps) {
   const { database: databaseId } = query;
+
+  const {
+    data: database,
+    isLoading: isDatabaseLoading,
+    error: databaseError,
+  } = useGetDatabaseQuery(databaseId ? { id: databaseId } : skipToken);
+
   const {
     data: schemas = [],
-    isLoading,
-    error,
+    isLoading: isSchemasLoading,
+    error: schemasError,
   } = useListDatabaseSchemasQuery(
     databaseId ? { id: databaseId, include_hidden: true } : skipToken,
   );
+
+  const isLoading = isDatabaseLoading || isSchemasLoading;
+  const error = databaseError ?? schemasError;
+
   const [createTransform] = useCreateTransformMutation();
+  const supportsSchemas = database && hasFeature(database, "schemas");
 
   const initialValues: NewTransformValues = useMemo(
     () => getInitialValues(schemas),
@@ -119,8 +136,12 @@ function CreateTransformForm({
             label={t`Description`}
             placeholder={t`This is optional`}
           />
-          {schemas.length > 1 && (
-            <FormSelect name="targetSchema" label={t`Schema`} data={schemas} />
+          {supportsSchemas && (
+            <SchemaFormSelect
+              name="targetSchema"
+              label={t`Schema`}
+              data={schemas}
+            />
           )}
           <FormTextInput
             name="targetName"

enterprise/frontend/src/metabase-enterprise/transforms/pages/PythonLibraryEditorPage/PythonLibraryEditorPage.module.css

@@ -0,0 +1,5 @@
+.editor {
+  /* HACK: 133px is the height of the headers */
+  height: calc(100vh - 133px);
+  flex-grow: 1;
+}

enterprise/frontend/src/metabase-enterprise/transforms/pages/PythonLibraryEditorPage/PythonLibraryEditorPage.tsx

@@ -0,0 +1,119 @@
+import { useEffect, useState } from "react";
+import { t } from "ttag";
+
+import Button from "metabase/common/components/Button";
+import EditBar from "metabase/common/components/EditBar";
+import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
+import { useMetadataToasts } from "metabase/metadata/hooks";
+import { Box, Flex } from "metabase/ui";
+import {
+  useGetPythonLibraryQuery,
+  useUpdatePythonLibraryMutation,
+} from "metabase-enterprise/api/python-transform-library";
+import { PythonEditor } from "metabase-enterprise/transforms/components/PythonEditor";
+import type { PythonLibraryEditorPageParams } from "metabase-enterprise/transforms/types";
+
+import S from "./PythonLibraryEditorPage.module.css";
+
+type PythonLibraryEditorPageProps = {
+  params: PythonLibraryEditorPageParams;
+};
+
+export function PythonLibraryEditorPage({
+  params,
+}: PythonLibraryEditorPageProps) {
+  const { name } = params;
+  const [source, setSource] = useState("");
+
+  const {
+    data: library,
+    isLoading,
+    error,
+  } = useGetPythonLibraryQuery({ name });
+  const [updatePythonLibrary, { isLoading: isSaving }] =
+    useUpdatePythonLibraryMutation();
+  const { sendSuccessToast, sendErrorToast } = useMetadataToasts();
+
+  useEffect(() => {
+    setSource(library?.source || "");
+  }, [library?.source]);
+
+  const isDirty = source !== library?.source;
+
+  function handleRevert() {
+    setSource(library?.source || "");
+  }
+
+  async function handleSave() {
+    try {
+      await updatePythonLibrary({ name, source }).unwrap();
+      sendSuccessToast(t`Python library saved`);
+    } catch (error) {
+      sendErrorToast(t`Python library could not be saved`);
+    }
+  }
+
+  if (isLoading || error) {
+    return (
+      <Box p="md">
+        <LoadingAndErrorWrapper loading={isLoading} error={error} />
+      </Box>
+    );
+  }
+
+  return (
+    <Flex h="100%" w="100%" bg="bg-light" gap={0} direction="column">
+      <LibraryEditorHeader
+        onSave={handleSave}
+        onRevert={handleRevert}
+        isDirty={isDirty}
+        isSaving={isSaving}
+      />
+      <PythonEditor
+        value={source}
+        onChange={setSource}
+        withPandasCompletions
+        className={S.editor}
+      />
+    </Flex>
+  );
+}
+
+export function LibraryEditorHeader({
+  isDirty,
+  isSaving,
+  onSave,
+  onRevert,
+}: {
+  isDirty?: boolean;
+  isSaving?: boolean;
+  onSave: () => void;
+  onRevert: () => void;
+}) {
+  return (
+    <EditBar
+      title={t`You are editing the shared Python library`}
+      admin
+      buttons={[
+        <Button
+          key="save"
+          onClick={onRevert}
+          primary
+          small
+          disabled={!isDirty || isSaving}
+        >
+          {t`Revert`}
+        </Button>,
+        <Button
+          key="save"
+          onClick={onSave}
+          primary
+          small
+          disabled={!isDirty || isSaving}
+        >
+          {t`Save`}
+        </Button>,
+      ]}
+    />
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/pages/PythonLibraryEditorPage/index.ts

@@ -0,0 +1 @@
+export * from "./PythonLibraryEditorPage";

enterprise/frontend/src/metabase-enterprise/transforms/pages/RunListPage/FilterList/FilterList.tsx

@@ -13,12 +13,12 @@ import type {
   TransformTagId,
 } from "metabase-types/api";
 
+import { TagFilterWidget } from "../../../components/TagFilterWidget";
+import { TimeFilterWidget } from "../../../components/TimeFilterWidget";
 import type { RunListParams } from "../../../types";
 
 import { RunMethodFilterWidget } from "./RunMethodFilterWidget";
 import { StatusFilterWidget } from "./StatusFilterWidget";
-import { TagFilterWidget } from "./TagFilterWidget";
-import { TimeFilterWidget } from "./TimeFilterWidget";
 import { TransformFilterWidget } from "./TransformFilterWidget";
 
 type FilterListProps = {