support bigints

# Conflicts:
#	enterprise/backend/src/metabase_enterprise/transforms/api.clj
#	enterprise/backend/src/metabase_enterprise/transforms/canceling.clj
#	enterprise/backend/src/metabase_enterprise/transforms/jobs.clj
#	enterprise/backend/src/metabase_enterprise/transforms/ordering.clj
#	enterprise/backend/src/metabase_enterprise/transforms/util.clj
#	enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
#	enterprise/backend/test/metabase_enterprise/transforms/test_util.clj

.clj-kondo/config.edn

@@ -1,7 +1,7 @@
 ;; -*- comment-column: 100; -*-
 {:config-paths ["macros" "src" "config"]
  :linters
- { ;;
+ {;;
   ;; enabled optional linters (these are disabled by default)
   ;;
 
@@ -173,6 +173,7 @@
      metabase.driver.sql-jdbc.execute/execute-prepared-statement!
      metabase.notification.models/create-notification!
      metabase.pulse.send/send-pulse!
+     metabase-enterprise.transforms-python.python-runner-test/execute!
      ;; Safe in tests, where we typically use a private, temporary index per thread.
      metabase.search.core/reindex!
      metabase.util/run-count!
@@ -186,7 +187,7 @@
   :metabase/disallow-hardcoded-driver-names-in-tests
   ;; don't include `:h2` since that will probably lead to a mountain of tests where we are specifically testing
   ;; middleware or compilation behavior with just H2 since it's the default.
-  {:drivers #{ ;; core drivers
+  {:drivers #{;; core drivers
               #_:h2
               :postgres
               :mysql
@@ -418,6 +419,7 @@
   metabase-enterprise.transforms.api-test/with-transform-tags                           clojure.core/let
   metabase-enterprise.transforms.api-test/with-worker-runs                              clojure.core/let
   metabase-enterprise.transforms.test-util/with-transform-cleanup!                      clojure.core/let
+  metabase-enterprise.transforms.execute/with-transform-lifecycle                       clojure.core/let
   metabase.actions.test-util/with-actions                                               clojure.core/let
   metabase.api.common/let-404                                                           clojure.core/let
   metabase.app-db.custom-migrations.util/with-temp-scheduler!                           clojure.core/fn
@@ -519,6 +521,7 @@
    metabase-enterprise.cache.config-test/with-temp-persist-models                                                            hooks.common/with-seven-bindings
    metabase-enterprise.semantic-search.db.connection/with-migrate-tx                                                         hooks.common/let-one-with-optional-value
    metabase-enterprise.serialization.test-util/with-temp-dpc                                                                 hooks.toucan2.tools.with-temp/with-temp
+   metabase-enterprise.transforms-python.drivers-test/with-test-table                                                        hooks.common/with-two-bindings
    metabase.api.common.internal-test/with-jetty-server                                                                       hooks.common/with-one-binding
    metabase.api.macros/defendpoint                                                                                           hooks.metabase.api.macros/defendpoint
    metabase.app-db.schema-migrations-test.impl/test-migrations                                                               hooks.metabase.app-db.schema-migrations-test.impl/test-migrations
@@ -624,7 +627,7 @@
    metabase.test/with-temp                                                                                                   hooks.toucan2.tools.with-temp/with-temp
    metabase.test/with-temp-file                                                                                              hooks.metabase.test.util/with-temp-file
    metabase.test/with-temporary-setting-values                                                                               hooks.metabase.test.util/with-temporary-setting-values
-   metabase.users.api-test/with-temp-user-email!                                                                              hooks.common/with-one-binding
+   metabase.users.api-test/with-temp-user-email!                                                                             hooks.common/with-one-binding
    metabase.util.i18n/deferred-trs                                                                                           hooks.metabase.util.i18n/deferred-trs
    metabase.util.i18n/deferred-tru                                                                                           hooks.metabase.util.i18n/deferred-tru
    metabase.util.i18n/trs                                                                                                    hooks.metabase.util.i18n/trs
@@ -739,7 +742,9 @@
   ;; `metabase-enterprise.sandbox.query-processor*-test`
   ;;
   {:pattern "^metabase(?:((?:-enterprise\\.sandbox)?\\.query-processor)|(?:\\.driver\\.sql(?!server)(?!ite))).*-test$"
-   :name    general-qp-and-driver-test-namespaces}]
+   :name    general-qp-and-driver-test-namespaces}
+  {:pattern "^metabase\\.python-runner\\..*"
+   :name    python-runner-namespaces}]
 
  :config-in-ns
  {test-namespaces
@@ -764,8 +769,8 @@
   source-namespaces
   {:linters
    {:metabase/validate-escaped-single-quotes-in-i18n {:level :warning}
-    :metabase/validate-string-or-str-args-to-i18n {:level :warning}
-    :metabase/discourage-millis-duration {:level :warning}
+    :metabase/validate-string-or-str-args-to-i18n    {:level :warning}
+    :metabase/discourage-millis-duration             {:level :warning}
 
     :discouraged-var
     {clojure.core/with-redefs {:message "Don't use with-redefs outside of tests"}

.clj-kondo/config/modules/config.edn

@@ -885,6 +885,19 @@
            enterprise/advanced-permissions
            enterprise/sandbox}}
 
+  python-runner
+  {:team "Workflows"
+   :api  #{metabase.python-runner.api
+           metabase.python-runner.core
+           metabase.python-runner.init}
+   :uses #{api
+           driver
+           models
+           query-processor
+           request
+           util
+           enterprise/transforms}}
+
   queries
   {:team "Querying"
    :api  #{metabase.queries.api
@@ -1969,6 +1982,7 @@
    :api  #{metabase-enterprise.transforms.api
            metabase-enterprise.transforms.core
            metabase-enterprise.transforms.init
+           metabase-enterprise.transforms.instrumentation
            metabase-enterprise.transforms.util}
    :uses #{analytics
            api
@@ -1994,6 +2008,7 @@
   enterprise/transforms-python
   {:team "Querying"
    :api  #{metabase-enterprise.transforms-python.api
+           metabase-enterprise.transforms-python.execute
            metabase-enterprise.transforms-python.init
            metabase-enterprise.transforms-python.settings}
    :uses #{api

.github/actions/await-port/action.yml

@@ -11,6 +11,18 @@ runs:
     - name: Wait for port ${{ inputs.port }} to be ready
       shell: bash
       run: |
+        echo "Waiting for port ${{ inputs.port }} to be ready..."
+        timeout=300  # 5 minutes timeout
+        elapsed=0
         while ! nc -z localhost ${{ inputs.port }}; do
-          sleep 0.1;
+          if [ $elapsed -ge $timeout ]; then
+            echo "❌ Timeout waiting for port ${{ inputs.port }} after ${timeout} seconds"
+            exit 1
+          fi
+          sleep 1
+          elapsed=$((elapsed + 1))
+          if [ $((elapsed % 10)) -eq 0 ]; then
+            echo "  Still waiting for port ${{ inputs.port }}... ($elapsed seconds)"
+          fi
         done
+        echo "✅ Port ${{ inputs.port }} is ready!"

.github/scripts/setup-python-runner.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+set -e
+
+# Parse command line arguments
+SKIP_START=false
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --skip-start)
+      SKIP_START=true
+      shift
+      ;;
+    *)
+      echo "Unknown option: $1"
+      echo "Usage: $0 [--skip-start]"
+      exit 1
+      ;;
+  esac
+done
+
+echo "Cloning python-runner repository..."
+git clone --depth 1 https://${METABASE_AUTOMATION_USER_TOKEN}@github.com/metabase/python-runner-container.git python-runner
+
+if [ "$SKIP_START" = false ]; then
+  echo "Starting python-runner..."
+  cd python-runner
+  make run-ci
+  cd ..
+fi

.github/workflows/app-db.yml

@@ -35,21 +35,21 @@ jobs:
             build-static-viz: false
             test-args: >-
               :only '"enterprise/backend/test"'
-            exclude-tag: ':mb/driver-tests'
+            exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
           - name: EE App DB Tests (Part 1)
             build-static-viz: true
             test-args: >-
               :only '"test"'
               :partition/total 2
               :partition/index 0
-            exclude-tag: ':mb/driver-tests'
+            exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
           - name: EE App DB Tests (Part 2)
             build-static-viz: true
             test-args: >-
               :only '"test"'
               :partition/total 2
               :partition/index 1
-            exclude-tag: ':mb/driver-tests'
+            exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
 
     services:
       mysql:
@@ -147,21 +147,21 @@ jobs:
             build-static-viz: false
             test-args: >-
               :only '"enterprise/backend/test"'
-            exclude-tag: ':mb/driver-tests'
+            exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
           - name: EE App DB Tests (Part 1)
             build-static-viz: true
             test-args: >-
               :only '"test"'
               :partition/total 2
               :partition/index 0
-            exclude-tag: ':mb/driver-tests'
+            exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
           - name: EE App DB Tests (Part 2)
             build-static-viz: true
             test-args: >-
               :only '"test"'
               :partition/total 2
               :partition/index 1
-            exclude-tag: ':mb/driver-tests'
+            exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
 
     services:
       mysql:
@@ -259,11 +259,13 @@ jobs:
             build-static-viz: false
             test-args: >-
               :only-tags [:mb/driver-tests]
+              :exclude-tags [:mb/transforms-python-test]
+            exclude-tag: ':mb/transforms-python-test'
           - name: Enterprise Tests
             build-static-viz: false
             test-args: >-
               :only '"enterprise/backend/test"'
-            exclude-tag: ':mb/driver-tests'
+            exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
           - name: EE App DB Tests (Part 1)
             build-static-viz: true
             test-args: >-
@@ -271,14 +273,14 @@ jobs:
               :exclude-tags [:mb/driver-tests]
               :partition/total 2
               :partition/index 0
-            exclude-tag: ':mb/driver-tests'
+            exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
           - name: EE App DB Tests (Part 2)
             build-static-viz: true
             test-args: >-
               :only '"test"'
               :partition/total 2
               :partition/index 1
-            exclude-tag: ':mb/driver-tests'
+            exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
     name: "${{ matrix.version.name }} ${{ matrix.job.name }}"
     env:
       # Only run old migrations tests on pushes to master or release branches. All other branches should skip tests
@@ -328,6 +330,10 @@ jobs:
           test-args: >-
             ${{ matrix.job.test-args }}
             :exclude-tags '[ ${{ matrix.job.exclude-tag }} ${{ env.__ADDITIONAL_EXCLUDED_TAG__ }}]'
+      - name: Setup python-runner
+        env:
+          METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+        run: ./.github/scripts/setup-python-runner.sh
       - name: Upload Test Results
         uses: ./.github/actions/upload-test-results
         if: always()

.github/workflows/backend.yml

@@ -195,6 +195,7 @@ jobs:
       - name: Run cljfmt
         run: ./bin/mage cljfmt-all --force-check
 
+
   be-tests-result:
     needs:
       - be-linter-eastwood

.github/workflows/drivers.yml

@@ -90,6 +90,7 @@ jobs:
         junit-name: 'be-tests-athena-ee'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -106,6 +107,18 @@ jobs:
     if: ${{ !inputs.skip }}
     runs-on: ubuntu-22.04
     timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        job:
+          - name: Driver Tests
+            test-args: >-
+              :only-tags [:mb/driver-tests]
+              :exclude-tags [:mb/transforms-python-test]
+          - name: Transforms Python Tests
+            test-args: >-
+              :only-tags [:mb/transforms-python-test]
+            needs-python-runner: true
     env:
       CI: 'true'
       DRIVERS: bigquery-cloud-sdk
@@ -116,15 +129,19 @@ jobs:
       MB_BIGQUERY_TEST_REFRESH_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_REFRESH_TOKEN }}
       MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON }}
       MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING }}
-    name: BigQuery
+    name: BigQuery ${{ matrix.job.name }}
     steps:
     - uses: actions/checkout@v4
+    - name: Setup python-runner
+      if: ${{ matrix.job.needs-python-runner == true }}
+      env:
+        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      run: ./.github/scripts/setup-python-runner.sh
     - name: Test BigQuery Cloud SDK driver
       uses: ./.github/actions/test-driver
       with:
         junit-name: 'be-tests-bigquery-cloud-sdk-ee'
-        test-args: >-
-          :only-tags [:mb/driver-tests]
+        test-args: ${{ matrix.job.test-args }}
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -141,6 +158,18 @@ jobs:
     if: ${{ !inputs.skip }}
     runs-on: ubuntu-22.04
     timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        job:
+          - name: Driver Tests
+            test-args: >-
+              :only-tags [:mb/driver-tests]
+              :exclude-tags [:mb/transforms-python-test]
+          - name: Transforms Python Tests
+            test-args: >-
+              :only-tags [:mb/transforms-python-test]
+            needs-python-runner: true
     env:
       CI: 'true'
       DRIVERS: clickhouse
@@ -148,9 +177,14 @@ jobs:
       MB_CLICKHOUSE_TEST_PORT: 8123
       MB_CLICKHOUSE_TEST_OLD_PORT: 8124
       MB_CLICKHOUSE_TEST_NGINX_PORT: 8127
-    name: Clickhouse
+    name: Clickhouse ${{ matrix.job.name }}
     steps:
     - uses: actions/checkout@v4
+    - name: Setup python-runner
+      if: ${{ matrix.job.needs-python-runner == true }}
+      env:
+        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      run: ./.github/scripts/setup-python-runner.sh
     - name: Add ClickHouse TLS instance to /etc/hosts
       run: |
         sudo echo "127.0.0.1 server.clickhouseconnect.test" | sudo tee -a /etc/hosts
@@ -170,8 +204,7 @@ jobs:
       uses: ./.github/actions/test-driver
       with:
         junit-name: 'be-tests-clickhouse-ee'
-        test-args: >-
-          :only-tags [:mb/driver-tests]
+        test-args: ${{ matrix.job.test-args }}
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -208,6 +241,7 @@ jobs:
         junit-name: 'be-tests-druid-ee'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -244,6 +278,7 @@ jobs:
         junit-name: 'be-tests-druid-jdbc-ee'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -280,6 +315,7 @@ jobs:
         junit-name: 'be-tests-databricks-ee'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -317,6 +353,7 @@ jobs:
         junit-name: 'be-tests-databricks-multi-catalog-ee'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -338,12 +375,17 @@ jobs:
     name: H2
     steps:
     - uses: actions/checkout@v4
+    - name: Setup python-runner
+      env:
+        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      run: ./.github/scripts/setup-python-runner.sh
     - name: Test H2 driver
       uses: ./.github/actions/test-driver
       with:
         junit-name: 'be-tests-h2'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -388,6 +430,11 @@ jobs:
           - name: Driver Tests
             test-args: >-
               :only-tags [:mb/driver-tests]
+              :exclude-tags [:mb/transforms-python-test]
+          - name: Transforms Python Tests
+            test-args: >-
+              :only-tags [:mb/transforms-python-test]
+            needs-python-runner: true
 
     services:
       mysql:
@@ -442,6 +489,11 @@ jobs:
     name: "${{ matrix.version.name }} ${{ matrix.job.name }}"
     steps:
       - uses: actions/checkout@v4
+      - name: Setup python-runner
+      #if: ${{ matrix.job.needs-python-runner == true }}
+        env:
+          METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+        run: ./.github/scripts/setup-python-runner.sh
       - name: Test ${{ matrix.version.name }}
         uses: ./.github/actions/test-driver
         with:
@@ -476,6 +528,15 @@ jobs:
           - name: MongoDB Latest
             junit-name: be-tests-mongo-latest-ee
             image: mongo:latest
+        job:
+          - name: Driver Tests
+            test-args: >-
+              :exclude-tags '[:mongo-sharded-cluster-tests :mb/transforms-python-test]'
+              :only-tags [:mb/driver-tests]
+          - name: Transforms Python Tests
+            test-args: >-
+              :only-tags [:mb/transforms-python-test]
+            needs-python-runner: true
     env:
       CI: 'true'
       DRIVERS: mongo
@@ -489,16 +550,19 @@ jobs:
         env:
           MONGO_INITDB_ROOT_USERNAME: metabase
           MONGO_INITDB_ROOT_PASSWORD: metasample123
-    name: ${{ matrix.version.name }}
+    name: ${{ matrix.version.name }} ${{ matrix.job.name }}
     steps:
     - uses: actions/checkout@v4
+    - name: Setup python-runner
+      if: ${{ matrix.job.needs-python-runner == true }}
+      env:
+        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      run: ./.github/scripts/setup-python-runner.sh
     - name: Test ${{ matrix.version.name }}
       uses: ./.github/actions/test-driver
       with:
         junit-name: ${{ matrix.version.junit-name }}
-        test-args: >-
-          :exclude-tags [:mongo-sharded-cluster-tests]
-          :only-tags [:mb/driver-tests]
+        test-args: ${{ matrix.job.test-args }}
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -562,7 +626,7 @@ jobs:
         with:
           junit-name: ${{ matrix.version.junit-name }}
           test-args: >-
-            :exclude-tags [:mongo-sharded-cluster-tests]
+            :exclude-tags '[:mongo-sharded-cluster-tests :mb/transforms-python-test]'
             :only-tags [:mb/driver-tests]
       - name: Upload Test Results
         uses: ./.github/actions/upload-test-results
@@ -670,6 +734,7 @@ jobs:
         junit-name: ${{ matrix.version.junit-name }}
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -704,6 +769,11 @@ jobs:
           - name: Driver Tests
             test-args: >-
               :only-tags [:mb/driver-tests]
+              :exclude-tags [:mb/transforms-python-test]
+          - name: Transforms Python Tests
+            test-args: >-
+              :only-tags [:mb/transforms-python-test]
+            needs-python-runner: true
     name: "${{ matrix.version.name }} ${{ matrix.job.name }}"
     env:
       # Only run old migrations tests on pushes to master or release branches. All other branches should skip tests
@@ -745,6 +815,11 @@ jobs:
           POSTGRES_HOST_AUTH_METHOD: trust
     steps:
       - uses: actions/checkout@v4
+      - name: Setup python-runner
+      #if: ${{ matrix.job.needs-python-runner == true }}
+        env:
+          METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+        run: ./.github/scripts/setup-python-runner.sh
       - name: Test ${{ matrix.version.name }} (${{ matrix.job.name }})
         uses: ./.github/actions/test-driver
         with:
@@ -821,6 +896,7 @@ jobs:
         junit-name: 'be-tests-presto-jdbc-ee'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -850,8 +926,14 @@ jobs:
             skip: false
             test-args: >-
               :only-tags [:mb/driver-tests]
-              :exclude-tags [:mb/upload-tests]
+              :exclude-tags '[:mb/upload-tests :mb/transforms-python-test]'
               :fail-fast? true
+          - name: Redshift Transforms Python Tests
+            skip: false
+            test-args: >-
+              :only-tags [:mb/transforms-python-test]
+              :fail-fast? true
+            needs-python-runner: true
 
     env:
       CI: 'true'
@@ -864,6 +946,11 @@ jobs:
     name: ${{ matrix.job.name }}
     steps:
     - uses: actions/checkout@v4
+    - name: Setup python-runner
+      if: ${{ matrix.job.needs-python-runner == true }}
+      env:
+        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      run: ./.github/scripts/setup-python-runner.sh
     - name: Test ${{ matrix.job.name }}
       if: ${{ !matrix.job.skip }}
       uses: ./.github/actions/test-driver
@@ -887,6 +974,18 @@ jobs:
     if: ${{ !inputs.skip }}
     runs-on: ubuntu-22.04
     timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        job:
+          - name: Driver Tests
+            test-args: >-
+              :only-tags [:mb/driver-tests]
+              :exclude-tags [:mb/transforms-python-test]
+          - name: Transforms Python Tests
+            test-args: >-
+              :only-tags [:mb/transforms-python-test]
+            needs-python-runner: true
     env:
       CI: 'true'
       DRIVERS: snowflake
@@ -905,15 +1004,19 @@ jobs:
       MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_CUSTOM_USER: RSA_ROLE_TEST_CUSTOM_USER
       MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_ROLE: RSA_ROLE_TEST_ROLE
       MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_DB: RSA_ROLE_TEST_DB
-    name: Snowflake
+    name: Snowflake ${{ matrix.job.name }}
     steps:
     - uses: actions/checkout@v4
+    - name: Setup python-runner
+      if: ${{ matrix.job.needs-python-runner == true }}
+      env:
+        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      run: ./.github/scripts/setup-python-runner.sh
     - name: Test Snowflake driver
       uses: ./.github/actions/test-driver
       with:
         junit-name: 'be-tests-snowflake-ee'
-        test-args: >-
-          :only-tags [:mb/driver-tests]
+        test-args: ${{ matrix.job.test-args }}
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -948,6 +1051,7 @@ jobs:
         junit-name: 'be-tests-sparksql-ee'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -977,6 +1081,7 @@ jobs:
         junit-name: 'be-tests-sqlite-ee'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -1004,6 +1109,15 @@ jobs:
           - name: SQL Server 2022
             junit-name: be-tests-sqlserver-2022-ee
             image: mcr.microsoft.com/mssql/server:2022-latest
+        job:
+          - name: Driver Tests
+            test-args: >-
+              :only-tags [:mb/driver-tests]
+              :exclude-tags [:mb/transforms-python-test]
+          - name: Transforms Python Tests
+            test-args: >-
+              :only-tags [:mb/transforms-python-test]
+            needs-python-runner: true
     env:
       CI: 'true'
       DRIVERS: sqlserver
@@ -1019,15 +1133,19 @@ jobs:
           ACCEPT_EULA: Y
           SA_PASSWORD: 'P@ssw0rd'
           MSSQL_MEMORY_LIMIT_MB: 1024
-    name: ${{ matrix.version.name }}
+    name: ${{ matrix.version.name }} ${{ matrix.job.name }}
     steps:
     - uses: actions/checkout@v4
+    - name: Setup python-runner
+      if: ${{ matrix.job.needs-python-runner == true }}
+      env:
+        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      run: ./.github/scripts/setup-python-runner.sh
     - name: Test ${{ matrix.version.name }}
       uses: ./.github/actions/test-driver
       with:
         junit-name: ${{ matrix.version.junit-name }}
-        test-args: >-
-          :only-tags [:mb/driver-tests]
+        test-args: ${{ matrix.job.test-args }}
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()

.github/workflows/e2e-test.yml

@@ -60,6 +60,14 @@ jobs:
       CYPRESS_IS_EMBEDDING_SDK: "${{ inputs.name == 'embedding-sdk' }}"
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
+          token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+
+      - name: Setup python-runner
+        env:
+          METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+        run: ./.github/scripts/setup-python-runner.sh
 
       - name: Prepare Docker containers
         uses: ./.github/actions/e2e-prepare-containers

.gitignore

@@ -163,3 +163,4 @@ CLAUDE.local.md
 
 /logs
 /.clojure-mcp
+__pycache__/

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "python-runner"]
+	path = python-runner
+	url = https://github.com/metabase/python-runner-container.git
+	branch = prototype

deps.edn

@@ -36,7 +36,7 @@
   com.clearspring.analytics/stream          {:mvn/version "2.9.8"               ; Various sketching algorithms
                                              :exclusions  [it.unimi.dsi/fastutil
                                                            org.slf4j/slf4j-api]}
-  com.clojure-goes-fast/jvm-alloc-rate-meter{:mvn/version "0.1.4"}              ; Monitor heap allocation rate.
+  com.clojure-goes-fast/jvm-alloc-rate-meter {:mvn/version "0.1.4"}              ; Monitor heap allocation rate.
   com.clojure-goes-fast/jvm-hiccup-meter    {:mvn/version "0.1.1"}              ; Monitor GC pauses and other system-induced pauses.
   com.draines/postal                        {:mvn/version "2.0.5"               ; SMTP library
                                              :exclusions [com.sun.mail/jakarta.mail]}
@@ -45,7 +45,7 @@
   com.github.albfernandez/juniversalchardet {:mvn/version "2.5.0"}
   com.github.blagerweij/liquibase-sessionlock {:mvn/version "1.6.9"}            ; session lock for liquibase migrations
   com.github.jknack/handlebars              ^:antq/exclude                      ; 4.4.0 requires java 17+
-                                            {:mvn/version "4.3.1"}              ; templating engine
+  {:mvn/version "4.3.1"}              ; templating engine
   com.github.oliyh/martian-clj-http         {:mvn/version "0.1.31"}             ; openapi client
   com.github.pangloss/transducers           {:git/sha "c91d39046c8c64dc31b84ebb27154a8084fcda4c" ; like medley for transducers
                                              :git/url "https://github.com/pangloss/transducers"}
@@ -55,14 +55,14 @@
   com.github.vertical-blank/sql-formatter   {:mvn/version "2.0.5"}              ; Java SQL formatting library https://github.com/vertical-blank/sql-formatter
   com.google.guava/guava                    {:mvn/version "33.4.8-jre"}         ; dep for BigQuery, Spark, and GA. Require here rather than letting different dep versions stomp on each other — see comments on #9697
   com.h2database/h2                         ^:antq/exclude                      ; https://metaboat.slack.com/archives/CKZEMT1MJ/p1727191010259979
-                                            {:mvn/version "2.1.214"}            ; embedded SQL database
+  {:mvn/version "2.1.214"}            ; embedded SQL database
   com.ibm.icu/icu4j                         {:mvn/version "77.1"}               ; Used to transliterate Unicode characters into Latin characters
   com.knuddels/jtokkit                      {:mvn/version "1.1.0"}              ; Java tokenizer library for OpenAI models, used for token counting in semantic search
   com.mchange/c3p0                          {:mvn/version "0.10.1"}             ; DB connection pool
   com.snowplowanalytics/snowplow-java-tracker ^:antq/exclude                    ; 2+ depend on apache httpclient 5.x, but saml and clj-http depend on 4.x
-                                            {:mvn/version "1.0.1"}              ; Snowplow analytics
+  {:mvn/version "1.0.1"}              ; Snowplow analytics
   com.sun.mail/jakarta.mail                 ^:antq/exclude                      ; 2+ changes imports from javax.mail to jakarta.mail
-                                            {:mvn/version "1.6.8"}              ; SMTP lib, required by postal
+  {:mvn/version "1.6.8"}              ; SMTP lib, required by postal
   com.taoensso/nippy                        {:mvn/version "3.6.0"}              ; Fast serialization (i.e., GZIP) library for Clojure
   com.vladsch.flexmark/flexmark             {:mvn/version "0.64.8"}             ; Markdown parsing
   com.vladsch.flexmark/flexmark-ext-autolink
@@ -92,7 +92,7 @@
   ;; TODO: replace back to normal artifact
   io.github.metabase/macaw                  {:mvn/version "0.2.27"}             ; Parse native SQL queries
   io.github.resilience4j/resilience4j-retry ^:antq/exclude                      ; 2.x do not support Java 8
-                                            {:mvn/version "1.7.1"}              ; Support for retrying operations
+  {:mvn/version "1.7.1"}              ; Support for retrying operations
   io.prometheus/simpleclient_hotspot        {:mvn/version "0.16.0"}             ; prometheus jvm collector
   junegunn/grouper                          {:mvn/version "0.1.1"}              ; Batch processing helper
   kixi/stats                                {:mvn/version "0.5.7"               ; Various statistic measures implemented as transducers
@@ -198,6 +198,8 @@
                                                            org.eclipse.jetty.ee9/jetty-ee9-servlet
                                                            org.eclipse.jetty.ee9.websocket/jetty-ee9-websocket-jetty-server]}
   slingshot/slingshot                       {:mvn/version "0.12.2"}             ; enhanced throw/catch, used by other deps
+  software.amazon.awssdk/s3                 {:mvn/version "2.33.0"}
+  software.amazon.awssdk/sts                {:mvn/version "2.33.0"}           ; AWS SDK v2 for S3
   stencil/stencil                           {:mvn/version "0.5.0"}              ; Mustache templates for Clojure
   user-agent/user-agent                     {:mvn/version "0.1.1"}              ; User-Agent string parser, for Login History page & elsewhere
   weavejester/dependency                    {:mvn/version "1.0.0"}              ; Dependency graphs and topological sorting
@@ -836,8 +838,14 @@
   :test/lib
   {:exec-args {:only ["test/metabase/lib"
                       "test/metabase/lib_be"]}}
+  ;; test the python transforms
+  ;;    clj -X:dev:ee:ee-dev:driver:test:test/python-transform
+  :test/python-transform
+  {:exec-args {:only [metabase-enterprise.transforms.api-test
+                      "enterprise/backend/test/metabase_enterprise/transforms"
+                      "enterprise/backend/test/metabase_enterprise/transforms_python"]}}
 
-  ;; test the Query Processor
+;; test the Query Processor
   ;;
   ;;    clj -X:dev:ee:ee-dev:test:test/qp
   :test/qp

docs/developers-guide/driver-changelog.md

@@ -20,7 +20,19 @@ title: Driver interface changelog
 - Added `metabase.driver/rename-tables!*` multimethod for atomic table renaming operations. Takes a map of {from-table to-table}
   pairs that has been topologically sorted.
 
-- Added the driver multi-methods `metabase.driver/schema-exists?` and `metabase.driver/create-schema-if-needed!` which should be implemented by drivers that support `:schemas` and `:transforms/table`.
+- Added the driver multi-methods `metabase.driver/schema-exists?` and `metabase.driver/create-schema-if-needed!`
+  which should be implemented by drivers that support `:schemas` and `:transforms/table`.
+
+- Added `metabase.driver/type->database-type` multimethod that returns the database type for a given Metabase
+  type (from the type hierarchy) as a HoneySQL spec. This method handles general Metabase base types.
+
+- Added `metabase.driver/insert-from-source!` multimethod that abstracts data insertion from various sources
+  into existing tables. This multimethod dispatches on both the driver and the data source type
+  (`:rows` or `:jsonl-file`). It allows drivers to optimize based on the data source type and returns the number
+  of rows inserted. Default implementations are provided for both `:rows` and `:jsonl-file` source types.
+
+- Added `metabase.driver/insert-col->val` multimethod to parse values for insertion based on driver, data source type
+  and column definition. Drivers should implement this when their insertion mechanism needs values converted to proper types.
 
 - `metabase.driver/humanize-connection-error-message` now takes a list of all messages in the exception chain,
   instead of just the top-level message as a string.

e2e/support/helpers/e2e-python-helpers.ts

@@ -0,0 +1,173 @@
+function pythonEditor() {
+  cy.findAllByTestId("loading-indicator").should("not.exist");
+  return cy.get("[data-testid=python-editor] .cm-content");
+}
+
+function focusPythonEditor() {
+  pythonEditor().should("be.visible").click();
+
+  pythonEditor().get(".cm-editor").should("have.class", "cm-focused");
+
+  return pythonEditor();
+}
+
+function blurPythonEditor() {
+  pythonEditor().get(".cm-editor").blur();
+}
+
+function pythonEditorCompletions() {
+  return cy.get(".cm-tooltip-autocomplete").should("be.visible");
+}
+
+function pythonEditorCompletion(label: string) {
+  return cy.get(".cm-completionLabel").contains(label).parent();
+}
+
+function pythonEditorSelectAll() {
+  const isMac = Cypress.platform === "darwin";
+  const metaKey = isMac ? "Meta" : "Control";
+  focusPythonEditor().realPress([metaKey, "A"]);
+  cy.get(".cm-selectionBackground").should("exist");
+}
+
+function clearPythonEditor() {
+  pythonEditorSelectAll();
+  cy.realPress(["Backspace"]);
+}
+
+type TypeOptions = {
+  delay?: number;
+  focus?: boolean;
+};
+
+function pythonEditorType(
+  text: string,
+  { delay = 10, focus = true }: TypeOptions = {},
+) {
+  if (focus) {
+    focusPythonEditor();
+  }
+
+  const isMac = Cypress.platform === "darwin";
+  const metaKey = isMac ? "Meta" : "Control";
+
+  const parts = text.replaceAll("{{", "{{}{{}").split(/(\{[^}]+\})/);
+
+  function type(text: string) {
+    cy.realType(text, {
+      pressDelay: delay,
+    });
+  }
+
+  // HACK: realType does not accept a lot of common escape sequences,
+  // so we implement them manually here for the python editor.
+  parts.forEach((part) => {
+    switch (part.toLowerCase()) {
+      case "":
+        return;
+
+      case "{clear}":
+        return clearPythonEditor();
+
+      case "{selectall}":
+        return pythonEditorSelectAll();
+
+      case "{leftarrow}":
+        return cy.realPress(["ArrowLeft"]);
+
+      case "{rightarrow}":
+        return cy.realPress(["ArrowRight"]);
+
+      case "{downarrow}":
+        return cy.realPress(["ArrowDown"]);
+
+      case "{uparrow}":
+        return cy.realPress(["ArrowUp"]);
+
+      case "{enter}":
+        return cy.realPress(["Enter"]);
+
+      case "{home}":
+      case "{movetostart}":
+        return cy.realPress(["Home"]);
+
+      case "{end}":
+      case "{movetoend}":
+        return cy.realPress(["End"]);
+
+      case "{backspace}":
+        return cy.realPress(["Backspace"]);
+
+      case "{tab}":
+        return cy.realPress(["Tab"]);
+
+      case "{nextcompletion}":
+        cy.wait(50);
+        return cy.realPress([metaKey, "j"]);
+
+      case "{prevcompletion}":
+        cy.wait(50);
+        return cy.realPress([metaKey, "k"]);
+
+      case "{{}":
+        return cy.realType("{");
+    }
+
+    if (part.startsWith("{") && part.endsWith("}")) {
+      // unknown escape sequence, let's try typing it
+      type("{");
+      type(part.slice(1));
+      return;
+    }
+
+    type(part);
+  });
+
+  return pythonEditor();
+}
+
+export const PythonEditor = {
+  get: pythonEditor,
+  type(text: string, options: TypeOptions = {}) {
+    pythonEditorType(text, options);
+    return PythonEditor;
+  },
+  focus() {
+    focusPythonEditor();
+    return PythonEditor;
+  },
+  blur() {
+    blurPythonEditor();
+    return PythonEditor;
+  },
+  selectAll() {
+    pythonEditorSelectAll();
+    return PythonEditor;
+  },
+  clear() {
+    clearPythonEditor();
+    return PythonEditor;
+  },
+  textbox() {
+    return PythonEditor.get().get("[role='textbox']");
+  },
+  value() {
+    // Get the multiline text content of the editor
+    return PythonEditor.textbox()
+      .get(".cm-line")
+      .then((lines) => {
+        const text: string[] = [];
+        lines.each((_, line) => {
+          text.push(line.textContent ?? "");
+        });
+        const value = text.join("\n");
+        const placeholder = "SELECT * FROM TABLE_NAME";
+        if (value === placeholder) {
+          return "";
+        }
+        return value;
+      });
+  },
+  completions: pythonEditorCompletions,
+  completion: pythonEditorCompletion,
+};

e2e/support/helpers/index.ts

@@ -35,6 +35,7 @@ export * from "./e2e-native-editor-helpers";
 export * from "./e2e-notebook-helpers";
 export * from "./e2e-notification-helpers";
 export * from "./e2e-permissions-helpers";
+export * from "./e2e-python-helpers";
 export * from "./e2e-qa-databases-helpers";
 export * from "./e2e-relative-date-picker-helpers";
 export * from "./e2e-request-helpers";

e2e/test/scenarios/admin/transforms.cy.spec.ts

@@ -1,5 +1,7 @@
 const { H } = cy;
 
+import dedent from "ts-dedent";
+
 import { SAMPLE_DB_ID, WRITABLE_DB_ID } from "e2e/support/cypress_data";
 import { SAMPLE_DATABASE } from "e2e/support/cypress_sample_database";
 import type {
@@ -181,6 +183,123 @@ H.describeWithSnowplowEE("scenarios > admin > transforms", () => {
       testSnippets();
     });
 
+    it(
+      "should be possible to create and run a Python transform",
+      { tags: ["@transforms-python"] },
+      () => {
+        setPythonRunnerSettings();
+        cy.log("create a new transform");
+        visitTransformListPage();
+        getTransformListPage().button("Create a transform").click();
+        H.popover().findByText("Python script").click();
+
+        H.expectUnstructuredSnowplowEvent({
+          event: "transform_create",
+          event_detail: "python",
+          triggered_from: "transform-page-create-menu",
+        });
+
+        cy.findByTestId("python-data-picker")
+          .findByText("Select a database")
+          .click();
+
+        cy.log("Unsupported databases should be disabled");
+        H.popover()
+          .findByRole("option", { name: "Sample Database" })
+          .should("have.attr", "aria-disabled", "true");
+
+        cy.log("Select database");
+        H.popover().findByText(DB_NAME).click();
+
+        getPythonDataPicker().button("Select a table…").click();
+        H.entityPickerModal().findByText("Animals").click();
+
+        getPythonDataPicker().within(() => {
+          cy.findByText("Writable Postgres12 / Schema A").should("be.visible");
+          cy.findByText("Animals").should("be.visible");
+          cy.findByPlaceholderText("Enter alias").should(
+            "have.value",
+            "animals",
+          );
+          cy.findByPlaceholderText("Enter alias")
+            .clear()
+            .type("foo bar")
+            .blur();
+          cy.findByPlaceholderText("Enter alias").should(
+            "have.value",
+            "foo_bar",
+          );
+        });
+
+        H.PythonEditor.value()
+          .should("contain", "def transform(foo_bar):")
+          .should(
+            "contain",
+            'foo_bar: DataFrame containing the data from the "Writable Postgres12.Schema A.Animals" table',
+          );
+
+        getPythonDataPicker().within(() => {
+          cy.findByText("Add a table").click();
+          cy.button("Select a table…").click();
+        });
+
+        H.entityPickerModal().within(() => {
+          cy.log("Selecting the same table should not be possible");
+          cy.findByText("Animals")
+            .parent()
+            .parent()
+            .parent()
+            .should("have.attr", "data-disabled", "true");
+
+          cy.findByText("Schema B").click();
+          cy.findByText("Animals").click();
+        });
+
+        getPythonDataPicker().within(() => {
+          cy.icon("refresh").click();
+          cy.findAllByPlaceholderText("Enter alias")
+            .first()
+            .should("have.value", "animals_1")
+            .clear()
+            .type("foo bar")
+            .blur()
+            .should("have.value", "foo_bar");
+          cy.findAllByPlaceholderText("Enter alias")
+            .eq(1)
+            .should("have.value", "animals")
+            .clear()
+            .type("foo bar")
+            .blur()
+            .should("have.value", "foo_bar_1");
+        });
+
+        getQueryEditor().button("Save").click();
+
+        H.modal().within(() => {
+          cy.findByLabelText("Name").clear().type("Python transform");
+          cy.findByLabelText("Table name").clear().type("python_transform");
+          cy.button("Save").click();
+        });
+
+        H.expectUnstructuredSnowplowEvent({
+          event: "transform_created",
+        });
+
+        cy.log("run the transform and make sure its table can be queried");
+        runTransformAndWaitForSuccess();
+        H.expectUnstructuredSnowplowEvent({
+          event: "transform_trigger_manual_run",
+          triggered_from: "transform-page",
+        });
+
+        getRunSection().should("contain", "Executing Python transform");
+
+        getTableLink().click();
+        H.queryBuilderHeader().findByText(DB_NAME).should("be.visible");
+        H.assertQueryBuilderRowCount(3);
+      },
+    );
+
     it("should be able to create and run a transform from a question or a model", () => {
       function testCardSource({
         type,
@@ -1321,6 +1440,107 @@ H.describeWithSnowplowEE("scenarios > admin > transforms", () => {
       H.main().findByText("Dependencies").should("not.exist");
     });
   });
+
+  describe("python > common library", () => {
+    it(
+      "should be possible to edit and save the common library",
+      { tags: ["@transforms-python"] },
+      () => {
+        visitCommonLibrary();
+
+        cy.log("updating the library should be possible");
+        H.PythonEditor.clear().type(
+          dedent`
+          def useful_calculation(a, b):
+          return a + b
+        `,
+        );
+        getLibraryEditorHeader().findByText("Save").click();
+
+        cy.log("the contents should be saved properly");
+        visitCommonLibrary();
+        H.PythonEditor.value().should(
+          "eq",
+          dedent`
+          def useful_calculation(a, b):
+              return a + b
+          `,
+        );
+
+        cy.log("reverting the changes should be possible");
+        H.PythonEditor.clear().type("# oops");
+        getLibraryEditorHeader().findByText("Revert").click();
+        H.PythonEditor.value().should(
+          "eq",
+          dedent`
+          def useful_calculation(a, b):
+              return a + b
+          `,
+        );
+      },
+    );
+
+    it(
+      "should be possible to use the common library",
+      { tags: ["@transforms-python"] },
+      () => {
+        createPythonLibrary(
+          "common.py",
+          dedent`
+          def useful_calculation(a, b):
+          return a + b
+      `,
+        );
+
+        visitTransformListPage();
+        getTransformListPage().button("Create a transform").click();
+        H.popover().findByText("Python script").click();
+
+        H.PythonEditor.clear().type(
+          dedent`
+          import pandas as pd
+
+          def transform():
+          return pd.DataFrame({"foo": common.useful_calculation(1, 2)})
+        `,
+        );
+
+        getQueryEditor().findByText("Import common library").click();
+        H.PythonEditor.value().should("contain", "import common");
+
+        cy.findByTestId("python-data-picker")
+          .findByText("Select a database")
+          .click();
+
+        H.popover().findByText(DB_NAME).click();
+
+        cy.findByTestId("python-data-picker")
+          .findByText("Select a table…")
+          .click();
+
+        H.entityPickerModal().within(() => {
+          cy.findByText("Schema a").click();
+          cy.findByText("Animals").click();
+        });
+
+        getQueryEditor().button("Save").click();
+
+        H.modal().within(() => {
+          cy.findByLabelText("Name").clear().type("Python transform");
+          cy.findByLabelText("Table name").clear().type("python_transform");
+          cy.button("Save").click();
+        });
+      },
+    );
+
+    function visitCommonLibrary(path = "common.py") {
+      cy.visit(`/admin/transforms/library/${path}`);
+    }
+
+    function getLibraryEditorHeader() {
+      return cy.findByTestId("library-editor-header");
+    }
+  });
 });
 
 describe("scenarios > admin > transforms > databases without :schemas", () => {
@@ -2302,6 +2522,7 @@ function createMbqlTransform({
           },
         },
         target: {
+          database: WRITABLE_DB_ID,
           type: "table",
           name: targetTable,
           schema: targetSchema,
@@ -2340,6 +2561,7 @@ function createSqlTransform({
         },
       },
       target: {
+        database: WRITABLE_DB_ID,
         type: "table",
         name: targetTable,
         schema: targetSchema,
@@ -2404,3 +2626,33 @@ function assertOptionNotSelected(name: string) {
 function editorSidebar() {
   return cy.findByTestId("editor-sidebar");
 }
+
+function getRunSection() {
+  return getTransformPage().findByTestId("run-section");
+}
+
+function getPythonDataPicker() {
+  return cy.findByTestId("python-data-picker");
+}
+
+function createPythonLibrary(path: string, source: string) {
+  cy.request("PUT", `/api/ee/transforms-python/library/${path}`, {
+    source,
+  });
+}
+
+function setPythonRunnerSettings() {
+  H.updateSetting("python-runner-url", "http://localhost:5001");
+  H.updateSetting("python-runner-api-token", "dev-token-12345");
+  H.updateSetting("python-storage-s-3-endpoint", "http://localhost:4566");
+  H.updateSetting("python-storage-s-3-region", "us-east-1");
+  H.updateSetting("python-storage-s-3-bucket", "metabase-python-runner");
+  H.updateSetting("python-storage-s-3-prefix", "test-prefix");
+  H.updateSetting("python-storage-s-3-access-key", "test");
+  H.updateSetting("python-storage-s-3-secret-key", "test");
+  H.updateSetting(
+    "python-storage-s-3-container-endpoint",
+    "http://localhost:4566",
+  );
+  H.updateSetting("python-storage-s-3-path-style-access", true);
+}

enterprise/backend/src/metabase_enterprise/metabot_v3/core.clj

@@ -6,7 +6,6 @@
    [metabase-enterprise.metabot-v3.client]
    [metabase-enterprise.metabot-v3.table-utils]
    [metabase-enterprise.metabot-v3.util]
-   [metabase.premium-features.core :refer [defenterprise]]
    [potemkin :as p]
    [toucan2.core :as t2]))
 
@@ -23,24 +22,20 @@
   used-tables
   schema-sample])
 
-(defenterprise metabot-stats
+(defn metabot-stats
   "Calculate total Metabot token usage over a window of the the previous UTC day 00:00-23:59"
-  :feature :none
   []
-  (let [yesterday-utc (-> (t/offset-date-time (t/zone-offset "+00"))
-                          (t/minus (t/days 1)))]
-    {:metabot-tokens     (or (t2/select-one-fn :sum
-                                               [:model/MetabotMessage [:%sum.total_tokens :sum]]
-                                               {:where [:= [:cast :created_at :date] [:cast yesterday-utc :date]]})
-                             0)
-     :metabot-queries    (t2/select-one-fn :cnt
-                                           [:model/MetabotMessage [:%count.id :cnt]]
-                                           ;; there are requests from users and responses from ai-service, in theory
-                                           ;; counting requests from users should be good enough
-                                           :role "user"
-                                           {:where [:= [:cast :created_at :date] [:cast yesterday-utc :date]]})
-     :metabot-users      (:cnt (t2/query-one {:select [[[:count [:distinct :c.user_id]] :cnt]]
-                                              :from   [[:metabot_message :m]]
-                                              :join   [[:metabot_conversation :c] [:= :c.id :m.conversation_id]]
-                                              :where  [:= [:cast :m.created_at :date] [:cast yesterday-utc :date]]}))
-     :metabot-usage-date (str (t/local-date yesterday-utc))}))
+  (let [yesterday-utc (t/minus (t/offset-date-time (t/zone-offset "+00")) (t/days 1))]
+    {:metabot-tokens  (t2/select-one-fn :sum
+                                        [:model/MetabotMessage [:%sum.total :sum]]
+                                        {:where [:= [:cast :created_at :date] [:cast yesterday-utc :date]]})
+     :metabot-queries (t2/select-one-fn :cnt
+                                        [:model/MetabotMessage [:%count.id :cnt]]
+                                        ;; there are requests from users and responses from ai-service, in theory
+                                        ;; counting requests from users should be good enough
+                                        :role "user"
+                                        {:where [:= [:cast :created_at :date] [:cast yesterday-utc :date]]})
+     :metabot-users   (:cnt (t2/query-one {:select [[[:count [:distinct :c.user_id]] :cnt]]
+                                           :from   [[:metabot_message :m]]
+                                           :join   [[:metabot_conversation :c] [:= :c.id :m.conversation_id]]
+                                           :where [:= [:cast :m.created_at :date] [:cast yesterday-utc :date]]}))}))

enterprise/backend/src/metabase_enterprise/transforms/api.clj

@@ -1,7 +1,9 @@
 (ns metabase-enterprise.transforms.api
   (:require
+   [metabase-enterprise.transforms-python.execute :as transforms-python.execute]
    [metabase-enterprise.transforms.api.transform-job]
    [metabase-enterprise.transforms.api.transform-tag]
+   [metabase-enterprise.transforms.canceling :as transforms.canceling]
    [metabase-enterprise.transforms.execute :as transforms.execute]
    [metabase-enterprise.transforms.models.transform :as transform.model]
    [metabase-enterprise.transforms.models.transform-run :as transform-run]
@@ -29,12 +31,21 @@
 (set! *warn-on-reflection* true)
 
 (mr/def ::transform-source
-  [:map
-   [:type [:= "query"]]
-   [:query [:map [:database :int]]]])
+  [:multi {:dispatch (comp keyword :type)}
+   [:query
+    [:map
+     [:type [:= "query"]]
+     [:query [:map [:database :int]]]]]
+   [:python
+    [:map {:closed true}
+     [:source-database {:optional true} :int]
+     [:source-tables   [:map-of :string :int]]
+     [:type [:= "python"]]
+     [:body :string]]]])
 
 (mr/def ::transform-target
   [:map
+   [:database {:optional true} :int]
    [:type [:enum "table"]]
    [:schema {:optional true} [:or ms/NonBlankString :nil]]
    [:name :string]])
@@ -55,14 +66,10 @@
              :schema "transforms"
              :name "gadget_products"}}])
 
-(defn- source-database-id
-  [transform]
-  (-> transform :source :query :database))
-
 (defn- check-database-feature
   [transform]
-  (let [database (api/check-400 (t2/select-one :model/Database (source-database-id transform))
-                                (deferred-tru "The source database cannot be found."))
+  (let [database (api/check-400 (t2/select-one :model/Database (transforms.util/target-database-id transform))
+                                (deferred-tru "The target database cannot be found."))
         feature (transforms.util/required-database-feature transform)]
     (api/check-400 (not (:is_sample database))
                    (deferred-tru "Cannot run transforms on the sample database."))
@@ -73,6 +80,11 @@
     (api/check-400 (not (transforms.util/db-routing-enabled? database))
                    (deferred-tru "Transforms are not supported on databases with DB routing enabled."))))
 
+(defn- check-feature-enabled!
+  [transform]
+  (api/check (transforms.util/check-feature-enabled transform)
+             [402 (deferred-tru "Premium features required for this transform type are not enabled.")]))
+
 (api.macros/defendpoint :get "/"
   "Get a list of transforms."
   [_route-params
@@ -95,6 +107,7 @@
             [:tag_ids {:optional true} [:sequential ms/PositiveInt]]]]
   (api/check-superuser)
   (check-database-feature body)
+  (check-feature-enabled! body)
   (api/check (not (transforms.util/target-table-exists? body))
              403
              (deferred-tru "A table with that name already exists."))
@@ -115,8 +128,7 @@
   (log/info "get transform" id)
   (api/check-superuser)
   (let [{:keys [target] :as transform} (api/check-404 (t2/select-one :model/Transform id))
-        database-id (source-database-id transform)
-        target-table (transforms.util/target-table database-id target :active true)]
+        target-table (transforms.util/target-table (transforms.util/target-database-id transform) target :active true)]
     (-> transform
         (t2/hydrate :last_run :transform_tag_ids)
         (u/update-some :last_run transforms.util/localize-run-timestamps)
@@ -173,10 +185,13 @@
     (let [old (t2/select-one :model/Transform id)
           new (merge old body)
           target-fields #(-> % :target (select-keys [:schema :name]))]
+      ;; we must validate on a full transform object
+      (check-feature-enabled! new)
       (check-database-feature new)
-      (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]
-        (throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)
-                        {:status-code 400})))
+      (when (transforms.util/query-transform? old)
+        (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]
+          (throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)
+                          {:status-code 400}))))
       (api/check (not (and (not= (target-fields old) (target-fields new))
                            (transforms.util/target-table-exists? new)))
                  403
@@ -212,8 +227,11 @@
                     [:id ms/PositiveInt]]]
   (log/info "canceling transform " id)
   (api/check-superuser)
-  (let [run (api/check-404 (transform-run/running-run-for-transform-id id))]
-    (transform-run-cancelation/mark-cancel-started-run! (:id run)))
+  (let [transform (api/check-404 (t2/select-one :model/Transform id))
+        run (api/check-404 (transform-run/running-run-for-transform-id id))]
+    (transform-run-cancelation/mark-cancel-started-run! (:id run))
+    (when (transforms.util/python-transform? transform)
+      (transforms.canceling/cancel-run! (:id run))))
   nil)
 
 (api.macros/defendpoint :post "/:id/run"
@@ -223,10 +241,15 @@
   (log/info "run transform" id)
   (api/check-superuser)
   (let [transform (api/check-404 (t2/select-one :model/Transform id))
+        _         (check-feature-enabled! transform)
         start-promise (promise)]
-    (u.jvm/in-virtual-thread*
-     (transforms.execute/run-mbql-transform! transform {:start-promise start-promise
-                                                        :run-method :manual}))
+    (if (transforms.util/python-transform? transform)
+      (u.jvm/in-virtual-thread*
+       (transforms-python.execute/execute-python-transform! transform {:start-promise start-promise
+                                                                       :run-method :manual}))
+      (u.jvm/in-virtual-thread*
+       (transforms.execute/run-mbql-transform! transform {:start-promise start-promise
+                                                          :run-method :manual})))
     (when (instance? Throwable @start-promise)
       (throw @start-promise))
     (let [result @start-promise

enterprise/backend/src/metabase_enterprise/transforms/canceling.clj

@@ -49,7 +49,9 @@
    (chan-signal-cancel! run-id)
    (transform-run/timeout-run! run-id)))
 
-(defn- cancel-run! [run-id]
+(defn cancel-run!
+  "Cancel a run with id."
+  [run-id]
   (when (chan-signal-cancel! run-id)
     (transform-run/cancel-run! run-id)))
 

enterprise/backend/src/metabase_enterprise/transforms/execute.clj

@@ -1,5 +1,6 @@
 (ns metabase-enterprise.transforms.execute
   (:require
+   [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
    [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.driver :as driver]
    [metabase.driver.util :as driver.u]
@@ -54,7 +55,8 @@
          (transforms.util/run-cancelable-transform!
           run-id driver transform-details
           (fn [_cancel-chan] (driver/run-transform! driver transform-details opts)))
-         (transforms.util/sync-target! target database run-id)))
+         (transforms.instrumentation/with-stage-timing [run-id :table-sync]
+           (transforms.util/sync-target! target database run-id))))
      (catch Throwable t
        (log/error t "Error executing transform")
        (when start-promise

enterprise/backend/src/metabase_enterprise/transforms/instrumentation.clj

@@ -0,0 +1,193 @@
+(ns metabase-enterprise.transforms.instrumentation
+  (:require
+   [metabase.analytics.prometheus :as prometheus]
+   [metabase.util :as u]
+   [metabase.util.log :as log]
+   [metabase.util.malli :as mu]
+   [metabase.util.malli.registry :as mr]))
+
+(set! *warn-on-reflection* true)
+
+(mr/def ::stage-type
+  [:enum :export :computation :import])
+
+(def ^:private label-to-stage
+  {:dwh-to-file      :export
+   :file-to-s3       :export
+   :file-to-dwh      :import
+   :python-execution :computation
+   :mbql-query       :computation
+   :table-sync       :import})
+
+(mr/def ::stage-label
+  [:enum
+   :dwh-to-file
+   :file-to-s3
+   :file-to-dwh
+
+   :python-execution
+   :mbql-query
+   :table-sync])
+
+(mu/defn record-stage-start!
+  "Record the start of a transform stage."
+  [job-run-id :- pos-int?
+   stage-label :- ::stage-label]
+  (let [stage-type (label-to-stage stage-label)]
+    (log/infof "Transform stage started: run-id=%d type=%s label=%s" job-run-id (name stage-type) (name stage-label))
+    (prometheus/inc! :metabase-transforms/stage-started
+                     {:job-run-id (str job-run-id)
+                      :stage-type (name stage-type)
+                      :stage-label (name stage-label)})))
+
+(mu/defn record-stage-completion!
+  "Record the successful completion of a transform stage."
+  [job-run-id :- pos-int?
+   stage-label :- ::stage-label
+   duration-ms :- int?]
+  (let [stage-type (label-to-stage stage-label)]
+    (log/infof "Transform stage completed: run-id=%d type=%s label=%s duration=%dms" job-run-id (name stage-type) (name stage-label) duration-ms)
+    (let [labels {:job-run-id (str job-run-id)
+                  :stage-type (name stage-type)
+                  :stage-label (name stage-label)}]
+      (prometheus/inc! :metabase-transforms/stage-completed labels)
+      (prometheus/observe! :metabase-transforms/stage-duration-ms labels duration-ms))))
+
+(mu/defn record-stage-failure!
+  "Record the failure of a transform stage."
+  [job-run-id :- pos-int?
+   stage-label :- ::stage-label
+   duration-ms :- int?]
+  (let [stage-type (label-to-stage stage-label)]
+    (log/warnf "Transform stage failed: run-id=%d type=%s label=%s duration=%s" job-run-id (name stage-type) (name stage-label) (str duration-ms "ms"))
+    (let [labels {:job-run-id (str job-run-id)
+                  :stage-type (name stage-type)
+                  :stage-label (name stage-label)}]
+      (prometheus/inc! :metabase-transforms/stage-failed labels)
+      (when duration-ms
+        (prometheus/observe! :metabase-transforms/stage-duration-ms labels duration-ms)))))
+
+(defn with-timing
+  "Generic timing function that executes body while recording metrics.
+
+   Takes a map of optional functions:
+   - :start-fn - function to call when starting
+   - :success-fn - function to call on success (receives duration-ms as last arg)
+   - :error-fn - function to call on error (receives duration-ms as last arg)
+
+   Usage:
+   (with-timing {:start-fn #(record-stage-start! %1 %3)
+                 :success-fn #(record-stage-completion! %1 %2 %3)
+                 :error-fn #(record-stage-failure! %1 %2 %3)}
+                [job-run-id stage-label]
+                #(do-work)) "
+  [callbacks args body-fn]
+  (let [start-time (u/start-timer)]
+    (when-let [start-fn (:start-fn callbacks)]
+      (apply start-fn args))
+    (try
+      (let [result (body-fn)]
+        (when-let [success-fn (:success-fn callbacks)]
+          (apply success-fn (conj (vec args) (long (u/since-ms start-time)))))
+        result)
+      (catch Throwable t
+        (when-let [error-fn (:error-fn callbacks)]
+          (apply error-fn (conj (vec args) (long (u/since-ms start-time)))))
+        (throw t)))))
+
+(defmacro with-stage-timing
+  "Execute body while timing a transform stage. Automatically records start, completion or failure.
+
+   Usage:
+   (with-stage-timing [job-run-id :dwh-to-file]
+     ;; stage implementation
+     result)"
+  [args & body]
+  `(with-timing {:start-fn record-stage-start!
+                 :success-fn record-stage-completion!
+                 :error-fn record-stage-failure!}
+     ~args
+     (^:once fn* [] ~@body)))
+
+(mu/defn record-data-transfer!
+  "Record metrics about data transfer (size and rows)."
+  [job-run-id :- pos-int?
+   stage-label :- ::stage-label
+   bytes :- [:maybe int?]
+   rows :- [:maybe int?]]
+  (if (and (nil? bytes) (nil? rows))
+    (log/warnf "Data transfer recorded: run-id=%d stage=%s but no bytes or rows provided"
+               job-run-id
+               (name stage-label))
+    (log/infof "Data transfer recorded: run-id=%d stage=%s%s%s"
+               job-run-id
+               (name stage-label)
+               (if bytes (str " bytes=" bytes) "")
+               (if rows (str " rows=" rows) "")))
+  (let [labels {:job-run-id (str job-run-id)
+                :stage-label (name stage-label)}]
+    (when bytes
+      (prometheus/observe! :metabase-transforms/data-transfer-bytes labels bytes))
+    (when rows
+      (prometheus/observe! :metabase-transforms/data-transfer-rows labels rows))))
+
+(defn record-job-start!
+  "Record the start of a transform job run."
+  [job-id run-method]
+  (log/infof "Transform job started: job-id=%d run-method=%s" job-id (name run-method))
+  (prometheus/inc! :metabase-transforms/job-runs-total
+                   {:job-id (str job-id)
+                    :run-method (name run-method)}))
+
+(defn record-job-completion!
+  "Record the successful completion of a transform job run."
+  [job-id run-method duration-ms]
+  (log/infof "Transform job completed: job-id=%d run-method=%s duration=%dms" job-id (name run-method) duration-ms)
+  (let [labels {:job-id (str job-id)
+                :run-method (name run-method)}]
+    (prometheus/inc! :metabase-transforms/job-runs-completed labels)
+    (prometheus/observe! :metabase-transforms/job-run-duration-ms labels duration-ms)))
+
+(defn record-job-failure!
+  "Record the failure of a transform job run."
+  [job-id run-method duration-ms]
+  (log/warnf "Transform job failed: job-id=%d run-method=%s duration=%dms" job-id (name run-method) duration-ms)
+  (let [labels {:job-id (str job-id)
+                :run-method (name run-method)}]
+    (prometheus/inc! :metabase-transforms/job-runs-failed labels)
+    (prometheus/observe! :metabase-transforms/job-run-duration-ms labels duration-ms)))
+
+(defmacro with-job-timing
+  "Execute body while timing a transform job run. Automatically records start, completion or failure.
+
+   Usage:
+   (with-job-timing [job-id run-method]
+     ;; job implementation
+     result)"
+  [[job-id run-method] & body]
+  `(with-timing {:start-fn record-job-start!
+                 :success-fn record-job-completion!
+                 :error-fn record-job-failure!}
+     [~job-id ~run-method]
+     (^:once fn* [] ~@body)))
+
+(mu/defn record-python-api-call!
+  "Record metrics about Python API calls."
+  [job-run-id :- pos-int?
+   duration-ms :- int?
+   status :- [:enum :success :error :timeout]]
+  (log/infof "Python API call %s: run-id=%d duration=%dms" (name status) job-run-id duration-ms)
+  (let [labels {:job-run-id (str job-run-id)}]
+    (prometheus/inc! :metabase-transforms/python-api-calls-total
+                     (assoc labels :status (name status)))
+    (prometheus/observe! :metabase-transforms/python-api-call-duration-ms labels duration-ms)))
+
+(defmacro with-python-api-timing
+  "Execute body while timing a Python API call."
+  [[job-run-id] & body]
+  `(with-timing {:success-fn (fn [job-run-id# duration-ms#]
+                               (record-python-api-call! job-run-id# duration-ms# :success))
+                 :error-fn (fn [job-run-id# duration-ms#]
+                             (record-python-api-call! job-run-id# duration-ms# :error))}
+     [~job-run-id]
+     (^:once fn* [] ~@body)))

enterprise/backend/src/metabase_enterprise/transforms/jobs.clj

@@ -5,11 +5,14 @@
    [clojurewerkz.quartzite.schedule.calendar-interval :as calendar-interval]
    [clojurewerkz.quartzite.triggers :as triggers]
    [flatland.ordered.set :as ordered-set]
+   [metabase-enterprise.transforms-python.execute :as transforms-python.execute]
    [metabase-enterprise.transforms.execute :as transforms.execute]
+   [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
    [metabase-enterprise.transforms.models.job-run :as transforms.job-run]
    [metabase-enterprise.transforms.models.transform-run :as transform-run]
    [metabase-enterprise.transforms.ordering :as transforms.ordering]
    [metabase-enterprise.transforms.settings :as transforms.settings]
+   [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.task.core :as task]
    [metabase.util.log :as log]
    [toucan2.core :as t2]))
@@ -64,18 +67,22 @@
         (map transforms-by-id complete)))))
 
 (defn- run-transform! [run-id run-method {transform-id :id :as transform}]
-  (when
-   (transform-run/running-run-for-transform-id transform-id)
-    (log/warn "Transform" (pr-str transform-id) "already running, waiting")
-    (loop []
-      (Thread/sleep 2000)
+  (if-not (transforms.util/check-feature-enabled transform)
+    (log/warnf "Skip running transform %d due to lacking premium features" transform-id)
+    (do
       (when (transform-run/running-run-for-transform-id transform-id)
-        (recur))))
-
-  (log/info "Executing job transform" (pr-str transform-id))
-  (transforms.execute/run-mbql-transform! transform {:run-method run-method})
-
-  (transforms.job-run/add-run-activity! run-id))
+        (log/warn "Transform" (pr-str transform-id) "already running, waiting")
+        (loop []
+          (Thread/sleep 2000)
+          (when (transform-run/running-run-for-transform-id transform-id)
+            (recur))))
+      (log/info "Executing job transform" (pr-str transform-id))
+      (if (transforms.util/python-transform? transform)
+        (transforms.instrumentation/with-stage-timing [run-id :python-execution]
+          (transforms-python.execute/execute-python-transform! transform {:run-method run-method}))
+        (transforms.instrumentation/with-stage-timing [run-id :mbql-query]
+          (transforms.execute/run-mbql-transform! transform {:run-method run-method})))
+      (transforms.job-run/add-run-activity! run-id))))
 
 (defn run-transforms!
   "Run a series of transforms and their dependencies.
@@ -113,12 +120,13 @@
     (let [transforms (job-transform-ids job-id)]
       (log/info "Executing transform job" (pr-str job-id) "with transforms" (pr-str transforms))
       (let [{run-id :id} (transforms.job-run/start-run! job-id run-method)]
-        (try
-          (run-transforms! run-id transforms opts)
-          (transforms.job-run/succeed-started-run! run-id)
-          (catch Throwable t
-            (transforms.job-run/fail-started-run! run-id {:message (.getMessage t)})
-            (throw t)))))))
+        (transforms.instrumentation/with-job-timing [job-id run-method]
+          (try
+            (run-transforms! run-id transforms opts)
+            (transforms.job-run/succeed-started-run! run-id)
+            (catch Throwable t
+              (transforms.job-run/fail-started-run! run-id {:message (.getMessage t)})
+              (throw t))))))))
 
 (def ^:private job-key "metabase-enterprise.transforms.jobs.timeout-job")
 

enterprise/backend/src/metabase_enterprise/transforms/ordering.clj

@@ -16,23 +16,29 @@
 (set! *warn-on-reflection* true)
 
 (defn- transform-deps [transform]
-  (let [query (-> (get-in transform [:source :query])
-                  transforms.util/massage-sql-query
-                  qp.preprocess/preprocess
-                  ;; legacy usage -- don't do things like this going forward
-                  #_{:clj-kondo/ignore [:discouraged-var]}
-                  lib/->legacy-MBQL)]
-    (case (:type query)
-      :native (driver/native-query-deps (-> (qp.store/metadata-provider)
-                                            lib.metadata/database
-                                            :engine)
-                                        (get-in query [:native :query]))
-      :query (into #{}
-                   (keep #(clojure.core.match/match %
-                            [:source-table source] (when (int? source)
-                                                     {:table source})
-                            _ nil))
-                   (tree-seq coll? seq query)))))
+  (if (transforms.util/python-transform? transform)
+    ;; Python transforms have dependencies via source-tables mapping
+    (into #{}
+          (map #(hash-map :table %))
+          (vals (get-in transform [:source :source-tables])))
+    ;; Query transforms have dependencies via SQL query analysis
+    (let [query (-> (get-in transform [:source :query])
+                    transforms.util/massage-sql-query
+                    qp.preprocess/preprocess
+                    ;; legacy usage -- don't do things like this going forward
+                    #_{:clj-kondo/ignore [:discouraged-var]}
+                    lib/->legacy-MBQL)]
+      (case (:type query)
+        :native (driver/native-query-deps (-> (qp.store/metadata-provider)
+                                              lib.metadata/database
+                                              :engine)
+                                          (get-in query [:native :query]))
+        :query (into #{}
+                     (keep #(clojure.core.match/match %
+                              [:source-table source] (when (int? source)
+                                                       {:table source})
+                              _ nil))
+                     (tree-seq coll? seq query))))))
 
 (defn- dependency-map [transforms]
   (into {}
@@ -56,12 +62,15 @@
 
   The result is a map of transform id -> #{transform ids the transform depends on}. Dependencies are limited to just
   the transforms in the original list -- if a transform depends on some transform not in the list, the 'extra'
-  dependency is ignored."
+  dependency is ignored. Both query and Python transforms can have dependencies on tables produced by other transforms."
   [transforms]
-  (let [transforms-by-db (->> transforms
+  (let [;; Group all transforms by their database
+        transforms-by-db (->> transforms
                               (map (fn [transform]
-                                     {(get-in transform [:source :query :database]) [transform]}))
+                                     (let [db-id (transforms.util/target-database-id transform)]
+                                       {db-id [transform]})))
                               (apply merge-with into))
+
         transform-ids (into #{} (map :id) transforms)
         {:keys [output-tables dependencies]} (->> transforms-by-db
                                                   (map (fn [[db-id db-transforms]]

enterprise/backend/src/metabase_enterprise/transforms/schedule.clj

@@ -101,6 +101,7 @@
         trigger (build-trigger job-id schedule)]
     (task/schedule-task! job trigger)))
 
+;; TODO: should this be part of transformjob hooks?
 (defn update-job!
   "Update the trigger for a transform job."
   [job-id schedule]

enterprise/backend/src/metabase_enterprise/transforms/settings.clj

@@ -7,11 +7,11 @@
 
 (setting/defsetting transform-timeout
   (deferred-tru "The timeout for a transform job, in minutes.")
-  :type       :integer
+  :type :integer
   :visibility :internal
-  :default    (* 4 60)
-  :feature    :transforms
-  :doc        false
-  :export?    false
+  :default (* 4 60)
+  :feature :transforms
+  :doc false
+  :export? false
   :encryption :no
-  :audit      :getter)
+  :audit :getter)

enterprise/backend/src/metabase_enterprise/transforms/util.clj

@@ -7,12 +7,16 @@
    [metabase-enterprise.transforms.models.transform-run :as transform-run]
    [metabase-enterprise.transforms.settings :as transforms.settings]
    [metabase.driver :as driver]
+   [metabase.lib.schema.common :as lib.schema.common]
+   [metabase.premium-features.core :as premium-features]
    [metabase.query-processor.compile :as qp.compile]
    [metabase.query-processor.pipeline :as qp.pipeline]
    [metabase.sync.core :as sync]
    [metabase.util :as u]
    [metabase.util.date-2 :as u.date]
    [metabase.util.log :as log]
+   [metabase.util.malli :as mu]
+   [metabase.util.malli.registry :as mr]
    [toucan2.core :as t2])
   (:import
    (java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)
@@ -27,6 +31,24 @@
     (keyword schema name)
     (keyword name)))
 
+(defn query-transform?
+  "Check if this is a query transform: native query / mbql query."
+  [transform]
+  (= :query (-> transform :source :type keyword)))
+
+(defn python-transform?
+  "Check if this is a Python transform."
+  [transform]
+  (= :python (-> transform :source :type keyword)))
+
+(defn check-feature-enabled
+  "Checking whether we have proper feature flags for using a given transform."
+  [transform]
+  (if (python-transform? transform)
+    (and (premium-features/has-feature? :transforms)
+         (premium-features/has-feature? :transforms-python))
+    (premium-features/has-feature? :transforms)))
+
 (defn try-start-unless-already-running
   "Start a transform run, throwing an informative error if already running."
   [id run-method]
@@ -76,10 +98,18 @@
    (log/info "Syncing target" (pr-str target) "for transform")
    (activate-table-and-mark-computed! database target)))
 
+;; TODO this and target-database-id can be transforms multimethods?
+(defn target-database-id
+  "Return the target database id of a transform"
+  [transform]
+  (if (python-transform? transform)
+    (-> transform :target :database)
+    (-> transform :source :query :database)))
+
 (defn target-table-exists?
   "Test if the target table of a transform already exists."
-  [{:keys [source target] :as _transform}]
-  (let [db-id (-> source :query :database)
+  [{:keys [target] :as transform}]
+  (let [db-id (target-database-id transform)
         {driver :engine :as database} (t2/select-one :model/Database db-id)]
     (driver/table-exists? driver database target)))
 
@@ -118,10 +148,10 @@
 
 (defn delete-target-table!
   "Delete the target table of a transform and sync it from the app db."
-  [{:keys [id target source], :as _transform}]
+  [{:keys [id target], :as transform}]
   (when target
     (let [target (update target :type keyword)
-          database-id (-> source :query :database)
+          database-id (target-database-id transform)
           {driver :engine :as database} (t2/select-one :model/Database database-id)]
       (driver/drop-transform-target! driver database target)
       (log/info "Deactivating  target " (pr-str target) "for transform" id)
@@ -178,6 +208,56 @@
       (u/update-some :start_time utc-timestamp-string)
       (u/update-some :end_time   utc-timestamp-string)))
 
+(mr/def ::column-definition
+  [:map
+   [:name :string]
+   [:type ::lib.schema.common/base-type]
+   [:nullable? {:optional true} :boolean]])
+
+(mr/def ::table-definition
+  [:map
+   [:name :keyword]
+   [:columns [:sequential ::column-definition]]
+   [:primary-key {:optional true} [:sequential :string]]])
+
+(mu/defn create-table-from-schema!
+  "Create a table from a table-schema"
+  [driver :- :keyword
+   database-id :- pos-int?
+   table-schema :- ::table-definition]
+  (let [{:keys [columns] table-name :name} table-schema
+        column-definitions (into {} (map (fn [{:keys [name type database-type]}]
+                                           (let [db-type (if database-type
+                                                           [[:raw database-type]]
+                                                           (try
+                                                             (driver/type->database-type driver type)
+                                                             (catch IllegalArgumentException _
+                                                               (log/warnf "Couldn't determine database type for type %s, fallback to Text" type)
+                                                               (driver/type->database-type driver :type/Text))))]
+                                             [name db-type])))
+                                 columns)
+        primary-key-opts (select-keys table-schema [:primary-key])]
+    (log/infof "Creating table %s with %d columns" table-name (count columns))
+    (driver/create-table! driver database-id table-name column-definitions primary-key-opts)))
+
+(defn drop-table!
+  "Drop a table in the database."
+  [driver database-id table-name]
+  (log/infof "Dropping table %s" table-name)
+  (driver/drop-table! driver database-id table-name))
+
+(defn temp-table-name
+  "Generate a temporary table name with the given suffix and current timestamp in seconds."
+  [base-table-name suffix]
+  (keyword (str (u/qualified-name base-table-name) "_" suffix "_" (quot (System/currentTimeMillis) 1000))))
+
+(defn rename-tables!
+  "Rename multiple tables atomically within a transaction using the new driver/rename-tables method.
+   This is a simpler, composable operation that only handles renaming."
+  [driver database-id rename-map]
+  (log/infof "Renaming tables: %s" (pr-str rename-map))
+  (driver/rename-tables! driver database-id rename-map))
+
 (defn db-routing-enabled?
   "Returns whether or not the given database is either a router or destination database"
   [db-or-id]

enterprise/backend/src/metabase_enterprise/transforms_python/core.clj

@@ -0,0 +1,5 @@
+(ns metabase-enterprise.transforms-python.core
+  (:require
+   [metabase-enterprise.transforms-python.models.python-library]))
+
+(comment metabase-enterprise.transforms-python.models.python-library/keep-me)

enterprise/backend/src/metabase_enterprise/transforms_python/data_transfer.clj

@@ -0,0 +1 @@
+(ns metabase-enterprise.transforms-python.data-transfer)

enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj

@@ -0,0 +1,257 @@
+(ns metabase-enterprise.transforms-python.execute
+  (:require
+   [clojure.core.async :as a]
+   [clojure.java.io :as io]
+   [clojure.string :as str]
+   [metabase-enterprise.transforms-python.python-runner :as python-runner]
+   [metabase-enterprise.transforms-python.s3 :as s3]
+   [metabase-enterprise.transforms-python.settings :as transforms-python.settings]
+   [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
+   [metabase-enterprise.transforms.util :as transforms.util]
+   [metabase.driver :as driver]
+   [metabase.util :as u]
+   [metabase.util.jvm :as u.jvm]
+   [metabase.util.log :as log]
+   [toucan2.core :as t2])
+  (:import
+   (java.io Closeable File)
+   (java.net SocketException)
+   (java.time Duration)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:const temp-table-suffix-new
+  "Suffix used for temporary tables containing new data during swap operations."
+  "new")
+
+(def ^:const temp-table-suffix-old
+  "Suffix used for temporary tables containing old data during swap operations."
+  "old")
+
+(defn- empty-message-log
+  "Returns a new message log.
+
+  Python transforms produce output while they run. The message log is a stateful structure that buffers log outputs during the run.
+  The goal is to assist in debugging in the case of python runtime or syntax errors, and to provide some immediate feedback for longer running transforms.
+
+  Important Note: in memory buffering is considered acceptable as we expect /logs output to be truncated or small.
+
+  The intended use of this structure is to produce a string suitable for `transform_run.message`, saving it every few seconds."
+  []
+  (atom {:pre-python  []                                    ; log! outputs previous to the python execution, i.e table read progress
+         :python      nil                                   ; events json structured logs from the /logs endpoint
+         :post-python []}))                                    ; log! outputs after the python execution, i.e. output reads, writes to target
+
+(defn- log!
+  "Appends a string to the message log, the string is user facing and should be suitable for presentation as part of the `transform_run.message` field."
+  [message-log s]
+  (swap! message-log (fn [m] (update m (if (:python m) :post-python :pre-python) conj s)))
+  nil)
+
+(defn- replace-python-logs!
+  "Called when events output is received from a /logs call to copy those events into the message-log."
+  [message-log events]
+  (swap! message-log assoc :python events)
+  nil)
+
+(defn- message-log->transform-run-message
+  "Presents the content of the log as a string suitable for storage in transform_run.message."
+  [message-log]
+  (let [{:keys [pre-python python post-python]} @message-log]
+    (str/join "\n" (concat pre-python
+                           (map :message python)
+                           post-python))))
+
+(defn- save-log-to-transform-run-message!
+  "Saves the content of the log into the transform_run's message field. Lossy, string contains user facing information only."
+  [run-id message-log]
+  (t2/update! :model/TransformRun
+              :id run-id
+              {:message (message-log->transform-run-message message-log)}))
+
+(def ^:private ^Duration python-message-loop-sleep-duration
+  (Duration/ofMillis 1000))
+
+(defn- python-message-update-loop!
+  "Block while relevant log data is replicated from the runner into the message log.
+  When new logs are received, the log data will be flushed to the transform_run.message field as a string.
+
+  The loop will exit on error or when interrupted."
+  [run-id message-log]
+  (try
+    (loop []
+      (if (.isInterrupted (Thread/currentThread))
+        (log/debug "Message update loop interrupted")
+        (do (let [sleep-ms (.toMillis python-message-loop-sleep-duration)]
+              (when (pos? sleep-ms) (Thread/sleep sleep-ms)))
+            (let [{:keys [status body]} (python-runner/get-logs run-id)]
+              (cond
+                (<= 200 status 299)
+                (let [{:keys [execution_id events]} body]
+                  (if-not (= run-id execution_id)
+                    (do (log/debugf "Run id did not match expected: %s actual: %s" run-id execution_id)
+                        (recur))
+                    (do
+                      (replace-python-logs! message-log events)
+                      (save-log-to-transform-run-message! run-id message-log)
+                      (recur))))
+                (= 404 status)
+                (do
+                  (log/debugf "No logs yet (or run finished), run-id: %s" run-id)
+                  (recur))
+                :else
+                (do
+                  (log/warnf "Unexpected status polling for logs %s %s, run-id: %s" status body run-id)
+                  (log/debug "Exiting due to poll error")))))))
+    (catch SocketException se (when-not (= "Closed by interrupt" (ex-message se)) (throw se)))
+    (catch InterruptedException _)
+    (catch Throwable e
+      (log/errorf e "An exception was caught during msg update loop, run-id: %s" run-id))))
+
+(defn- open-python-message-update-future! ^Closeable [run-id message-log]
+  (let [cleanup (fn [fut]
+                  (future-cancel fut)
+                  (if (= ::timeout (try (deref fut 10000 ::timeout) (catch Throwable _)))
+                    (log/fatalf "Log polling task did not respond to interrupt, run-id: %s" run-id)
+                    (log/debugf "Log polling task done, run-id: %s" run-id)))
+        fut     (u.jvm/in-virtual-thread*
+                 (python-message-update-loop! run-id message-log))]
+    (reify Closeable
+      (close [_] (cleanup fut)))))
+
+(defn- create-table-and-insert-data!
+  "Create a table from metadata and insert data from source."
+  [driver db-id table-name metadata data-source]
+  (let [table-schema {:name (if (keyword? table-name) table-name (keyword table-name))
+                      :columns (mapv (fn [{:keys [name base_type #_database_type]}]
+                                       {:name name
+                                        :type (python-runner/restricted-insert-type base_type)
+                                        ;; :database-type database_type
+                                        :nullable? true})
+                                     (:fields metadata))}
+        data-source (assoc data-source :table-schema table-schema)]
+    (transforms.util/create-table-from-schema! driver db-id table-schema)
+    (driver/insert-from-source! driver db-id table-schema data-source)))
+
+(defn- transfer-file-to-db [driver db {:keys [target] :as transform} metadata temp-file]
+  (let [table-name (transforms.util/qualified-table-name driver target)
+        table-exists? (transforms.util/target-table-exists? transform)
+        data-source {:type :jsonl-file
+                     :file temp-file}]
+    (if table-exists?
+      ;; Table exists - use temp table + atomic swap pattern
+      (let [source-table-name (transforms.util/temp-table-name table-name temp-table-suffix-new)
+            temp-table-name (transforms.util/temp-table-name table-name temp-table-suffix-old)]
+        (log/info "Existing table detected, Create then swap")
+        (try
+          (create-table-and-insert-data! driver (:id db) source-table-name metadata data-source)
+          ;; Use the new atomic rename-tables! function: target <- source (using temp)
+          (transforms.util/rename-tables! driver (:id db) {table-name temp-table-name
+                                                           source-table-name table-name})
+          ;; Drop the old table (now stored in temp-table-name) separately
+          (transforms.util/drop-table! driver (:id db) temp-table-name)
+          (catch Exception e
+            (log/error e "Failed to transfer data to table")
+            (try
+              (transforms.util/drop-table! driver (:id db) source-table-name)
+              (catch Exception _))
+            (throw e))))
+      ;; Table doesn't exist - create directly with target name
+      (do
+        (log/info "New table")
+        (create-table-and-insert-data! driver (:id db) table-name metadata data-source)))))
+
+(defn- start-cancellation-process!
+  "Starts a core.async process that optimistically sends a cancellation request to the python executor if cancel-chan receives a value.
+  Returns a channel that will receive either the async http call j.u.c.FutureTask in the case of cancellation, or nil when the cancel-chan is closed."
+  [server-url run-id cancel-chan]
+  (a/go (when (a/<! cancel-chan)
+          (python-runner/cancel-python-code-http-call! server-url run-id))))
+
+(defn- run-python-transform! [{:keys [source] :as transform} db run-id cancel-chan message-log]
+  ;; TODO restructure things such that s3 can we swapped out for other transfer mechanisms
+  (with-open [log-future-ref     (open-python-message-update-future! run-id message-log)
+              shared-storage-ref (s3/open-shared-storage! (:source-tables source))]
+    (let [driver     (:engine db)
+          server-url (transforms-python.settings/python-runner-url)
+          _          (python-runner/copy-tables-to-s3! {:run-id         run-id
+                                                        :shared-storage @shared-storage-ref
+                                                        :table-name->id (:source-tables source)
+                                                        :cancel-chan    cancel-chan})
+          _          (start-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan
+          {:keys [status body] :as response}
+          (python-runner/execute-python-code-http-call!
+           {:server-url     server-url
+            :code           (:body source)
+            :run-id         run-id
+            :table-name->id (:source-tables source)
+            :shared-storage @shared-storage-ref})
+          ;; TODO temporary to keep more code stable while refactoring
+          ;; no need to materialize these early (i.e output we can stream directly into a tmp file or db if small)
+          {:keys [output output-manifest events]} (python-runner/read-output-objects @shared-storage-ref)]
+      (.close log-future-ref)                               ; early close to force any writes to flush
+      (when (seq events)
+        (replace-python-logs! message-log events))
+      (if (not= 200 status)
+        (throw (ex-info "Python runner call failed"
+                        {:status-code           400
+                         :api-status-code       status
+                         :body                  body
+                         :events                events
+                         :transform-run-message (message-log->transform-run-message message-log)}))
+        (try
+          (let [temp-file (File/createTempFile "transform-output-" ".jsonl")]
+            (when-not (seq (:fields output-manifest))
+              (throw (ex-info "No fields in metadata"
+                              {:metadata               output-manifest
+                               :raw-body               body
+                               :events                 events
+                               :transform-run-message  (message-log->transform-run-message message-log)})))
+            (try
+              (with-open [writer (io/writer temp-file)]
+                (.write writer ^String output))
+              (let [file-size (.length temp-file)]
+                (transforms.instrumentation/with-stage-timing [run-id :file-to-dwh]
+                  (transfer-file-to-db driver db transform output-manifest temp-file))
+                (transforms.instrumentation/record-data-transfer! run-id :file-to-dwh file-size nil))
+              (finally
+                (.delete temp-file))))
+          response
+          (catch Exception e
+            (log/error e "Failed to to create resulting table")
+            (throw (ex-info "Failed to create the resulting table"
+                            {:transform-run-message (message-log->transform-run-message message-log)}
+                            e))))))))
+
+(defn execute-python-transform!
+  "Execute a Python transform by calling the python runner.
+
+  Blocks until the transform returns."
+  [transform {:keys [run-method start-promise]}]
+  (assert (transforms.util/python-transform? transform) "Transform must be a python transform")
+  (try
+    (let [message-log (empty-message-log)
+          {:keys [target] transform-id :id} transform
+          {driver :engine :as db} (t2/select-one :model/Database (:database target))
+          {run-id :id} (transforms.util/try-start-unless-already-running transform-id run-method)]
+      (some-> start-promise (deliver [:started run-id]))
+      (log! message-log "Executing Python transform")
+      (log/info "Executing Python transform" transform-id "with target" (pr-str target))
+      (let [start-ms          (u/start-timer)
+            transform-details {:db-id          (:id db)
+                               :transform-type (keyword (:type target))
+                               :conn-spec      (driver/connection-spec driver db)
+                               :output-schema  (:schema target)
+                               :output-table   (transforms.util/qualified-table-name driver target)}
+            run-fn            (fn [cancel-chan]
+                                (run-python-transform! transform db run-id cancel-chan message-log)
+                                (log! message-log (format "Python execution finished successfully in %s" (Duration/ofMillis (u/since-ms start-ms))))
+                                (save-log-to-transform-run-message! run-id message-log))
+            result            (transforms.util/run-cancelable-transform! run-id driver transform-details run-fn)]
+        (transforms.instrumentation/with-stage-timing [run-id :table-sync]
+          (transforms.util/sync-target! target db run-id))
+        {:run_id run-id
+         :result result}))
+    (catch Throwable t
+      (log/error t "Error executing Python transform")
+      (throw t))))

enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj

@@ -0,0 +1,283 @@
+(ns metabase-enterprise.transforms-python.python-runner
+  (:require
+   [clj-http.client :as http]
+   [clojure.java.io :as io]
+   [clojure.string :as str]
+   [medley.core :as m]
+   [metabase-enterprise.transforms-python.s3 :as s3]
+   [metabase-enterprise.transforms-python.settings :as transforms-python.settings]
+   [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
+   [metabase.config.core :as config]
+   [metabase.driver :as driver]
+   [metabase.query-processor.compile :as qp.compile]
+   [metabase.query-processor.pipeline :as qp.pipeline]
+   ;; TODO check that querying team are ok with us accessing this directly, otherwise make another plan
+   ^{:clj-kondo/ignore [:discouraged-namespace]}
+   [metabase.query-processor.store :as qp.store]
+   [metabase.util.json :as json]
+   [metabase.util.log :as log]
+   [toucan2.core :as t2])
+  (:import
+   (clojure.lang PersistentQueue)
+   (java.io BufferedWriter File OutputStream OutputStreamWriter)
+   (java.nio.charset StandardCharsets)))
+
+(set! *warn-on-reflection* true)
+
+(defn- authorization-headers
+  "Returns HTTP headers with Authorization bearer token if configured.
+  Throws configuration error in production if token is not set."
+  []
+  (let [api-token (transforms-python.settings/python-runner-api-token)]
+    (if api-token
+      {"Authorization" (str "Bearer " api-token)}
+      (if config/is-prod?
+        (throw (ex-info "Python runner API token is required in production but not configured"
+                        {:error-type :configuration-error}))
+        {}))))
+
+(defn- python-runner-request
+  "Helper function for making HTTP requests to the python runner service."
+  [server-url method endpoint & [request-options]]
+  (let [url          (str server-url "/v1" endpoint)
+        base-options {:content-type     :json
+                      :accept           :json
+                      :throw-exceptions false
+                      :as               :json
+                      :headers          (authorization-headers)}]
+    (http/request (merge base-options request-options {:method method, :url url}))))
+
+(defn- write-jsonl-to-stream! [^OutputStream os col-names reducible-rows]
+  (let [none? (volatile! true)
+        writer (-> os
+                   (OutputStreamWriter. StandardCharsets/UTF_8)
+                   (BufferedWriter.))]
+
+    (run! (fn [row]
+            (when @none? (vreset! none? false))
+            (let [row-map (zipmap col-names row)]
+              (json/encode-to row-map writer {})
+              (.newLine writer)))
+          reducible-rows)
+
+    ;; Workaround for LocalStack, which doesn't support zero byte files.
+    (when @none?
+      (.write writer " "))
+
+    (doto writer
+      (.flush)
+      (.close))))
+
+(defn- execute-mbql-query
+  [driver db-id query respond cancel-chan]
+  (driver/with-driver driver
+    (let [native (qp.compile/compile {:type :query, :database db-id :query query})
+          query  {:database db-id
+                  :type     :native
+                  :native   native}]
+      (qp.store/with-metadata-provider db-id
+        (binding [qp.pipeline/*canceled-chan* cancel-chan]
+          (driver/execute-reducible-query driver query {:canceled-chan cancel-chan} respond))))))
+
+(defn root-type
+  "Supported type for roundtrip/insertion"
+  [base-type]
+  (when base-type
+    (some #(when (isa? base-type %) %)
+          [:type/Number
+           :type/Date
+           :type/DateTime
+           :type/Instant
+           :type/DateTimeWithTZ
+           :type/Text
+           :type/Boolean])))
+
+(defn restricted-insert-type
+  "Type for insertion restricted to supported"
+  [base_type]
+  (if base_type
+    (let [base-type-kw (keyword "type" base_type)]
+      (if (root-type base-type-kw)
+        base-type-kw
+        :type/Text))
+    :type/Text))
+
+(defn- closest-ancestor [t pred]
+  (loop [remaining (conj PersistentQueue/EMPTY [t])]
+    (when-let [t (first remaining)]
+      (if (pred t)
+        t
+        (recur (into (pop t) (parents t)))))))
+
+(defn- effective-semantic-type-i-think
+  "Kinda sketchy but maybe reasonable way to infer the effective-semantic-type"
+  [{:keys [base_type effective_type semantic_type]}]
+  (or semantic_type (closest-ancestor (or effective_type base_type) #(isa? % :Semantic/*))))
+
+(defn- generate-manifest
+  "Generate a manifest to communicate schema and metadata information around the table."
+  [table-id cols-meta]
+  {:schema_version 1
+   :data_format    "jsonl"
+   :data_version   1
+   :table_metadata {:table_id table-id}
+   :fields         (mapv (fn [col-meta]
+                           {:name           (:name col-meta)
+                            :base_type      (some-> (:base_type col-meta) name)
+                            :database_type  (some-> (:database_type col-meta) name)
+                            :root_type      (some-> (root-type (:base_type col-meta)) name)
+                            :semantic_type  (some-> (effective-semantic-type-i-think col-meta) name)
+                            :effective_type (some-> (or (:effective_type col-meta) (:database_type col-meta)) name)
+                            :field_id       (:id col-meta)})
+                         cols-meta)})
+
+(defn- maybe-fixup-value [col v]
+  (cond
+    (nil? (root-type (:base_type col)))
+    ;; we're not a supported base type, so we just stringify it
+    (when v (json/encode v))
+
+    ;; the clickhouse driver returns bigdecimals for int64 values
+    (and (isa? (:base_type col) :type/Integer)
+         (or (instance? BigDecimal v)
+             (float? v)))
+    (bigint v)
+
+    :else
+    v))
+
+(defn- write-table-data-to-file! [{:keys [db-id driver table-id fields-meta temp-file cancel-chan]}]
+  (let [query    {:source-table table-id}]
+    (execute-mbql-query driver db-id query
+                        (fn [{cols-meta :cols} reducible-rows]
+                          (with-open [os (io/output-stream temp-file)]
+                            (let [filtered-col-meta (m/index-by :name fields-meta)
+                                  col-names         (map :name cols-meta)
+                                  filtered-rows     (eduction (map (fn [row]
+                                                                     (->>
+                                                                      (map vector col-names row)
+                                                                      (filter (fn [[n _]]
+                                                                                (contains? filtered-col-meta n)))
+                                                                      (map (fn [[n v]]
+                                                                             (maybe-fixup-value (filtered-col-meta n) v))))))
+                                                              reducible-rows)]
+                              (write-jsonl-to-stream! os (filter filtered-col-meta col-names) filtered-rows))))
+                        cancel-chan)))
+
+(defn get-logs
+  "Return the logs of the current running python process"
+  [run-id]
+  (let [server-url (transforms-python.settings/python-runner-url)]
+    (python-runner-request server-url :get "/logs" {:query-params {:request_id run-id}})))
+
+(defn execute-python-code-http-call!
+  "Calls the /execute endpoint of the python runner. Blocks until the run either succeeds or fails and returns
+  the response from the server."
+  [{:keys [server-url code run-id table-name->id shared-storage]}]
+  (let [{:keys [objects]} shared-storage
+        {:keys [output output-manifest events]} objects
+
+        url-for-path             (fn [path] (:url (get objects path)))
+        table-name->url          (update-vals table-name->id #(url-for-path [:table % :data]))
+        table-name->manifest-url (update-vals table-name->id #(url-for-path [:table % :manifest]))
+
+        payload                  {:code                code
+                                  :library             (t2/select-fn->fn :path :source :model/PythonLibrary)
+                                  :timeout             30
+                                  :request_id          run-id
+                                  :output_url          (:url output)
+                                  :output_manifest_url (:url output-manifest)
+                                  :events_url          (:url events)
+                                  :table_mapping       table-name->url
+                                  :manifest_mapping    table-name->manifest-url}
+
+        response                 (transforms.instrumentation/with-python-api-timing [run-id]
+                                   (python-runner-request server-url :post "/execute" {:body (json/encode payload)}))]
+    ;; when a 500 is returned we observe a string in the body (despite the python returning json)
+    ;; always try to parse the returned string as json before yielding (could tighten this up at some point)
+    (update response :body (fn [string-if-error]
+                             (if (string? string-if-error)
+                               (try
+                                 (json/decode+kw string-if-error)
+                                 (catch Exception _
+                                   {:error string-if-error}))
+                               string-if-error)))))
+
+;; temporary, we should not need to realize data/events files into memory longer term
+(defn read-output-objects
+  "Temporary function that strings/jsons stuff in S3 and returns it for compatibility."
+  [{:keys [s3-client bucket-name objects]}]
+  (let [{:keys [output output-manifest events]} objects
+        output-content          (s3/read-to-string s3-client bucket-name (:path output) nil)
+        output-manifest-content (s3/read-to-string s3-client bucket-name (:path output-manifest) "{}")
+        events-content          (s3/read-to-string s3-client bucket-name (:path events))]
+    {:output          output-content
+     :output-manifest (json/decode+kw output-manifest-content)
+     :events          (mapv json/decode+kw (str/split-lines events-content))}))
+
+(defn cancel-python-code-http-call!
+  "Calls the /cancel endpoint of the python runner. Returns immediately."
+  [server-url run-id]
+  (python-runner-request server-url :post "/cancel" {:body   (json/encode {:request_id run-id})
+                                                     :async? true}
+                         #_success #(log/debug %)
+                         #_failure #(log/error %)))
+
+(defn- safe-delete
+  "Safely delete a file."
+  [^File file]
+  (try (.delete file) (catch Exception _)))
+
+(defn- fields-metadata [_driver table-id]
+  (t2/select [:model/Field :id :name :base_type :effective_type :semantic_type :database_type :database_position]
+             :table_id table-id
+             :active true
+             ;; we are only interested in top-level objects, so filter out nested fields (parent or path)
+             :parent_id nil
+             :nfc_path nil
+             {:order-by [[:database_position :asc]]}))
+
+;; TODO break this up such that s3 can be swapped out for other transfer mechanisms.
+(defn copy-tables-to-s3!
+  "Writes table content to their corresponding objects named in shared-storage, see (open-shared-storage!).
+  Blocks until all tables are fully written and committed to shared storage."
+  [{:keys [run-id
+           shared-storage
+           table-name->id
+           cancel-chan]}]
+  ;; TODO there's scope for some parallelism here, in particular across different databases
+  (doseq [table-id (vals table-name->id)
+          :let [{:keys [s3-client bucket-name objects]} shared-storage
+                {data-path :path}                       (get objects [:table table-id :data])
+                {manifest-path :path}                   (get objects [:table table-id :manifest])]]
+    (let [tmp-data-file (File/createTempFile data-path "")
+          tmp-meta-file (File/createTempFile manifest-path "")]
+      (try
+        (let [db-id       (t2/select-one-fn :db_id (t2/table-name :model/Table) :id table-id)
+              driver      (t2/select-one-fn :engine :model/Database db-id)
+              fields-meta (fields-metadata driver table-id)
+              manifest    (generate-manifest table-id fields-meta)]
+
+          (transforms.instrumentation/with-stage-timing [run-id :dwh-to-file]
+            (write-table-data-to-file!
+             {:db-id       db-id
+              :driver      driver
+              :table-id    table-id
+              :fields-meta fields-meta
+              :temp-file   tmp-data-file
+              :cancel-chan cancel-chan}))
+
+          (with-open [writer (io/writer tmp-meta-file)]
+            (json/encode-to manifest writer {}))
+          (let [data-size (.length tmp-data-file)
+                meta-size (.length tmp-meta-file)]
+            (transforms.instrumentation/record-data-transfer! run-id :dwh-to-file data-size nil)
+
+            (transforms.instrumentation/with-stage-timing [run-id :file-to-s3]
+              (s3/upload-file s3-client bucket-name data-path tmp-data-file)
+              (s3/upload-file s3-client bucket-name manifest-path tmp-meta-file))
+
+            (transforms.instrumentation/record-data-transfer! run-id :file-to-s3 (+ data-size meta-size) nil)))
+        (finally
+          (safe-delete tmp-data-file)
+          (safe-delete tmp-meta-file))))))

enterprise/backend/src/metabase_enterprise/transforms_python/s3.clj

@@ -0,0 +1,215 @@
+(ns metabase-enterprise.transforms-python.s3
+  (:require
+   [metabase-enterprise.transforms-python.settings :as transforms-python.settings]
+   [metabase.util.log :as log])
+  (:import
+   (clojure.lang IDeref)
+   (java.io Closeable File)
+   (java.net URI)
+   (java.time Duration)
+   (software.amazon.awssdk.auth.credentials AwsBasicCredentials StaticCredentialsProvider)
+   (software.amazon.awssdk.auth.credentials AwsCredentialsProvider DefaultCredentialsProvider)
+   (software.amazon.awssdk.core.sync RequestBody)
+   (software.amazon.awssdk.regions Region)
+   (software.amazon.awssdk.services.s3 S3Client S3ClientBuilder S3Configuration)
+   (software.amazon.awssdk.services.s3.model DeleteObjectRequest GetObjectRequest NoSuchKeyException PutObjectRequest)
+   (software.amazon.awssdk.services.s3.presigner S3Presigner S3Presigner$Builder)
+   (software.amazon.awssdk.services.s3.presigner.model GetObjectPresignRequest PutObjectPresignRequest)))
+
+(set! *warn-on-reflection* true)
+
+;; TODO We should make these durations configurable in future.
+
+;; Longer duration for inputs than for outputs, to compensate for the duration of the code execution itself.
+(def ^:private ^Duration presigned-get-duration (Duration/ofMinutes 30))
+(def ^:private ^Duration presigned-put-duration (Duration/ofHours 5))
+
+;; We should consider namespacing these paths, for example, using the run id to simplify cleanup.
+;; Another idea is to have deterministic paths, so that it's easier to resume incomplete runs.
+(defn- working-dir-for-run
+  "The path within the target bucket within which to store all the files for this run."
+  []
+  (let [shared-prefix (some-> (transforms-python.settings/python-storage-s-3-prefix) (str "/"))]
+    (str shared-prefix "run-" (System/nanoTime) "-" (rand-int 10000))))
+
+(defmacro ^:private maybe-with-endpoint* [builder endpoint]
+  `(let [builder# ~builder]
+     (when-let [region# (transforms-python.settings/python-storage-s-3-region)]
+       (.region builder# (Region/of region#)))
+     (when ~endpoint (.endpointOverride builder# (URI/create ~endpoint)))
+     builder#))
+
+(defn- maybe-with-endpoint-s3-client [^S3ClientBuilder builder endpoint]
+  (maybe-with-endpoint* builder endpoint))
+
+(defn- maybe-with-endpoint-s3-presigner [^S3Presigner$Builder builder endpoint]
+  (maybe-with-endpoint* builder endpoint))
+
+(defn- s3-configuration ^S3Configuration []
+  (-> (S3Configuration/builder)
+      (.pathStyleAccessEnabled (transforms-python.settings/python-storage-s-3-path-style-access))
+      (.build)))
+
+(defn- put-object-request ^PutObjectRequest [^String bucket-name ^String key]
+  (-> (PutObjectRequest/builder) (.bucket bucket-name) (.key key) .build))
+
+(defn- get-object-request ^GetObjectRequest [^String bucket-name ^String key]
+  (-> (GetObjectRequest/builder) (.bucket bucket-name) (.key key) .build))
+
+(defn- delete-object-request ^DeleteObjectRequest [^String bucket-name ^String key]
+  (-> (DeleteObjectRequest/builder) (.bucket bucket-name) (.key key) .build))
+
+(defn- maybe-with-credentials*
+  [credentials-provider]
+  (let [access-key (transforms-python.settings/python-storage-s-3-access-key)
+        secret-key (transforms-python.settings/python-storage-s-3-secret-key)]
+    (if (or access-key secret-key)
+      (if-not (and access-key secret-key)
+        (do (log/warnf "Ignoring %s because %s is not defined"
+                       (if access-key "access-key" "secret-key")
+                       (if (not access-key) "access-key" "secret-key"))
+            (credentials-provider (DefaultCredentialsProvider/create)))
+        (credentials-provider
+         (StaticCredentialsProvider/create
+          (AwsBasicCredentials/create access-key secret-key))))
+      (credentials-provider (DefaultCredentialsProvider/create)))))
+
+(defn- maybe-with-credentials-s3-client [^S3ClientBuilder builder]
+  (maybe-with-credentials* #(.credentialsProvider builder ^AwsCredentialsProvider %)))
+
+(defn- maybe-with-credentials-s3-presigner [^S3Presigner$Builder builder]
+  (maybe-with-credentials* #(.credentialsProvider builder ^AwsCredentialsProvider %)))
+
+;; We just recreate the client every time, to keep things simple if config is changed.
+(defn create-s3-client
+  "Create S3 client for transferring table data and manifests."
+  ^S3Client []
+  (.build
+   (doto (S3Client/builder)
+     (maybe-with-endpoint-s3-client (transforms-python.settings/python-storage-s-3-endpoint))
+     maybe-with-credentials-s3-client
+     (.serviceConfiguration (s3-configuration)))))
+
+(defn- create-s3-presigner-for-container
+  "Create S3 presigner for container operations (presigned URLs). Uses distinct container-endpoint if relevant."
+  ^S3Presigner []
+  (let [container-endpoint (transforms-python.settings/python-storage-s-3-container-endpoint)
+        endpoint           (or container-endpoint (transforms-python.settings/python-storage-s-3-endpoint))]
+    (.build
+     (doto (S3Presigner/builder)
+       (maybe-with-endpoint-s3-presigner endpoint)
+       maybe-with-credentials-s3-presigner
+       (.serviceConfiguration (s3-configuration))))))
+
+(defn- generate-presigned-get-url
+  "Generate GET URL using container presigner"
+  [^S3Presigner presigner ^String bucket-name ^String key]
+  (let [request (-> (GetObjectPresignRequest/builder)
+                    (.signatureDuration presigned-get-duration)
+                    (.getObjectRequest (get-object-request bucket-name key))
+                    (.build))]
+    (.toString (.url (.presignGetObject presigner request)))))
+
+(defn- generate-presigned-put-url
+  "Generate PUT URL using container presigner"
+  [^S3Presigner presigner ^String bucket-name ^String key]
+  (let [request (-> (PutObjectPresignRequest/builder)
+                    (.signatureDuration presigned-put-duration)
+                    (.putObjectRequest (put-object-request bucket-name key))
+                    (.build))]
+    (.toString (.url (.presignPutObject presigner request)))))
+
+(defn- s3-shared-storage [table-name->id]
+  (let [work-dir-name       (working-dir-for-run)
+        container-presigner (create-s3-presigner-for-container)
+        bucket-name         (transforms-python.settings/python-storage-s-3-bucket)
+        ref                 (fn [method relative-path]
+                              (let [path (str work-dir-name "/" relative-path)]
+                                {:path   path
+                                 :method method
+                                 :url    (case method
+                                           :put (generate-presigned-put-url container-presigner bucket-name path)
+                                           :get (generate-presigned-get-url container-presigner bucket-name path))}))]
+    ;; It feels dirty mixing the S3 client object with the other pure values, but it is convenient for the caller
+    {:s3-client   (create-s3-client)
+     :bucket-name bucket-name
+     :objects
+     (into
+      {:output          (ref :put "output.csv")
+       :output-manifest (ref :put "output-manifest.json")
+       :events          (ref :put "events.jsonl")}
+      (for [[table-name id] table-name->id]
+        {[:table id :manifest] (ref :get (str "table-" (name table-name) "-" id ".manifest.json"))
+         [:table id :data]     (ref :get (str "table-" (name table-name) "-" id ".jsonl"))}))}))
+
+(declare delete-many)
+
+;; TODO this feels like it's a hair's width from abstracting away s3 versus other transfer mechanisms - go all the way
+(defn open-shared-storage!
+  "Returns a map wrapped in a deref-able auto-closable, to be used in a with-open.
+
+  The map contains:
+  - :s3-client   - an S3Client that can be used to upload/download files
+  - :bucket-name - the S3 bucket name
+  - :objects     - a map of keys to {:path :method :url} maps, where:
+    - :path   is the S3 key
+    - :method is either :get or :put, indicating whether the python-runner should fetch or upload the file.
+    - :url    is the presigned URL to be used by the python-runner to fetch or upload the file.
+
+  The objects are broken into the following transform inputs:
+
+  - [:table id :manifest] - for each input table, the manifest JSON file
+  - [:table id :data]     - for each input table, the data file, in format specified by the manifest.
+
+  And the following transform outputs:
+  - :output-manifest - the output manifest JSON file
+  - :output          - the output data file, in format specified by the output manifest.
+  - :events          - a JSONL file containing the events logged during python execution.
+
+  When the value is closed, all the relevant keys will be deleted, but this is best-effort only.
+  We rely on the bucket retention policy to ensure any stragglers are eventually deleted."
+  ^Closeable [table-name->id]
+  (let [shared-storage (s3-shared-storage table-name->id)]
+    (reify IDeref
+      (deref [_] shared-storage)
+      Closeable
+      (close [_] (delete-many (:s3-client shared-storage)
+                              (:bucket-name shared-storage)
+                              (map :path (vals (:objects shared-storage))))))))
+
+(defn upload-file
+  "Upload the given file to s3"
+  [^S3Client s3-client ^String bucket-name ^String key ^File file]
+  (let [^PutObjectRequest request (put-object-request bucket-name key)]
+    (.putObject s3-client request (RequestBody/fromFile file))))
+
+;; TODO optimize our ingestion to stream the data. reading a string will not work in any case for binary files.
+(defn read-to-string
+  "Get back the contents of the given key as a string."
+  ([s3-client bucket-name key] (read-to-string s3-client bucket-name key ::throw))
+  ([^S3Client s3-client ^String bucket-name ^String key not-found]
+   (try
+     (let [^GetObjectRequest request (get-object-request bucket-name key)
+           response                  (.getObject s3-client request)]
+       (slurp response))
+     (catch NoSuchKeyException e
+       (if (identical? ::throw not-found)
+         (throw e)
+         not-found)))))
+
+(defn delete
+  ;; TODO better error handling
+  "Delete the given key from s3, ignoring any errors for now"
+  [^S3Client s3-client ^String bucket-name ^String key]
+  (try
+    (.deleteObject s3-client (delete-object-request bucket-name key))
+    (catch Exception e
+      (log/debugf e "Error deleting s3 object %s" key)
+      ;; Ignore deletion errors - object might not exist, or we might not have permissions
+      ;; NOTE: we plan to put general retention on the bucket so that objects will eventually be deleted
+      nil)))
+
+(defn delete-many
+  "Best effort delete the given s3 keys"
+  [^S3Client s3-client bucket-name s3-keys]
+  (run! (partial delete s3-client bucket-name) s3-keys))

enterprise/backend/src/metabase_enterprise/transforms_python/settings.clj

@@ -52,7 +52,7 @@
   (deferred-tru "S3 bucket name for storing Python execution artifacts.")
   :type       :string
   :visibility :admin
-  :default    (if (not config/is-prod?) "metabase-python-runner" "metabase-cloud-python-runner-staging")
+  :default    (when (not config/is-prod?) "metabase-python-runner")
   :feature    :transforms-python
   :doc        false
   :export?    false
@@ -64,7 +64,7 @@
   :type       :string
   :visibility :admin
   :feature    :transforms-python
-  :default    (if (not config/is-prod?) "test-prefix" "hosting-4b9351bb22b84791")
+  :default    (when (not config/is-prod?) "test-prefix")
   :doc        false
   :export?    false
   :encryption :no

enterprise/backend/test/metabase_enterprise/models/entity_id_test.clj

@@ -66,6 +66,7 @@
     :model/PermissionsGroupMembership
     :model/PermissionsRevision
     :model/PersistedInfo
+    :model/PythonLibrary
     :model/Pulse
     :model/PulseCard
     :model/PulseChannel
@@ -91,7 +92,6 @@
     :model/TransformJobTransformTag
     :model/TransformTag
     :model/TransformTransformTag
-    :model/PythonLibrary
     :model/Undo
     :model/User
     :model/UserParameterValue

enterprise/backend/test/metabase_enterprise/transforms/api_test.clj

@@ -8,7 +8,8 @@
    [metabase-enterprise.transforms.models.transform-transform-tag]
    [metabase-enterprise.transforms.query-test-util :as query-test-util]
    [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
-   [metabase-enterprise.transforms.test-util :refer [parse-instant with-transform-cleanup! utc-timestamp]]
+   [metabase-enterprise.transforms.test-util :as transforms.tu]
+   [metabase-enterprise.transforms.test-util :refer [parse-instant with-transform-cleanup! utc-timestamp get-test-schema]]
    [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.driver :as driver]
    [metabase.legacy-mbql.normalize :as mbql.normalize]
@@ -52,12 +53,6 @@
       ;; Convert to legacy MBQL which the transform API expects
     (lib.convert/->legacy-MBQL query)))
 
-(defn- get-test-schema
-  "Get the schema from the products table in the test dataset.
-   This is needed for databases like BigQuery that require a schema/dataset."
-  []
-  (t2/select-one-fn :schema :model/Table (mt/id :transforms_products)))
-
 (deftest create-transform-test
   (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
     (mt/with-premium-features #{:transforms}
@@ -73,6 +68,80 @@
                                             :schema schema
                                             :name   table-name}})))))))
 
+(deftest create-transform-feature-flag-test
+  (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+    (testing "Creating a query transform without :transforms feature returns 403"
+      (mt/with-premium-features #{}
+        (mt/dataset transforms-dataset/transforms-test
+          (let [query  (make-query "Gadget")
+                schema (get-test-schema)
+                response (mt/user-http-request :crowberto :post 402 "ee/transform"
+                                               {:name   "Test Transform"
+                                                :source {:type  "query"
+                                                         :query query}
+                                                :target {:type   "table"
+                                                         :schema schema
+                                                         :name   "test_transform"}})]
+            (is (= "error-premium-feature-not-available" (:status response)))))))
+
+    (testing "Creating a query transform with :transforms feature succeeds"
+      (mt/with-premium-features #{:transforms}
+        (mt/dataset transforms-dataset/transforms-test
+          (with-transform-cleanup! [table-name "test_query_transform"]
+            (let [query  (make-query "Gadget")
+                  schema (get-test-schema)
+                  response (mt/user-http-request :crowberto :post 200 "ee/transform"
+                                                 {:name   "Test Transform"
+                                                  :source {:type  "query"
+                                                           :query query}
+                                                  :target {:type   "table"
+                                                           :schema schema
+                                                           :name   table-name}})]
+              (is (some? (:id response))))))))))
+
+(deftest update-transform-feature-flag-test
+  (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+    (testing "Updating a query transform requires :transforms feature"
+      (mt/with-premium-features #{:transforms}
+        (mt/dataset transforms-dataset/transforms-test
+          (with-transform-cleanup! [table-name "test_update"]
+            (let [query  (make-query "Gadget")
+                  schema (get-test-schema)
+                  transform-payload {:name   "Original Transform"
+                                     :source {:type  "query"
+                                              :query query}
+                                     :target {:type   "table"
+                                              :schema schema
+                                              :name   table-name}}
+                  created (mt/user-http-request :crowberto :post 200 "ee/transform" transform-payload)]
+              ;; Now test update without feature flag
+              (mt/with-premium-features #{}
+                (let [response (mt/user-http-request :crowberto :put
+                                                     (format "ee/transform/%d" (:id created))
+                                                     (assoc transform-payload :name "Updated Transform"))]
+                  (is (= "error-premium-feature-not-available" (:status response))))))))))))
+
+(deftest run-transform-feature-flag-test
+  (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+    (testing "Running a query transform requires :transforms feature"
+      (mt/with-premium-features #{:transforms}
+        (mt/dataset transforms-dataset/transforms-test
+          (with-transform-cleanup! [table-name "test_run"]
+            (let [query  (make-query "Gadget")
+                  schema (get-test-schema)
+                  transform-payload {:name   "Test Run Transform"
+                                     :source {:type  "query"
+                                              :query query}
+                                     :target {:type   "table"
+                                              :schema schema
+                                              :name   table-name}}
+                  created (mt/user-http-request :crowberto :post 200 "ee/transform" transform-payload)]
+              ;; Now test run without feature flag
+              (mt/with-premium-features #{}
+                (let [response (mt/user-http-request :crowberto :post
+                                                     (format "ee/transform/%d/run" (:id created)))]
+                  (is (= "error-premium-feature-not-available" (:status response))))))))))))
+
 (deftest list-transforms-test
   (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
     (testing "Can list without query parameters"
@@ -213,24 +282,6 @@
                                                      :name   table-name}})]
             (mt/user-http-request :crowberto :delete 204 (format "ee/transform/%s/table" (:id resp)))))))))
 
-(defn- test-run
-  [transform-id]
-  (let [resp      (mt/user-http-request :crowberto :post 202 (format "ee/transform/%s/run" transform-id))
-        timeout-s 10 ; 10 seconds is our timeout to finish execution and sync
-        limit     (+ (System/currentTimeMillis) (* timeout-s 1000))]
-    (is (=? {:message "Transform run started"}
-            resp))
-    (loop []
-      (when (> (System/currentTimeMillis) limit)
-        (throw (ex-info (str "Transform run timed out after " timeout-s " seconds") {})))
-      (let [resp   (mt/user-http-request :crowberto :get 200 (format "ee/transform/%s" transform-id))
-            status (some-> resp :last_run :status keyword)]
-        (when-not (contains? #{:started :succeeded} status)
-          (throw (ex-info (str "Transform run failed with status " status) {:resp resp})))
-        (when-not (some? (:table resp))
-          (Thread/sleep 100)
-          (recur))))))
-
 (defn- check-query-results
   "Verifies that a transform successfully created a table with expected data.
 
@@ -269,19 +320,6 @@
           (str "Expected " (count ids) " rows with category " category
                " in table " table-name ", but got " actual-count)))))
 
-(defn- wait-for-table
-  "Wait for a table to appear in metadata, with timeout.
-   Copied from execute_test.clj - will consolidate later."
-  [table-name timeout-ms]
-  (let [mp    (mt/metadata-provider)
-        limit (+ (System/currentTimeMillis) timeout-ms)]
-    (loop []
-      (Thread/sleep 200)
-      (when (> (System/currentTimeMillis) limit)
-        (throw (ex-info "table has not been created" {:table-name table-name, :timeout-ms timeout-ms})))
-      (or (m/find-first (comp #{table-name} :name) (lib.metadata/tables mp))
-          (recur)))))
-
 (deftest execute-transform-test
   (testing "transform execution with :transforms/table target"
     (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
@@ -301,8 +339,8 @@
                                         :target target1}
                     {transform-id :id} (mt/user-http-request :crowberto :post 200 "ee/transform"
                                                              original)
-                    _                  (do (test-run transform-id)
-                                           (wait-for-table table1-name 5000))
+                    _                  (do (transforms.tu/test-run transform-id)
+                                           (transforms.tu/wait-for-table table1-name 5000))
                     _                  (is (true? (transforms.util/target-table-exists? original)))
                     _                  (check-query-results table1-name [5 11 16] "Gadget")
                     updated            {:name        "Doohickey Products"
@@ -314,8 +352,8 @@
                         (->
                          (mt/user-http-request :crowberto :put 200 (format "ee/transform/%s" transform-id) updated)
                          (update-in [:source :query] mbql.normalize/normalize))))
-                (test-run transform-id)
-                (wait-for-table table2-name 5000)
+                (transforms.tu/test-run transform-id)
+                (transforms.tu/wait-for-table table2-name 5000)
                 (is (true? (transforms.util/target-table-exists? original)))
                 (is (true? (transforms.util/target-table-exists? updated)))
                 (check-query-results table2-name [2 3 4 13] "Doohickey")))))))))

enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj

@@ -1,11 +1,11 @@
 (ns ^:mb/driver-tests metabase-enterprise.transforms.execute-test
   (:require
    [clojure.test :refer :all]
-   [medley.core :as m]
    [metabase-enterprise.transforms.execute :as transforms.execute]
    [metabase-enterprise.transforms.query-test-util :as query-test-util]
    [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
-   [metabase-enterprise.transforms.test-util :refer [with-transform-cleanup!]]
+   [metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]
+   [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.driver :as driver]
    [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
    [metabase.lib.core :as lib]
@@ -13,6 +13,7 @@
    [metabase.query-processor :as qp]
    [metabase.sync.core :as sync]
    [metabase.test :as mt]
+   [metabase.test.data.sql :as sql.tx]
    [toucan2.core :as t2]
    [toucan2.util :as u]))
 
@@ -34,19 +35,6 @@
   [mp table-name]
   (->> table-name mt/id (lib.metadata/table mp)))
 
-(defn- wait-for-table
-  "Wait for a table to appear in metadata, with timeout.
-   Copied from execute_test.clj - will consolidate later."
-  [table-name timeout-ms]
-  (let [mp    (mt/metadata-provider)
-        limit (+ (System/currentTimeMillis) timeout-ms)]
-    (loop []
-      (Thread/sleep 200)
-      (when (> (System/currentTimeMillis) limit)
-        (throw (ex-info "table has not been created" {:table-name table-name, :timeout-ms timeout-ms})))
-      (or (m/find-first (comp #{table-name} :name) (lib.metadata/tables mp))
-          (recur)))))
-
 (deftest execute-test
   (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
     (mt/dataset transforms-dataset/transforms-test
@@ -65,14 +53,14 @@
                                                          :query t1-query}
                                                 :target target1}]
               (transforms.execute/run-mbql-transform! t1 {:run-method :manual})
-              (let [table1       (wait-for-table table1-name 10000)
+              (let [table1       (transforms.tu/wait-for-table table1-name 10000)
                     t2-query     (make-query (->> table1 :name (table-name->qp-table (mt/metadata-provider))) "category" lib/= "Gizmo")]
                 (mt/with-temp [:model/Transform t2 {:name   "transform2"
                                                     :source {:type  :query
                                                              :query t2-query}
                                                     :target target2}]
                   (transforms.execute/run-mbql-transform! t2 {:run-method :cron})
-                  (let [table2      (wait-for-table table2-name 10000)
+                  (let [table2      (transforms.tu/wait-for-table table2-name 10000)
                         check-query (lib/aggregate (make-query (->> table2 :name (table-name->qp-table (mt/metadata-provider)))) (lib/count))
                         query-result (qp/process-query check-query)]
                     ;; The transforms-test dataset has exactly 4 Gizmo products (IDs 6, 8, 12, 14)
@@ -123,7 +111,7 @@
               (doseq [transform [transform-no-limit transform-limit]]
                 (transforms.execute/run-mbql-transform! transform {:run-method :manual})
                 (let [table-name (-> transform :target :name)
-                      _            (wait-for-table table-name 10000)
+                      _            (transforms.tu/wait-for-table table-name 10000)
                       table-result (lib.metadata/table mp (mt/id (keyword table-name)))
                       query-result (->> (lib/query mp table-result)
                                         (qp/process-query)
@@ -137,6 +125,24 @@
                           ["2024-01-23T00:00:00Z" 14.99]]
                          query-result)))))))))))
 
+(deftest create-table-from-schema!-test
+  (mt/test-drivers (mt/normal-drivers-with-feature :uploads)
+    (let [driver       driver/*driver*
+          db-id        (mt/id)
+          table-name   (mt/random-name)
+          schema-name  (sql.tx/session-schema driver)
+          table-schema {:name    (if schema-name
+                                   (keyword schema-name table-name)
+                                   (keyword table-name))
+                        :columns [{:name "id" :type :type/Integer :nullable? false}
+                                  {:name "name" :type :type/Text :nullable? true}]}]
+      (mt/as-admin
+        (testing "create-table-from-schema! should create the table successfully"
+          (transforms.util/create-table-from-schema! driver db-id table-schema)
+          (let [table-exists? (driver/table-exists? driver db-id {:schema schema-name :name table-name})]
+            (is (some? table-exists?) "Table should exist in the database schema")
+            (driver/drop-table! driver db-id (:name table-schema))))))))
+
 (deftest transform-schema-created-if-needed-test
   (mt/test-drivers (mt/normal-driver-select {:+features [:transforms/table :schemas]})
     (mt/dataset transforms-dataset/transforms-test
@@ -155,7 +161,8 @@
                                                               :query query}
                                                      :target target-table}]
             (transforms.execute/run-mbql-transform! transform {:run-method :manual})
-            (let [table-result      (wait-for-table (:name target-table) 10000)
+            (let [_            (transforms.tu/wait-for-table (:name target-table) 10000)
+                  table-result (lib.metadata/table mp (mt/id (keyword (:name target-table))))
                   query-result (->> (lib/query mp table-result)
                                     (qp/process-query)
                                     (mt/formatted-rows [int str str 2.0 str])
@@ -203,7 +210,7 @@
                                                                         :query query}
                                                                :target target-table}]
                       (transforms.execute/run-mbql-transform! transform {:run-method :manual})
-                      (let [_            (wait-for-table (:name target-table) 10000)
+                      (let [_            (transforms.tu/wait-for-table (:name target-table) 10000)
                             table-result (lib.metadata/table mp (mt/id (keyword (:name target-table))))
                             query-result (->> (lib/query mp table-result)
                                               (qp/process-query)

enterprise/backend/test/metabase_enterprise/transforms/jobs_test.clj

@@ -1,7 +1,12 @@
 (ns metabase-enterprise.transforms.jobs-test
+  #_{:clj-kondo/ignore [:discouraged-namespace]}
   (:require
    [clojure.test :refer :all]
-   [metabase-enterprise.transforms.jobs :as jobs]))
+   [clojure.tools.logging :as log]
+   [metabase-enterprise.transforms.execute :as transforms.execute]
+   [metabase-enterprise.transforms.jobs :as jobs]
+   [metabase-enterprise.transforms.models.transform-run :as transform-run]
+   [metabase.test :as mt]))
 
 (deftest basic-deps-test
   (let [ordering {1 #{2 3}
@@ -75,3 +80,64 @@
            (-> (#'jobs/next-transform sorted-ordering transforms-by-id #{2 3})
                :id)))
     (is (nil? (#'jobs/next-transform sorted-ordering transforms-by-id #{1 2 3})))))
+
+(def ^:private query-source {:type "query"})
+(def ^:private python-source {:type "python"})
+
+(deftest run-transform-feature-flag-test
+  (testing "Query transforms are skipped without :transforms feature"
+    (mt/with-premium-features #{}
+      (let [query-transform {:id 1
+                             :source query-source
+                             :name "Test Query Transform"}
+            run-id 100
+            logged-messages (atom [])]
+        (with-redefs [log/log* (fn [_ level _ message]
+                                 (swap! logged-messages conj {:level level :message message}))
+                      transform-run/running-run-for-transform-id (constantly nil)]
+          (#'jobs/run-transform! run-id :scheduled query-transform)
+          (is (= 1 (count @logged-messages))
+              "Should log exactly one warning")
+          (is (= :warn (:level (first @logged-messages)))
+              "Should log at warn level")
+          (is (re-matches #".*Skip running transform 1 due to lacking premium features.*"
+                          (:message (first @logged-messages)))
+              "Warning message should indicate transform was skipped due to missing features")))))
+
+  (testing "Python transforms are skipped without :transforms-python feature"
+    (mt/with-premium-features #{:transforms}
+      (let [python-transform {:id 2
+                              :source python-source
+                              :name "Test Python Transform"}
+            run-id 101
+            logged-messages (atom [])]
+        (with-redefs [log/log* (fn [_ level _ message]
+                                 (swap! logged-messages conj {:level level :message message}))
+                      transform-run/running-run-for-transform-id (constantly nil)]
+          (#'jobs/run-transform! run-id :scheduled python-transform)
+          (is (= 1 (count @logged-messages))
+              "Should log exactly one warning")
+          (is (= :warn (:level (first @logged-messages)))
+              "Should log at warn level")
+          (is (re-matches #".*Skip running transform 2 due to lacking premium features.*"
+                          (:message (first @logged-messages)))
+              "Warning message should indicate transform was skipped due to missing features")))))
+
+  (testing "Query transforms run with :transforms feature"
+    (mt/with-premium-features #{:transforms}
+      (let [query-transform {:id 3
+                             :source query-source
+                             :name "Test Query Transform"}
+            run-id 102
+            logged-messages (atom [])
+            run-called? (atom false)]
+        (with-redefs [log/log* (fn [_ level _ message]
+                                 (swap! logged-messages conj {:level level :message message}))
+                      transform-run/running-run-for-transform-id (constantly nil)
+                      transforms.execute/run-mbql-transform! (fn [_ _]
+                                                               (reset! run-called? true))]
+          (#'jobs/run-transform! run-id :scheduled query-transform)
+          (is (empty? (filter (comp #{:warn} :level) @logged-messages))
+              "Should not log warnings when feature is enabled")
+          (is @run-called?
+              "Should call run-mbql-transform! when feature is enabled"))))))

enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj

@@ -21,6 +21,20 @@
               :name name
               :type :table}}))
 
+(defn- make-python-transform [source-tables & [name schema target-db]]
+  (let [name (or name (mt/random-name))
+        default-schema (some-> driver/*driver* driver.sql/default-schema)
+        schema (or schema default-schema "public")
+        target-db (or target-db (mt/id))]
+    {:source {:type "python"
+              :source-tables source-tables
+              :body "# Python code here\nresult = df"}
+     :name (str "python_transform_" name)
+     :target {:database target-db
+              :schema schema
+              :name name
+              :type "table"}}))
+
 (deftest basic-ordering-test
   (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
     (mt/with-temp [:model/Transform {t1 :id} (make-transform
@@ -354,4 +368,105 @@
                                                 "table_1")]
         (is (= {:cycle-str "transform_table_2 -> transform_table_1 -> transform_table_3",
                 :cycle [t1 t3 t2]}
-               (ordering/get-transform-cycle (t2/select-one :model/Transform :id t1))))))))
+               (ordering/get-transform-cycle (t2/select-one :model/Transform :id t1)))))))
+
+  (deftest python-transform-basic-dependencies-test
+    (testing "Python transforms with source-tables dependencies are extracted correctly"
+      (mt/with-temp [:model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)
+                                                                       "products" (mt/id :products)})]
+        (is (= #{{:table (mt/id :orders)}
+                 {:table (mt/id :products)}}
+               (transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))
+
+  (deftest python-transform-ordering-test
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+      (testing "Python transforms are ordered correctly based on source-tables dependencies"
+        (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                  :name "output_1"}
+                       :model/Field _ {:table_id table1
+                                       :name "foo"}
+                       :model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)} "output_1")
+                       :model/Transform {t2 :id} (make-python-transform {"output_1" table1} "output_2")]
+          (is (= {t1 #{}
+                  t2 #{t1}}
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2]]))))))))
+
+  (deftest python-transform-multiple-dependencies-test
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+      (testing "Python transforms with multiple table dependencies"
+        (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                  :name "output_1"}
+                       :model/Field _ {:table_id table1
+                                       :name "foo"}
+                       :model/Table {table2 :id} {:schema "public"
+                                                  :name "output_2"}
+                       :model/Field _ {:table_id table2
+                                       :name "bar"}
+                       :model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)} "output_1")
+                       :model/Transform {t2 :id} (make-python-transform {"products" (mt/id :products)} "output_2")
+                       :model/Transform {t3 :id} (make-python-transform {"output_1" table1
+                                                                         "output_2" table2} "final_output")]
+          (is (= {t1 #{}
+                  t2 #{}
+                  t3 #{t1 t2}}
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]]))))))))
+
+  (deftest mixed-transform-ordering-test
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+      (testing "Python and query transforms are ordered together correctly"
+        (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                  :name "sql_output"}
+                       :model/Field _ {:table_id table1
+                                       :name "foo"}
+                       :model/Table {table2 :id} {:schema "public"
+                                                  :name "python_output"}
+                       :model/Field _ {:table_id table2
+                                       :name "bar"}
+                       ;; SQL transform that depends on orders table
+                       :model/Transform {t1 :id} (make-transform
+                                                  {:database (mt/id),
+                                                   :type "query",
+                                                   :query {:source-table (mt/id :orders)}}
+                                                  "sql_output")
+                       ;; Python transform that depends on the SQL transform's output
+                       :model/Transform {t2 :id} (make-python-transform {"sql_output" table1} "python_output")
+                       ;; Another SQL transform that depends on the Python transform's output
+                       :model/Transform {t3 :id} (make-transform
+                                                  {:database (mt/id)
+                                                   :type "query"
+                                                   :query {:source-table table2}}
+                                                  "final_output")]
+          (is (= {t1 #{}
+                  t2 #{t1}
+                  t3 #{t2}}
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]])))))))))
+
+#_(deftest mixed-database-transform-ordering-test
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+      (testing "Python and query transforms are ordered together correctly, across multiple databases"
+      ;; orders (db-1) -> sql_output (db-1) -> python_output (db-2) -> final_output (db-2)
+        (mt/with-temp [:model/Database db-2    {}
+                       :model/Table    table-1 {:db_id (mt/id),    :schema "public", :name "sql_output"}
+                       :model/Table    table-2 {:db_id (:id db-2), :schema "public", :name "python_output"}
+                       :model/Field    _       {:table_id (:id table-1), :name "foo"}
+                       :model/Field    _       {:table_id (:id table-2), :name "bar"}
+
+                     ;; SQL transform that depends on orders table
+                       :model/Transform {t1 :id} (make-transform
+                                                  {:database (mt/id)
+                                                   :type     "query"
+                                                   :query    {:source-table (mt/id :orders)}}
+                                                  (:name table-1))
+                     ;; Python transform that depends on the SQL transform's output
+                       :model/Transform {t2 :id} (make-python-transform {"sql_output" (:id table-1)} (:name table-2))
+                     ;; Another SQL transform that depends on the Python transform's output
+                       :model/Transform {t3 :id} (make-transform
+                                                  {:database (:id db-2)
+                                                   :type     "query"
+                                                   :query    {:source-table (:id table-2)}}
+                                                  "final_output")]
+          (is (= {t1 #{}
+                  t2 #{t1}
+                  t3 #{t2}}
+
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]]))))))))

enterprise/backend/test/metabase_enterprise/transforms/test_util.clj

@@ -1,6 +1,7 @@
 (ns metabase-enterprise.transforms.test-util
   (:require
    [clojure.string :as str]
+   [clojure.test :refer :all]
    [metabase.api.common :as api]
    [metabase.driver :as driver]
    [metabase.test :as mt]
@@ -52,6 +53,12 @@
     `(mt/with-model-cleanup [:model/Transform]
        ~@body)))
 
+(defn table-rows
+  [table-name]
+  (mt/rows (mt/process-query {:database (mt/id)
+                              :query    {:source-table (t2/select-one-pk :model/Table :name table-name)}
+                              :type     :query})))
+
 (defn parse-timestamp
   "Parse a local datetime and convert it to a ZonedDateTime in the default timezone."
   ^ZonedDateTime [timestamp-string]
@@ -68,3 +75,42 @@
   "Parse a local datetime and convert it to a string encoding a ZonedDateTime in the default timezone."
   ^String [timestamp-string]
   (-> timestamp-string parse-instant str))
+
+(defn wait-for-table
+  "Wait for a table to appear in metadata, with timeout."
+  [^String table-name timeout-ms]
+  (let [timer (u/start-timer)]
+    (loop []
+      (let [table (t2/select-one :model/Table :name table-name)
+            fields (t2/select :model/Field :table_id (:id table))]
+        (cond
+          (and table (seq fields)) table
+          (> (u/since-ms timer) timeout-ms)
+          (throw (ex-info (format "Table %s did not appear after %dms" table-name timeout-ms)
+                          {:table-name table-name :timeout-ms timeout-ms}))
+          :else (do (Thread/sleep 100)
+                    (recur)))))))
+
+(defn test-run
+  [transform-id]
+  (let [resp      (mt/user-http-request :crowberto :post 202 (format "ee/transform/%s/run" transform-id))
+        timeout-s 10 ; 10 seconds is our timeout to finish execution and sync
+        limit     (+ (System/currentTimeMillis) (* timeout-s 1000))]
+    (is (=? {:message "Transform run started"}
+            resp))
+    (loop [last-resp nil]
+      (when (> (System/currentTimeMillis) limit)
+        (throw (ex-info (str "Transform run timed out after " timeout-s " seconds") {:resp last-resp})))
+      (let [resp   (mt/user-http-request :crowberto :get 200 (format "ee/transform/%s" transform-id))
+            status (some-> resp :last_run :status keyword)]
+        (when-not (contains? #{:started :succeeded} status)
+          (throw (ex-info (str "Transform run failed with status " status) {:resp resp :status status})))
+        (when-not (some? (:table resp))
+          (Thread/sleep 100)
+          (recur resp))))))
+
+(defn get-test-schema
+  "Get the schema from the products table in the test dataset.
+   This is needed for databases like BigQuery that require a schema/dataset."
+  []
+  (t2/select-one-fn :schema :model/Table (mt/id :transforms_products)))

enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj

@@ -0,0 +1,524 @@
+(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.drivers-test
+  "Comprehensive tests for Python transforms across all supported drivers with all base and exotic types."
+  (:require
+   [clojure.core.async :as a]
+   [clojure.string :as str]
+   [clojure.test :refer :all]
+   [metabase-enterprise.transforms-python.execute :as transforms.execute]
+   [metabase-enterprise.transforms-python.python-runner :as python-runner]
+   [metabase-enterprise.transforms-python.s3 :as s3]
+   [metabase-enterprise.transforms-python.settings :as transforms-python.settings]
+   [metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]
+   [metabase-enterprise.transforms.util :as transforms.util]
+   [metabase.driver :as driver]
+   [metabase.driver.mysql :as mysql]
+   [metabase.sync.core :as sync]
+   [metabase.test :as mt]
+   [metabase.test.data.interface :as tx]
+   [metabase.test.data.sql :as sql.tx]
+   [metabase.util :as u]
+   [metabase.util.date-2 :as u.date]
+   [metabase.util.json :as json]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(def test-id 42)
+
+(defn- execute!
+  "Execute a Python transform with the given code and tables"
+  [{:keys [code tables]}]
+  (with-open [shared-storage-ref (s3/open-shared-storage! (or tables {}))]
+    (let [server-url (transforms-python.settings/python-runner-url)
+          cancel-chan (a/promise-chan)
+          table-name->id (or tables {})
+          _ (python-runner/copy-tables-to-s3! {:run-id test-id
+                                               :shared-storage @shared-storage-ref
+                                               :table-name->id table-name->id
+                                               :cancel-chan cancel-chan})
+          response (python-runner/execute-python-code-http-call! {:server-url server-url
+                                                                  :code code
+                                                                  :run-id test-id
+                                                                  :table-name->id table-name->id
+                                                                  :shared-storage @shared-storage-ref})
+          {:keys [output output-manifest events]} (python-runner/read-output-objects @shared-storage-ref)]
+      (merge (:body response)
+             {:output output
+              :output-manifest output-manifest
+              :stdout (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
+              :stderr (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))}))))
+
+(defn- execute-e2e-transform!
+  "Execute an e2e Python transform test using execute-python-transform!"
+  [table-name transform-code source-tables]
+  (let [schema (when (#{:postgres :bigquery-cloud-sdk} driver/*driver*)
+                 (sql.tx/session-schema driver/*driver*))
+        target {:type "table"
+                :schema schema
+                :name table-name
+                :database (mt/id)}
+        transform-def {:name (str "E2E Transform Test " table-name)
+                       :source {:type "python"
+                                :source-tables source-tables
+                                :body transform-code}
+                       :target target}]
+    (with-transform-cleanup! [_target target]
+      (mt/with-temp [:model/Transform transform transform-def]
+        (transforms.execute/execute-python-transform! transform {:run-method :manual})
+        (let [table (transforms.tu/wait-for-table table-name 10000)
+              columns (t2/select :model/Field :table_id (:id table) {:order-by [:position]})
+              column-names (mapv :name columns)
+              rows (transforms.tu/table-rows table-name)]
+          {:columns column-names
+           :rows rows})))))
+
+(def ^:private base-type-test-data
+  "Base types that all drivers should support with test data."
+  {:columns [{:name "id" :type :type/Integer :nullable? false}
+             {:name "name" :type :type/Text :nullable? true}
+             {:name "price" :type :type/Float :nullable? true}
+             {:name "active" :type :type/Boolean :nullable? true}
+             {:name "created_date" :type :type/Date :nullable? true}
+             {:name "created_at" :type :type/DateTime :nullable? true}]
+   :data [[1 "Product A" 19.99 true "2024-01-01" "2024-01-01T12:00:00"]
+          [2 "Product B" 15.50 false "2024-02-01" "2024-02-01T09:15:30"]
+          [3 nil nil nil nil nil]]})
+
+(def ^:private driver-exotic-types
+  "Driver-specific exotic types with test data."
+  {:postgres {:columns [{:name "id" :type :type/Integer :nullable? false}
+                        {:name "created_tz" :type :type/DateTimeWithTZ :nullable? true}
+                        {:name "uuid_field" :type :type/UUID :nullable? true}
+                        {:name "json_field" :type :type/JSON :nullable? true}
+                        {:name "ip_field" :type :type/IPAddress :nullable? true}
+                        {:name "int_array" :type :type/* :nullable? true :database-type "integer[]"}
+                        {:name "text_array" :type :type/* :nullable? true :database-type "text[]"}
+                        {:name "uuid_array" :type :type/* :nullable? true :database-type "uuid[]"}]
+              :data [[1 "2024-01-01T12:00:00Z" "550e8400-e29b-41d4-a716-446655440000" "{\"key\": \"value\"}" "192.168.1.1"
+                      "{1,2,3,4,5}" "{\"hello\",\"world\",\"test\"}" "{550e8400-e29b-41d4-a716-446655440000,123e4567-e89b-12d3-a456-426614174000}"]
+                     [2 nil nil nil nil nil nil nil]
+                     [3 "2024-02-01T09:15:30-05:00" nil nil nil "{}" "{}" "{}"]]}
+
+   :mysql {:columns [{:name "id" :type :type/Integer :nullable? false}
+                     {:name "json_field" :type :type/JSON :nullable? true}
+                     {:name "timestamp" :type :type/DateTimeWithLocalTZ :nullable? true :database-type "timestamp"}]
+           :data [[1 "{\"key\": \"value\"}" "2024-01-01 12:00:00"]
+                  [2 nil
+                   nil]]}
+   :mariadb {:columns [{:name "id" :type :type/Integer :nullable? false}
+                       {:name "json_field" :type :type/JSON :nullable? true}]
+             :data [[1 "{\"key\": \"mariadb_value\"}"]
+                    [2 nil]]}
+
+   :bigquery-cloud-sdk {:columns [{:name "id" :type :type/Integer :nullable? false}
+                                  {:name "json_field" :type :type/JSON :nullable? true}
+                                  {:name "dict_field" :type :type/Dictionary :nullable? true :database-type "STRUCT<key STRING, value INT64>"}]
+                        :data [[1 "{\"key\": \"value\"}" {"key" "test", "value" 42}]
+                               [2 nil nil]]}
+   :sqlserver {:columns [{:name "id" :type :type/Integer :nullable? false}
+                         {:name "uuid_field" :type :type/UUID :nullable? true}
+                         {:name "datetimeoffset_field" :type :type/DateTimeWithTZ :nullable? true :database-type "datetimeoffset"}]
+               :data [[1 "550e8400-e29b-41d4-a716-446655440000" "2024-01-01 12:00:00 -05:00"]
+                      [2 nil nil]]}
+   :mongo {:columns [{:name "id" :type :type/Integer :nullable? false}
+                     {:name "uuid_field" :type :type/UUID :nullable? true}
+                     {:name "array_field" :type :type/Array :nullable? true :database-type "array"}
+                     {:name "dict_field" :type :type/Dictionary :nullable? true :database-type "object"}]
+           :data [[1 #uuid "550e8400-e29b-41d4-a716-446655440000" [1, 2, 3] {"nested" "object"}]
+                  [2 nil nil nil]]}})
+
+(defn- create-test-table-with-data!
+  "Create a test table with the given schema and data for the current driver."
+  [table-name schema data]
+  (let [driver driver/*driver*
+        db-id (mt/id)
+        schema-name (case driver
+
+                      :clickhouse (-> (mt/db) :details :db)
+
+                      :mongo nil
+
+                      (sql.tx/session-schema driver))
+        qualified-table-name (if schema-name
+                               (keyword schema-name table-name)
+                               (keyword table-name))
+        table-schema {:name qualified-table-name
+                      :columns (:columns schema)}]
+    (mt/as-admin
+      (transforms.util/create-table-from-schema! driver db-id table-schema))
+
+    (when (seq data)
+      (driver/insert-from-source! driver db-id table-schema
+                                  {:type :rows
+                                   :data (map (fn [row]
+                                                (map #(cond
+                                                        (and (= :sqlserver driver) (boolean? %))
+                                                        (if % 1 0)
+
+                                                        (and (= :mongo driver) (string? %) (try (u.date/parse %) (catch Exception _)))
+                                                        (u.date/parse %)
+
+                                                        :else
+                                                        %) row))
+                                              data)}))
+
+    (sync/sync-database! (mt/db) {:scan :schema})
+
+    (t2/select-one-pk :model/Table :name (name qualified-table-name) :db_id db-id)))
+
+(defn- cleanup-table!
+  "Drop the test table by table ID."
+  [table-id]
+  (try
+    (when-let [table (t2/select-one :model/Table :id table-id)]
+      (let [table-name (:name table)
+            qualified-name (case driver/*driver*
+                             :bigquery-cloud-sdk
+                             (let [schema (sql.tx/session-schema driver/*driver*)]
+                               (if schema
+                                 (keyword (str schema "." table-name))
+                                 (keyword table-name)))
+
+                             (let [schema (sql.tx/session-schema driver/*driver*)]
+                               (if schema
+                                 (keyword schema table-name)
+                                 (keyword table-name))))]
+        (driver/drop-table! driver/*driver* (mt/id) qualified-name)))
+    (catch Exception _e
+      nil)))
+
+(defn- validate-transform-output
+  "Validate that the Python transform output preserves types and data correctly,
+   assuming output is in JSONL format."
+  [result expected-columns expected-row-count]
+  (testing "Transform execution succeeded"
+    (is (some? result))
+    (is (contains? result :output))
+    (is (contains? result :output-manifest)))
+
+  (when result
+    (let [lines (str/split-lines (:output result))
+          rows (map json/decode lines)
+          metadata (:output-manifest result)
+          headers (map :name (:fields metadata))]
+
+      (testing "Column headers are correct"
+        (is (= (set expected-columns) (disj (set headers) "_id"))))
+
+      (testing "Row count is correct"
+        (is (= expected-row-count (count rows))))
+
+      (testing "Metadata contains all expected columns"
+        (is (= (set expected-columns)
+               (disj (set (map :name (:fields metadata))) "_id"))))
+
+      {:headers headers
+       :rows rows
+       :metadata metadata})))
+
+#_:clj-kondo/ignore
+(defmacro with-test-table
+  [[table-id table-name] [schema data] & body]
+  `(let [table-name# (if (= :redshift driver/*driver*)
+                       (tx/db-qualified-table-name (get-in (mt/db) [:settings :database-source-dataset-name]) (mt/random-name))
+                       (mt/random-name))
+         table-id# (create-test-table-with-data! table-name# ~schema ~data)]
+     (try
+       (let [~table-id table-id#
+             ~table-name table-name#]
+         ~@body)
+       (finally
+         (cleanup-table! table-id#)))))
+
+(defn- simple-identity-transform-code
+  "Generate Python code for a simple identity transform."
+  [table-name]
+  (str "import pandas as pd\n"
+       "\n"
+       "def transform(" table-name "):\n"
+       "    df = " table-name ".copy()\n"
+       "    return df"))
+
+(defn- execute-and-validate-transform!
+  "Execute a Python transform and validate the output, returning validation results."
+  [transform-code table-name table-id expected-columns expected-row-count]
+  (let [result (execute! {:code transform-code
+                          :tables {table-name table-id}})]
+    (validate-transform-output result expected-columns expected-row-count)))
+
+(defn- test-exotic-types-for-driver!
+  "Helper to test exotic types for the current driver."
+  [driver-key]
+  (when-let [exotic-config (get driver-exotic-types (if (mysql/mariadb? (mt/db)) :mariadb driver-key))]
+    (with-test-table [table-id table-name] [exotic-config (:data exotic-config)]
+      (let [transform-code (simple-identity-transform-code table-name)
+            expected-columns (map :name (:columns exotic-config))
+            expected-row-count (count (:data exotic-config))
+            validation (execute-and-validate-transform!
+                        transform-code table-name table-id
+                        expected-columns expected-row-count)]
+
+        (when validation
+          (testing (str "Exotic types for " driver-key)
+            (let [{:keys [metadata]} validation
+                  type-map (u/for-map [{:keys [name base_type]} (:fields metadata)]
+                             [name (python-runner/restricted-insert-type base_type)])]
+
+              (is (isa? :type/Integer (type-map "id")))
+
+              (case driver-key
+                :postgres (do
+                            (is (isa? (type-map "uuid_field") :type/Text #_:type/UUID))
+                            (is (isa? (type-map "json_field") :type/Text #_:type/JSON))
+                            (is (isa? (type-map "ip_field") :type/Text #_:type/IPAddress)))
+                :mysql (if (mysql/mariadb? (mt/db))
+                         (is (isa? (type-map "json_field") :type/Text #_:type/JSON))
+                         (do
+                           (is (isa? (type-map "json_field") :type/Text #_:type/JSON))
+                           (is (isa? (type-map "timestamp") :type/DateTimeWithLocalTZ))))
+
+                :bigquery-cloud-sdk (do
+                                      (is (isa? (type-map "json_field") :type/Text #_:type/JSON))
+                                      (is (isa? (type-map "dict_field") :type/Text #_:type/JSON)))
+
+                :mongo (do
+                         (is (isa? (type-map "array_field") :type/Text #_:type/Array))
+                         (is (isa? (type-map "dict_field") :type/Text #_:type/Dictionary)))
+
+                :sqlserver (do
+                             (is (isa? (type-map "uuid_field") :type/Text #_:type/UUID))
+                             (is (isa? (type-map "datetimeoffset_field") :type/DateTimeWithTZ)))))))
+        validation))))
+
+(deftest create-table-test
+  (testing "Test we can create base table"
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
+      (mt/with-empty-db
+        (with-test-table [table-id _table-name] [base-type-test-data (:data base-type-test-data)]
+          (is table-id "Table should be created and have an ID"))))))
+
+(deftest base-types-python-transform-test
+  (testing "Test Python transforms with base types across all supported drivers"
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
+      (mt/with-empty-db
+        (with-test-table [table-id table-name] [base-type-test-data (:data base-type-test-data)]
+          (let [transform-code (simple-identity-transform-code table-name)
+                expected-columns ["id" "name" "price" "active" "created_date" "created_at"]
+                validation (execute-and-validate-transform!
+                            transform-code table-name table-id expected-columns 3)]
+
+            (when validation
+              (let [{:keys [metadata]} validation]
+                (testing "Base type preservation"
+                  (let [type-map (u/for-map [{:keys [name base_type]} (:fields metadata)]
+                                   [name (python-runner/restricted-insert-type base_type)])]
+                    (is (isa? (type-map "id") (if (= driver/*driver* :snowflake) :type/Number :type/Integer)))
+                    (is (isa? (type-map "name") :type/Text))
+                    (is (isa? (type-map "price") :type/Float))
+                    (is (isa? (type-map "active") :type/Boolean))
+                    (is (isa? (type-map "created_date") (if (= driver/*driver* :mongo) :type/Instant :type/Date)))
+                    (is (isa? (type-map "created_at") :type/DateTime))))))))))))
+
+(deftest exotic-types-python-transform-test
+  (testing "Test Python transforms with driver-specific exotic types"
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
+      (mt/with-empty-db
+        (test-exotic-types-for-driver! driver/*driver*)))))
+
+(deftest edge-cases-python-transform-test
+  (testing "Test Python transforms with edge cases: null values, empty strings, extreme values"
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
+      (mt/with-empty-db
+        (let [edge-case-schema {:columns [{:name "id" :type :type/Integer :nullable? false}
+                                          {:name "text_field" :type :type/Text :nullable? true}
+                                          {:name "int_field" :type :type/Integer :nullable? true}
+                                          {:name "float_field" :type :type/Float :nullable? true}
+                                          {:name "bool_field" :type :type/Boolean :nullable? true}
+                                          {:name "date_field" :type :type/Date :nullable? true}]
+                                :data [[1 "" 0 0.0 false "2024-01-01"]
+                                       [2 "Very long text with special chars: !@#$%^&*(){}[]|\\:;\"'<>,.?/~`"
+                                        2147483647 1.79769313486 true "2222-12-31"]
+                                       [3 nil nil nil nil nil]]}]
+          (with-test-table [table-id table-name] [edge-case-schema (:data edge-case-schema)]
+            (let [transform-code (str "import pandas as pd\n"
+                                      "import numpy as np\n"
+                                      "\n"
+                                      "def transform(" table-name "):\n"
+                                      "    df = " table-name ".copy()\n"
+                                      "    \n"
+                                      "    # Handle text operations safely\n"
+                                      "    df['text_length'] = df['text_field'].fillna(\"\").astype(str).str.len()"
+                                      "    \n"
+                                      "    # Handle numeric operations with null safety\n"
+                                      "    df['int_doubled'] = df['int_field'] * 2\n"
+                                      "    df['float_squared'] = df['float_field'] ** 2\n"
+                                      "    \n"
+                                      "    # Boolean operations\n"
+                                      "    df['bool_inverted'] = ~df['bool_field'].fillna(False)\n"
+                                      "    \n"
+                                      "    return df")
+                  expected-columns ["id" "text_field" "int_field" "float_field" "bool_field" "date_field"
+                                    "text_length" "int_doubled" "float_squared" "bool_inverted"]
+                  validation (execute-and-validate-transform!
+                              transform-code table-name table-id expected-columns 3)]
+
+              (when validation
+                (let [{:keys [rows metadata]} validation
+                      type-map (u/for-map [{:keys [name base_type]} (:fields metadata)]
+                                 [name (python-runner/restricted-insert-type base_type)])]
+
+                  (testing "Original columns preserved"
+                    (is (isa? (type-map "id") (if (= driver/*driver* :snowflake) :type/Number :type/Integer)))
+                    (is (isa? (type-map "text_field") :type/Text))
+                    (is (isa? (type-map "int_field") (if (= driver/*driver* :snowflake) :type/Number :type/Integer)))
+                    (is (isa? (type-map "float_field") :type/Float))
+                    (is (isa? (type-map "bool_field") :type/Boolean))
+                    (is (isa? (type-map "date_field") (if (= driver/*driver* :mongo) :type/Instant :type/Date))))
+
+                  (testing "Computed columns have correct types"
+                    (is (isa? (type-map "text_length") :type/Integer))
+                    (is (isa? (type-map "int_doubled") :type/Integer))
+                    (is (isa? (type-map "float_squared") :type/Float))
+                    (is (isa? (type-map "bool_inverted") :type/Boolean)))
+
+                  (testing "Edge case data handling"
+                    (let [[row1 row2 row3] rows]
+                      (is (= 1 (get row1 "id")))
+                      (is (= 0 (get row1 "text_length"))) ; empty string length
+                      (is (= 0 (get row1 "int_doubled"))) ; 0 * 2
+
+                      (is (= 2 (get row2 "id")))
+                      (is (not= "" (get row2 "text_length"))) ; long string has length
+
+                      (is (= 3 (get row3 "id"))))))))))))))
+
+(deftest idempotent-transform-test
+  (testing "Test that running the same transform multiple times produces identical results"
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
+      (mt/with-empty-db
+        (with-test-table [table-id table-name] [base-type-test-data (:data base-type-test-data)]
+          (let [transform-code (str "import pandas as pd\n"
+                                    "\n"
+                                    "def transform(" table-name "):\n"
+                                    "    df = " table-name ".copy()\n"
+                                    "    df['computed_field'] = df['price'] * 1.1  # 10% markup\n"
+                                    "    df['name_upper'] = df['name'].str.upper()\n"
+                                    "    return df")
+                ;; Run the transform twice
+                result1 (execute!
+                         {:code transform-code
+                          :tables {table-name table-id}})
+
+                result2 (execute!
+                         {:code transform-code
+                          :tables {table-name table-id}})]
+            (testing "Both transforms succeeded"
+              (is (some? result1))
+              (is (some? result2)))
+
+            (when (and result1 result2)
+              (testing "Results are identical"
+                (is (= (:output result1) (:output result2)))
+                (is (= (count (:fields (:output-manifest result1)))
+                       (count (:fields (:output-manifest result2))))))
+
+              (let [expected-columns ["id" "name" "price" "active" "created_date" "created_at"
+                                      "computed_field" "name_upper"]
+                    validation (validate-transform-output result1 expected-columns 3)]
+
+                (when validation
+                  (testing "Computed columns are added correctly"
+                    (let [{:keys [metadata]} validation
+                          type-map (u/for-map [{:keys [name base_type]} (:fields metadata)]
+                                     [name (python-runner/restricted-insert-type base_type)])]
+                      (is (= :type/Float (type-map "computed_field")))
+                      (is (= :type/Text (type-map "name_upper"))))))))))))))
+
+(deftest comprehensive-e2e-python-transform-test
+  (testing "End-to-end test using execute-python-transform! across all supported drivers with comprehensive type coverage"
+    (mt/test-drivers (disj (mt/normal-drivers-with-feature :transforms/python)
+                           ;; we sometimes get I/O error in CI due to it taking too long. it's too slow, too flakey to keep enabled
+                           :redshift)
+      (mt/with-empty-db
+        (mt/with-premium-features #{:transforms-python}
+          (with-test-table [source-table-id source-table-name] [base-type-test-data (:data base-type-test-data)]
+            (let [table-name (mt/random-name)
+                  exotic-config (get driver-exotic-types driver/*driver*)
+                  exotic-table-id (when exotic-config
+                                    (create-test-table-with-data!
+                                     (str source-table-name "_exotic")
+                                     exotic-config
+                                     (:data exotic-config)))]
+
+              (try
+                (let [transform-code (str "import pandas as pd\n"
+                                          "\n"
+                                          "def transform(" source-table-name
+                                          (when exotic-config (str ", " source-table-name "_exotic"))
+                                          "):\n"
+                                          "    # Start with base types table\n"
+                                          "    df = " source-table-name ".copy()\n"
+                                          "    \n"
+                                          "    # Add computed columns to test type handling\n"
+                                          "    df['price_with_tax'] = df['price'] * 1.08  # Float computation\n"
+                                          "    df['name_length'] = df['name'].str.len()   # String operations\n"
+                                          "    df['is_expensive'] = df['price'] > 18.0    # Boolean computation\n"
+                                          "    \n"
+                                          "    # Date operations\n"
+                                          "    df['created_year'] = pd.to_datetime(df['created_date']).dt.year\n"
+                                          "    \n"
+                                          (when exotic-config
+                                            (str "    # Merge with exotic types if available\n"
+                                                 "    exotic_df = " source-table-name "_exotic.copy()\n"
+                                                 "    df = df.merge(exotic_df, on='id', how='left', suffixes=('', '_exotic'))\n"
+                                                 "    \n"))
+                                          "    # Return processed dataframe\n"
+                                          "    return df")
+
+                      source-tables (cond-> {source-table-name source-table-id}
+                                      exotic-table-id
+                                      (assoc (str source-table-name "_exotic") exotic-table-id))
+
+                      result (execute-e2e-transform! table-name transform-code source-tables)
+                      {:keys [columns] result-rows :rows} result]
+
+                  (testing "E2E transform execution succeeded"
+                    (is (seq result-rows) "Transform should produce results"))
+
+                  (testing "Expected data transformations"
+                    (let [first-row (first result-rows)
+                          second-row (second result-rows)
+                          third-row (nth result-rows 2)
+                          get-column (fn [row col-name] (get (zipmap columns row) col-name))]
+                      (testing "Computed columns are present and have reasonable values"
+                        (is (> (count columns) 7) "Should have additional computed columns")
+                        (is (= 3 (count result-rows)) "Should have 3 rows from source data"))
+
+                      (testing "Computed column values are mathematically correct"
+                        (testing "Row 1 price_with_tax calculation"
+                          (let [price-with-tax (get-column first-row "price_with_tax")]
+                            (is (and (number? price-with-tax) (> price-with-tax 21.5) (< price-with-tax 21.7))
+                                "First row price_with_tax should be ~21.59")))
+
+                        (testing "Row 2 price_with_tax calculation"
+                          (let [price-with-tax (get-column second-row "price_with_tax")]
+                            (is (and (number? price-with-tax) (> price-with-tax 16.7) (< price-with-tax 16.8))
+                                "Second row price_with_tax should be ~16.74")))
+
+                        (testing "Name length calculations"
+                          (is (== 9 (get-column first-row "name_length")) "First row name_length should be 9")
+                          (is (== 9 (get-column second-row "name_length")) "Second row name_length should be 9"))
+
+                        (testing "Boolean expense calculations"
+                          (is (true? (get-column first-row "is_expensive")) "First row is_expensive should be true")
+                          (is (false? (get-column second-row "is_expensive")) "Second row is_expensive should be false"))
+
+                        (testing "Date year extraction"
+                          (is (== 2024 (get-column first-row "created_year")) "First row created_year should be 2024")
+                          (is (== 2024 (get-column second-row "created_year")) "Second row created_year should be 2024"))
+
+                        (testing "Null value handling"
+                          (is (seq third-row) "Third row should contain values (nulls handled gracefully)"))))))
+                (finally
+                  (when exotic-table-id
+                    (cleanup-table! exotic-table-id)))))))))))

enterprise/backend/test/metabase_enterprise/transforms_python/e2e_test.clj

@@ -0,0 +1,86 @@
+(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.e2e-test
+  "End-to-end tests for Python library functionality."
+  (:require
+   [clojure.test :refer :all]
+   [metabase-enterprise.transforms-python.models.python-library :as python-library]
+   [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
+   [metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]
+   [metabase.driver :as driver]
+   [metabase.test :as mt]
+   [metabase.test.util :as mt.util]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(deftest transforms-python-with-library-test
+  (testing "Python transform execution with common library"
+    (mt/test-drivers #{:postgres}
+      (mt/with-premium-features #{:transforms :transforms-python}
+        (mt/dataset transforms-dataset/transforms-test
+          (mt.util/with-discard-model-updates! [:model/PythonLibrary]
+            ;; Create or update the python library
+            (python-library/update-python-library-source! "common"
+                                                          (str "def multiply_by_two(x):\n"
+                                                               "    return x * 2\n"
+                                                               "\n"
+                                                               "def add_greeting(name):\n"
+                                                               "    return f'Hello, {name}!'\n"))
+            (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
+              ;; Create and run a transform that uses the library
+              (with-transform-cleanup! [{table-name :name :as target} {:type   "table"
+                                                                       :schema schema
+                                                                       :name   "library_test_output"}]
+                (let [transform-body (str "import pandas as pd\n"
+                                          "from common import multiply_by_two, add_greeting\n"
+                                          "\n"
+                                          "def transform():\n"
+                                          "    data = {\n"
+                                          "        'number': [1, 2, 3, 4, 5],\n"
+                                          "        'doubled': [multiply_by_two(x) for x in [1, 2, 3, 4, 5]],\n"
+                                          "        'name': ['Alice', 'Bob', 'Charlie', 'David', 'Eve'],\n"
+                                          "        'greeting': [add_greeting(name) for name in ['Alice', 'Bob', 'Charlie', 'David', 'Eve']]\n"
+                                          "    }\n"
+                                          "    return pd.DataFrame(data)")
+                      transform-payload {:name   "Library Test Transform"
+                                         :source {:type  "python"
+                                                  :source-database (mt/id)
+                                                  :source-tables {}
+                                                  :body transform-body}
+                                         :target (assoc target :database (mt/id))}
+                      {transform-id :id} (mt/user-http-request :crowberto :post 200 "ee/transform"
+                                                               transform-payload)]
+                  (transforms.tu/test-run transform-id)
+                  (transforms.tu/wait-for-table table-name 5000)
+                  (is (true? (driver/table-exists? driver/*driver* (mt/db) target)))
+
+                  (let [rows (transforms.tu/table-rows table-name)]
+                    (is (= 5 (count rows)))
+                    (is (every? #(= (* 2 (first %)) (second %)) rows))
+                    (is (= ["Hello, Alice!" "Hello, Bob!" "Hello, Charlie!" "Hello, David!" "Hello, Eve!"]
+                           (map #(nth % 3) rows)))))))))))))
+
+(deftest execute-transforms-python-test
+  (testing "transform execution with :transforms/table target"
+    (mt/test-drivers #{:postgres}
+      (mt/with-premium-features #{:transforms :transforms-python}
+        (mt/dataset transforms-dataset/transforms-test
+          (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
+            (with-transform-cleanup! [{table-name :name :as target} {:type   "table"
+                                                                     :schema schema
+                                                                     :name   "target_table"}]
+              (let [original           {:name   "Gadget Products"
+                                        :source {:type  "python"
+                                                 :source-database (mt/id)
+                                                 :source-tables {"transforms_customers" (mt/id :transforms_customers)}
+                                                 :body  (str "import pandas as pd\n"
+                                                             "\n"
+                                                             "def transform():\n"
+                                                             "    return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}
+                                        :target  (assoc target :database (mt/id))}
+                    {transform-id :id} (mt/user-http-request :crowberto :post 200 "ee/transform"
+                                                             original)]
+                (transforms.tu/test-run transform-id)
+                (transforms.tu/wait-for-table table-name 5000)
+                (is (true? (driver/table-exists? driver/*driver* (mt/db) target)))
+                (is (= [["Alice" 25] ["Bob" 30]]
+                       (transforms.tu/table-rows table-name)))))))))))

enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj

@@ -0,0 +1,97 @@
+(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.execute-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase-enterprise.transforms-python.execute :as transforms-python.execute]
+   [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
+   [metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]
+   [metabase-enterprise.transforms.util :as transforms.util]
+   [metabase.test :as mt]
+   [toucan2.core :as t2])
+  (:import
+   (java.util.concurrent CountDownLatch)))
+
+(set! *warn-on-reflection* true)
+
+(deftest atomic-python-transform-swap-test
+  (testing "Python transform execution with atomic table swap"
+    (mt/test-drivers #{:mysql :postgres}
+      (mt/with-premium-features #{:transforms-python}
+        (mt/dataset transforms-dataset/transforms-test
+          (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
+            (with-transform-cleanup! [{table-name :name :as target} {:type   "table"
+                                                                     :schema schema
+                                                                     :name   "swap_tbl"}]
+              (let [initial-transform {:name   "Python Transform Initial"
+                                       :source {:type  "python"
+                                                :source-tables {}
+                                                :body  (str "import pandas as pd\n"
+                                                            "\n"
+                                                            "def transform():\n"
+                                                            "    return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}
+                                       :target (assoc target :database (mt/id))}]
+                (mt/with-temp [:model/Transform transform initial-transform]
+                  (transforms-python.execute/execute-python-transform! transform {:run-method :manual})
+                  (transforms.tu/wait-for-table table-name 10000)
+
+                  (let [initial-rows (transforms.tu/table-rows table-name)]
+                    (is (= [["Alice" 25] ["Bob" 30]] initial-rows) "Initial data should be Alice and Bob")
+
+                    (t2/update! :model/Transform (:id transform)
+                                {:source {:type "python"
+                                          :source-tables {}
+                                          :body (str "import pandas as pd\n"
+                                                     "\n"
+                                                     "def transform():\n"
+                                                     "    return pd.DataFrame({'name': ['Charlie', 'Diana', 'Eve'], 'age': [35, 40, 45]})")}}))
+
+                  (let [swap-latch (CountDownLatch. 1)
+                        original-rename-tables-atomic! transforms.util/rename-tables!]
+                    (with-redefs [transforms.util/rename-tables! (fn [driver db-id rename-pairs]
+                                                                   (.await swap-latch)
+                                                                   (original-rename-tables-atomic! driver db-id rename-pairs))]
+                      (let [transform-future (future
+                                               (transforms-python.execute/execute-python-transform!
+                                                (t2/select-one :model/Transform (:id transform))
+                                                {:run-method :manual}))]
+                        (is (= [["Alice" 25] ["Bob" 30]]
+                               (transforms.tu/table-rows table-name))
+                            "Original data should still be accessible during transform")
+                        (.countDown swap-latch)
+                        @transform-future
+                        (is (= [["Charlie" 35] ["Diana" 40] ["Eve" 45]]
+                               (transforms.tu/table-rows table-name))
+                            "Table should now contain Charlie, Diana, and Eve")))))))))))))
+
+(deftest python-transform-temp-table-cleanup-test
+  (testing "Python transform cleans up temp tables on success"
+    (mt/test-drivers #{:mysql :postgres}
+      (mt/with-premium-features #{:transforms-python}
+        (mt/dataset transforms-dataset/transforms-test
+          (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
+            (with-transform-cleanup! [{table-name :name :as target} {:type   "table"
+                                                                     :schema schema
+                                                                     :name   "cleanup_"}]
+              (let [transform-def {:name   "Python Transform Cleanup"
+                                   :source {:type  "python"
+                                            :source-tables {}
+                                            :body  (str "import pandas as pd\n"
+                                                        "\n"
+                                                        "def transform():\n"
+                                                        "    return pd.DataFrame({'col1': [1, 2, 3], 'col2': ['a', 'b', 'c']})")}
+                                   :target (assoc target :database (mt/id))}]
+                (mt/with-temp [:model/Transform transform transform-def]
+                  (transforms-python.execute/execute-python-transform! transform {:run-method :manual})
+                  (transforms.tu/wait-for-table table-name 10000)
+
+                  (transforms-python.execute/execute-python-transform! transform {:run-method :manual})
+
+                  (let [db-id (mt/id)
+                        tables (t2/select :model/Table :db_id db-id :active true)
+                        new-table-pattern (re-pattern (str ".*" table-name "_" transforms-python.execute/temp-table-suffix-new "_.*"))
+                        old-table-pattern (re-pattern (str ".*" table-name "_" transforms-python.execute/temp-table-suffix-old "_.*"))]
+                    (is (not-any? #(or (re-matches new-table-pattern (:name %))
+                                       (re-matches old-table-pattern (:name %))) tables)
+                        (str "No temp tables (_" transforms-python.execute/temp-table-suffix-new "_ or _" transforms-python.execute/temp-table-suffix-old "_) should remain after successful Python transform"))
+
+                    (is (= [[1 "a"] [2 "b"] [3 "c"]] (transforms.tu/table-rows table-name))
+                        "Table should contain the expected data after swap")))))))))))

enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj

@@ -0,0 +1,46 @@
+(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.job-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase-enterprise.transforms.schedule :as transforms.schedule]
+   [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
+   [metabase-enterprise.transforms.test-util :as transforms.tu]
+   [metabase.driver :as driver]
+   [metabase.task.core :as task]
+   [metabase.test :as mt]
+   [metabase.util :as u]
+   [toucan2.core :as t2]))
+
+(deftest python-transform-scheduled-job-test
+  (mt/test-helpers-set-global-values!
+    (mt/with-temp-scheduler!
+      (task/init! ::transforms.schedule/RunTransform)
+      (mt/test-drivers #{:postgres}
+        (mt/with-premium-features #{:transforms-python :transforms}
+          (mt/dataset transforms-dataset/transforms-test
+            (transforms.tu/with-transform-cleanup! [target {:type   "table"
+                                                            :schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))
+                                                            :name   "target_table"}]
+              (mt/with-temp
+                [:model/TransformTag {tag-id :id}       {:name "every second"}
+                 :model/Transform    {transform-id :id} {:name   "Gadget Products"
+                                                         :source {:type  "python"
+                                                                  :source-database (mt/id)
+                                                                  :source-tables {"transforms_customers" (mt/id :transforms_customers)}
+                                                                  :body  (str "import pandas as pd\n"
+                                                                              "\n"
+                                                                              "def transform():\n"
+                                                                              "    return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}
+                                                         :target  (assoc target :database (mt/id))}
+                 :model/TransformTransformTag   _  {:transform_id transform-id :tag_id tag-id :position 0}
+                 :model/TransformJob {job-id :id :as job} {:schedule "* * * * * ? *"}
+                 :model/TransformJobTransformTag _ {:job_id job-id :tag_id tag-id :position 0}]
+                (transforms.schedule/initialize-job! job)
+                (transforms.schedule/update-job! job-id "* * * * * ? *")
+                (is (true? (u/poll {:thunk   (fn [] (driver/table-exists? driver/*driver* (mt/db) target))
+                                    :done?   true?
+                                    :timeout-ms 20000
+                                    :interval-ms 1000})))
+                (is (true? (u/poll {:thunk   (fn [] (t2/exists? :model/Table :name (:name target)))
+                                    :done?   true?
+                                    :timeout-ms 10000
+                                    :interval-ms 1000})))))))))))

enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj

@@ -0,0 +1,492 @@
+(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.python-runner-test
+  (:require
+   [clojure.core.async :as a]
+   [clojure.java.jdbc :as jdbc]
+   [clojure.string :as str]
+   [clojure.test :refer :all]
+   [java-time.api :as t]
+   [metabase-enterprise.transforms-python.python-runner :as python-runner]
+   [metabase-enterprise.transforms-python.s3 :as s3]
+   [metabase-enterprise.transforms-python.settings :as transforms-python.settings]
+   [metabase-enterprise.transforms.util :as transforms.util]
+   [metabase.driver :as driver]
+   [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
+   [metabase.sync.core :as sync]
+   [metabase.test :as mt]
+   [metabase.test.data.interface :as tx]
+   [metabase.test.data.sql :as sql.tx]
+   [metabase.util :as u]
+   [metabase.util.json :as json]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private last-job-run-id (atom 0))
+
+(defn next-job-run-id [] (swap! last-job-run-id inc))
+
+(defn- parse-jsonl [s] (map json/decode+kw (str/split-lines s)))
+
+(defn template->regex
+  "Convert a template string with $var$ placeholders to a regex pattern.
+   Example: template->regex 'File ___PATH___/script.py, line ___LINE___'
+   will match 'File /tmp/unique-path/script.py, line 123'"
+  [template]
+  (-> template
+      (str/replace #"(?m)^\d{13}\s*" "")
+      (str/replace "\\" "\\\\")
+      (str/replace "." "\\.")
+      (str/replace "*" "\\*")
+      (str/replace "+" "\\+")
+      (str/replace "?" "\\?")
+      (str/replace "^" "\\^")
+      (str/replace "$" "\\$")
+      (str/replace "{" "\\{")
+      (str/replace "}" "\\}")
+      (str/replace "(" "\\(")
+      (str/replace ")" "\\)")
+      (str/replace "|" "\\|")
+      (str/replace "[" "\\[")
+      (str/replace "]" "\\]")
+      (str/replace #"___PATH___" "(/[\\\\w-]+)+")
+      (str/replace #"___LINE___" "\\\\d+")
+      re-pattern))
+
+(defn- jsonl-output [expected] #(= expected (parse-jsonl %)))
+
+(defn- execute! [{:keys [code tables]}]
+  (with-open [shared-storage-ref (s3/open-shared-storage! (or tables {}))]
+    (let [server-url     (transforms-python.settings/python-runner-url)
+          cancel-chan    (a/promise-chan)
+          table-name->id (or tables {})
+          test-id        (next-job-run-id)
+          _              (python-runner/copy-tables-to-s3! {:run-id         test-id
+                                                            :shared-storage @shared-storage-ref
+                                                            :table-name->id table-name->id
+                                                            :cancel-chan    cancel-chan})
+          response       (python-runner/execute-python-code-http-call! {:server-url     server-url
+                                                                        :code           code
+                                                                        :run-id         test-id
+                                                                        :table-name->id table-name->id
+                                                                        :shared-storage @shared-storage-ref})
+          {:keys [output output-manifest events]} (python-runner/read-output-objects @shared-storage-ref)]
+      ;; not sure about munging this all together but its what tests expect for now
+      (merge (:body response)
+             {:output          output
+              :output-manifest output-manifest
+              :stdout          (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
+              :stderr          (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))}))))
+
+(defn ok-stdout [num-rows num-cols]
+  (format (str "Successfully saved %d rows to S3\n"
+               "Successfully saved output manifest with %d fields to S3")
+          num-rows num-cols))
+
+(deftest ^:parallel transform-function-basic-test
+  (testing "executes transform function and returns CSV output"
+    (let [transform-code (str "import pandas as pd\n"
+                              "\n"
+                              "def transform():\n"
+                              "    return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")
+          result         (execute! {:code transform-code})]
+      (is (=? {:output (jsonl-output [{:name "Alice", :age 25}
+                                      {:name "Bob", :age 30}])
+               :stdout (ok-stdout 2 2)}
+              result)))))
+
+(deftest ^:parallel transform-function-missing-test
+  (testing "handles missing transform function"
+    (let [result (execute! {:code (str "import pandas as pd\n"
+                                       "\n"
+                                       "# No transform function defined")})]
+      (is (=? {:exit_code 1
+               :stderr    "ERROR: User script must define a 'transform()' function"
+               :stdout    ""}
+              result)))))
+
+(deftest ^:parallel transform-function-wrong-return-type-test
+  (testing "handles transform function returning non-DataFrame"
+    (let [result (execute! {:code (str "def transform():\n"
+                                       "    return 'not a dataframe'")})]
+      (is (=? {:exit_code 1
+               :stderr    "ERROR: Transform function must return a pandas DataFrame, got <class 'str'>"
+               :stdout    ""}
+              result)))))
+
+(deftest ^:parallel transform-function-error-test
+  (testing "handles transform function with error"
+    (let [result            (execute! {:code (str "def transform():\n"
+                                                  "    raise ValueError('Something went wrong')")})
+          expected-template (str "ERROR: Transform function failed: Something went wrong\n"
+                                 "Traceback (most recent call last):\n"
+                                 "  File \"___PATH___/transform_runner.py\", line ___LINE___, in main\n"
+                                 "    result = script.transform()\n"
+                                 "             ^^^^^^^^^^^^^^^^^^\n"
+                                 "  File \"___PATH___/script.py\", line 2, in transform\n"
+                                 "    raise ValueError('Something went wrong')\n"
+                                 "ValueError: Something went wrong")
+          stderr-pattern    (template->regex expected-template)]
+      (is (=? {:exit_code 1
+               :stderr    #(re-matches stderr-pattern %)
+               :stdout    ""
+               :timeout   false}
+              result)))))
+
+(deftest ^:parallel transform-function-complex-dataframe-test
+  (testing "can create complex DataFrames with transform"
+    (let [transform-code (str "import pandas as pd\n"
+                              "\n"
+                              "def transform():\n"
+                              "    data = {'x': [1, 2, 3], 'y': [10, 20, 30], 'z': ['a', 'b', 'c']}\n"
+                              "    return pd.DataFrame(data)")
+          result         (execute! {:code transform-code})]
+      (is (=? {:output (jsonl-output [{:x 1, :y 10, :z "a"}
+                                      {:x 2, :y 20, :z "b"}
+                                      {:x 3, :y 30, :z "c"}])
+               :stdout (ok-stdout 3 3)}
+              result)))))
+
+(deftest ^:parallel transform-function-with-db-parameter-test
+  (mt/test-drivers #{:postgres}
+    (testing "transform function can accept db parameter for forward compatibility"
+      (let [transform-code (str "import pandas as pd\n"
+                                "\n"
+                                "def transform():\n"
+                                "    data = {'name': ['Charlie', 'Dana'], 'score': [85, 92]}\n"
+                                "    return pd.DataFrame(data)")
+            result         (execute! {:code transform-code})]
+        (is (=? {:output (jsonl-output [{:name "Charlie", :score 85}
+                                        {:name "Dana", :score 92}])
+                 :stdout (ok-stdout 2 2)}
+                result))))))
+
+(deftest transform-function-with-pass-thru
+  (testing "transform function successfully connects to PostgreSQL database and reads data"
+    (mt/test-drivers #{:postgres}
+      (mt/with-empty-db
+        (let [db-spec        (sql-jdbc.conn/db->pooled-connection-spec (mt/db))
+              _              (jdbc/execute! db-spec ["DROP TABLE IF EXISTS students"])
+              _              (jdbc/execute! db-spec ["CREATE TABLE students (id INTEGER PRIMARY KEY, name VARCHAR(100), score INTEGER)"])
+              _              (jdbc/execute! db-spec ["INSERT INTO students (id, name, score) VALUES (1, 'Alice', 85), (2, 'Bob', 92), (3, 'Charlie', 88), (4, 'Dana', 90)"])
+
+              _              (sync/sync-database! (mt/db))
+
+              transform-code (str "import pandas as pd\n"
+                                  "\n"
+                                  "def transform(students):\n"
+                                  "    return students")
+              result         (execute! {:code   transform-code
+                                        :tables {"students" (mt/id :students)}})]
+
+          (is (=? {:output          (jsonl-output [{:id 1 :name "Alice" :score 85}
+                                                   {:id 2 :name "Bob" :score 92}
+                                                   {:id 3 :name "Charlie" :score 88}
+                                                   {:id 4 :name "Dana" :score 90}])
+                   :output-manifest {:fields          [{:base_type      "Integer",
+                                                        :database_type  "int4",
+                                                        :effective_type "Integer",
+                                                        :name           "id",
+                                                        :root_type      "Number",
+                                                        :semantic_type  "PK"}
+                                                       {:base_type      "Text",
+                                                        :database_type  "varchar",
+                                                        :effective_type "Text",
+                                                        :name           "name",
+                                                        :root_type      "Text",
+                                                        :semantic_type  "Name"}
+                                                       {:base_type      "Integer",
+                                                        :database_type  "int4",
+                                                        :effective_type "Integer",
+                                                        :name           "score",
+                                                        :root_type      "Number",
+                                                        :semantic_type  "Score"}],
+                                     :source_metadata {:fields         [{:base_type      "Integer",
+                                                                         :database_type  "int4",
+                                                                         :effective_type "Integer",
+                                                                         :name           "id",
+                                                                         :root_type      "Number",
+                                                                         :semantic_type  "PK"}
+                                                                        {:base_type      "Text",
+                                                                         :database_type  "varchar",
+                                                                         :effective_type "Text",
+                                                                         :name           "name",
+                                                                         :root_type      "Text",
+                                                                         :semantic_type  "Name"}
+                                                                        {:base_type      "Integer",
+                                                                         :database_type  "int4",
+                                                                         :effective_type "Integer",
+                                                                         :name           "score",
+                                                                         :root_type      "Number",
+                                                                         :semantic_type  "Score"}],
+                                                       :table_metadata {:table_id (mt/malli=? int?)},
+                                                       :schema_version 1
+                                                       :data_format    "jsonl"
+                                                       :data_version   1}
+                                     :schema_version  1
+                                     :data_format     "jsonl"
+                                     :data_version    1}
+                   :stdout          (ok-stdout 4 3)}
+                  result)))))))
+
+(deftest transform-function-with-empty-table-test
+  (testing "transform function successfully connects to PostgreSQL database and reads data"
+    (mt/test-drivers #{:postgres}
+      (mt/with-empty-db
+        (let [db-spec        (sql-jdbc.conn/db->pooled-connection-spec (mt/db))
+              _              (jdbc/execute! db-spec ["DROP TABLE IF EXISTS students"])
+              _              (jdbc/execute! db-spec ["CREATE TABLE students (id INTEGER PRIMARY KEY, name VARCHAR(100), score INTEGER)"])
+
+              _              (sync/sync-database! (mt/db))
+
+              transform-code (str "import pandas as pd\n"
+                                  "\n"
+                                  "def transform(students):\n"
+                                  "    # Calculate average score\n"
+                                  "    avg_score = students['score'].mean()\n"
+                                  "    result = pd.DataFrame({\n"
+                                  "        'student_count': [len(students)],\n"
+                                  "        'average_score': [round(avg_score, 2)]\n"
+                                  "    })\n"
+                                  "    return result")
+              result         (execute! {:code  transform-code
+                                        :tables {"students" (mt/id :students)}})]
+
+          (is (=? {:output          "{\"student_count\":0,\"average_score\":null}\n"
+                   :output-manifest {:schema_version 1
+                                     :data_format    "jsonl"
+                                     :data_version   1
+                                     :fields         [{:name "student_count", :base_type "Integer"}
+                                                      {:name "average_score", :base_type "Float"}]}
+                   :stdout          (ok-stdout 1 2)}
+                  result)))))))
+
+(deftest transform-function-with-working-database-test
+  (testing "transform function successfully connects to PostgreSQL database and reads data"
+    (mt/test-drivers #{:postgres}
+      (mt/with-empty-db
+        (let [db-spec        (sql-jdbc.conn/db->pooled-connection-spec (mt/db))
+              _              (jdbc/execute! db-spec ["DROP TABLE IF EXISTS students"])
+              _              (jdbc/execute! db-spec ["CREATE TABLE students (id INTEGER PRIMARY KEY, name VARCHAR(100), score INTEGER)"])
+              _              (jdbc/execute! db-spec ["INSERT INTO students (id, name, score) VALUES (1, 'Alice', 85), (2, 'Bob', 92), (3, 'Charlie', 88), (4, 'Dana', 90)"])
+
+              _              (sync/sync-database! (mt/db))
+
+              transform-code (str "import pandas as pd\n"
+                                  "\n"
+                                  "def transform(students):\n"
+                                  "    # Calculate average score\n"
+                                  "    avg_score = students['score'].mean()\n"
+                                  "    result = pd.DataFrame({\n"
+                                  "        'student_count': [len(students)],\n"
+                                  "        'average_score': [round(avg_score, 2)]\n"
+                                  "    })\n"
+                                  "    return result")
+              result         (execute! {:code  transform-code
+                                        :tables {"students" (mt/id :students)}})]
+
+          (is (=? {:output          "{\"student_count\":4,\"average_score\":88.75}\n"
+                   :output-manifest {:schema_version 1
+                                     :data_format    "jsonl"
+                                     :data_version   1
+                                     :fields         [{:name "student_count", :base_type "Integer"}
+                                                      {:name "average_score", :base_type "Float"}]}
+                   :stdout          (ok-stdout 1 2)}
+                  result)))))))
+
+(defn- datetime-equal?
+  [expected-iso-str actual-pandas-str]
+  (let [expected-dt (t/zoned-date-time expected-iso-str)
+        actual-dt   (t/zoned-date-time actual-pandas-str)]
+    (= (t/to-millis-from-epoch expected-dt)
+       (t/to-millis-from-epoch actual-dt))))
+
+(deftest transform-type-test
+  (mt/test-drivers #{:postgres}
+    (mt/with-empty-db
+      (let [db-spec (sql-jdbc.conn/db->pooled-connection-spec (mt/db))
+            _ (jdbc/execute! db-spec ["DROP TABLE IF EXISTS sample_table"])
+            _ (jdbc/execute! db-spec ["CREATE TABLE sample_table (
+                                                        id BIGSERIAL PRIMARY KEY,
+                                                        name VARCHAR(255),
+                                                        description TEXT,
+                                                        count BIGINT,
+                                                        price FLOAT,
+                                                        is_active BOOLEAN,
+                                                        created_date DATE,
+                                                        updated_at TIMESTAMP,
+                                                        scheduled_for TIMESTAMP WITH TIME ZONE
+                                                    )"])
+            _ (jdbc/execute! db-spec ["INSERT INTO sample_table
+                                                        (name, description, count, price, is_active, created_date, updated_at, scheduled_for)
+                                                      VALUES
+                                                        ('Product A', 'A sample product description', 100, 29.99, true, '2024-01-15', '2024-01-15 10:30:00', '2024-01-16 14:00:00+00'),
+                                                        ('Product B', 'Another product with longer description text', 50, 15.50, false, '2024-02-01', '2024-02-01 09:15:30', '2024-02-02 16:30:00-05'),
+                                                        ('Product C', NULL, 0, 0.0, true, '2024-03-10', '2024-03-10 18:45:15', '2024-03-11 08:00:00+02')"])
+
+            _ (sync/sync-database! (mt/db))
+
+            transform-code (str "import pandas as pd\n"
+                                "\n"
+                                "def transform(sample_table):\n"
+                                "    df = sample_table.copy()\n"
+                                "    return df")
+            result (execute! {:code  transform-code
+                              :tables {"sample_table" (mt/id :sample_table)}})
+            rows (parse-jsonl (:output result))
+            headers (map name (keys (first rows)))
+            [row1 row2 row3] rows
+            get-col (fn [row col-name] (get row (keyword col-name)))
+            metadata (:output-manifest result)]
+
+        (is (= (set ["id" "name" "description" "count" "price" "is_active" "created_date" "updated_at" "scheduled_for"])
+               (set headers)))
+
+        (is (= 1 (get-col row1 "id")))
+        (is (= "Product A" (get-col row1 "name")))
+        (is (datetime-equal? "2024-01-16T14:00:00Z" (get-col row1 "scheduled_for")))
+
+        (is (= 2 (get-col row2 "id")))
+        (is (= "Product B" (get-col row2 "name")))
+        (is (datetime-equal? "2024-02-02T21:30:00Z" (get-col row2 "scheduled_for")))
+
+        (is (= 3 (get-col row3 "id")))
+        (is (= "Product C" (get-col row3 "name")))
+        (is (datetime-equal? "2024-03-11T06:00:00Z" (get-col row3 "scheduled_for")))
+        (testing "types are preserved correctly"
+          (is (= {"id"            :type/BigInteger
+                  "name"          :type/Text
+                  "description"   :type/Text
+                  "count"         :type/BigInteger
+                  "price"         :type/Float
+                  "is_active"     :type/Boolean
+                 ;; Our hack works
+                  "created_date"  :type/Date
+                  "updated_at"    :type/DateTime
+                  "scheduled_for" :type/DateTimeWithLocalTZ}
+                 (u/for-map [{:keys [name base_type]} (:fields metadata)]
+                   [name (python-runner/restricted-insert-type base_type)]))))))))
+
+(deftest transform-function-with-library-test
+  (testing "transform function can use libraries"
+    (mt/test-drivers #{:postgres}
+      (mt/with-temp [:model/PythonLibrary _ {:path   "circle"
+                                             :source "import math\n\ndef calculate_circle_area(radius):\n    return math.pi * radius ** 2"}
+                     :model/PythonLibrary _ {:path   "utils"
+                                             :source "def format_currency(amount):\n    return f\"${amount:,.2f}\""}]
+        (let [transform-code (str "import pandas as pd\n"
+                                  "from circle import calculate_circle_area\n"
+                                  "from utils import format_currency\n"
+                                  "\n"
+                                  "def transform():\n"
+                                  "    data = [\n"
+                                  "        {'radius': 5, 'area': calculate_circle_area(5), 'price': format_currency(78.54)},\n"
+                                  "        {'radius': 10, 'area': calculate_circle_area(10), 'price': format_currency(314.16)}\n"
+                                  "    ]\n"
+                                  "    return pd.DataFrame(data)")
+              result         (execute! {:code transform-code})]
+          (is (=? {:output (jsonl-output [{:radius 5,  :area 78.5398163397, :price "$78.54"}
+                                          {:radius 10, :area 314.159265359, :price "$314.16"}])
+                   :stdout (ok-stdout 2 3)
+                   #_#_:stderr ""}
+                  result)))))))
+
+(deftest transform-function-without-libraries-test
+  (testing "transform function works when no libraries exist"
+    (mt/test-drivers #{:postgres}
+      (with-redefs [t2/select-fn->fn (fn [k v model]
+                                       (when (and (= k :path)
+                                                  (= v :source)
+                                                  (= model :model/PythonLibrary))
+                                         {}))]
+        (let [transform-code (str "import pandas as pd\n"
+                                  "\n"
+                                  "def transform():\n"
+                                  "    return pd.DataFrame({'status': ['ok']})")
+              result         (execute! {:code transform-code})]
+          (is (=? {:output (jsonl-output [{:status "ok"}])
+                   :stdout (ok-stdout 1 1)}
+                  result)))))))
+
+(deftest transform-function-library-import-error-test
+  (testing "transform function handles missing library gracefully"
+    (mt/test-drivers #{:postgres}
+      (with-redefs [t2/select-fn->fn (fn [k v model]
+                                       (when (and (= k :path)
+                                                  (= v :source)
+                                                  (= model :model/PythonLibrary))
+                                         {"utils" "def helper():\n    return 42"}))]
+        (let [transform-code (str "import pandas as pd\n"
+                                  "from common import some_function  # This library doesn't exist\n"
+                                  "\n"
+                                  "def transform():\n"
+                                  "    return pd.DataFrame({'value': [some_function()]})")
+              result         (execute! {:code transform-code})]
+          ;; TODO this error message could still be improved a lot
+          (is (=? {#_#_:error     "Execution failed"
+                   :exit_code 1
+                   :stderr    #(str/includes? % "No module named 'common'")}
+                  result)))))))
+
+(deftest transform-type-roundtrip-test
+  (mt/test-drivers #{:postgres :h2 :mysql :bigquery-cloud-sdk :redshift :snowflake :sqlserver :clickhouse}
+    (mt/with-empty-db
+      (let [driver       driver/*driver*
+            db-id        (mt/id)
+            table-name   (if (= :redshift driver)
+                           (tx/db-qualified-table-name (get-in (mt/db) [:settings :database-source-dataset-name]) (mt/random-name))
+                           (mt/random-name))
+            schema-name  (if  (= :clickhouse driver)
+                           (-> (mt/db) :details :db)
+                           (sql.tx/session-schema driver))
+            qualified-table-name (if schema-name
+                                   (keyword schema-name table-name)
+                                   (keyword table-name))
+            table-schema {:name    qualified-table-name
+                          :columns [{:name "id"            :type :type/Integer  :nullable? false}
+                                    {:name "price"         :type :type/Float    :nullable? true}
+                                    {:name "active"        :type :type/Boolean  :nullable? true}
+                                    {:name "created_tz"    :type :type/DateTimeWithTZ :nullable? true}
+                                    {:name "created_at"    :type :type/DateTime :nullable? true}
+                                    {:name "created_date"  :type :type/Date     :nullable? true}
+                                    {:name "description"   :type :type/Text     :nullable? true}]}
+            _ (mt/as-admin
+                (transforms.util/create-table-from-schema! driver db-id table-schema))
+
+            row-values   [[1
+                           19.99
+                           (if (= driver/*driver* :sqlserver) 1 true)
+                           (if (= driver/*driver* :mysql) "2024-01-01 12:00:00" "2024-01-01T12:00:00Z")
+                           (if (= driver/*driver* :mysql) "2024-01-01 12:00:00" "2024-01-01T12:00:00")
+                           "2024-01-01"
+                           "Sample product"]]
+            _ (driver/insert-from-source! driver db-id table-schema
+                                          {:type :rows :data row-values})
+
+            _ (sync/sync-database! (mt/db) {:scan :schema})
+
+            transform-code (str "import pandas as pd\n"
+                                "\n"
+                                "def transform(" table-name "):\n"
+                                "    return " table-name)
+            result (execute! {:code  transform-code
+                              :tables {table-name (mt/id qualified-table-name)}})
+
+            metadata (:output-manifest result)]
+
+        (testing "All expected columns are present"
+          (is (= #{"id" "price" "active" "created_tz" "created_at" "created_date" "description"}
+                 (set (map :name (:fields metadata))))))
+
+        (testing "types are preserved correctly"
+          (is (= {"id"           (if (= :snowflake driver) :type/Number :type/Integer)
+                  "price"        :type/Float
+                  "active"       :type/Boolean
+                  "created_tz"   (if (= :snowflake driver) :type/DateTimeWithTZ :type/DateTimeWithLocalTZ)
+                  "created_at"   :type/DateTime
+                  "created_date" :type/Date
+                  "description"  :type/Text}
+                 (u/for-map [{:keys [name base_type]} (:fields metadata)]
+                   [name (python-runner/restricted-insert-type base_type)]))))
+
+       ;; cleanup
+        (driver/drop-table! driver db-id qualified-table-name)))))

enterprise/backend/test/metabase_enterprise/transforms_python/s3_test.clj

@@ -0,0 +1,94 @@
+;; This is a driver test currently to benefit from the localstack S3 setup in driver tests.
+;; Perhaps we should create a new testing category?
+;; This code otherwise runs in the appdb section, but it does not touch the appdb.
+(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.s3-test
+  "Tests for S3 operations in transforms-python module."
+  (:require
+   [clj-http.client :as http]
+   [clojure.string :as str]
+   [clojure.test :refer :all]
+   [metabase-enterprise.transforms-python.s3 :as s3]
+   [metabase-enterprise.transforms-python.settings :as transforms-python.settings]
+   [metabase.test :as mt]
+   [metabase.util.log :as log])
+  (:import
+   (java.net ConnectException)
+   (java.nio.file Files)
+   (java.nio.file.attribute FileAttribute)
+   (software.amazon.awssdk.services.s3 S3Client)
+   (software.amazon.awssdk.services.s3.model NoSuchKeyException)))
+
+(set! *warn-on-reflection* true)
+
+(defn- s3-endpoint-running? []
+  (try
+    (http/head (transforms-python.settings/python-runner-url) {:throw-exceptions false})
+    (catch ConnectException _
+      false)))
+
+;; TODO remove this once CI situation is resolved
+(use-fixtures :each (fn [thunk]
+                      (if (s3-endpoint-running?)
+                        (thunk)
+                        (log/warn "Skipping S3 tests because localstack isn't running"))))
+
+(deftest s3-read-write-test
+  (testing "We can open an s3 connection, and read and write things"
+    (with-open [s3-client (s3/create-s3-client)]
+      (is (instance? S3Client s3-client))
+      (let [bucket (transforms-python.settings/python-storage-s-3-bucket)
+            key    (str "test-object-" (random-uuid) ".txt")
+            body   (str "Hello, S3! My secret is:" (random-uuid))]
+
+        (is (thrown? NoSuchKeyException (s3/read-to-string s3-client bucket key)))
+        (is (= :default (s3/read-to-string s3-client bucket key :default)))
+
+        (let [tmp-file (Files/createTempFile "s3-test" ".txt" (into-array FileAttribute []))]
+          (try
+            (spit (.toFile tmp-file) body)
+
+            (is (s3/upload-file s3-client bucket key (.toFile tmp-file)))
+
+            (is (= body (s3/read-to-string s3-client bucket key)))
+            (is (= body (s3/read-to-string s3-client bucket key :default)))
+
+            (s3/delete s3-client bucket key)
+
+            (is (thrown? NoSuchKeyException (s3/read-to-string s3-client bucket key)))
+            (is (= :default (s3/read-to-string s3-client bucket key :default)))
+
+            (finally
+              (Files/deleteIfExists tmp-file))))))))
+
+(deftest open-s3-shared-storage-test
+  (testing "open-s3-shared-storage! returns closeable derefable"
+    (mt/with-premium-features #{:transforms-python}
+      (let [table-name->id {"users" 1}
+            storage-ref    (s3/open-shared-storage! table-name->id)
+            {:keys [s3-client bucket-name objects]} @storage-ref]
+        (testing "can be dereferenced"
+          (doseq [[k {:keys [method path url]}] objects
+                  :let [url (str/replace url "localstack" "localhost")]]
+            (testing (format "%s (%s)" k method)
+              (is (= :not-created (s3/read-to-string s3-client bucket-name path :not-created)))
+              (let [content (str (random-uuid))]
+                (case method
+                  ;; Files that we should write, and python-runner should fetch
+                  :get (let [tmp-file (Files/createTempFile "s3-test" ".txt" (into-array FileAttribute []))]
+                         (try
+                           (spit (.toFile tmp-file) content)
+                           (is (s3/upload-file s3-client bucket-name path (.toFile tmp-file)))
+                           (finally
+                             (Files/deleteIfExists tmp-file)))
+                         (is (= content (s3/read-to-string s3-client bucket-name path :not-created)))
+                         (is (= content (:body (http/get url)))))
+                  ;; Files that python-runner should post, and we should fetch
+                  :put (do
+                         (is (http/put url {:body content}))
+                         (is (= content (s3/read-to-string s3-client bucket-name path :not-created)))))))))
+
+        (testing "closing the ref deletes the files"
+          (.close storage-ref)
+          (doseq [[k {:keys [path]}] objects]
+            (testing (str k)
+              (is (= :not-created (s3/read-to-string s3-client bucket-name path :not-created))))))))))

enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj

@@ -0,0 +1,316 @@
+(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.transforms-api-test
+  (:require
+   [clojure.string :as str]
+   [clojure.test :refer :all]
+   [metabase-enterprise.transforms-python.execute :as transforms-python.execute]
+   [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
+   [metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup! get-test-schema]]
+   [metabase.driver :as driver]
+   [metabase.test :as mt]
+   [toucan2.core :as t2])
+  (:import
+   (clojure.lang IDeref)
+   (java.io Closeable)
+   (java.time Duration)))
+
+(set! *warn-on-reflection* true)
+
+(deftest create-python-transform-test
+  (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+    (mt/dataset transforms-dataset/transforms-test
+      (letfn [(create-transform! []
+                (let [schema            (get-test-schema)
+                      transform-payload {:name   "My beautiful python runner"
+                                         :source {:type          "python"
+                                                  :body          "print('hello world')"
+                                                  :source-tables {}}
+                                         :target {:type     "table"
+                                                  :schema   schema
+                                                  :name     "gadget_products"
+                                                  :database (mt/id)}}]
+                  (mt/user-http-request :crowberto :post "ee/transform"
+                                        transform-payload)))]
+
+        (testing "without any feature flags"
+          (mt/with-premium-features #{}
+            (testing "creating python transform without any features fails"
+              (is (= "error-premium-feature-not-available"
+                     (:status (mt/user-http-request :crowberto :post 402 "ee/transform"
+                                                    {:name   "My beautiful python runner"
+                                                     :source {:type          "python"
+                                                              :body          "print('hello world')"
+                                                              :source-tables {}}
+                                                     :target {:type     "table"
+                                                              :schema   (get-test-schema)
+                                                              :name     "gadget_products"
+                                                              :database (mt/id)}})))))))
+
+        (testing "with only transforms feature flag (no transforms-python)"
+          (mt/with-premium-features #{:transforms}
+            (testing "creating python transform without transforms-python feature fails"
+              (is (= "Premium features required for this transform type are not enabled."
+                     (mt/user-http-request :crowberto :post 402 "ee/transform"
+                                           {:name   "My beautiful python runner"
+                                            :source {:type          "python"
+                                                     :body          "print('hello world')"
+                                                     :source-tables {}}
+                                            :target {:type     "table"
+                                                     :schema   (get-test-schema)
+                                                     :name     "gadget_products"
+                                                     :database (mt/id)}}))))))
+
+        (testing "with transforms-python feature flag"
+          (mt/with-premium-features #{:transforms :transforms-python}
+            (with-transform-cleanup! [table-name "gadget_products"]
+              (let [transform         (create-transform!)]
+                (is (= "print('hello chris')"
+                       (-> (mt/user-http-request :crowberto :put 200 (format "ee/transform/%s" (:id transform))
+                                                 {:name   "My beautiful python runner"
+                                                  :source {:type          "python"
+                                                           :body          "print('hello chris')"
+                                                           :source-tables {}}
+                                                  :target {:type     "table"
+                                                           :schema   (get-test-schema)
+                                                           :name     table-name
+                                                           :database (mt/id)}})
+                           :source :body)))))))))))
+
+(deftest update-python-transform-feature-flag-test
+  (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+    (testing "Updating a python transform requires both :transforms and :transforms-python features"
+      (mt/with-temp [:model/Transform {id :id
+                                       :as transform} {:name   "Original Python Transform"
+                                                       :source {:type          "python"
+                                                                :body          "print('original')"
+                                                                :source-tables {}}
+                                                       :target {:type     "table"
+                                                                :schema   "scheam"
+                                                                :name     "table"
+                                                                :database (mt/id)}}]
+        (mt/with-premium-features #{}
+          (let [response (mt/user-http-request :crowberto :put
+                                               (format "ee/transform/%d" id)
+                                               (assoc-in transform [:source :body] "print('no features')"))]
+            (is (= "error-premium-feature-not-available" (:status response))
+                "Should return 403 without any features")))))))
+
+(deftest run-python-transform-feature-flag-test
+  (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+    (testing "Running a python transform requires both :transforms and :transforms-python features"
+      (mt/with-premium-features #{:transforms :transforms-python}
+        (mt/dataset transforms-dataset/transforms-test
+          (with-transform-cleanup! [table-name "test_run_python"]
+            (let [schema (get-test-schema)
+                  transform-payload {:name   "Test Run Python Transform"
+                                     :source {:type          "python"
+                                              :body          "def transform():\n    pass"
+                                              :source-tables {}}
+                                     :target {:type     "table"
+                                              :schema   schema
+                                              :name     table-name
+                                              :database (mt/id)}}
+                  created (mt/user-http-request :crowberto :post 200 "ee/transform" transform-payload)]
+              (mt/with-premium-features #{}
+                (let [response (mt/user-http-request :crowberto :post 402
+                                                     (format "ee/transform/%d/run" (:id created)))]
+                  (is (= "error-premium-feature-not-available" (:status response)))))
+              (mt/with-premium-features #{:transforms}
+                (let [response (mt/user-http-request :crowberto :post
+                                                     (format "ee/transform/%d/run" (:id created)))]
+                  (is (= "Premium features required for this transform type are not enabled." response)
+                      "Should return 403 without :transforms-python feature"))))))))))
+
+(deftest execute-python-transform-test
+  (testing "transform execution with :transforms/table target"
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+      (mt/with-premium-features #{:transforms :transforms-python}
+        (mt/dataset transforms-dataset/transforms-test
+          (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
+            (with-transform-cleanup! [{table-name :name :as target} {:type   "table"
+                                                                     :schema schema
+                                                                     :name   "target_table"}]
+              (let [original           {:name   "Gadget Products"
+                                        :source {:type  "python"
+                                                 :source-database (mt/id)
+                                                 :source-tables {"transforms_customers" (mt/id :transforms_customers)}
+                                                 :body  (str "import pandas as pd\n"
+                                                             "\n"
+                                                             "def transform():\n"
+                                                             "    return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}
+                                        :target  (assoc target :database (mt/id))}
+                    {transform-id :id} (mt/user-http-request :crowberto :post 200 "ee/transform" original)]
+                (transforms.tu/test-run transform-id)
+                (transforms.tu/wait-for-table table-name 5000)
+                (is (true? (driver/table-exists? driver/*driver* (mt/db) target)))
+                (is (= [["Alice" 25] ["Bob" 30]]
+                       (transforms.tu/table-rows table-name)))))))))))
+
+(defn- subsequence?
+  "Returns true if sequence ys is a subsequence of xs:
+  every element of ys appears in xs in the same order,
+  though not necessarily contiguously."
+  [xs ys]
+  (cond
+    (empty? ys) true
+    (empty? xs) false
+    (= (first xs) (first ys)) (recur (rest xs) (rest ys))
+    :else (recur (rest xs) ys)))
+
+(deftest python-transform-logging-test
+  (letfn [(program->source [program]
+            (->> (concat ["import pandas as pd"
+                          "def transform():"]
+                         (for [s program] (str "  " s))
+                         ["  return pd.DataFrame({'x': [42]})"])
+                 (str/join "\n")))
+
+          (create-transform [{:keys [program]} target]
+            {:post [(integer? %)]}
+            (:id (mt/user-http-request :crowberto :post 200 "ee/transform"
+                                       {:name   "Python logging test"
+                                        :source {:type            "python"
+                                                 :body            (program->source program)
+                                                 :source-tables   {}}
+                                        :target (assoc target :database (mt/id))})))
+
+          (block-on-run [{:keys [expect-status]} target transform-id]
+            (try
+              (transforms.tu/test-run transform-id)
+              (catch Throwable e
+                ;; test-run throws for non success, but we want to test failures too
+                (when-not (= expect-status (:status (ex-data e)))
+                  (throw e))))
+            (when (= :succeeded expect-status)
+              (transforms.tu/wait-for-table (:name target) 5000)))
+
+          (get-last-run [transform-id]
+            (:last_run (mt/user-http-request :crowberto :get 200 (format "ee/transform/%d" transform-id))))
+
+          (open-message-value-observer [transform-id]
+            (let [states (atom [])
+                  fut    (future
+                           (try
+                             (loop []
+                               (let [{:keys [message]} (get-last-run transform-id)]
+                                 (cond
+                                   (.isInterrupted (Thread/currentThread))
+                                   nil
+
+                                  ;; same message as last time
+                                   (= message (peek @states))
+                                   (recur)
+
+                                  ;; new message value
+                                   :else (do (swap! states conj message)
+                                             (recur)))))
+                             (catch InterruptedException _ nil)))]
+              (reify IDeref
+                (deref [_] @states)
+                Closeable
+                (close [_]
+                  (future-cancel fut)
+                  (assert (not= :timeout (try (deref fut 1000 :timeout) (catch Throwable _))) "Observation thread did not exit!")))))
+
+          (run-scenario [scenario schema]
+            (with-redefs [transforms-python.execute/python-message-loop-sleep-duration Duration/ZERO
+                          transforms-python.execute/transfer-file-to-db                (if-some [e (:writeback-ex scenario)]
+                                                                                         (fn [& _] (throw e))
+                                                                                         @#'transforms-python.execute/transfer-file-to-db)]
+              (with-transform-cleanup! [target {:type   "table"
+                                                :schema schema
+                                                :name   "result"}]
+                (let [transform-id      (create-transform scenario target)
+                      observed-messages (with-open [observer ^Closeable (open-message-value-observer transform-id)]
+                                          (block-on-run scenario target transform-id)
+                                          @observer)
+                      last-run          (get-last-run transform-id)]
+                  {:observed-messages observed-messages
+                   :last-run          last-run}))))]
+    (let [scenarios [{:desc          "stdin"
+                      :program       ["print(\"hello, world\")"]
+                      :expect-status :succeeded
+                      :expected      ["hello, world"]}
+                     {:desc          "stderr"
+                      :program       ["import sys" "print(\"hello, world\", file=sys.stderr)"]
+                      :expect-status :succeeded
+                      :expected      ["hello, world"]}
+                     {:desc          "interleaved streams"
+                      :program       ["import sys"
+                                      "print(\"1\", file=sys.stderr)"
+                                      "print(\"2\", file=sys.stderr)"
+                                      "print(\"3\", file=sys.stdout)"
+                                      "print(\"4\", file=sys.stderr)"]
+                      :expect-status :succeeded
+                      :expected      ["1" "2" "3" "4"]}
+                     {:desc          "syntax error"
+                      :program       ["print(40 + 2)"
+                                      "this is not valid code"]
+                      :expect-status :failed
+                      :expected      ["SyntaxError: invalid syntax"]}
+                     {:desc                  "takes time, early feedback possible"
+                      :program               ["import time"
+                                              "print(\"a\")"
+                                              "time.sleep(0.1)"
+                                              "print(\"b\")"
+                                              "time.sleep(0.1)"
+                                              "print(\"c\")"]
+                      :expect-early-feedback true
+                      :expect-status         :succeeded
+                      :expected              ["a" "b" "c"]}
+                     {:desc "crash during writeback"
+                      :program ["print(42)" "print(\"is the answer\")"]
+                      :expect-status :failed
+                      :expected ["42" "is the answer" "Failed to create the resulting table"]
+                      :writeback-ex (Exception. "Boom!")}]]
+      (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+        (mt/with-premium-features #{:transforms :transforms-python}
+          (mt/dataset transforms-dataset/transforms-test
+            (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
+              (doseq [{:keys [expected
+                              expect-early-feedback
+                              expect-status
+                              desc] :as scenario}
+                      scenarios]
+                (testing desc
+                  (let [{:keys [last-run observed-messages]} (run-scenario scenario schema)
+                        {:keys [status message]} last-run]
+                    (testing "sanity: status is what we expect"
+                      (is (= expect-status (some-> status keyword))))
+                    (testing "sanity: message has a value"
+                      (is message))
+                    (when message
+                      (is (str/starts-with? message "Executing Python transform"))
+                      (testing "all observed values of message reflect a prefix of the final message (ordering consistent)"
+                        (doseq [observed-message observed-messages]
+                          ;; hmmm, sometimes the order seems flipped
+                          (is (str/starts-with? message observed-message))))
+                      (when expect-early-feedback
+                        (testing "scenario takes time, we should see partial messages for immediate feedback"
+                          (is (< 1 (count observed-messages)))))
+                      (testing "message includes the expected lines"
+                        (is (subsequence? (str/split-lines message) expected))))))))))))))
+
+(deftest get-python-transform-with-different-target-database-test
+  (testing "GET /api/ee/transform/:id correctly fetches target table from different database"
+    (mt/with-premium-features #{:transforms :transforms-python}
+      (mt/with-temp [:model/Database target-db {:engine :h2
+                                                :details {:db "mem:target-db"}}
+                     :model/Table target-table {:db_id (:id target-db)
+                                                :schema "PUBLIC"
+                                                :name "python_target_table"
+                                                :active true}
+                     :model/Transform transform {:name "Python Transform Cross DB"
+                                                 :source {:type "python"
+                                                          :source-database (mt/id)
+                                                          :source-tables {}
+                                                          :body "def transform():\n    pass"}
+                                                 :target {:type "table"
+                                                          :schema "PUBLIC"
+                                                          :name "python_target_table"
+                                                          :database (:id target-db)}}]
+        (let [response (mt/user-http-request :crowberto :get 200 (format "ee/transform/%s" (:id transform)))]
+          (is (=? {:id     (:id target-table)
+                   :name   "python_target_table"
+                   :schema "PUBLIC"
+                   :db_id  (:id target-db)}
+                  (:table response))))))))

enterprise/frontend/src/metabase-enterprise/api/python-transform-library.ts

@@ -0,0 +1,33 @@
+import type {
+  GetPythonLibraryRequest,
+  PythonLibrary,
+  UpdatePythonLibraryRequest,
+} from "metabase-types/api";
+
+import { EnterpriseApi } from "./api";
+import { idTag, invalidateTags, providePythonLibraryTags } from "./tags";
+
+export const pythonLibraryApi = EnterpriseApi.injectEndpoints({
+  endpoints: (builder) => ({
+    getPythonLibrary: builder.query<PythonLibrary, GetPythonLibraryRequest>({
+      query: ({ path }) => ({
+        url: `/api/ee/transforms-python/library/${path}`,
+        method: "GET",
+      }),
+      providesTags: (library) =>
+        library ? providePythonLibraryTags(library) : [],
+    }),
+    updatePythonLibrary: builder.mutation<void, UpdatePythonLibraryRequest>({
+      query: ({ path, source }) => ({
+        url: `/api/ee/transforms-python/library/${path}`,
+        method: "PUT",
+        body: { source },
+      }),
+      invalidatesTags: (_, error, { path }) =>
+        invalidateTags(error, [idTag("python-transform-library", path)]),
+    }),
+  }),
+});
+
+export const { useGetPythonLibraryQuery, useUpdatePythonLibraryMutation } =
+  pythonLibraryApi;

enterprise/frontend/src/metabase-enterprise/api/tags.ts

@@ -3,6 +3,7 @@ import type { TagDescription } from "@reduxjs/toolkit/query";
 import { TAG_TYPES, provideUserTags } from "metabase/api/tags";
 import type {
   Comment,
+  PythonLibrary,
   Transform,
   TransformJob,
   TransformRun,
@@ -23,6 +24,7 @@ export const ENTERPRISE_TAG_TYPES = [
   "transform-job",
   "transform-job-via-tag",
   "transform-run",
+  "python-transform-library",
 ] as const;
 
 export type EnterpriseTagType = (typeof ENTERPRISE_TAG_TYPES)[number];
@@ -111,6 +113,12 @@ export function provideTransformJobListTags(
   return [listTag("transform-job"), ...jobs.flatMap(provideTransformJobTags)];
 }
 
+export function providePythonLibraryTags(
+  library: PythonLibrary,
+): TagDescription<EnterpriseTagType>[] {
+  return [idTag("python-transform-library", library.path)];
+}
+
 export function provideCommentListTags(
   comments: Comment[],
 ): TagDescription<EnterpriseTagType>[] {

enterprise/frontend/src/metabase-enterprise/api/transform-python.ts

@@ -0,0 +1,34 @@
+import type {
+  ExecutePythonTransformRequest,
+  ExecutePythonTransformResponse,
+} from "metabase-types/api";
+
+import { EnterpriseApi } from "./api";
+
+export const pythonRunnerApi = EnterpriseApi.injectEndpoints({
+  endpoints: (builder) => ({
+    executePython: builder.mutation<
+      ExecutePythonTransformResponse,
+      ExecutePythonTransformRequest
+    >({
+      query: ({ code, tables }) => ({
+        url: "/api/ee/transform/test-python",
+        method: "POST",
+        body: {
+          code,
+          tables,
+        },
+      }),
+    }),
+    cancelPython: builder.mutation<void, void>({
+      query: () => ({
+        url: "/api/ee/transform/test-python/cancel",
+        method: "POST",
+      }),
+    }),
+  }),
+  overrideExisting: true,
+});
+
+export const { useExecutePythonMutation, useCancelPythonMutation } =
+  pythonRunnerApi;

enterprise/frontend/src/metabase-enterprise/plugins.js

@@ -42,3 +42,4 @@ import "./table-editing";
 import "./documents";
 import "./semantic_search";
 import "./transforms";
+import "./transforms-python";

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/PythonEditor.module.css

@@ -0,0 +1,4 @@
+.editor {
+  width: 100%;
+  height: 100%;
+}

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/PythonEditor.tsx

@@ -0,0 +1,33 @@
+import cx from "classnames";
+
+import { CodeEditor } from "metabase/common/components/CodeEditor";
+
+import S from "./PythonEditor.module.css";
+import { completion } from "./utils";
+
+export function PythonEditor({
+  value,
+  onChange,
+  withPandasCompletions,
+  className,
+  readOnly,
+  ...rest
+}: {
+  value: string;
+  onChange?: (value: string) => void;
+  withPandasCompletions?: boolean;
+  className?: string;
+  readOnly?: boolean;
+}) {
+  return (
+    <CodeEditor
+      className={cx(S.editor, className)}
+      value={value}
+      onChange={onChange}
+      language="python"
+      extensions={withPandasCompletions ? completion : undefined}
+      readOnly={readOnly}
+      {...rest}
+    />
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/index.ts

@@ -0,0 +1 @@
+export * from "./PythonEditor";

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/utils.ts

@@ -0,0 +1,676 @@
+import {
+  type Completion,
+  type CompletionContext,
+  autocompletion,
+} from "@codemirror/autocomplete";
+import { pythonLanguage } from "@codemirror/lang-python";
+import { t } from "ttag";
+
+const pandasDefinitions = (): Record<string, Completion[]> => ({
+  pd: [
+    {
+      label: "DataFrame",
+      type: "class",
+      info: t`Two-dimensional, size-mutable, potentially heterogeneous tabular data`,
+    },
+    {
+      label: "Series",
+      type: "class",
+      info: t`One-dimensional ndarray with axis labels`,
+    },
+    {
+      label: "read_csv",
+      type: "function",
+      info: t`Read a comma-separated values (csv) file into DataFrame`,
+    },
+    {
+      label: "read_excel",
+      type: "function",
+      info: t`Read an Excel file into a pandas DataFrame`,
+    },
+    {
+      label: "read_json",
+      type: "function",
+      info: t`Convert a JSON string to pandas object`,
+    },
+    {
+      label: "read_sql",
+      type: "function",
+      info: t`Read SQL query or database table into a DataFrame`,
+    },
+    {
+      label: "read_parquet",
+      type: "function",
+      info: t`Load a parquet object from file path`,
+    },
+    {
+      label: "concat",
+      type: "function",
+      info: t`Concatenate pandas objects along a particular axis`,
+    },
+    {
+      label: "merge",
+      type: "function",
+      info: t`Merge DataFrame or named Series objects with a database-style join`,
+    },
+    {
+      label: "pivot_table",
+      type: "function",
+      info: t`Create a spreadsheet-style pivot table as a DataFrame`,
+    },
+    {
+      label: "crosstab",
+      type: "function",
+      info: t`Compute a simple cross tabulation of two (or more) factors`,
+    },
+    {
+      label: "get_dummies",
+      type: "function",
+      info: t`Convert categorical variable(s) into dummy/indicator variables`,
+    },
+    {
+      label: "date_range",
+      type: "function",
+      info: t`Return a fixed frequency DatetimeIndex`,
+    },
+    {
+      label: "to_datetime",
+      type: "function",
+      info: t`Convert argument to datetime`,
+    },
+    {
+      label: "to_numeric",
+      type: "function",
+      info: t`Convert argument to a numeric type`,
+    },
+    {
+      label: "cut",
+      type: "function",
+      info: t`Bin values into discrete intervals`,
+    },
+    {
+      label: "qcut",
+      type: "function",
+      info: t`Quantile-based discretization function`,
+    },
+    {
+      label: "isna",
+      type: "function",
+      info: t`Detect missing values for an array-like object`,
+    },
+    {
+      label: "isnull",
+      type: "function",
+      info: t`Detect missing values for an array-like object`,
+    },
+    {
+      label: "notna",
+      type: "function",
+      info: t`Detect non-missing values for an array-like object`,
+    },
+    {
+      label: "notnull",
+      type: "function",
+      info: t`Detect non-missing values for an array-like object`,
+    },
+  ],
+
+  // DataFrame methods and properties
+  DataFrame: [
+    // Viewing/Inspection
+    { label: "head", type: "method", info: "Return the first n rows" },
+    { label: "tail", type: "method", info: "Return the last n rows" },
+    {
+      label: "info",
+      type: "method",
+      info: t`Print a concise summary of a DataFrame`,
+    },
+    {
+      label: "describe",
+      type: "method",
+      info: t`Generate descriptive statistics`,
+    },
+    {
+      label: "shape",
+      type: "property",
+      info: t`Return a tuple representing the dimensionality`,
+    },
+    {
+      label: "size",
+      type: "property",
+      info: t`Return an int representing the number of elements`,
+    },
+    {
+      label: "columns",
+      type: "property",
+      info: t`The column labels of the DataFrame`,
+    },
+    {
+      label: "index",
+      type: "property",
+      info: t`The index (row labels) of the DataFrame`,
+    },
+    {
+      label: "dtypes",
+      type: "property",
+      info: t`Return the dtypes in the DataFrame`,
+    },
+    {
+      label: "values",
+      type: "property",
+      info: t`Return a Numpy representation of the DataFrame`,
+    },
+
+    // Selection/Indexing
+    {
+      label: "loc",
+      type: "property",
+      info: t`Access a group of rows and columns by label(s)`,
+    },
+    {
+      label: "iloc",
+      type: "property",
+      info: t`Purely integer-location based indexing`,
+    },
+    {
+      label: "at",
+      type: "property",
+      info: t`Access a single value for a row/column label pair`,
+    },
+    {
+      label: "iat",
+      type: "property",
+      info: t`Access a single value for a row/column pair by integer position`,
+    },
+    {
+      label: "query",
+      type: "method",
+      info: t`Query the columns of a DataFrame with a boolean expression`,
+    },
+    {
+      label: "filter",
+      type: "method",
+      info: t`Subset the dataframe rows or columns according to labels`,
+    },
+    {
+      label: "where",
+      type: "method",
+      info: t`Replace values where the condition is False`,
+    },
+    {
+      label: "mask",
+      type: "method",
+      info: t`Replace values where the condition is True`,
+    },
+
+    // Data Cleaning
+    {
+      label: "drop",
+      type: "method",
+      info: t`Drop specified labels from rows or columns`,
+    },
+    {
+      label: "drop_duplicates",
+      type: "method",
+      info: t`Return DataFrame with duplicate rows removed`,
+    },
+    { label: "dropna", type: "method", info: "Remove missing values" },
+    {
+      label: "fillna",
+      type: "method",
+      info: t`Fill NA/NaN values using the specified method`,
+    },
+    {
+      label: "replace",
+      type: "method",
+      info: t`Replace values given in to_replace with value`,
+    },
+    {
+      label: "interpolate",
+      type: "method",
+      info: t`Fill NaN values using an interpolation method`,
+    },
+    {
+      label: "duplicated",
+      type: "method",
+      info: t`Return boolean Series denoting duplicate rows`,
+    },
+    { label: "isnull", type: "method", info: "Detect missing values" },
+    { label: "isna", type: "method", info: "Detect missing values" },
+    {
+      label: "notnull",
+      type: "method",
+      info: t`Detect existing (non-missing) values`,
+    },
+    {
+      label: "notna",
+      type: "method",
+      info: t`Detect existing (non-missing) values`,
+    },
+
+    // Transformations
+    {
+      label: "sort_values",
+      type: "method",
+      info: t`Sort by the values along either axis`,
+    },
+    {
+      label: "sort_index",
+      type: "method",
+      info: t`Sort object by labels (along an axis)`,
+    },
+    {
+      label: "reset_index",
+      type: "method",
+      info: t`Reset the index, or a level of it`,
+    },
+    {
+      label: "set_index",
+      type: "method",
+      info: t`Set the DataFrame index using existing columns`,
+    },
+    {
+      label: "reindex",
+      type: "method",
+      info: t`Conform Series/DataFrame to new index`,
+    },
+    { label: "rename", type: "method", info: "Alter axes labels" },
+    {
+      label: "assign",
+      type: "method",
+      info: t`Assign new columns to a DataFrame`,
+    },
+    {
+      label: "pipe",
+      type: "method",
+      info: t`Apply chainable functions that expect Series or DataFrames`,
+    },
+
+    // Aggregation/GroupBy
+    {
+      label: "groupby",
+      type: "method",
+      info: t`Group DataFrame using a mapper or by a Series of columns`,
+    },
+    {
+      label: "agg",
+      type: "method",
+      info: t`Aggregate using one or more operations`,
+    },
+    {
+      label: "aggregate",
+      type: "method",
+      info: t`Aggregate using one or more operations`,
+    },
+    {
+      label: "apply",
+      type: "method",
+      info: t`Apply a function along an axis of the DataFrame`,
+    },
+    {
+      label: "applymap",
+      type: "method",
+      info: t`Apply a function to a Dataframe elementwise`,
+    },
+    {
+      label: "transform",
+      type: "method",
+      info: t`Call func on self producing a DataFrame with transformed values`,
+    },
+    {
+      label: "rolling",
+      type: "method",
+      info: t`Provide rolling window calculations`,
+    },
+    {
+      label: "expanding",
+      type: "method",
+      info: t`Provide expanding transformations`,
+    },
+    { label: "resample", type: "method", info: "Resample time-series data" },
+
+    // Statistical Methods
+    {
+      label: "sum",
+      type: "method",
+      info: t`Return the sum of the values for the requested axis`,
+    },
+    {
+      label: "mean",
+      type: "method",
+      info: t`Return the mean of the values for the requested axis`,
+    },
+    {
+      label: "median",
+      type: "method",
+      info: t`Return the median of the values for the requested axis`,
+    },
+    {
+      label: "mode",
+      type: "method",
+      info: t`Get the mode(s) of each element along the selected axis`,
+    },
+    {
+      label: "std",
+      type: "method",
+      info: t`Return sample standard deviation over requested axis`,
+    },
+    {
+      label: "var",
+      type: "method",
+      info: t`Return unbiased variance over requested axis`,
+    },
+    {
+      label: "min",
+      type: "method",
+      info: t`Return the minimum of the values for the requested axis`,
+    },
+    {
+      label: "max",
+      type: "method",
+      info: t`Return the maximum of the values for the requested axis`,
+    },
+    {
+      label: "count",
+      type: "method",
+      info: t`Count non-NA cells for each column or row`,
+    },
+    {
+      label: "nunique",
+      type: "method",
+      info: t`Count number of distinct elements in specified axis`,
+    },
+    {
+      label: "value_counts",
+      type: "method",
+      info: t`Return a Series containing counts of unique values`,
+    },
+    {
+      label: "corr",
+      type: "method",
+      info: t`Compute pairwise correlation of columns`,
+    },
+    {
+      label: "cov",
+      type: "method",
+      info: t`Compute pairwise covariance of columns`,
+    },
+    {
+      label: "rank",
+      type: "method",
+      info: t`Compute numerical data ranks (1 through n) along axis`,
+    },
+    {
+      label: "quantile",
+      type: "method",
+      info: t`Return values at the given quantile over requested axis`,
+    },
+    {
+      label: "clip",
+      type: "method",
+      info: t`Trim values at input threshold(s)`,
+    },
+
+    // Merging/Joining
+    {
+      label: "merge",
+      type: "method",
+      info: t`Merge DataFrame or named Series objects`,
+    },
+    {
+      label: "join",
+      type: "method",
+      info: t`Join columns of another DataFrame`,
+    },
+    {
+      label: "concat",
+      type: "method",
+      info: t`Concatenate pandas objects along a particular axis`,
+    },
+    {
+      label: "append",
+      type: "method",
+      info: t`Append rows of other to the end of caller`,
+    },
+    {
+      label: "update",
+      type: "method",
+      info: t`Modify in place using non-NA values from another DataFrame`,
+    },
+
+    // Reshaping
+    {
+      label: "pivot",
+      type: "method",
+      info: t`Return reshaped DataFrame organized by given index / column values`,
+    },
+    {
+      label: "pivot_table",
+      type: "method",
+      info: t`Create a spreadsheet-style pivot table`,
+    },
+    {
+      label: "melt",
+      type: "method",
+      info: t`Unpivot a DataFrame from wide to long format`,
+    },
+    {
+      label: "stack",
+      type: "method",
+      info: t`Stack the prescribed level(s) from columns to index`,
+    },
+    {
+      label: "unstack",
+      type: "method",
+      info: t`Pivot a level of the (necessarily hierarchical) index labels`,
+    },
+    { label: "transpose", type: "method", info: "Transpose index and columns" },
+    { label: "T", type: "property", info: "Transpose index and columns" },
+
+    // I/O
+    {
+      label: "to_csv",
+      type: "method",
+      info: t`Write object to a comma-separated values (csv) file`,
+    },
+    {
+      label: "to_excel",
+      type: "method",
+      info: t`Write object to an Excel sheet`,
+    },
+    {
+      label: "to_json",
+      type: "method",
+      info: t`Convert the object to a JSON string`,
+    },
+    {
+      label: "to_sql",
+      type: "method",
+      info: t`Write records stored in a DataFrame to a SQL database`,
+    },
+    {
+      label: "to_parquet",
+      type: "method",
+      info: t`Write a DataFrame to the binary parquet format`,
+    },
+    {
+      label: "to_dict",
+      type: "method",
+      info: t`Convert the DataFrame to a dictionary`,
+    },
+    {
+      label: "to_html",
+      type: "method",
+      info: t`Render a DataFrame as an HTML table`,
+    },
+    {
+      label: "to_string",
+      type: "method",
+      info: t`Render a DataFrame to a console-friendly tabular output`,
+    },
+    {
+      label: "to_numpy",
+      type: "method",
+      info: t`Convert the DataFrame to a NumPy array`,
+    },
+
+    // Time Series
+    {
+      label: "shift",
+      type: "method",
+      info: t`Shift index by desired number of periods`,
+    },
+    { label: "tshift", type: "method", info: "Shift the time index" },
+    {
+      label: "diff",
+      type: "method",
+      info: t`First discrete difference of element`,
+    },
+    {
+      label: "pct_change",
+      type: "method",
+      info: t`Percentage change between the current and a prior element`,
+    },
+
+    // Plotting (if matplotlib available)
+    {
+      label: "plot",
+      type: "method",
+      info: t`Make plots of Series or DataFrame`,
+    },
+    {
+      label: "hist",
+      type: "method",
+      info: t`Make a histogram of the DataFrame's columns`,
+    },
+    {
+      label: "boxplot",
+      type: "method",
+      info: t`Make a box plot of the DataFrame columns`,
+    },
+  ],
+
+  // Series methods (subset of DataFrame methods that make sense for Series)
+  Series: [
+    { label: "head", type: "method", info: "Return the first n values" },
+    { label: "tail", type: "method", info: "Return the last n values" },
+    {
+      label: "describe",
+      type: "method",
+      info: t`Generate descriptive statistics`,
+    },
+    {
+      label: "value_counts",
+      type: "method",
+      info: t`Return a Series containing counts of unique values`,
+    },
+    {
+      label: "unique",
+      type: "method",
+      info: t`Return unique values of Series object`,
+    },
+    {
+      label: "nunique",
+      type: "method",
+      info: t`Return number of unique elements in the object`,
+    },
+    { label: "sort_values", type: "method", info: "Sort by the values" },
+    {
+      label: "sort_index",
+      type: "method",
+      info: t`Sort Series by index labels`,
+    },
+    {
+      label: "reset_index",
+      type: "method",
+      info: t`Generate a new DataFrame or Series with the index reset`,
+    },
+    {
+      label: "dropna",
+      type: "method",
+      info: t`Return a new Series with missing values removed`,
+    },
+    {
+      label: "fillna",
+      type: "method",
+      info: t`Fill NA/NaN values using the specified method`,
+    },
+    {
+      label: "apply",
+      type: "method",
+      info: t`Invoke function on values of Series`,
+    },
+    {
+      label: "map",
+      type: "method",
+      info: t`Map values of Series according to input correspondence`,
+    },
+    {
+      label: "replace",
+      type: "method",
+      info: t`Replace values given in to_replace with value`,
+    },
+    {
+      label: "str",
+      type: "property",
+      info: t`Vectorized string functions for Series and Index`,
+    },
+    {
+      label: "dt",
+      type: "property",
+      info: t`Accessor object for datetimelike properties`,
+    },
+    {
+      label: "cat",
+      type: "property",
+      info: t`Accessor object for categorical properties`,
+    },
+  ],
+});
+
+function genericPandasCompletions(context: CompletionContext) {
+  const definitions = pandasDefinitions();
+
+  const word = context.matchBefore(/[\w.]*/);
+  if (!word || (word.from === word.to && !context.explicit)) {
+    return null;
+  }
+
+  const text = word.text;
+
+  // Match patterns like "pd.", "df.", etc.
+  if (text.includes(".")) {
+    const parts = text.split(".");
+    const obj = parts.at(-2);
+    const partial = parts.at(-1) ?? "";
+
+    if (!obj || !definitions[obj]) {
+      return null;
+    }
+
+    const options = definitions[obj]
+      .filter(
+        (item) =>
+          !partial ||
+          item.label.toLowerCase().startsWith(partial.toLowerCase()),
+      )
+      .map((comp) => ({
+        ...comp,
+        info: undefined,
+        boost: comp.type === "method" ? 2 : comp.type === "function" ? 1 : 0,
+      }));
+
+    return {
+      filter: false,
+      from: word.from + text.length - partial.length,
+      options,
+    };
+  }
+}
+
+export const completion = [
+  pythonLanguage.data.of({
+    autocomplete: genericPandasCompletions,
+  }),
+  autocompletion(),
+];

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/AliasInput/AliasInput.tsx

@@ -0,0 +1,65 @@
+import { useEffect, useState } from "react";
+import { t } from "ttag";
+
+import { ActionIcon, Icon, TextInput } from "metabase/ui";
+import type { Table } from "metabase-types/api";
+
+import type { TableSelection } from "../types";
+import { slugify } from "../utils";
+
+export function AliasInput({
+  table,
+  selection,
+  onChange,
+  usedAliases,
+  disabled,
+}: {
+  selection: TableSelection;
+  table?: Table;
+  onChange: (value: string) => void;
+  usedAliases: Set<string>;
+  disabled?: boolean;
+}) {
+  const [value, setValue] = useState(selection.alias);
+
+  useEffect(() => {
+    setValue(selection.alias);
+  }, [selection.alias]);
+
+  const defaultAlias = table
+    ? slugify(table.name, usedAliases, selection.alias)
+    : "";
+  const isManualAlias = selection.alias !== defaultAlias;
+  const showReset = table && isManualAlias;
+
+  function handleChange(evt: React.ChangeEvent<HTMLInputElement>) {
+    setValue(evt.target.value);
+  }
+
+  function handleBlur() {
+    onChange(value);
+  }
+
+  return (
+    <TextInput
+      w="100%"
+      value={value}
+      onChange={handleChange}
+      onBlur={handleBlur}
+      placeholder={t`Enter alias`}
+      disabled={disabled}
+      rightSection={
+        showReset && (
+          <ActionIcon
+            onClick={() => onChange(defaultAlias)}
+            aria-label={t`Reset alias to default`}
+            color="gray"
+            variant="subtle"
+          >
+            <Icon name="refresh" size={12} />
+          </ActionIcon>
+        )
+      }
+    />
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/AliasInput/index.ts

@@ -0,0 +1 @@
+export * from "./AliasInput";

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.module.css

@@ -0,0 +1,11 @@
+.dataPicker {
+  border-right: 1px solid var(--mb-color-border);
+  overflow: auto;
+}
+
+.databaseSelector {
+  padding: var(--mantine-spacing-sm) var(--mantine-spacing-md);
+  border: 1px solid var(--mb-color-border);
+  border-radius: var(--mantine-radius-xs);
+  cursor: pointer;
+}

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.tsx

@@ -0,0 +1,299 @@
+import { useState } from "react";
+import { t } from "ttag";
+
+import { hasFeature } from "metabase/admin/databases/utils";
+import {
+  skipToken,
+  useListDatabasesQuery,
+  useListTablesQuery,
+} from "metabase/api";
+import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
+import { DatabaseDataSelector } from "metabase/query_builder/components/DataSelector";
+import { Box, Button, Icon, Stack, Text } from "metabase/ui";
+import type {
+  Database,
+  DatabaseId,
+  PythonTransformTableAliases,
+  Table,
+} from "metabase-types/api";
+
+import { AliasInput } from "./AliasInput";
+import S from "./PythonDataPicker.module.css";
+import { TableSelector } from "./TableSelector";
+import type { TableSelection } from "./types";
+import {
+  getInitialTableSelections,
+  selectionsToTableAliases,
+  slugify,
+} from "./utils";
+
+type PythonDataPickerProps = {
+  database?: DatabaseId;
+  tables: PythonTransformTableAliases;
+  onChange: (
+    database: number,
+    tables: PythonTransformTableAliases,
+    tableInfo: Table[],
+  ) => void;
+};
+
+export function PythonDataPicker({
+  database,
+  tables,
+  onChange,
+}: PythonDataPickerProps) {
+  const [tableSelections, setTableSelections] = useState<TableSelection[]>(
+    getInitialTableSelections(tables),
+  );
+
+  const {
+    data: databases,
+    isLoading: isLoadingDatabases,
+    error: databaseError,
+  } = useListDatabasesQuery();
+
+  const {
+    data: tablesData,
+    isLoading: isLoadingTables,
+    error: tablesError,
+  } = useListTablesQuery(
+    database
+      ? {
+          dbId: database,
+          include_hidden: false,
+          include_editable_data_model: true,
+        }
+      : skipToken,
+  );
+
+  const handleChange = (
+    database: DatabaseId | undefined,
+    selections: TableSelection[],
+  ) => {
+    if (database) {
+      const tableAliases = selectionsToTableAliases(selections);
+      onChange(database, tableAliases, tablesData ?? []);
+    }
+  };
+
+  if (databaseError || tablesError) {
+    return <LoadingAndErrorWrapper error={databaseError || tablesError} />;
+  }
+
+  if (isLoadingDatabases) {
+    return <LoadingAndErrorWrapper loading />;
+  }
+
+  const availableTables = (tablesData || []).filter(
+    (tbl: Table) => tbl.db_id === database && tbl.active,
+  );
+
+  const usedAliases = new Set(tableSelections.map((s) => s.alias));
+
+  const handleDatabaseChange = (value: string | null) => {
+    const newDatabase = value ? parseInt(value) : undefined;
+
+    if (newDatabase === database) {
+      return;
+    }
+
+    // Clear selections since they won't make sense
+    const newSelections = [
+      {
+        tableId: undefined,
+        alias: "",
+      },
+    ];
+    setTableSelections(newSelections);
+    handleChange(newDatabase, newSelections);
+  };
+
+  const handleSelectionChange = (
+    selectionIndex: number,
+    selection: TableSelection,
+  ) => {
+    const newSelections = Array.from(tableSelections);
+    newSelections[selectionIndex] = selection;
+    setTableSelections(newSelections);
+    handleChange(database, newSelections);
+  };
+
+  const handleAddTable = () => {
+    setTableSelections([
+      ...tableSelections,
+      {
+        tableId: undefined,
+        alias: "",
+      },
+    ]);
+  };
+
+  const handleRemoveTable = (selectionIndex: number) => {
+    const newSelections = Array.from(tableSelections);
+    newSelections.splice(selectionIndex, 1);
+    setTableSelections(newSelections);
+    handleChange(database, newSelections);
+  };
+
+  return (
+    <Stack
+      p="md"
+      gap="md"
+      h="100%"
+      w="30%"
+      miw="300px"
+      className={S.dataPicker}
+      data-testid="python-data-picker"
+    >
+      <Box>
+        <Text fw="bold">{t`Source database`}</Text>
+        <Text size="sm" c="text-light" mb="sm">
+          {t`Select the database that contains your source data.`}
+        </Text>
+
+        <DatabaseDataSelector
+          className={S.databaseSelector}
+          selectedDatabaseId={database}
+          setDatabaseFn={handleDatabaseChange}
+          databases={databases?.data ?? []}
+          databaseIsDisabled={(database: Database) =>
+            !hasFeature(database, "transforms/python")
+          }
+        />
+      </Box>
+      {database && (
+        <Box>
+          <Text fw="bold">{t`Pick tables and alias them`}</Text>
+          <Text size="sm" c="text-light" mb="sm">
+            {t`Select tables to use as data sources and provide aliases that can be referenced in your Python script.`}
+          </Text>
+          <Stack gap="md">
+            {tableSelections.map((selection, index) => (
+              <SelectionInput
+                key={index}
+                selection={selection}
+                database={database}
+                tables={tables}
+                usedAliases={usedAliases}
+                availableTables={availableTables}
+                onChange={(selection) =>
+                  handleSelectionChange(index, selection)
+                }
+                onRemove={() => handleRemoveTable(index)}
+                disabled={isLoadingTables}
+              />
+            ))}
+            <AddTableButton
+              onClick={handleAddTable}
+              disabled={availableTables.length === 0}
+            />
+          </Stack>
+        </Box>
+      )}
+    </Stack>
+  );
+}
+
+function SelectionInput({
+  database,
+  tables,
+  selection,
+  availableTables,
+  usedAliases,
+  onChange,
+  onRemove,
+  disabled,
+}: {
+  database: DatabaseId | undefined;
+  tables: PythonTransformTableAliases;
+  selection: TableSelection;
+  availableTables: Table[];
+  usedAliases: Set<string>;
+  onChange: (selection: TableSelection) => void;
+  onRemove: () => void;
+  disabled?: boolean;
+}) {
+  const table = availableTables.find((table) => table.id === selection.tableId);
+
+  function handleAliasChange(newAlias: string) {
+    const newSelection = {
+      ...selection,
+      alias: slugify(newAlias, usedAliases, selection.alias),
+    };
+
+    onChange(newSelection);
+  }
+
+  function handleTableChange(table: Table | undefined) {
+    if (!table) {
+      onChange({
+        ...selection,
+        tableId: undefined,
+      });
+      return;
+    }
+
+    const oldTable = availableTables.find(
+      (table) => table.id === selection?.tableId,
+    );
+
+    const wasOldAliasManuallySet =
+      selection.alias !== "" &&
+      (!oldTable ||
+        selection.alias !==
+          slugify(oldTable.name, usedAliases, selection.alias));
+
+    const newAlias = wasOldAliasManuallySet
+      ? selection.alias
+      : slugify(table.name, usedAliases);
+
+    onChange({
+      ...selection,
+      tableId: table.id,
+      alias: newAlias,
+    });
+  }
+
+  return (
+    <Stack gap="xs" align="center" w="100%">
+      <TableSelector
+        database={database}
+        table={table}
+        selectedTableIds={Object.values(tables)}
+        onChange={handleTableChange}
+        onRemove={onRemove}
+        availableTables={availableTables}
+        disabled={disabled}
+      />
+      <AliasInput
+        selection={selection}
+        table={table}
+        onChange={handleAliasChange}
+        usedAliases={usedAliases}
+        disabled={disabled}
+      />
+    </Stack>
+  );
+}
+
+function AddTableButton({
+  onClick,
+  disabled,
+}: {
+  disabled?: boolean;
+  onClick: () => void;
+}) {
+  return (
+    <Box>
+      <Button
+        leftSection={<Icon name="add_data" />}
+        variant="subtle"
+        px={0}
+        onClick={onClick}
+        disabled={disabled}
+      >
+        {t`Add a table`}
+      </Button>
+    </Box>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/TableSelector/TableSelector.module.css

@@ -0,0 +1,7 @@
+.tableSelector {
+  border: 1px solid var(--border-color);
+}
+
+.tableSelectorButtonInner {
+  justify-content: start;
+}

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/TableSelector/TableSelector.tsx

@@ -0,0 +1,123 @@
+import { useDisclosure } from "@mantine/hooks";
+import { t } from "ttag";
+
+import type { CollectionPickerItem } from "metabase/common/components/Pickers/CollectionPicker";
+import {
+  type DataPickerItem,
+  DataPickerModal,
+} from "metabase/common/components/Pickers/DataPicker";
+import {
+  ActionIcon,
+  Box,
+  Button,
+  Group,
+  Icon,
+  Stack,
+  Tooltip,
+} from "metabase/ui";
+import type {
+  DatabaseId,
+  RecentItem,
+  Table,
+  TableId,
+} from "metabase-types/api";
+
+import S from "./TableSelector.module.css";
+
+export function TableSelector({
+  database,
+  availableTables,
+  selectedTableIds,
+  disabled,
+  onChange,
+  onRemove,
+  table,
+}: {
+  database: DatabaseId | undefined;
+  table: Table | undefined;
+  availableTables: Table[];
+  selectedTableIds: TableId[];
+  disabled?: boolean;
+  onChange: (table: Table | undefined) => void;
+  onRemove: () => void;
+}) {
+  const [isOpened, { open, close }] = useDisclosure();
+
+  function handleChange(tableId: TableId | undefined) {
+    const table = availableTables.find((table) => table.id === tableId);
+    if (table) {
+      onChange(table);
+    }
+  }
+
+  function shouldDisableItem(
+    item: DataPickerItem | CollectionPickerItem | RecentItem,
+  ) {
+    if (item.model === "table") {
+      // Filter available tables to exclude already selected ones (except current selection)
+      return selectedTableIds.includes(item.id);
+    }
+    if (item.model === "database") {
+      return item.id !== database;
+    }
+    return true;
+  }
+
+  return (
+    <>
+      <Group w="100%" bdrs="xs" gap="xs" className={S.tableSelector}>
+        <Button
+          flex="1 1 auto"
+          onClick={open}
+          disabled={disabled}
+          classNames={{ inner: S.tableSelectorButtonInner }}
+          px="sm"
+          py="lg"
+          variant="subtle"
+        >
+          <Stack gap={0} align="start" justify="center">
+            {table ? (
+              <>
+                <Box fz="sm" c="text-medium" fw="normal">
+                  {table?.db?.name} / {table?.schema}
+                </Box>
+                <Box c="text-dark">{table?.display_name}</Box>
+              </>
+            ) : (
+              <Box c="text-dark">{t`Select a table…`}</Box>
+            )}
+          </Stack>
+        </Button>
+
+        <Tooltip label={t`Remove this table`}>
+          <ActionIcon onClick={onRemove} pr="sm">
+            <Icon name="close" c="text-dark" />
+          </ActionIcon>
+        </Tooltip>
+      </Group>
+      {isOpened && (
+        <DataPickerModal
+          title={t`Pick a table`}
+          value={getDataPickerValue(table)}
+          databaseId={database}
+          onChange={handleChange}
+          onClose={close}
+          shouldDisableItem={shouldDisableItem}
+        />
+      )}
+    </>
+  );
+}
+
+function getDataPickerValue(table: Table | undefined) {
+  if (!table) {
+    return undefined;
+  }
+  return {
+    model: "table" as const,
+    id: table.id,
+    name: table.display_name,
+    db_id: table.db_id,
+    schema: table.schema,
+  };
+}

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/TableSelector/index.ts

@@ -0,0 +1 @@
+export * from "./TableSelector";

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/index.ts

@@ -0,0 +1 @@
+export * from "./PythonDataPicker";

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/types.ts

@@ -0,0 +1,6 @@
+import type { TableId } from "metabase-types/api";
+
+export type TableSelection = {
+  tableId: TableId | undefined;
+  alias: string;
+};

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/utils.ts

@@ -0,0 +1,61 @@
+import { slugify as toSlug } from "metabase/lib/formatting";
+import type { PythonTransformTableAliases } from "metabase-types/api";
+
+import type { TableSelection } from "./types";
+
+export function getInitialTableSelections(
+  tables: PythonTransformTableAliases | undefined,
+) {
+  if (tables && Object.keys(tables).length > 0) {
+    return Object.entries(tables).map(([alias, tableId]) => ({
+      tableId,
+      alias,
+    }));
+  }
+  // Start with one empty row by default
+  return [
+    {
+      tableId: undefined,
+      alias: "",
+    },
+  ];
+}
+
+export function selectionsToTableAliases(
+  selections: TableSelection[],
+): PythonTransformTableAliases {
+  const tableAliases: PythonTransformTableAliases = {};
+
+  for (const selection of selections) {
+    const { alias, tableId } = selection;
+    if (tableId !== undefined && alias !== "") {
+      tableAliases[alias] = tableId;
+    }
+  }
+  return tableAliases;
+}
+
+export function slugify(
+  name: string,
+  usedNames: Set<string> = new Set(),
+  allowThisAliasIfItExists?: string,
+) {
+  if (name === "") {
+    return "";
+  }
+
+  const plain = toSlug(name).replace(/\./, "_");
+  if (plain === allowThisAliasIfItExists) {
+    return plain;
+  }
+  if (!usedNames.has(plain)) {
+    return plain;
+  }
+  for (let index = 1; index <= 100; index++) {
+    const alias = `${plain}_${index}`;
+    if (!usedNames.has(alias)) {
+      return alias;
+    }
+  }
+  return name;
+}

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/PythonEditorBody.module.css

@@ -0,0 +1,10 @@
+.root {
+  border-bottom: 1px solid var(--mb-color-border);
+  position: relative;
+}
+
+.libraryActions {
+  position: absolute;
+  top: 0;
+  right: 0;
+}

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/PythonEditorBody.tsx

@@ -0,0 +1,146 @@
+import type { ReactNode } from "react";
+import { ResizableBox } from "react-resizable";
+import { t } from "ttag";
+
+import Link from "metabase/common/components/Link";
+import RunButtonWithTooltip from "metabase/query_builder/components/RunButtonWithTooltip";
+import { Box, Checkbox, Flex, Icon, Stack } from "metabase/ui";
+import { SHARED_LIB_IMPORT_PATH } from "metabase-enterprise/transforms-python/constants";
+import { getPythonLibraryUrl } from "metabase-enterprise/transforms-python/urls";
+
+import { PythonEditor } from "../../PythonEditor";
+
+import S from "./PythonEditorBody.module.css";
+import { ResizableBoxHandle } from "./ResizableBoxHandle";
+import { hasImport, insertImport, removeImport } from "./utils";
+
+type PythonEditorBodyProps = {
+  source: string;
+  isRunnable: boolean;
+  onChange: (source: string) => void;
+  onRun?: () => void;
+  onCancel?: () => void;
+  isRunning?: boolean;
+  isDirty?: boolean;
+  tables?: Record<string, number>;
+  withDebugger?: boolean;
+};
+
+const EDITOR_HEIGHT = 400;
+
+export function PythonEditorBody({
+  source,
+  onChange,
+  isRunnable,
+  onRun,
+  onCancel,
+  isRunning,
+  isDirty,
+  withDebugger,
+}: PythonEditorBodyProps) {
+  return (
+    <MaybeResizableBox resizable={withDebugger}>
+      <Flex h="100%" align="end" bg="bg-light" pos="relative">
+        <PythonEditor
+          value={source}
+          onChange={onChange}
+          withPandasCompletions
+          data-testid="python-editor"
+        />
+
+        {withDebugger && (
+          <Box p="md">
+            <RunButtonWithTooltip
+              disabled={!isRunnable}
+              isRunning={isRunning}
+              isDirty={isDirty}
+              onRun={onRun}
+              onCancel={onCancel}
+              getTooltip={() => t`Run Python script`}
+            />
+          </Box>
+        )}
+        <SharedLibraryActions source={source} onChange={onChange} />
+      </Flex>
+    </MaybeResizableBox>
+  );
+}
+
+function MaybeResizableBox({
+  resizable,
+  children,
+}: {
+  resizable?: boolean;
+  children?: ReactNode;
+}) {
+  if (!resizable) {
+    return <Box h="100%">{children}</Box>;
+  }
+
+  return (
+    <ResizableBox
+      axis="y"
+      height={EDITOR_HEIGHT}
+      handle={<ResizableBoxHandle />}
+      resizeHandles={["s"]}
+    >
+      {children}
+    </ResizableBox>
+  );
+}
+
+function SharedLibraryActions({
+  source,
+  onChange,
+}: {
+  source: string;
+  onChange: (source: string) => void;
+}) {
+  return (
+    <Stack className={S.libraryActions} p="md" gap="sm">
+      <SharedLibraryImportToggle source={source} onChange={onChange} />
+      <SharedLibraryEditLink />
+    </Stack>
+  );
+}
+
+function SharedLibraryImportToggle({
+  source,
+  onChange,
+}: {
+  source: string;
+  onChange: (source: string) => void;
+}) {
+  const hasSharedLib = hasImport(source, SHARED_LIB_IMPORT_PATH);
+
+  function handleToggleSharedLib() {
+    if (hasImport(source, SHARED_LIB_IMPORT_PATH)) {
+      onChange(removeImport(source, SHARED_LIB_IMPORT_PATH));
+    } else {
+      onChange(insertImport(source, SHARED_LIB_IMPORT_PATH));
+    }
+  }
+
+  return (
+    <Checkbox
+      label={t`Import common library`}
+      checked={hasSharedLib}
+      onChange={handleToggleSharedLib}
+      size="sm"
+    />
+  );
+}
+
+function SharedLibraryEditLink() {
+  return (
+    <Flex
+      component={Link}
+      target="_blank"
+      to={getPythonLibraryUrl({ path: SHARED_LIB_IMPORT_PATH })}
+      gap="sm"
+    >
+      <Icon name="pencil" />
+      {t`Edit common library`}
+    </Flex>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/ResizableBoxHandle/ResizableBoxHandle.module.css

@@ -0,0 +1,3 @@
+.root {
+  cursor: row-resize;
+}

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/ResizableBoxHandle/ResizableBoxHandle.tsx

@@ -0,0 +1,27 @@
+import { type HTMLAttributes, type Ref, forwardRef } from "react";
+import _ from "underscore";
+
+import { Box, Flex, rem } from "metabase/ui";
+
+import S from "./ResizableBoxHandle.module.css";
+
+export const ResizableBoxHandle = forwardRef(function ResizableBoxHandle(
+  props: HTMLAttributes<HTMLDivElement>,
+  ref: Ref<HTMLDivElement>,
+) {
+  return (
+    <Flex
+      ref={ref}
+      className={S.root}
+      align="center"
+      justify="center"
+      w="100%"
+      h="sm"
+      pos="absolute"
+      bottom={rem(-4)}
+      {..._.omit(props, "handleAxis")}
+    >
+      <Box w="6.25rem" h="xs" bg="border" />
+    </Flex>
+  );
+});

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/ResizableBoxHandle/index.ts

@@ -0,0 +1 @@
+export * from "./ResizableBoxHandle";

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/index.ts

@@ -0,0 +1 @@
+export * from "./PythonEditorBody";

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorBody/utils.ts

@@ -0,0 +1,33 @@
+export function insertImport(source: string, path: string) {
+  const lines = source.split("\n");
+
+  // Find the first line that is not a comment
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    if (line.trim().startsWith("#")) {
+      continue;
+    }
+    lines.splice(i, 0, `import ${pathToImportSpecifier(path)}`);
+    break;
+  }
+
+  return lines.join("\n");
+}
+
+function pathToImportSpecifier(path: string) {
+  return path.replace(/\.py$/, "").split("/").join(".");
+}
+
+function libImportRegex(path: string) {
+  return new RegExp(`import ${pathToImportSpecifier(path)}`, "g");
+}
+
+export function removeImport(source: string, path: string) {
+  const lines = source.split("\n");
+  return lines.filter((line) => !libImportRegex(path).test(line)).join("\n");
+}
+
+export function hasImport(source: string, path: string) {
+  const regex = libImportRegex(path);
+  return regex.test(source);
+}

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/ExecutionOutputTable.tsx

@@ -0,0 +1,25 @@
+import { useMemo } from "react";
+
+import { DataGrid, useDataGridInstance } from "metabase/data-grid";
+
+import { type Row, parseOutput } from "./utils";
+
+export function ExecutionOutputTable({ output }: { output?: string }) {
+  const { headers, rows } = useMemo(() => parseOutput(output ?? ""), [output]);
+
+  const tableProps = useDataGridInstance<Row, unknown>({
+    data: rows,
+    columnsOptions: headers.map((header) => ({
+      id: header,
+      name: header,
+      accessorFn: (row) => row[header],
+    })),
+    minGridWidth: 100,
+  });
+
+  if (!output || headers.length === 0) {
+    return null;
+  }
+
+  return <DataGrid {...tableProps} />;
+}

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/PythonEditorResults.module.css

@@ -0,0 +1,57 @@
+.logs {
+  white-space: pre;
+  border: 1px solid var(--mb-color-border);
+  font-family: var(--mb-default-monospace-font-family);
+}
+
+.visualization {
+  flex: 1 0 auto;
+}
+
+.results {
+  width: 100%;
+  border-collapse: collapse;
+  border: 1px solid var(--mb-color-border);
+  border-radius: var(--mantine-radius-md);
+}
+
+.empty {
+  border-top: 1px solid var(--mb-color-border);
+}
+
+.tableHeader {
+  background-color: var(--mb-color-bg-light);
+  padding: var(--mantine-spacing-sm);
+}
+
+.tableCell {
+  border-top: 1px solid var(--mb-color-border);
+  padding: var(--mantine-spacing-sm);
+}
+
+.error {
+  color: var(--mb-color-error);
+  background-color: var(--mb-color-bg-error);
+  border-bottom: 1px solid var(--mb-color-border);
+}
+
+.info {
+  color: var(--mb-color-text-dark);
+  border: 1px solid var(--mb-color-bg-medium);
+  background-color: var(--mb-color-bg-light);
+}
+
+.success {
+  color: var(--mb-color-success);
+  background-color: var(--mb-color-bg-success);
+  border-bottom: 1px solid var(--mb-color-border);
+}
+
+.loading {
+  position: absolute;
+  top: 0;
+  left: 0;
+  bottom: 0;
+  right: 0;
+  background-color: rgba(255, 255, 255, 0.5);
+}

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/PythonEditorResults.tsx

@@ -0,0 +1,175 @@
+import type { ReactNode } from "react";
+import { c, t } from "ttag";
+
+import EmptyCodeResult from "assets/img/empty-states/code.svg";
+import DebouncedFrame from "metabase/common/components/DebouncedFrame";
+import { LoadingSpinner } from "metabase/common/components/MetadataInfo/MetadataInfo.styled";
+import { isMac } from "metabase/lib/browser";
+import { Box, Flex, Icon, Stack, Text, Title } from "metabase/ui";
+
+import type { ExecutionResult } from "../utils";
+
+import { ExecutionOutputTable } from "./ExecutionOutputTable";
+import S from "./PythonEditorResults.module.css";
+
+type PythonEditorProps = {
+  isRunning?: boolean;
+  executionResult?: ExecutionResult | null;
+};
+
+export function PythonEditorResults({
+  executionResult,
+  isRunning,
+}: PythonEditorProps) {
+  return (
+    <DebouncedFrame className={S.visualization}>
+      <ExecutionResultHeader executionResult={executionResult} />
+      <ExecutionResults executionResult={executionResult} />
+      {isRunning && <LoadingState />}
+    </DebouncedFrame>
+  );
+}
+
+function ExecutionResultHeader({
+  executionResult,
+}: {
+  executionResult?: ExecutionResult | null;
+}) {
+  const message = getMessageForExecutionResult(executionResult);
+  return <Flex w="100%">{message}</Flex>;
+}
+
+function getRunQueryShortcut() {
+  return isMac() ? t`⌘ + return` : t`Ctrl + enter`;
+}
+
+function LoadingState() {
+  return (
+    <Flex p="md" className={S.loading}>
+      <LoadingSpinner />
+    </Flex>
+  );
+}
+
+function EmptyState() {
+  const keyboardShortcut = getRunQueryShortcut();
+
+  return (
+    <Flex h="100%" align="center" justify="center" className={S.empty}>
+      <Stack maw="25rem" gap={0} ta="center" align="center">
+        <Box maw="3rem" mb="0.75rem">
+          <img src={EmptyCodeResult} alt="Code prompt icon" />
+        </Box>
+        <Text c="text-medium">
+          {c("{0} refers to the keyboard shortcut")
+            .jt`To run your code, click on the Run button or type ${(
+            <b key="shortcut">({keyboardShortcut})</b>
+          )}`}
+        </Text>
+      </Stack>
+    </Flex>
+  );
+}
+
+function ExecutionResults({
+  executionResult,
+}: {
+  executionResult?: ExecutionResult | null;
+}) {
+  if (!executionResult) {
+    return <EmptyState />;
+  }
+
+  return (
+    <Stack gap={0}>
+      <ExecutionLogs
+        label={t`Standard output:`}
+        content={executionResult.stdout}
+      />
+      <ExecutionLogs
+        label={t`Standard error:`}
+        content={executionResult.stderr}
+      />
+
+      <ExecutionOutputTable output={executionResult.output} />
+    </Stack>
+  );
+}
+
+function getMessageForExecutionResult(
+  executionResult?: ExecutionResult | null,
+) {
+  if (!executionResult) {
+    return null;
+  }
+  if (executionResult.error) {
+    return (
+      <Box className={S.error} p="sm" w="100%">
+        <Flex gap="sm">
+          <Icon name="warning" />
+          <Stack gap={0}>
+            <Title order={5} mb="xs">
+              {t`An error occurred while executing your Python script`}
+            </Title>
+            {executionResult.error}
+          </Stack>
+        </Flex>
+      </Box>
+    );
+  }
+
+  if (!executionResult.output) {
+    return (
+      <Flex className={S.info} p="sm" gap="sm" align="center" w="100%">
+        <Icon name="info" />
+        {t`No results to display.`}
+      </Flex>
+    );
+  }
+
+  return (
+    <Flex className={S.success} p="sm" gap="sm" align="center" w="100%">
+      <Icon name="check" />
+      {t`Script executed successfully.`}
+    </Flex>
+  );
+}
+
+function ExecutionLogs({
+  label,
+  content,
+}: {
+  label: string;
+  content?: string | null;
+}) {
+  if (!content) {
+    return null;
+  }
+
+  return (
+    <Section title={label}>
+      <Box p="sm" bg="bg-light" mah="150px" bdrs="xs" className={S.logs}>
+        {content}
+      </Box>
+    </Section>
+  );
+}
+
+function Section({
+  title,
+  children,
+}: {
+  title: ReactNode;
+  children: ReactNode;
+}) {
+  return (
+    <Box>
+      {title && (
+        <Title order={5} mb="xs">
+          {title}
+        </Title>
+      )}
+      {children}
+    </Box>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/index.ts

@@ -0,0 +1 @@
+export * from "./PythonEditorResults";

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonEditorResults/utils.ts

@@ -0,0 +1,29 @@
+import type { RowValue } from "metabase-types/api";
+
+export type Row = Record<string, RowValue>;
+
+export function parseOutput(output: string): {
+  headers: string[];
+  rows: Row[];
+} {
+  if (!output?.trim()) {
+    return { headers: [], rows: [] };
+  }
+
+  const lines = output.trim().split("\n");
+  const headers = new Set<string>();
+  const rows: Row[] = [];
+
+  for (const line of lines) {
+    const data = JSON.parse(line) as Row;
+    for (const key in data) {
+      headers.add(key);
+    }
+    rows.push(data);
+  }
+
+  return {
+    headers: Array.from(headers),
+    rows,
+  };
+}

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonTransformEditor.tsx

@@ -0,0 +1,153 @@
+import { useHotkeys } from "@mantine/hooks";
+import { useState } from "react";
+
+import { Flex, Stack } from "metabase/ui";
+import { EditorHeader } from "metabase-enterprise/transforms/components/QueryEditor/EditorHeader";
+import type {
+  DatabaseId,
+  PythonTransformSource,
+  PythonTransformTableAliases,
+  Table,
+} from "metabase-types/api";
+
+import { PythonDataPicker } from "./PythonDataPicker";
+import { PythonEditorBody } from "./PythonEditorBody";
+import { PythonEditorResults } from "./PythonEditorResults";
+import {
+  isPythonTransformSource,
+  updateTransformSignature,
+  useTestPythonTransform,
+} from "./utils";
+
+export type PythonTransformSourceDraft = {
+  type: "python";
+  body: string;
+  "source-database": DatabaseId | undefined;
+  "source-tables": PythonTransformTableAliases;
+};
+
+export type PythonTransformEditorProps = {
+  initialSource: PythonTransformSourceDraft;
+  isNew?: boolean;
+  isSaving?: boolean;
+  isRunnable?: boolean;
+  onSave: (newSource: PythonTransformSource) => void;
+  onCancel: () => void;
+};
+
+export function PythonTransformEditor({
+  initialSource,
+  isNew = true,
+  isSaving = false,
+  isRunnable = true,
+  onSave,
+  onCancel,
+}: PythonTransformEditorProps) {
+  const [source, setSource] = useState(initialSource);
+  const [isSourceDirty, setIsSourceDirty] = useState(false);
+
+  const { isRunning, isDirty, cancel, run, executionResult } =
+    useTestPythonTransform(source);
+
+  const handleScriptChange = (body: string) => {
+    const newSource = {
+      ...source,
+      body,
+    };
+    setSource(newSource);
+    setIsSourceDirty(true);
+  };
+
+  const handleDataChange = (
+    database: number,
+    sourceTables: PythonTransformTableAliases,
+    tableInfo: Table[],
+  ) => {
+    const updatedScript = updateTransformSignature(
+      source.body,
+      sourceTables,
+      tableInfo,
+    );
+
+    const newSource = {
+      ...source,
+      body: updatedScript,
+      "source-database": database,
+      "source-tables": sourceTables,
+    };
+    setSource(newSource);
+    setIsSourceDirty(true);
+  };
+
+  const handleSave = () => {
+    if (isPythonTransformSource(source)) {
+      onSave(source);
+    }
+  };
+
+  const showDebugger =
+    new URLSearchParams(window.location.search).get("debugger") === "1";
+
+  const handleCmdEnter = () => {
+    if (!showDebugger) {
+      return;
+    }
+    if (isRunning) {
+      cancel();
+    } else if (isRunnable && isPythonTransformSource(source)) {
+      run();
+    }
+  };
+
+  useHotkeys([["mod+Enter", handleCmdEnter]], []);
+
+  const canSave = Boolean(
+    source.body.trim() &&
+      source["source-database"] &&
+      source["source-tables"] &&
+      Object.keys(source["source-tables"]).length > 0,
+  );
+
+  return (
+    <Stack
+      w="100%"
+      h="100%"
+      bg="bg-white"
+      data-testid="transform-query-editor"
+      gap={0}
+    >
+      <EditorHeader
+        isNew={isNew}
+        isSaving={isSaving}
+        canSave={canSave && (isNew || isSourceDirty)}
+        onSave={handleSave}
+        onCancel={onCancel}
+      />
+      <Flex h="100%" w="100%">
+        <PythonDataPicker
+          database={source["source-database"]}
+          tables={source["source-tables"]}
+          onChange={handleDataChange}
+        />
+        <Stack w="100%" h="100%" gap={0}>
+          <PythonEditorBody
+            isRunnable={isRunnable && isPythonTransformSource(source)}
+            isRunning={isRunning}
+            isDirty={isDirty}
+            onRun={run}
+            onCancel={cancel}
+            source={source.body}
+            onChange={handleScriptChange}
+            withDebugger={showDebugger}
+          />
+          {showDebugger && (
+            <PythonEditorResults
+              isRunning={isRunning}
+              executionResult={executionResult}
+            />
+          )}
+        </Stack>
+      </Flex>
+    </Stack>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/index.ts

@@ -0,0 +1 @@
+export * from "./PythonTransformEditor";

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/utils.ts

@@ -0,0 +1,234 @@
+import { useRef, useState } from "react";
+import { t } from "ttag";
+
+import { useExecutePythonMutation } from "metabase-enterprise/api/transform-python";
+import type {
+  ExecutePythonTransformResponse,
+  PythonTransformSource,
+  PythonTransformTableAliases,
+  Table,
+  TableId,
+} from "metabase-types/api";
+
+import type { PythonTransformSourceDraft } from "./PythonTransformEditor";
+
+export function updateTransformSignature(
+  script: string,
+  tables: PythonTransformTableAliases,
+  tableInfo: Table[],
+): string {
+  const tableAliases = Object.keys(tables);
+
+  const transformRegex = /^def\s+transform\s*\([^)]*\)\s*:\s*\n(\s*)/m;
+
+  const signatureOneLine = `def transform(${tableAliases.join(", ")}):`;
+  const newSignature =
+    signatureOneLine.length > 80 && tableAliases.length > 0
+      ? `def transform(\n${tableAliases.map((alias) => `    ${alias}`).join(",\n")},\n):`
+      : signatureOneLine;
+
+  function getTableName(tableId: TableId) {
+    const table = tableInfo.find((t) => t.id === tableId);
+    if (!table) {
+      return undefined;
+    }
+    return [table.db?.name, table.schema, table.name].filter(Boolean).join(".");
+  }
+
+  if (transformRegex.test(script)) {
+    // Capture the existing indentation after the colon
+    const match = script.match(transformRegex);
+    const originalIndent = match ? match[1] : "    ";
+
+    let updatedScript = script.replace(
+      transformRegex,
+      newSignature + "\n" + originalIndent,
+    );
+
+    const argsRegex =
+      /(\s+"""[\s\S]*?)\s*Args:\s*\n([\s\S]*?)\s*(\n\s+Returns:|\n\s+""")/;
+    const argsMatch = updatedScript.match(argsRegex);
+
+    if (tableAliases.length > 0) {
+      const maxAliasLength = Math.max(...tableAliases.map((a) => a.length));
+      const newArgsSection = tableAliases
+        .map((alias) => {
+          const padding = " ".repeat(maxAliasLength - alias.length);
+          const tableId = tables[alias];
+          const tableName = getTableName(tableId);
+          return `        ${alias}:${padding} DataFrame containing the data from the "${tableName}" table`;
+        })
+        .join("\n");
+
+      if (argsMatch) {
+        // Replace existing Args section
+        updatedScript = updatedScript.replace(
+          argsRegex,
+          `    $1
+
+    Args:
+${newArgsSection}
+$3`,
+        );
+      } else {
+        // No Args section exists, look for the pattern before Returns and insert Args
+        // Also remove the instructional text about selecting tables
+        const insertRegex =
+          /(\s+"""[\s\S]*?)\s*Select tables above to add them as function parameters\.\s*(\n\s+Returns:|\n\s+""")/;
+        const insertMatch = updatedScript.match(insertRegex);
+        if (insertMatch) {
+          updatedScript = updatedScript.replace(
+            insertRegex,
+            `$1
+
+    Args:
+${newArgsSection}
+$2`,
+          );
+        } else {
+          // Fallback: just look for Returns section without the specific text
+          // Remove any extra whitespace before Returns and ensure single blank line
+          const fallbackRegex = /(\s+"""[\s\S]*?)\s*(\n\s+Returns:|\n\s+""")/;
+          const fallbackMatch = updatedScript.match(fallbackRegex);
+          if (fallbackMatch) {
+            updatedScript = updatedScript.replace(
+              fallbackRegex,
+              `$1
+
+    Args:
+${newArgsSection}
+$2`,
+            );
+          }
+        }
+      }
+    } else if (argsMatch) {
+      // Remove existing Args section when no tables
+      updatedScript = updatedScript.replace(argsRegex, "$1$3");
+    }
+
+    return updatedScript;
+  }
+
+  const maxAliasLength =
+    tableAliases.length > 0
+      ? Math.max(...tableAliases.map((a) => a.length))
+      : 0;
+  const functionTemplate = `
+${newSignature}
+    """
+    Transform function that processes the input data.
+    ${
+      tableAliases.length > 0
+        ? `
+
+    Args:
+${tableAliases
+  .map((alias) => {
+    const padding = " ".repeat(maxAliasLength - alias.length);
+    const tableId = tables[alias];
+    const tableName = getTableName(tableId);
+    if (alias === tableName) {
+      return `        ${alias}:${padding} DataFrame containing the data from the corresponding table`;
+    }
+    return `        ${alias}:${padding} DataFrame containing the data from table "${tableName}"`;
+  })
+  .join("\n")}`
+        : ""
+    }
+
+    Returns:
+        DataFrame: The transformed data
+    """
+    import pandas as pd
+    return pd.DataFrame()
+`;
+
+  return script + functionTemplate;
+}
+
+export type ExecutionResult = {
+  output?: string;
+  stdout?: string;
+  stderr?: string;
+  error?: string;
+};
+
+type TestPythonScriptState = {
+  isRunning: boolean;
+  isDirty: boolean;
+  executionResult: ExecutionResult | null;
+  run: () => void;
+  cancel: () => void;
+};
+
+export function useTestPythonTransform(
+  source: PythonTransformSourceDraft,
+): TestPythonScriptState {
+  const [executePython, { isLoading: isRunning, originalArgs }] =
+    useExecutePythonMutation();
+  const abort = useRef<(() => void) | null>(null);
+  const [executionResult, setData] =
+    useState<ExecutePythonTransformResponse | null>(null);
+
+  const isDirty = originalArgs?.code !== source.body;
+
+  const run = async () => {
+    if (source["source-database"] === undefined) {
+      return null;
+    }
+    const request = executePython({
+      code: source.body,
+      tables: source["source-tables"],
+    });
+    abort.current = () => request.abort();
+
+    try {
+      const data = await request.unwrap();
+      setData(data);
+    } catch (error) {
+      if (typeof error === "object" && error !== null) {
+        if ("name" in error && error.name === "AbortError") {
+          setData({ error: t`Python script execution was canceled` });
+          return;
+        }
+
+        if ("message" in error && typeof error.message === "string") {
+          setData({ error: error.message });
+          return;
+        }
+
+        if (
+          "data" in error &&
+          typeof error.data === "object" &&
+          error.data !== null &&
+          "error" in error.data &&
+          typeof error.data.error === "string"
+        ) {
+          setData({ error: error?.data?.error });
+          return;
+        }
+      }
+
+      setData({ error: t`An unknown error occurred` });
+    }
+  };
+
+  const cancel = () => {
+    abort.current?.();
+  };
+
+  return {
+    isRunning,
+    isDirty,
+    cancel,
+    run,
+    executionResult,
+  };
+}
+
+export function isPythonTransformSource(
+  source: PythonTransformSourceDraft,
+): source is PythonTransformSource {
+  return source.type === "python" && source["source-database"] !== undefined;
+}

enterprise/frontend/src/metabase-enterprise/transforms-python/constants.ts

@@ -0,0 +1 @@
+export const SHARED_LIB_IMPORT_PATH = "common.py";

enterprise/frontend/src/metabase-enterprise/transforms-python/index.tsx

@@ -0,0 +1,53 @@
+import { t } from "ttag";
+
+import { AdminNavItem } from "metabase/admin/components/AdminNav";
+import { ForwardRefLink } from "metabase/common/components/Link";
+import { Route } from "metabase/hoc/Title";
+import { PLUGIN_TRANSFORMS_PYTHON } from "metabase/plugins";
+import { Icon, Menu } from "metabase/ui";
+import { hasPremiumFeature } from "metabase-enterprise/settings";
+import { trackTransformCreate } from "metabase-enterprise/transforms/analytics";
+import { FullWidthTransformPageLayout } from "metabase-enterprise/transforms/pages/TransformPageLayout";
+import { getNewTransformFromTypeUrl } from "metabase-enterprise/transforms/urls";
+
+import { PythonTransformEditor } from "./components/PythonTransformEditor";
+import { SHARED_LIB_IMPORT_PATH } from "./constants";
+import { PythonLibraryEditorPage } from "./pages/PythonLibraryEditorPage";
+import { PythonRunnerSettingsPage } from "./pages/PythonRunnerSettingsPage";
+import { getPythonLibraryUrl } from "./urls";
+
+if (hasPremiumFeature("transforms-python")) {
+  PLUGIN_TRANSFORMS_PYTHON.PythonRunnerSettingsPage = PythonRunnerSettingsPage;
+
+  PLUGIN_TRANSFORMS_PYTHON.getAdminRoutes = () => (
+    <Route component={FullWidthTransformPageLayout}>
+      <Route path="library/:path" component={PythonLibraryEditorPage} />
+    </Route>
+  );
+
+  PLUGIN_TRANSFORMS_PYTHON.getTransformsNavLinks = () => (
+    <AdminNavItem
+      label={t`Python library`}
+      path={getPythonLibraryUrl({ path: SHARED_LIB_IMPORT_PATH })}
+      icon="code_block"
+    />
+  );
+
+  PLUGIN_TRANSFORMS_PYTHON.getCreateTransformsMenuItems = () => (
+    <Menu.Item
+      component={ForwardRefLink}
+      to={getNewTransformFromTypeUrl("python")}
+      leftSection={<Icon name="code_block" />}
+      onClick={() => {
+        trackTransformCreate({
+          triggeredFrom: "transform-page-create-menu",
+          creationType: "python",
+        });
+      }}
+    >
+      {t`Python script`}
+    </Menu.Item>
+  );
+
+  PLUGIN_TRANSFORMS_PYTHON.TransformEditor = PythonTransformEditor;
+}

enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonLibraryEditorPage/PythonLibraryEditorPage.module.css

@@ -0,0 +1,5 @@
+.editor {
+  /* HACK: 133px is the height of the headers */
+  height: calc(100vh - 133px);
+  flex-grow: 1;
+}

enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonLibraryEditorPage/PythonLibraryEditorPage.tsx

@@ -0,0 +1,140 @@
+import { useEffect, useState } from "react";
+import { t } from "ttag";
+
+import Button from "metabase/common/components/Button";
+import EditBar from "metabase/common/components/EditBar";
+import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
+import { isResourceNotFoundError } from "metabase/lib/errors";
+import { useMetadataToasts } from "metabase/metadata/hooks";
+import { Box, Flex } from "metabase/ui";
+import {
+  useGetPythonLibraryQuery,
+  useUpdatePythonLibraryMutation,
+} from "metabase-enterprise/api/python-transform-library";
+
+import { PythonEditor } from "../../components/PythonEditor";
+import type { PythonLibraryEditorPageParams } from "../../types";
+
+import S from "./PythonLibraryEditorPage.module.css";
+
+type PythonLibraryEditorPageProps = {
+  params: PythonLibraryEditorPageParams;
+};
+
+const EMPTY_LIBRARY_SOURCE = `
+# This is your Python library.
+# You can add functions and classes here that can be reused in Python transforms.
+`
+  .trim()
+  .concat("\n");
+
+export function PythonLibraryEditorPage({
+  params,
+}: PythonLibraryEditorPageProps) {
+  const { path } = params;
+  const [source, setSource] = useState("");
+
+  const {
+    data: library,
+    isLoading,
+    error,
+  } = useGetPythonLibraryQuery({ path });
+  const [updatePythonLibrary, { isLoading: isSaving }] =
+    useUpdatePythonLibraryMutation();
+  const { sendSuccessToast, sendErrorToast } = useMetadataToasts();
+
+  function handleRevert() {
+    if (isLoading) {
+      return;
+    }
+
+    if (isResourceNotFoundError(error)) {
+      setSource(EMPTY_LIBRARY_SOURCE);
+      return;
+    }
+
+    if (library != null) {
+      setSource(library.source || "");
+    }
+  }
+
+  async function handleSave() {
+    try {
+      await updatePythonLibrary({ path, source }).unwrap();
+      sendSuccessToast(t`Python library saved`);
+    } catch (error) {
+      sendErrorToast(t`Python library could not be saved`);
+    }
+  }
+
+  // When the library loads, set the source to the current library source
+  useEffect(handleRevert, [isLoading, error, library]);
+
+  const isDirty = source !== (library?.source ?? EMPTY_LIBRARY_SOURCE);
+
+  if (isLoading || (error && !isResourceNotFoundError(error))) {
+    return (
+      <Box p="md">
+        <LoadingAndErrorWrapper loading={isLoading} error={error} />
+      </Box>
+    );
+  }
+
+  return (
+    <Flex h="100%" w="100%" bg="bg-light" gap={0} direction="column">
+      <LibraryEditorHeader
+        onSave={handleSave}
+        onRevert={handleRevert}
+        isDirty={isDirty}
+        isSaving={isSaving}
+      />
+      <PythonEditor
+        value={source}
+        onChange={setSource}
+        withPandasCompletions
+        className={S.editor}
+        data-testid="python-editor"
+      />
+    </Flex>
+  );
+}
+
+export function LibraryEditorHeader({
+  isDirty,
+  isSaving,
+  onSave,
+  onRevert,
+}: {
+  isDirty?: boolean;
+  isSaving?: boolean;
+  onSave: () => void;
+  onRevert: () => void;
+}) {
+  return (
+    <EditBar
+      title={t`You are editing the shared Python library`}
+      admin
+      data-testid="library-editor-header"
+      buttons={[
+        <Button
+          key="save"
+          onClick={onRevert}
+          primary
+          small
+          disabled={!isDirty || isSaving}
+        >
+          {t`Revert`}
+        </Button>,
+        <Button
+          key="save"
+          onClick={onSave}
+          primary
+          small
+          disabled={!isDirty || isSaving}
+        >
+          {t`Save`}
+        </Button>,
+      ]}
+    />
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonLibraryEditorPage/index.ts

@@ -0,0 +1 @@
+export * from "./PythonLibraryEditorPage";

enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonRunnerSettingsPage/PythonRunnerSettingsPage.tsx

@@ -0,0 +1,67 @@
+import { t } from "ttag";
+
+import {
+  SettingsPageWrapper,
+  SettingsSection,
+} from "metabase/admin/components/SettingsSection";
+import { AdminSettingInput } from "metabase/admin/settings/components/widgets/AdminSettingInput";
+
+export function PythonRunnerSettingsPage() {
+  return (
+    <SettingsPageWrapper title={t`Python Runner`}>
+      <SettingsSection title={t`Service Configuration`}>
+        <AdminSettingInput
+          name="python-execution-server-url"
+          title={t`Python Execution Server URL`}
+          description={t`URL for the Python execution server that runs transform functions.`}
+          placeholder="http://localhost:5001"
+          inputType="text"
+        />
+      </SettingsSection>
+      <SettingsSection title={t`S3 Storage Configuration`}>
+        <AdminSettingInput
+          name="python-storage-s-3-endpoint"
+          title={t`S3 Endpoint`}
+          description={t`S3 endpoint URL for storing Python execution artifacts.`}
+          inputType="text"
+        />
+        <AdminSettingInput
+          name="python-storage-s-3-region"
+          title={t`S3 Region`}
+          description={t`AWS region for S3 storage.`}
+          inputType="text"
+        />
+        <AdminSettingInput
+          name="python-storage-s-3-bucket"
+          title={t`S3 Bucket`}
+          description={t`S3 bucket name for storing Python execution artifacts.`}
+          inputType="text"
+        />
+        <AdminSettingInput
+          name="python-storage-s-3-access-key"
+          title={t`S3 Access Key ID`}
+          description={t`AWS access key ID for S3 authentication.`}
+          inputType="password"
+        />
+        <AdminSettingInput
+          name="python-storage-s-3-secret-key"
+          title={t`S3 Secret Access Key`}
+          description={t`AWS secret access key for S3 authentication.`}
+          inputType="password"
+        />
+        <AdminSettingInput
+          name="python-storage-s-3-container-endpoint"
+          title={t`S3 Container Endpoint (Optional)`}
+          description={t`Alternative S3 endpoint accessible from containers. Leave empty if same as main endpoint.`}
+          inputType="text"
+        />
+        <AdminSettingInput
+          name="python-storage-s-3-path-style-access"
+          title={t`Use Path-Style Access`}
+          description={t`Use path-style access for S3 requests (required for LocalStack and some S3-compatible services).`}
+          inputType="boolean"
+        />
+      </SettingsSection>
+    </SettingsPageWrapper>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms-python/pages/PythonRunnerSettingsPage/index.ts

@@ -0,0 +1 @@
+export * from "./PythonRunnerSettingsPage";

enterprise/frontend/src/metabase-enterprise/transforms-python/types.ts

@@ -0,0 +1,3 @@
+export type PythonLibraryEditorPageParams = {
+  path: string;
+};

enterprise/frontend/src/metabase-enterprise/transforms-python/urls.ts

@@ -0,0 +1,5 @@
+import type { PythonLibraryEditorPageParams } from "./types";
+
+export function getPythonLibraryUrl({ path }: PythonLibraryEditorPageParams) {
+  return `/admin/transforms/library/${path}`;
+}

enterprise/frontend/src/metabase-enterprise/transforms/analytics.ts

@@ -34,7 +34,7 @@ export function trackTransformCreate({
   creationType,
 }: {
   triggeredFrom: "transform-page-create-menu";
-  creationType: "query" | "native" | "saved-question";
+  creationType: "query" | "native" | "python" | "saved-question";
 }) {
   trackSimpleEvent({
     event: "transform_create",

enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/LogOutput.modules.css

@@ -0,0 +1,9 @@
+.logOutput {
+  background-color: var(--mb-color-bg-light);
+  font-family: var(--mb-default-monospace-font-family);
+  white-space: pre;
+  padding: 1rem;
+  overflow: auto;
+  line-height: 1.4;
+  border-top: 1px solid var(--mb-color-border);
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/LogOutput.tsx

@@ -0,0 +1,66 @@
+import * as React from "react";
+import { useEffect, useMemo, useRef, useState } from "react";
+import reactAnsiStyle from "react-ansi-style";
+import { t } from "ttag";
+
+import { AnsiLogs } from "metabase/common/components/AnsiLogs";
+
+import S from "./LogOutput.modules.css";
+
+interface LogOutputProps {
+  content?: string;
+}
+
+export function LogOutput({ content }: LogOutputProps) {
+  const containerRef = useRef<HTMLDivElement>(null);
+  const [isUserScrolled, setIsUserScrolled] = useState(false);
+  const [shouldAutoScroll, setShouldAutoScroll] = useState(true);
+
+  const displayContent = useMemo(() => {
+    if (!content || content.trim().length === 0) {
+      return t`No output`;
+    }
+
+    return reactAnsiStyle(React, content);
+  }, [content]);
+
+  // Auto-scroll to bottom when content changes
+  useEffect(() => {
+    if (containerRef.current && shouldAutoScroll && !isUserScrolled) {
+      containerRef.current.scrollTop = containerRef.current.scrollHeight;
+    }
+  }, [displayContent, shouldAutoScroll, isUserScrolled]);
+
+  // Handle scroll events to detect user scrolling
+  const handleScroll = () => {
+    if (!containerRef.current) {
+      return;
+    }
+
+    const { scrollTop, scrollHeight, clientHeight } = containerRef.current;
+    const isAtBottom = Math.abs(scrollHeight - clientHeight - scrollTop) < 5;
+
+    if (isAtBottom) {
+      setIsUserScrolled(false);
+      setShouldAutoScroll(true);
+    } else {
+      setIsUserScrolled(true);
+      setShouldAutoScroll(false);
+    }
+  };
+
+  return (
+    <AnsiLogs
+      ref={containerRef}
+      className={S.logOutput}
+      onScroll={handleScroll}
+      mah="12rem"
+      p="sm"
+      bg="bg-light"
+      fz="xs"
+      data-css-specificity-hack=""
+    >
+      {displayContent}
+    </AnsiLogs>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/index.ts

@@ -0,0 +1 @@
+export { LogOutput } from "./LogOutput";

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/QueryEditor.tsx

@@ -6,7 +6,10 @@ import type { SelectionRange } from "metabase/query_builder/components/NativeQue
 import { Center, Flex, Loader, Stack } from "metabase/ui";
 import * as Lib from "metabase-lib";
 import type Question from "metabase-lib/v1/Question";
-import type { DatasetQuery, NativeQuerySnippet } from "metabase-types/api";
+import type {
+  NativeQuerySnippet,
+  QueryTransformSource,
+} from "metabase-types/api";
 
 import { useQueryMetadata } from "../../hooks/use-query-metadata";
 import { useQueryResults } from "../../hooks/use-query-results";
@@ -20,21 +23,23 @@ import S from "./QueryEditor.module.css";
 import { useInsertSnippetHandler, useSelectedText } from "./util";
 
 type QueryEditorProps = {
-  initialQuery: DatasetQuery;
+  initialSource: QueryTransformSource;
   isNew?: boolean;
   isSaving?: boolean;
-  onSave: (newQuery: DatasetQuery) => void;
+  onSave: (source: QueryTransformSource) => void;
   onCancel: () => void;
 };
 
 export function QueryEditor({
-  initialQuery,
+  initialSource,
   isNew = true,
   isSaving = false,
   onSave,
   onCancel,
 }: QueryEditorProps) {
-  const { question, isQueryDirty, setQuestion } = useQueryState(initialQuery);
+  const { question, isQueryDirty, setQuestion } = useQueryState(
+    initialSource.query,
+  );
   const { isInitiallyLoaded } = useQueryMetadata(question);
   const {
     result,
@@ -53,7 +58,7 @@ export function QueryEditor({
   };
 
   const handleSave = () => {
-    onSave(question.datasetQuery());
+    onSave({ type: "query", query: question.datasetQuery() });
   };
 
   const handleCmdEnter = () => {

enterprise/frontend/src/metabase-enterprise/transforms/components/RunButton/RunButton.tsx

@@ -2,7 +2,9 @@ import { type ReactNode, type Ref, forwardRef, useState } from "react";
 import { useUpdateEffect } from "react-use";
 import { t } from "ttag";
 
+
 import { Button, Icon, Loader, Tooltip } from "metabase/ui";
+
 import type { TransformRun } from "metabase-types/api";
 
 const RECENT_TIMEOUT = 5000;

enterprise/frontend/src/metabase-enterprise/transforms/components/RunErrorInfo/index.ts

@@ -1 +0,0 @@
-export * from "./RunErrorInfo";

enterprise/frontend/src/metabase-enterprise/transforms/components/RunInfo/RunInfo.tsx

@@ -16,15 +16,17 @@ import {
   Stack,
   Tooltip,
 } from "metabase/ui";
+import type { TransformRunStatus } from "metabase-types/api";
 
-import S from "./RunErrorInfo.module.css";
+import S from "./RunInfo.module.css";
 
-type RunErrorInfoProps = {
+type RunInfoProps = {
+  status: TransformRunStatus;
   message: string;
   endTime: Date | null;
 };
 
-export function RunErrorInfo({ message, endTime }: RunErrorInfoProps) {
+export function RunInfo({ status, message, endTime }: RunInfoProps) {
   const [isOpened, { open, close }] = useDisclosure();
 
   const handleIconClick = (event: MouseEvent) => {
@@ -38,16 +40,20 @@ export function RunErrorInfo({ message, endTime }: RunErrorInfoProps) {
     event.stopPropagation();
   };
 
+  if (!message || (status !== "failed" && status !== "succeeded")) {
+    return null;
+  }
+
   return (
     <>
-      <Tooltip label={t`See error`}>
+      <Tooltip label={getTooltip(status)}>
         <ActionIcon aria-label={t`See error`} onClick={handleIconClick}>
           <Icon name="document" />
         </ActionIcon>
       </Tooltip>
       {isOpened && (
         <Modal
-          title={getTitle(endTime)}
+          title={getTitle(status, endTime)}
           size="xl"
           padding="xl"
           opened
@@ -55,17 +61,34 @@ export function RunErrorInfo({ message, endTime }: RunErrorInfoProps) {
           onClose={close}
         >
           <FocusTrap.InitialFocus />
-          <RunErrorModalContent message={message} onClose={close} />
+          <RunInfoModalContent message={message} onClose={close} />
         </Modal>
       )}
     </>
   );
 }
 
-function getTitle(endTime: Date | null) {
-  return endTime
-    ? t`Failed at ${dayjs(endTime).format("lll")}, with this error`
-    : t`Failed with this error`;
+function getTooltip(status: TransformRunStatus) {
+  if (status === "failed") {
+    return t`See error`;
+  }
+  if (status === "succeeded") {
+    return t`See logs`;
+  }
+}
+
+function getTitle(status: TransformRunStatus, endTime: Date | null) {
+  if (status === "failed") {
+    return endTime
+      ? t`Failed at ${dayjs(endTime).format("lll")}, with this error`
+      : t`Failed with this error`;
+  }
+  if (status === "succeeded") {
+    return endTime
+      ? t`Succeeded at ${dayjs(endTime).format("lll")}, with the following logs`
+      : t`Succeeded with the following logs`;
+  }
+  return null;
 }
 
 type RunErrorModalContentProps = {
@@ -73,7 +96,7 @@ type RunErrorModalContentProps = {
   onClose: () => void;
 };
 
-function RunErrorModalContent({ message, onClose }: RunErrorModalContentProps) {
+function RunInfoModalContent({ message, onClose }: RunErrorModalContentProps) {
   return (
     <Stack pt="md" gap="lg">
       <Box className={S.codeContainer} pos="relative" pr="lg">

enterprise/frontend/src/metabase-enterprise/transforms/components/RunInfo/RunInfo.unit.spec.tsx

@@ -2,7 +2,7 @@ import userEvent from "@testing-library/user-event";
 
 import { renderWithProviders, screen } from "__support__/ui";
 
-import { RunErrorInfo } from "./RunErrorInfo";
+import { RunInfo } from "./RunInfo";
 
 type SetupOpts = {
   message?: string;
@@ -13,10 +13,12 @@ function setup({
   message = "Test error message",
   endTime = null,
 }: SetupOpts = {}) {
-  renderWithProviders(<RunErrorInfo message={message} endTime={endTime} />);
+  renderWithProviders(
+    <RunInfo status="failed" message={message} endTime={endTime} />,
+  );
 }
 
-describe("RunErrorInfo", () => {
+describe("RunInfo", () => {
   it("should show the error message when there is a date", async () => {
     const message = "Test error with end time";
     const endTime = new Date("2023-10-15T14:30:00");

enterprise/frontend/src/metabase-enterprise/transforms/components/RunInfo/index.ts

@@ -0,0 +1 @@
+export * from "./RunInfo";

enterprise/frontend/src/metabase-enterprise/transforms/components/RunStatusInfo/RunStatusInfo.tsx

@@ -2,6 +2,7 @@ import { Box, Group } from "metabase/ui";
 import type { Transform, TransformRunStatus } from "metabase-types/api";
 
 import { formatStatus } from "../../utils";
+import { RunInfo } from "../RunInfo";
 import { RunCancelButton } from "../RunCancelButton/RunCancelButton";
 import { RunErrorInfo } from "../RunErrorInfo";
 
@@ -26,7 +27,7 @@ export function RunStatusInfo({
     <Group gap="xs" className={S.runStatusInfo} wrap="nowrap">
       <Box c={isError ? "error" : undefined}>{formatStatus(status)}</Box>
       {isError && message != null && (
-        <RunErrorInfo message={message} endTime={endTime} />
+        <RunInfo status={status} message={message} endTime={endTime} />
       )}
       <RunCancelButton transform={transform} status={status} />
     </Group>

enterprise/frontend/src/metabase-enterprise/transforms/index.tsx

@@ -3,13 +3,13 @@ import { t } from "ttag";
 
 import { createAdminRouteGuard } from "metabase/admin/utils";
 import { Route } from "metabase/hoc/Title";
-import { PLUGIN_TRANSFORMS } from "metabase/plugins";
+import { PLUGIN_TRANSFORMS, PLUGIN_TRANSFORMS_PYTHON } from "metabase/plugins";
 import { hasPremiumFeature } from "metabase-enterprise/settings";
 
 import { JobListPage } from "./pages/JobListPage";
 import { JobPage } from "./pages/JobPage";
 import { NewJobPage } from "./pages/NewJobPage";
-import { NewTransformQueryPage } from "./pages/NewTransformQueryPage";
+import { NewTransformPage } from "./pages/NewTransformPage";
 import { RunListPage } from "./pages/RunListPage";
 import { TransformListPage } from "./pages/TransformListPage";
 import { TransformPage } from "./pages/TransformPage";
@@ -32,8 +32,9 @@ if (hasPremiumFeature("transforms")) {
           <Route path="runs" component={RunListPage} />
           <Route path=":transformId" component={TransformPage} />
         </Route>
-        <Route path="new/:type" component={NewTransformQueryPage} />
-        <Route path="new/card/:cardId" component={NewTransformQueryPage} />
+        {PLUGIN_TRANSFORMS_PYTHON.getAdminRoutes()}
+        <Route path="new/:type" component={NewTransformPage} />
+        <Route path="new/card/:cardId" component={NewTransformPage} />
         <Route path=":transformId/query" component={TransformQueryPage} />
       </Route>
     </Route>

enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx

@@ -24,18 +24,18 @@ import { trackTransformCreated } from "metabase-enterprise/transforms/analytics"
 import { SchemaFormSelect } from "metabase-enterprise/transforms/components/SchemaFormSelect";
 import type {
   CreateTransformRequest,
-  DatasetQuery,
   Transform,
+  TransformSource,
 } from "metabase-types/api";
 
 type CreateTransformModalProps = {
-  query: DatasetQuery;
+  source: TransformSource;
   onCreate: (transform: Transform) => void;
   onClose: () => void;
 };
 
 export function CreateTransformModal({
-  query,
+  source,
   onCreate,
   onClose,
 }: CreateTransformModalProps) {
@@ -43,7 +43,7 @@ export function CreateTransformModal({
     <Modal title={t`Save your transform`} opened padding="xl" onClose={onClose}>
       <FocusTrap.InitialFocus />
       <CreateTransformForm
-        query={query}
+        source={source}
         onCreate={onCreate}
         onClose={onClose}
       />
@@ -52,7 +52,7 @@ export function CreateTransformModal({
 }
 
 type CreateTransformFormProps = {
-  query: DatasetQuery;
+  source: TransformSource;
   onCreate: (transform: Transform) => void;
   onClose: () => void;
 };
@@ -72,11 +72,12 @@ const NEW_TRANSFORM_SCHEMA = Yup.object({
 });
 
 function CreateTransformForm({
-  query,
+  source,
   onCreate,
   onClose,
 }: CreateTransformFormProps) {
-  const { database: databaseId } = query;
+  const databaseId =
+    source.type === "query" ? source.query.database : source["source-database"];
 
   const {
     data: database,
@@ -108,7 +109,10 @@ function CreateTransformForm({
   }
 
   const handleSubmit = async (values: NewTransformValues) => {
-    const request = getCreateRequest(query, values);
+    if (!databaseId) {
+      throw new Error("Database ID is required");
+    }
+    const request = getCreateRequest(source, values, databaseId);
     const transform = await createTransform(request).unwrap();
 
     trackTransformCreated({ transformId: transform.id });
@@ -171,20 +175,19 @@ function getInitialValues(schemas: string[]): NewTransformValues {
 }
 
 function getCreateRequest(
-  query: DatasetQuery,
+  source: TransformSource,
   { name, description, targetName, targetSchema }: NewTransformValues,
+  databaseId: number,
 ): CreateTransformRequest {
   return {
     name: name,
     description,
-    source: {
-      type: "query",
-      query,
-    },
+    source,
     target: {
       type: "table",
       name: targetName,
       schema: targetSchema,
+      database: databaseId,
     },
   };
 }