Parametrise PythonLibraryEditor

- python-runner no longer simulates a single http api that does
everything (incl s3 interaction)
- Separates file lifetimes (shared storage) from execution
- Easier to see how to avoid reading into strings (e.g. for piping into
tmp files, follow up)

Earlier on we had a service that managed much of the I/O, metabase now
coordinates it so it makes little sense pretending there is still
an overall http api.

.clj-kondo/config.edn

@@ -1,7 +1,7 @@
 ;; -*- comment-column: 100; -*-
 {:config-paths ["macros" "src" "config"]
  :linters
- { ;;
+ {;;
   ;; enabled optional linters (these are disabled by default)
   ;;
 
@@ -186,7 +186,7 @@
   :metabase/disallow-hardcoded-driver-names-in-tests
   ;; don't include `:h2` since that will probably lead to a mountain of tests where we are specifically testing
   ;; middleware or compilation behavior with just H2 since it's the default.
-  {:drivers #{ ;; core drivers
+  {:drivers #{;; core drivers
               #_:h2
               :postgres
               :mysql
@@ -418,6 +418,7 @@
   metabase-enterprise.transforms.api-test/with-transform-tags                           clojure.core/let
   metabase-enterprise.transforms.api-test/with-worker-runs                              clojure.core/let
   metabase-enterprise.transforms.test-util/with-transform-cleanup!                      clojure.core/let
+  metabase-enterprise.transforms.execute/with-transform-lifecycle                       clojure.core/let
   metabase.actions.test-util/with-actions                                               clojure.core/let
   metabase.api.common/let-404                                                           clojure.core/let
   metabase.app-db.custom-migrations.util/with-temp-scheduler!                           clojure.core/fn
@@ -734,7 +735,9 @@
   ;; `metabase-enterprise.sandbox.query-processor*-test`
   ;;
   {:pattern "^metabase(?:((?:-enterprise\\.sandbox)?\\.query-processor)|(?:\\.driver\\.sql(?!server)(?!ite))).*-test$"
-   :name    general-qp-and-driver-test-namespaces}]
+   :name    general-qp-and-driver-test-namespaces}
+  {:pattern "^metabase\\.python-runner\\..*"
+   :name    python-runner-namespaces}]
 
  :config-in-ns
  {test-namespaces

.clj-kondo/config/modules/config.edn

@@ -887,6 +887,19 @@
            enterprise/advanced-permissions
            enterprise/sandbox}}
 
+  python-runner
+  {:team "Workflows"
+   :api  #{metabase.python-runner.api
+           metabase.python-runner.core
+           metabase.python-runner.init}
+   :uses #{api
+           driver
+           models
+           query-processor
+           request
+           util
+           enterprise/transforms}}
+
   queries
   {:team "Querying"
    :api  #{metabase.queries.api
@@ -1948,8 +1961,11 @@
   {:team "Querying"
    :api  #{metabase-enterprise.transforms.api
            metabase-enterprise.transforms.core
-           metabase-enterprise.transforms.init}
-   :uses #{api
+           metabase-enterprise.transforms.init
+           ;; TODO: Remove this from API once we remove the python-runner callback to metabase
+           metabase-enterprise.transforms.settings}
+   :uses #{analytics
+           api
            app-db
            config
            driver
@@ -1965,6 +1981,8 @@
            sync
            task
            task-history
+           ;; TODO: Remove this once we're using a better file format than CSV
+           upload
            util}}
 
   enterprise/upload-management

.github/actions/await-port/action.yml

@@ -11,6 +11,18 @@ runs:
     - name: Wait for port ${{ inputs.port }} to be ready
       shell: bash
       run: |
+        echo "Waiting for port ${{ inputs.port }} to be ready..."
+        timeout=300  # 5 minutes timeout
+        elapsed=0
         while ! nc -z localhost ${{ inputs.port }}; do
-          sleep 0.1;
+          if [ $elapsed -ge $timeout ]; then
+            echo "❌ Timeout waiting for port ${{ inputs.port }} after ${timeout} seconds"
+            exit 1
+          fi
+          sleep 1
+          elapsed=$((elapsed + 1))
+          if [ $((elapsed % 10)) -eq 0 ]; then
+            echo "  Still waiting for port ${{ inputs.port }}... ($elapsed seconds)"
+          fi
         done
+        echo "✅ Port ${{ inputs.port }} is ready!"

.github/workflows/backend.yml

@@ -195,6 +195,7 @@ jobs:
       - name: Run cljfmt
         run: ./bin/mage cljfmt-all --force-check
 
+
   be-tests-result:
     needs:
       - be-linter-eastwood

.github/workflows/drivers.yml

@@ -338,6 +338,12 @@ jobs:
     name: H2
     steps:
     - uses: actions/checkout@v4
+      with:
+        submodules: true
+        token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+    - name: Start python-runner
+      working-directory: python-runner
+      run: make run-all
     - name: Test H2 driver
       uses: ./.github/actions/test-driver
       with:
@@ -442,6 +448,12 @@ jobs:
     name: "${{ matrix.version.name }} ${{ matrix.job.name }}"
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
+          token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      - name: Start python-runner
+        working-directory: python-runner
+        run: make run-all
       - name: Test ${{ matrix.version.name }}
         uses: ./.github/actions/test-driver
         with:
@@ -745,6 +757,12 @@ jobs:
           POSTGRES_HOST_AUTH_METHOD: trust
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
+          token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      - name: Start python-runner
+        working-directory: python-runner
+        run: make run-all
       - name: Test ${{ matrix.version.name }} (${{ matrix.job.name }})
         uses: ./.github/actions/test-driver
         with:

.gitignore

@@ -163,3 +163,4 @@ CLAUDE.local.md
 
 /logs
 /.clojure-mcp
+__pycache__/

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "python-runner"]
+	path = python-runner
+	url = https://github.com/metabase/python-runner-container.git
+	branch = prototype

deps.edn

@@ -36,7 +36,7 @@
   com.clearspring.analytics/stream          {:mvn/version "2.9.8"               ; Various sketching algorithms
                                              :exclusions  [it.unimi.dsi/fastutil
                                                            org.slf4j/slf4j-api]}
-  com.clojure-goes-fast/jvm-alloc-rate-meter{:mvn/version "0.1.4"}              ; Monitor heap allocation rate.
+  com.clojure-goes-fast/jvm-alloc-rate-meter {:mvn/version "0.1.4"}              ; Monitor heap allocation rate.
   com.clojure-goes-fast/jvm-hiccup-meter    {:mvn/version "0.1.1"}              ; Monitor GC pauses and other system-induced pauses.
   com.draines/postal                        {:mvn/version "2.0.5"               ; SMTP library
                                              :exclusions [com.sun.mail/jakarta.mail]}
@@ -45,7 +45,7 @@
   com.github.albfernandez/juniversalchardet {:mvn/version "2.5.0"}
   com.github.blagerweij/liquibase-sessionlock {:mvn/version "1.6.9"}            ; session lock for liquibase migrations
   com.github.jknack/handlebars              ^:antq/exclude                      ; 4.4.0 requires java 17+
-                                            {:mvn/version "4.3.1"}              ; templating engine
+  {:mvn/version "4.3.1"}              ; templating engine
   com.github.oliyh/martian-clj-http         {:mvn/version "0.1.31"}             ; openapi client
   com.github.pangloss/transducers           {:git/sha "c91d39046c8c64dc31b84ebb27154a8084fcda4c" ; like medley for transducers
                                              :git/url "https://github.com/pangloss/transducers"}
@@ -55,14 +55,14 @@
   com.github.vertical-blank/sql-formatter   {:mvn/version "2.0.5"}              ; Java SQL formatting library https://github.com/vertical-blank/sql-formatter
   com.google.guava/guava                    {:mvn/version "33.4.8-jre"}         ; dep for BigQuery, Spark, and GA. Require here rather than letting different dep versions stomp on each other — see comments on #9697
   com.h2database/h2                         ^:antq/exclude                      ; https://metaboat.slack.com/archives/CKZEMT1MJ/p1727191010259979
-                                            {:mvn/version "2.1.214"}            ; embedded SQL database
+  {:mvn/version "2.1.214"}            ; embedded SQL database
   com.ibm.icu/icu4j                         {:mvn/version "77.1"}               ; Used to transliterate Unicode characters into Latin characters
   com.knuddels/jtokkit                      {:mvn/version "1.1.0"}              ; Java tokenizer library for OpenAI models, used for token counting in semantic search
   com.mchange/c3p0                          {:mvn/version "0.10.1"}             ; DB connection pool
   com.snowplowanalytics/snowplow-java-tracker ^:antq/exclude                    ; 2+ depend on apache httpclient 5.x, but saml and clj-http depend on 4.x
-                                            {:mvn/version "1.0.1"}              ; Snowplow analytics
+  {:mvn/version "1.0.1"}              ; Snowplow analytics
   com.sun.mail/jakarta.mail                 ^:antq/exclude                      ; 2+ changes imports from javax.mail to jakarta.mail
-                                            {:mvn/version "1.6.8"}              ; SMTP lib, required by postal
+  {:mvn/version "1.6.8"}              ; SMTP lib, required by postal
   com.taoensso/nippy                        {:mvn/version "3.6.0"}              ; Fast serialization (i.e., GZIP) library for Clojure
   com.vladsch.flexmark/flexmark             {:mvn/version "0.64.8"}             ; Markdown parsing
   com.vladsch.flexmark/flexmark-ext-autolink
@@ -92,7 +92,7 @@
   ;; TODO: replace back to normal artifact
   io.github.metabase/macaw                  {:mvn/version "0.2.27"}             ; Parse native SQL queries
   io.github.resilience4j/resilience4j-retry ^:antq/exclude                      ; 2.x do not support Java 8
-                                            {:mvn/version "1.7.1"}              ; Support for retrying operations
+  {:mvn/version "1.7.1"}              ; Support for retrying operations
   io.prometheus/simpleclient_hotspot        {:mvn/version "0.16.0"}             ; prometheus jvm collector
   junegunn/grouper                          {:mvn/version "0.1.1"}              ; Batch processing helper
   kixi/stats                                {:mvn/version "0.5.7"               ; Various statistic measures implemented as transducers
@@ -198,6 +198,8 @@
                                                            org.eclipse.jetty.ee9/jetty-ee9-servlet
                                                            org.eclipse.jetty.ee9.websocket/jetty-ee9-websocket-jetty-server]}
   slingshot/slingshot                       {:mvn/version "0.12.2"}             ; enhanced throw/catch, used by other deps
+  software.amazon.awssdk/s3                 {:mvn/version "2.33.0"}
+  software.amazon.awssdk/sts                {:mvn/version "2.33.0"}           ; AWS SDK v2 for S3
   stencil/stencil                           {:mvn/version "0.5.0"}              ; Mustache templates for Clojure
   user-agent/user-agent                     {:mvn/version "0.1.1"}              ; User-Agent string parser, for Login History page & elsewhere
   weavejester/dependency                    {:mvn/version "1.0.0"}              ; Dependency graphs and topological sorting
@@ -836,8 +838,13 @@
   :test/lib
   {:exec-args {:only ["test/metabase/lib"
                       "test/metabase/lib_be"]}}
+  ;; test the python transforms
+  ;;    clj -X:dev:ee:ee-dev:driver:test:test/python-transform
+  :test/python-transform
+  {:exec-args {:only [metabase-enterprise.transforms.api-test
+                      "enterprise/backend/test/metabase_enterprise/transforms"]}}
 
-  ;; test the Query Processor
+;; test the Query Processor
   ;;
   ;;    clj -X:dev:ee:ee-dev:test:test/qp
   :test/qp

docs/developers-guide/driver-changelog.md

@@ -19,6 +19,12 @@ title: Driver interface changelog
 
 - Added the driver multi-methods `metabase.driver/schema-exists?` and `metabase.driver/create-schema-if-needed!` which should be implemented by drivers that support `:schemas` and `:transforms/table`.
 
+- Added `metabase.driver/type->database-type` multimethod that returns the database type for a given Metabase type (from the type hierarchy) as a HoneySQL spec. Unlike `upload-type->database-type` which is specific to upload types, this method handles general Metabase base types.
+
+- Added `metabase.driver/rename-table!` multimethod to rename a table from one name to another. Both table names may be qualified by schema (e.g., `schema.table`). This is currently used for transform operations and uploads.
+
+- Added `metabase.driver/insert-from-source!` multimethod that abstracts data insertion from various sources into existing tables. This multimethod dispatches on both the driver and the data source type (`:rows` or `:csv-file`). It allows drivers to optimize based on the data source type and returns the number of rows inserted. Default implementations are provided for both `:rows` and `:csv-file` source types.
+
 - `metabase.driver/humanize-connection-error-message` now takes a list of all messages in the exception chain,
   instead of just the top-level message as a string.
 

enterprise/backend/src/metabase_enterprise/transforms/api.clj

@@ -1,7 +1,10 @@
 (ns metabase-enterprise.transforms.api
   (:require
+   [clojure.core.async :as a]
+   [clojure.string :as str]
    [metabase-enterprise.transforms.api.transform-job]
    [metabase-enterprise.transforms.api.transform-tag]
+   [metabase-enterprise.transforms.canceling :as trasnforms.canceling]
    [metabase-enterprise.transforms.execute :as transforms.execute]
    [metabase-enterprise.transforms.models.transform :as transform.model]
    [metabase-enterprise.transforms.models.transform-run :as transform-run]
@@ -29,12 +32,21 @@
 (set! *warn-on-reflection* true)
 
 (mr/def ::transform-source
-  [:map
-   [:type [:= "query"]]
-   [:query [:map [:database :int]]]])
+  [:multi {:dispatch (comp keyword :type)}
+   [:query
+    [:map
+     [:type [:= "query"]]
+     [:query [:map [:database :int]]]]]
+   [:python
+    [:map {:closed true}
+     [:source-database {:optional true} :int]
+     [:source-tables   [:map-of :string :int]]
+     [:type [:= "python"]]
+     [:body :string]]]])
 
 (mr/def ::transform-target
   [:map
+   [:database {:optional true} :int]
    [:type [:enum "table"]]
    [:schema {:optional true} [:or ms/NonBlankString :nil]]
    [:name :string]])
@@ -57,19 +69,22 @@
 
 (defn- source-database-id
   [transform]
-  (-> transform :source :query :database))
+  (if (transforms.util/python-transform? transform)
+    (-> transform :source :source-database)
+    (-> transform :source :query :database)))
 
 (defn- check-database-feature
   [transform]
-  (let [database (api/check-400 (t2/select-one :model/Database (source-database-id transform))
-                                (deferred-tru "The source database cannot be found."))
-        feature (transforms.util/required-database-feature transform)]
-    (api/check-400 (not (:is_sample database))
-                   (deferred-tru "Cannot run transforms on the sample database."))
-    (api/check-400 (not (:is_audit database))
-                   (deferred-tru "Cannot run transforms on audit databases."))
-    (api/check-400 (driver.u/supports? (:engine database) feature database)
-                   (deferred-tru "The database does not support the requested transform target type."))))
+  (when (transforms.util/query-transform? transform)
+    (let [database (api/check-400 (t2/select-one :model/Database (source-database-id transform))
+                                  (deferred-tru "The source database cannot be found."))
+          feature (transforms.util/required-database-feature transform)]
+      (api/check-400 (not (:is_sample database))
+                     (deferred-tru "Cannot run transforms on the sample database."))
+      (api/check-400 (not (:is_audit database))
+                     (deferred-tru "Cannot run transforms on audit databases."))
+      (api/check-400 (driver.u/supports? (:engine database) feature database)
+                     (deferred-tru "The database does not support the requested transform target type.")))))
 
 (api.macros/defendpoint :get "/"
   "Get a list of transforms."
@@ -172,9 +187,10 @@
           new (merge old body)
           target-fields #(-> % :target (select-keys [:schema :name]))]
       (check-database-feature new)
-      (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]
-        (throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)
-                        {:status-code 400})))
+      (when (transforms.util/query-transform? old)
+        (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]
+          (throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)
+                          {:status-code 400}))))
       (api/check (not (and (not= (target-fields old) (target-fields new))
                            (transforms.util/target-table-exists? new)))
                  403
@@ -207,11 +223,14 @@
 (api.macros/defendpoint :post "/:id/cancel"
   "Cancel the current run for a given transform."
   [{:keys [id]} :- [:map
-                    [:id :string]]]
+                    [:id ms/PositiveInt]]]
   (log/info "canceling transform " id)
   (api/check-superuser)
-  (let [run (api/check-404 (transform-run/running-run-for-run-id id))]
-    (transform-run-cancelation/mark-cancel-started-run! (:id run)))
+  (let [transform (api/check-404 (t2/select-one :model/Transform id))
+        run (api/check-404 (transform-run/running-run-for-transform-id id))]
+    (transform-run-cancelation/mark-cancel-started-run! (:id run))
+    (when (transforms.util/python-transform? transform)
+      (trasnforms.canceling/cancel-run! (:id run))))
   nil)
 
 (api.macros/defendpoint :post "/:id/run"
@@ -222,9 +241,16 @@
   (api/check-superuser)
   (let [transform (api/check-404 (t2/select-one :model/Transform id))
         start-promise (promise)]
-    (u.jvm/in-virtual-thread*
-     (transforms.execute/run-mbql-transform! transform {:start-promise start-promise
-                                                        :run-method :manual}))
+    (if (transforms.util/python-transform? transform)
+      (u.jvm/in-virtual-thread*
+       (transforms.execute/execute-python-transform! transform {:start-promise start-promise
+                                                                :message-log   (atom {:pre-python  []
+                                                                                      :python      nil
+                                                                                      :post-python []})
+                                                                :run-method :manual}))
+      (u.jvm/in-virtual-thread*
+       (transforms.execute/run-mbql-transform! transform {:start-promise start-promise
+                                                          :run-method :manual})))
     (when (instance? Throwable @start-promise)
       (throw @start-promise))
     (let [result @start-promise
@@ -234,6 +260,53 @@
                               :run_id run-id})
           (assoc :status 202)))))
 
+;; hack
+(def ^:private python-test-run-id Integer/MAX_VALUE)
+
+(api.macros/defendpoint :post "/test-python"
+  "Test Python code execution without creating a transform."
+  [_route-params
+   _query-params
+   body :- [:map
+            [:code :string]
+            [:tables [:map-of :string :int]]]]
+  (log/info "test python code execution")
+  (api/check-superuser)
+  (let [run-id python-test-run-id
+        cancel-chan (a/promise-chan)]
+    (trasnforms.canceling/chan-start-run! run-id cancel-chan)
+    (try
+      (let [{:keys [response output events]} (transforms.execute/test-python-transform! (:code body) (:tables body) run-id cancel-chan)
+            {:keys [body status]} response]
+        (if (= status 200)
+          (do
+            (log/info "Python test execution succeeded")
+            (-> (response/response {:message (deferred-tru "Python code executed successfully")
+                                    :result  {:body (assoc body :output output)}})
+                (assoc :status 200)))
+          (do
+            (log/error "Error executing Python test code")
+            (-> (response/response {:message   (deferred-tru "Python code execution failed")
+                                    :error     body
+                                    :stdout    (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
+                                    :stderr    (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))
+                                    :exit_code (:exit_code (:exit_code body))})
+                (assoc :status status)))))
+      (finally
+        (trasnforms.canceling/chan-end-run! run-id)))))
+
+(api.macros/defendpoint :post "/test-python/cancel"
+  "Cancel the current test-python execution."
+  [_route-params
+   _query-params]
+  (log/info "canceling test python execution")
+  (api/check-superuser)
+  (if (trasnforms.canceling/chan-signal-cancel! python-test-run-id)
+    (-> (response/response {:message (deferred-tru "Python test canceled")})
+        (assoc :status 200))
+    (-> (response/response {:message (deferred-tru "No running Python test to cancel")})
+        (assoc :status 404))))
+
 (def ^{:arglists '([request respond raise])} routes
   "`/api/ee/transform` routes."
   (handlers/routes

enterprise/backend/src/metabase_enterprise/transforms/canceling.clj

@@ -49,7 +49,9 @@
    (chan-signal-cancel! run-id)
    (transform-run/timeout-run! run-id)))
 
-(defn- cancel-run! [run-id]
+(defn cancel-run!
+  "Cancel run"
+  [run-id]
   (when (chan-signal-cancel! run-id)
     (transform-run/cancel-run! run-id)))
 

enterprise/backend/src/metabase_enterprise/transforms/execute.clj

@@ -1,17 +1,27 @@
 (ns metabase-enterprise.transforms.execute
   (:require
    [clojure.core.async :as a]
+   [clojure.java.io :as io]
+   [clojure.string :as str]
    [metabase-enterprise.transforms.canceling :as canceling]
+   [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
    [metabase-enterprise.transforms.models.transform-run :as transform-run]
+   [metabase-enterprise.transforms.python-runner :as python-runner]
    [metabase-enterprise.transforms.settings :as transforms.settings]
    [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.driver :as driver]
    [metabase.driver.util :as driver.u]
    [metabase.lib.schema.common :as schema.common]
    [metabase.query-processor.pipeline :as qp.pipeline]
+   [metabase.util :as u]
+   [metabase.util.jvm :as u.jvm]
    [metabase.util.log :as log]
    [metabase.util.malli.registry :as mr]
-   [toucan2.core :as t2]))
+   [toucan2.core :as t2])
+  (:import
+   (clojure.lang IDeref)
+   (java.io Closeable File)
+   (java.time Duration)))
 
 (set! *warn-on-reflection* true)
 
@@ -37,24 +47,38 @@
    (log/info "Syncing target" (pr-str target) "for transform")
    (transforms.util/activate-table-and-mark-computed! database target)))
 
-(defn run-transform!
+(defn run-cancelable-transform!
   "Run a compiled transform"
-  [run-id driver {:keys [db-id conn-spec output-schema] :as transform-details} opts]
-  ;; local run is responsible for status
+  [run-id driver {:keys [db conn-spec output-schema]} run-transform!]
+  ;; local run is responsible for status, using canceling lifecycle
   (try
-    (when-not (driver/schema-exists? driver db-id output-schema)
-      (driver/create-schema-if-needed! driver conn-spec output-schema))
+    (when (driver.u/supports? driver :schemas db)
+      (when-not (driver/schema-exists? driver (:id db) output-schema)
+        (driver/create-schema-if-needed! driver conn-spec output-schema)))
     (canceling/chan-start-timeout-vthread! run-id (transforms.settings/transform-timeout))
-    (binding [qp.pipeline/*canceled-chan* (a/promise-chan)]
-      (canceling/chan-start-run! run-id qp.pipeline/*canceled-chan*)
-      (driver/run-transform! driver transform-details opts))
-    (transform-run/succeed-started-run! run-id)
+    (let [cancel-chan (a/promise-chan)
+          ret (binding [qp.pipeline/*canceled-chan* cancel-chan]
+                (canceling/chan-start-run! run-id cancel-chan)
+                (run-transform! cancel-chan))]
+      (transform-run/succeed-started-run! run-id)
+      ret)
     (catch Throwable t
       (transform-run/fail-started-run! run-id {:message (.getMessage t)})
       (throw t))
     (finally
       (canceling/chan-end-run! run-id))))
 
+(defn- try-start-unless-already-running [id run-method]
+  (try
+    (transform-run/start-run! id {:run_method run-method})
+    (catch java.sql.SQLException e
+      (if (= (.getSQLState e) "23505")
+        (throw (ex-info "Transform is already running"
+                        {:error :already-running
+                         :transform-id id}
+                        e))
+        (throw e)))))
+
 (defn run-mbql-transform!
   "Run `transform` and sync its target table.
 
@@ -67,7 +91,7 @@
      (let [db (get-in source [:query :database])
            {driver :engine :as database} (t2/select-one :model/Database db)
            feature (transforms.util/required-database-feature transform)
-           transform-details {:db-id db
+           transform-details {:db db
                               :transform-type (keyword (:type target))
                               :conn-spec (driver/connection-spec driver database)
                               :query (transforms.util/compile-source source)
@@ -78,20 +102,13 @@
          (throw (ex-info "The database does not support the requested transform target type."
                          {:driver driver, :database database, :feature feature})))
        ;; mark the execution as started and notify any observers
-       (let [{run-id :id} (try
-                            (transform-run/start-run! id {:run_method run-method})
-                            (catch java.sql.SQLException e
-                              (if (= (.getSQLState e) "23505")
-                                (throw (ex-info "Transform is already running"
-                                                {:error :already-running
-                                                 :transform-id id}
-                                                e))
-                                (throw e))))]
+       (let [{run-id :id} (try-start-unless-already-running id run-method)]
          (when start-promise
            (deliver start-promise [:started run-id]))
          (log/info "Executing transform" id "with target" (pr-str target))
-         (run-transform! run-id driver transform-details opts)
-         (sync-target! target database run-id)))
+         (run-cancelable-transform! run-id driver transform-details (fn [_cancel-chan] (driver/run-transform! driver transform-details opts)))
+         (transforms.instrumentation/with-stage-timing [run-id :sync :table-sync]
+           (sync-target! target database run-id))))
      (catch Throwable t
        (log/error t "Error executing transform")
        (when start-promise
@@ -99,3 +116,236 @@
          ;; but we assume nobody would catch the exception anyway
          (deliver start-promise t))
        (throw t)))))
+
+(defn- log! [message-log s]
+  (when message-log
+    (swap! message-log (fn [m] (update m (if (:python m) :post-python :pre-python) conj s))))
+  nil)
+
+(defn- replace-python-logs! [message-log events]
+  (when message-log (swap! message-log assoc :python events))
+  nil)
+
+(defn- save-log-as-message! [run-id message-log]
+  (when message-log
+    (let [{:keys [pre-python python post-python]} @message-log]
+      (t2/update! :model/TransformRun
+                  :id run-id
+                  {:message (str/join "\n" (concat pre-python
+                                                   (for [{:keys [stream message]} python]
+                                                     (if (= "stdout" stream)
+                                                       (str "\033[32m" message "\033[0m")
+                                                       (str "\033[31m" message "\033[0m")))
+                                                   post-python))}))))
+
+(defn- log-loop! [run-id message-log]
+  (let [poll #(python-runner/get-logs run-id)]
+    (try
+      (loop []
+        (if (.isInterrupted (Thread/currentThread))
+          (log/debug "Message update loop interrupted")
+          (do (Thread/sleep 1000)
+              (let [{:keys [status body]} (poll)]
+                (cond
+                  (<= 200 status 299)
+                  (let [{:keys [execution_id events]} body]
+                    (if-not (= run-id execution_id)
+                      (do (log/debugf "Run id did not match expected: %s actual: %s" run-id execution_id)
+                          (recur))
+                      (do
+                        (replace-python-logs! message-log events)
+                        (save-log-as-message! run-id message-log)
+                        (recur))))
+                  (= 404 status)
+                  (do
+                    (log/debugf "No logs yet (or run finished), run-id: %s" run-id)
+                    (recur))
+                  :else
+                  (do
+                    (log/warnf "Unexpected status polling for logs %s %s, run-id: %s" status body run-id)
+                    (log/debug "Exiting due to poll error")))))))
+      (catch InterruptedException _)
+      (catch Throwable e
+        (log/errorf e "An exception was caught during msg update loop, run-id: %s" run-id)))))
+
+(defn- open-log-thread! ^Closeable [run-id message-log]
+  (let [log-thread-promise (promise)
+        cleanup (fn []
+                  (let [^Thread log-thread (deref log-thread-promise 1000 nil)]
+                    (when-not log-thread (log/fatalf "Log thread reference not bound, run-id: %s" run-id))
+                    (when (.isAlive log-thread)
+                      (.interrupt log-thread)
+                      (when-not (.join log-thread (Duration/ofSeconds 10))
+                        (log/fatalf "Log thread could not be interrupted, run-id: %s" run-id)))))]
+    (u.jvm/in-virtual-thread*
+     (deliver log-thread-promise (Thread/currentThread))
+     (log-loop! run-id message-log))
+    (when-not (deref log-thread-promise 1000 nil) (log/fatalf "Log thread reference not bound, run-id: %s" run-id))
+    (reify
+      IDeref
+      (deref [_] @log-thread-promise)
+      Closeable
+      (close [_] (cleanup)))))
+
+(defn- debug-info-str [{:keys [exit-code events]}]
+  (str/join "\n"
+            ;; todo this was temporary before the log stuff, needs a rethink at some point
+            ["stdout"
+             "======"
+             (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
+             "stderr"
+             "======"
+             (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))
+             "======"
+             (format "exit code %d" exit-code)]))
+
+(defn- create-table-and-insert-data!
+  "Create a table from metadata and insert data from source."
+  [driver db-id table-name metadata data-source]
+  (let [table-schema {:name (if (keyword? table-name) table-name (keyword table-name))
+                      :columns (mapv (fn [{:keys [name dtype base_type database_type]}]
+                                       {:name name
+                                        :type (or (some->> base_type (keyword "type"))
+                                                  (transforms.util/dtype->base-type dtype))
+                                        :database-type database_type
+                                        :nullable? true})
+                                     (:fields metadata))}
+        data-source (assoc data-source :table-schema table-schema)]
+    (transforms.util/create-table-from-schema! driver db-id table-schema)
+    (driver/insert-from-source! driver db-id table-name (mapv :name (:columns table-schema)) data-source)))
+
+(defn- transfer-file-to-db [driver db {:keys [target] :as transform} metadata temp-file]
+  (let [table-name (transforms.util/qualified-table-name driver target)
+        table-exists? (transforms.util/target-table-exists? transform)
+        data-source {:type :csv-file
+                     :file temp-file}]
+    (if table-exists?
+      ;; Table exists - use temp table + atomic swap pattern
+      (let [temp-table-name (keyword (str (u/qualified-name table-name) "_temp_" (System/currentTimeMillis)))]
+        (log/info "Existing table deletected, Create then swap")
+        (try
+          (create-table-and-insert-data! driver (:id db) temp-table-name metadata data-source)
+          ;; TODO: These operations should be within a transaction for atomicity
+          (transforms.util/drop-table! driver (:id db) table-name)
+          (transforms.util/rename-table! driver (:id db) temp-table-name table-name)
+          (catch Exception e
+            (log/error e "Failed to transfer data to table")
+            (try
+              (transforms.util/drop-table! driver (:id db) temp-table-name)
+              (catch Exception _))
+            (throw e))))
+      ;; Table doesn't exist - create directly with target name
+      (do
+        (log/info "New table")
+        (create-table-and-insert-data! driver (:id db) table-name metadata data-source)))))
+
+(defn test-python-transform!
+  "Execute a transform in test mode (does not write result into a table)."
+  [code tables->id run-id cancel-chan]
+  (with-open [shared-storage-ref (python-runner/open-s3-shared-storage! tables->id)]
+    (let [server-url              (transforms.settings/python-execution-server-url)
+          _                       (python-runner/copy-tables-to-s3! {:run-id         run-id
+                                                                     :shared-storage @shared-storage-ref
+                                                                     :table-name->id tables->id
+                                                                     :cancel-chan    cancel-chan})
+          _                       (python-runner/open-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan
+          response
+          (python-runner/execute-python-code-http-call!
+           {:server-url     server-url
+            :code           code
+            :run-id         run-id
+            :table-name->id tables->id
+            :shared-storage @shared-storage-ref})
+          {:keys [output events]} (python-runner/read-output-objects @shared-storage-ref)]
+      {:response response
+       :events   events
+       :output   output})))
+
+(defn- run-python-transform! [{:keys [source] :as transform} db run-id cancel-chan message-log]
+  (with-open [log-thread-ref     (open-log-thread! run-id message-log)
+              shared-storage-ref (python-runner/open-s3-shared-storage! (:source-tables source))]
+    (let [driver     (:engine db)
+          server-url (transforms.settings/python-execution-server-url)
+          _          (python-runner/copy-tables-to-s3! {:run-id                  run-id
+                                                        :shared-storage @shared-storage-ref
+                                                        :table-name->id (:source-tables source)
+                                                        :cancel-chan    cancel-chan})
+          _          (python-runner/open-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan
+          {:keys [status body] :as response}
+          (python-runner/execute-python-code-http-call!
+           {:server-url     server-url
+            :code           (:body source)
+            :run-id         run-id
+            :table-name->id (:source-tables source)
+            :shared-storage @shared-storage-ref})
+          {:keys [exit_code]} body
+          ;; TODO temporary to keep more code stable while refactoring
+          ;; no need to materialize these early (i.e output we can stream directly into a tmp file or db if small)
+          {:keys [output output-manifest events]} (python-runner/read-output-objects @shared-storage-ref)]
+      (.close log-thread-ref)           ; early close to force any writes to flush
+      (when (seq events)
+        (replace-python-logs! message-log events))
+      (if (not= 200 status)
+        (throw (ex-info (debug-info-str {:exit-code exit_code :events events}) ;; todo do better here
+                        {:status-code     400
+                         :api-status-code status
+                         :body            body
+                         :events          events
+                         :error           (:error body)}))
+        (try
+          (let [temp-file (File/createTempFile "transform-output-" ".csv")]
+            (when-not (seq (:fields output-manifest))
+              (throw (ex-info "No fields in metadata"
+                              {:metadata output-manifest
+                               :raw-body body
+                               :events   events})))
+            (try
+              (with-open [writer (io/writer temp-file)]
+                (.write writer ^String output))
+              (let [file-size (.length temp-file)]
+                (transforms.instrumentation/with-stage-timing [run-id :data-transfer :file-to-dwh]
+                  (transfer-file-to-db driver db transform output-manifest temp-file))
+                (transforms.instrumentation/record-data-transfer! run-id :file-to-dwh file-size nil))
+              (finally
+                (.delete temp-file))))
+          response
+          (catch Exception e
+            (log/error e "Failed to to create resulting table")
+            (throw (ex-info "Failed to create the resulting table" {:error (.getMessage e)}))))))))
+
+(defn execute-python-transform!
+  "Execute a Python transform by calling the python runner.
+
+  This is executing synchronously, but supports being kicked off in the background
+  by delivering the `start-promise` just before the start when the beginning of the execution has been booked
+  in the database."
+  [transform {:keys [run-method start-promise message-log]}]
+  (when (transforms.util/python-transform? transform)
+    (try
+      (let [message-log (or message-log (atom {:pre-python  []
+                                               :python      nil
+                                               :post-python []}))
+            {:keys [target] transform-id :id} transform
+            {driver :engine :as db} (t2/select-one :model/Database (:database target))
+            {run-id :id} (try-start-unless-already-running transform-id run-method)]
+        (some-> start-promise (deliver [:started run-id]))
+        (log! message-log "Executing Python transform")
+        (log/info "Executing Python transform" transform-id "with target" (pr-str target))
+        (let [start-ms (u/start-timer)
+              transform-details
+              {:db db
+               :transform-type (keyword (:type target))
+               :conn-spec (driver/connection-spec driver db)
+               :output-schema (:schema target)
+               :output-table (transforms.util/qualified-table-name driver target)}
+              result   (run-cancelable-transform! run-id driver transform-details (fn [cancel-chan] (run-python-transform! transform db run-id cancel-chan message-log)))]
+          (transforms.instrumentation/with-stage-timing [run-id :sync :table-sync]
+            (sync-target! target db run-id))
+          (log! message-log (format "Python execution finished in %s" (Duration/ofMillis (u/since-ms start-ms))))
+          (save-log-as-message! run-id message-log)
+          {:run_id run-id
+           :result result}))
+      (catch Throwable t
+        (log/error t "Error executing Python transform")
+        (log! message-log "Error executing python transform")
+        (throw t)))))

enterprise/backend/src/metabase_enterprise/transforms/instrumentation.clj

@@ -0,0 +1,181 @@
+(ns metabase-enterprise.transforms.instrumentation
+  (:require
+   [metabase.analytics.prometheus :as prometheus]
+   [metabase.util :as u]
+   [metabase.util.log :as log]
+   [metabase.util.malli :as mu]
+   [metabase.util.malli.registry :as mr]))
+
+(set! *warn-on-reflection* true)
+
+(mr/def ::stage-type
+  [:enum :data-transfer :computation :sync])
+
+(mr/def ::stage-label
+  [:enum
+   :dwh-to-file
+   :file-to-s3
+   :file-to-dwh
+
+   :python-execution
+   :mbql-query
+   :table-sync])
+
+(mu/defn record-stage-start!
+  "Record the start of a transform stage."
+  [job-run-id :- pos-int?
+   stage-type :- ::stage-type
+   stage-label :- ::stage-label]
+  (log/infof "Transform stage started: run-id=%d type=%s label=%s" job-run-id (name stage-type) (name stage-label))
+  (prometheus/inc! :metabase-transforms/stage-started
+                   {:job-run-id (str job-run-id)
+                    :stage-type (name stage-type)
+                    :stage-label (name stage-label)}))
+
+(mu/defn record-stage-completion!
+  "Record the successful completion of a transform stage."
+  [job-run-id :- pos-int?
+   stage-type :- ::stage-type
+   stage-label :- ::stage-label
+   duration-ms :- int?]
+  (log/infof "Transform stage completed: run-id=%d type=%s label=%s duration=%dms" job-run-id (name stage-type) (name stage-label) duration-ms)
+  (let [labels {:job-run-id (str job-run-id)
+                :stage-type (name stage-type)
+                :stage-label (name stage-label)}]
+    (prometheus/inc! :metabase-transforms/stage-completed labels)
+    (prometheus/observe! :metabase-transforms/stage-duration-ms labels duration-ms)))
+
+(mu/defn record-stage-failure!
+  "Record the failure of a transform stage."
+  [job-run-id :- pos-int?
+   stage-type :- ::stage-type
+   stage-label :- ::stage-label
+   duration-ms :- [:maybe int?]]
+  (log/warnf "Transform stage failed: run-id=%d type=%s label=%s duration=%s" job-run-id (name stage-type) (name stage-label) (if duration-ms (str duration-ms "ms") "unknown"))
+  (let [labels {:job-run-id (str job-run-id)
+                :stage-type (name stage-type)
+                :stage-label (name stage-label)}]
+    (prometheus/inc! :metabase-transforms/stage-failed labels)
+    (when duration-ms
+      (prometheus/observe! :metabase-transforms/stage-duration-ms labels duration-ms))))
+
+(defn with-timing
+  "Generic timing function that executes body while recording metrics.
+
+   Takes a map of optional functions:
+   - :start-fn - function to call when starting
+   - :success-fn - function to call on success (receives duration-ms as last arg)
+   - :error-fn - function to call on error (receives duration-ms as last arg)
+
+   Usage:
+   (with-timing {:start-fn #(record-stage-start! %1 %2 %3)
+                 :success-fn #(record-stage-completion! %1 %2 %3 %4)
+                 :error-fn #(record-stage-failure! %1 %2 %3 %4)}
+                [job-run-id stage-type stage-label]
+                #(do-work)) "
+  [callbacks args body-fn]
+  (let [start-time (u/start-timer)]
+    (when-let [start-fn (:start-fn callbacks)]
+      (apply start-fn args))
+    (try
+      (let [result (body-fn)]
+        (when-let [success-fn (:success-fn callbacks)]
+          (apply success-fn (conj (vec args) (long (u/since-ms start-time)))))
+        result)
+      (catch Throwable t
+        (when-let [error-fn (:error-fn callbacks)]
+          (apply error-fn (conj (vec args) (long (u/since-ms start-time)))))
+        (throw t)))))
+
+(defmacro with-stage-timing
+  "Execute body while timing a transform stage. Automatically records start, completion or failure.
+
+   Usage:
+   (with-stage-timing [job-run-id :data-transfer :dwh-to-file]
+     ;; stage implementation
+     result)"
+  [args & body]
+  `(with-timing {:start-fn record-stage-start!
+                 :success-fn record-stage-completion!
+                 :error-fn record-stage-failure!}
+     ~args
+     (^:once fn* [] ~@body)))
+
+(mu/defn record-data-transfer!
+  "Record metrics about data transfer (size and rows)."
+  [job-run-id :- pos-int?
+   stage-label :- ::stage-label
+   bytes :- [:maybe int?]
+   rows :- [:maybe int?]]
+  (log/infof "Data transfer recorded: run-id=%d stage=%s%s%s"
+             job-run-id (name stage-label)
+             (if bytes (str " bytes=" bytes) "")
+             (if rows (str " rows=" rows) ""))
+  (let [labels {:job-run-id (str job-run-id)
+                :stage-label (name stage-label)}]
+    (when bytes
+      (prometheus/observe! :metabase-transforms/data-transfer-bytes labels bytes))
+    (when rows
+      (prometheus/observe! :metabase-transforms/data-transfer-rows labels rows))))
+
+(defn record-job-start!
+  "Record the start of a transform job run."
+  [job-id run-method]
+  (log/infof "Transform job started: job-id=%d run-method=%s" job-id (name run-method))
+  (prometheus/inc! :metabase-transforms/job-runs-total
+                   {:job-id (str job-id)
+                    :run-method (name run-method)}))
+
+(defn record-job-completion!
+  "Record the successful completion of a transform job run."
+  [job-id run-method duration-ms]
+  (log/infof "Transform job completed: job-id=%d run-method=%s duration=%dms" job-id (name run-method) duration-ms)
+  (let [labels {:job-id (str job-id)
+                :run-method (name run-method)}]
+    (prometheus/inc! :metabase-transforms/job-runs-completed labels)
+    (prometheus/observe! :metabase-transforms/job-run-duration-ms labels duration-ms)))
+
+(defn record-job-failure!
+  "Record the failure of a transform job run."
+  [job-id run-method duration-ms]
+  (log/warnf "Transform job failed: job-id=%d run-method=%s duration=%s" job-id (name run-method) (if duration-ms (str duration-ms "ms") "unknown"))
+  (let [labels {:job-id (str job-id)
+                :run-method (name run-method)}]
+    (prometheus/inc! :metabase-transforms/job-runs-failed labels)
+    (when duration-ms
+      (prometheus/observe! :metabase-transforms/job-run-duration-ms labels duration-ms))))
+
+(defmacro with-job-timing
+  "Execute body while timing a transform job run. Automatically records start, completion or failure.
+
+   Usage:
+   (with-job-timing [job-id run-method]
+     ;; job implementation
+     result)"
+  [[job-id run-method] & body]
+  `(with-timing {:start-fn record-job-start!
+                 :success-fn record-job-completion!
+                 :error-fn record-job-failure!}
+     [~job-id ~run-method]
+     (^:once fn* [] ~@body)))
+
+(mu/defn record-python-api-call!
+  "Record metrics about Python API calls."
+  [job-run-id :- pos-int?
+   duration-ms :- int?
+   status :- [:enum :success :error :timeout]]
+  (log/infof "Python API call %s: run-id=%d duration=%dms" (name status) job-run-id duration-ms)
+  (let [labels {:job-run-id (str job-run-id)}]
+    (prometheus/inc! :metabase-transforms/python-api-calls-total
+                     (assoc labels :status (name status)))
+    (prometheus/observe! :metabase-transforms/python-api-call-duration-ms labels duration-ms)))
+
+(defmacro with-python-api-timing
+  "Execute body while timing a Python API call."
+  [[job-run-id] & body]
+  `(with-timing {:success-fn (fn [job-run-id# duration-ms#]
+                               (record-python-api-call! job-run-id# duration-ms# :success))
+                 :error-fn (fn [job-run-id# duration-ms#]
+                             (record-python-api-call! job-run-id# duration-ms# :error))}
+     [~job-run-id]
+     (^:once fn* [] ~@body)))

enterprise/backend/src/metabase_enterprise/transforms/jobs.clj

@@ -6,10 +6,12 @@
    [clojurewerkz.quartzite.triggers :as triggers]
    [flatland.ordered.set :as ordered-set]
    [metabase-enterprise.transforms.execute :as transforms.execute]
+   [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
    [metabase-enterprise.transforms.models.job-run :as transforms.job-run]
    [metabase-enterprise.transforms.models.transform-run :as transform-run]
    [metabase-enterprise.transforms.ordering :as transforms.ordering]
    [metabase-enterprise.transforms.settings :as transforms.settings]
+   [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.task.core :as task]
    [metabase.util.log :as log]
    [toucan2.core :as t2]))
@@ -64,14 +66,21 @@
         (map transforms-by-id complete)))))
 
 (defn- run-transform! [run-id run-method {transform-id :id :as transform}]
-  (when (transform-run/running-run-for-run-id transform-id)
+  (when
+   (transform-run/running-run-for-transform-id transform-id)
     (log/warn "Transform" (pr-str transform-id) "already running, waiting")
     (loop []
       (Thread/sleep 2000)
-      (when (transform-run/running-run-for-run-id transform-id)
+      (when (transform-run/running-run-for-transform-id transform-id)
         (recur))))
+
   (log/info "Executing job transform" (pr-str transform-id))
-  (transforms.execute/run-mbql-transform! transform {:run-method run-method})
+  (if (transforms.util/python-transform? transform)
+    (transforms.instrumentation/with-stage-timing [run-id :computation :python-execution]
+      (transforms.execute/execute-python-transform! transform {:run-method run-method}))
+    (transforms.instrumentation/with-stage-timing [run-id :computation :mbql-query]
+      (transforms.execute/run-mbql-transform! transform {:run-method run-method})))
+
   (transforms.job-run/add-run-activity! run-id))
 
 (defn run-transforms!
@@ -110,12 +119,13 @@
     (let [transforms (job-transform-ids job-id)]
       (log/info "Executing transform job" (pr-str job-id) "with transforms" (pr-str transforms))
       (let [{run-id :id} (transforms.job-run/start-run! job-id run-method)]
-        (try
-          (run-transforms! run-id transforms opts)
-          (transforms.job-run/succeed-started-run! run-id)
-          (catch Throwable t
-            (transforms.job-run/fail-started-run! run-id {:message (.getMessage t)})
-            (throw t)))))))
+        (transforms.instrumentation/with-job-timing [job-id run-method]
+          (try
+            (run-transforms! run-id transforms opts)
+            (transforms.job-run/succeed-started-run! run-id)
+            (catch Throwable t
+              (transforms.job-run/fail-started-run! run-id {:message (.getMessage t)})
+              (throw t))))))))
 
 (def ^:private job-key "metabase-enterprise.transforms.jobs.timeout-job")
 

enterprise/backend/src/metabase_enterprise/transforms/models/transform_run.clj

@@ -147,11 +147,11 @@
                         :message   "Canceled by user but could not guarantee run stopped."})
     (cancel/delete-old-canceling-runs!)))
 
-(defn running-run-for-run-id
+(defn running-run-for-transform-id
   "Return a single active transform run or nil."
-  [id]
+  [transform-id]
   (t2/select-one :model/TransformRun
-                 :id id
+                 :transform_id transform-id
                  :is_active true))
 
 (defn- timestamp-constraint

enterprise/backend/src/metabase_enterprise/transforms/models/transform_run_cancelation.clj

@@ -14,14 +14,10 @@
   ([run-id]
    (mark-cancel-started-run! run-id {}))
   ([run-id properties]
-   (t2/insert! :model/TransformRunCancelation
-               (assoc properties
-                      :run_id run-id)
-               {:where [:exists {:select [1]
-                                 :from   [:transform_run]
-                                 :where  [:and
-                                          [:= :transform_run.id run-id]
-                                          :transform_run.is_active]}]})))
+   (when (t2/exists? :transform_run :id run-id :is_active true)
+     (t2/insert! :model/TransformRunCancelation
+                 (assoc properties
+                        :run_id run-id)))))
 
 (defn reducible-canceled-local-runs
   "Return a reducible sequence of local canceled runs."

enterprise/backend/src/metabase_enterprise/transforms/ordering.clj

@@ -16,23 +16,29 @@
 (set! *warn-on-reflection* true)
 
 (defn- transform-deps [transform]
-  (let [query (-> (get-in transform [:source :query])
-                  transforms.util/massage-sql-query
-                  qp.preprocess/preprocess
-                  ;; legacy usage -- don't do things like this going forward
-                  #_{:clj-kondo/ignore [:discouraged-var]}
-                  lib/->legacy-MBQL)]
-    (case (:type query)
-      :native (driver/native-query-deps (-> (qp.store/metadata-provider)
-                                            lib.metadata/database
-                                            :engine)
-                                        (get-in query [:native :query]))
-      :query (into #{}
-                   (keep #(clojure.core.match/match %
-                            [:source-table source] (when (int? source)
-                                                     {:table source})
-                            _ nil))
-                   (tree-seq coll? seq query)))))
+  (if (transforms.util/python-transform? transform)
+    ;; Python transforms have dependencies via source-tables mapping
+    (into #{}
+          (map #(hash-map :table %))
+          (vals (get-in transform [:source :source-tables])))
+    ;; Query transforms have dependencies via SQL query analysis
+    (let [query (-> (get-in transform [:source :query])
+                    transforms.util/massage-sql-query
+                    qp.preprocess/preprocess
+                    ;; legacy usage -- don't do things like this going forward
+                    #_{:clj-kondo/ignore [:discouraged-var]}
+                    lib/->legacy-MBQL)]
+      (case (:type query)
+        :native (driver/native-query-deps (-> (qp.store/metadata-provider)
+                                              lib.metadata/database
+                                              :engine)
+                                          (get-in query [:native :query]))
+        :query (into #{}
+                     (keep #(clojure.core.match/match %
+                              [:source-table source] (when (int? source)
+                                                       {:table source})
+                              _ nil))
+                     (tree-seq coll? seq query))))))
 
 (defn- dependency-map [transforms]
   (into {}
@@ -56,24 +62,32 @@
 
   The result is a map of transform id -> #{transform ids the transform depends on}. Dependencies are limited to just
   the transforms in the original list -- if a transform depends on some transform not in the list, the 'extra'
-  dependency is ignored."
+  dependency is ignored. Both query and Python transforms can have dependencies on tables produced by other transforms."
   [transforms]
-  (let [transforms-by-db (->> transforms
+
+  (let [;; Group all transforms by their database
+        transforms-by-db (->> transforms
                               (map (fn [transform]
-                                     {(get-in transform [:source :query :database]) [transform]}))
+                                     (let [db-id (if (transforms.util/python-transform? transform)
+                                                   (get-in transform [:target :database])
+                                                   (get-in transform [:source :query :database]))]
+                                       {db-id [transform]})))
                               (apply merge-with into))
+
         transform-ids (into #{} (map :id) transforms)
         {:keys [output-tables dependencies]} (->> transforms-by-db
                                                   (map (fn [[db-id db-transforms]]
                                                          (qp.store/with-metadata-provider db-id
                                                            {:output-tables (output-table-map db-transforms)
                                                             :dependencies (dependency-map db-transforms)})))
-                                                  (apply merge-with merge))]
-    (update-vals dependencies #(into #{}
-                                     (keep (fn [{:keys [table transform]}]
-                                             (or (output-tables table)
-                                                 (transform-ids transform))))
-                                     %))))
+                                                  (apply merge-with merge))
+
+        all-deps (update-vals dependencies #(into #{}
+                                                  (keep (fn [{:keys [table transform]}]
+                                                          (or (output-tables table)
+                                                              (transform-ids transform))))
+                                                  %))]
+    all-deps))
 
 (defn find-cycle
   "Finds a path containing a cycle in the directed graph `node->children`.

enterprise/backend/src/metabase_enterprise/transforms/python_runner.clj

@@ -0,0 +1,442 @@
+(ns metabase-enterprise.transforms.python-runner
+  (:require
+   [clj-http.client :as http]
+   [clojure.core.async :as a]
+   [clojure.java.io :as io]
+   [clojure.string :as str]
+   [medley.core :as m]
+   [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
+   [metabase-enterprise.transforms.settings :as transforms.settings]
+   [metabase.driver :as driver]
+   [metabase.query-processor.compile :as qp.compile]
+   [metabase.query-processor.pipeline :as qp.pipeline]
+   ^{:clj-kondo/ignore [:discouraged-namespace]}
+   [metabase.query-processor.store :as qp.store]
+   [metabase.util.json :as json]
+   [metabase.util.log :as log]
+   [toucan2.core :as t2])
+  (:import
+   (clojure.lang IDeref)
+   (java.io BufferedWriter Closeable File OutputStream OutputStreamWriter)
+   (java.net URI)
+   (java.nio.charset StandardCharsets)
+   (java.time Duration)
+   (software.amazon.awssdk.auth.credentials AwsBasicCredentials StaticCredentialsProvider)
+   (software.amazon.awssdk.auth.credentials DefaultCredentialsProvider)
+   (software.amazon.awssdk.core.sync RequestBody)
+   (software.amazon.awssdk.regions Region)
+   (software.amazon.awssdk.services.s3 S3Client S3ClientBuilder S3Configuration)
+   (software.amazon.awssdk.services.s3.model DeleteObjectRequest GetObjectRequest NoSuchKeyException PutObjectRequest)
+   (software.amazon.awssdk.services.s3.presigner S3Presigner S3Presigner$Builder)
+   (software.amazon.awssdk.services.s3.presigner.model GetObjectPresignRequest PresignedGetObjectRequest PresignedPutObjectRequest PutObjectPresignRequest)))
+
+(set! *warn-on-reflection* true)
+
+(defn- safe-delete
+  "Safely delete a file."
+  [^File file]
+  (try (.delete file) (catch Exception _)))
+
+(defn- write-to-stream! [^OutputStream os col-names reducible-rows]
+  (let [writer (-> os
+                   (OutputStreamWriter. StandardCharsets/UTF_8)
+                   (BufferedWriter.))]
+
+    (run! (fn [row]
+            (let [row-map (zipmap col-names row)]
+              (json/encode-to row-map writer {})
+              (.newLine writer)))
+          reducible-rows)
+
+    (doto writer
+      (.flush)
+      (.close))))
+
+(defn- execute-mbql-query
+  [driver db-id query respond cancel-chan]
+  (driver/with-driver driver
+    (let [native (qp.compile/compile {:type :query, :database db-id :query query})
+
+          query  {:database db-id
+                  :type     :native
+                  :native   native}]
+      (qp.store/with-metadata-provider db-id
+        (binding [qp.pipeline/*canceled-chan* cancel-chan]
+          (driver/execute-reducible-query driver query {:canceled-chan cancel-chan} respond))))))
+
+(defn- root-type
+  [base-type]
+  (when base-type
+    (some #(when (isa? base-type %) %)
+          [:type/Float
+           :type/Integer
+           :type/Boolean
+           :type/DateTimeWithTZ
+           :type/DateTime
+           :type/Dictionary
+           :type/JSON
+           :type/Date
+           :type/Text])))
+
+(defn- closest-ancestor [t pred]
+  (loop [remaining (conj clojure.lang.PersistentQueue/EMPTY [t])]
+    (when-let [t (first remaining)]
+      (if (pred t)
+        t
+        (recur (into (pop t) (parents t)))))))
+
+(defn- effective-semantic-type-i-think
+  "Kinda sketchy but maybe reasonable way to infer the effective-semantic-type"
+  [{:keys [base_type effective_type semantic_type]}]
+  (or semantic_type (closest-ancestor (or effective_type base_type) #(isa? % :Semantic/*))))
+
+(defn- generate-manifest
+  "Generate a metadata manifest for the table columns."
+  [table-id cols-meta]
+  {:version        "0.1.0"
+   :fields         (mapv (fn [col-meta]
+                           {:name           (:name col-meta)
+                            :base_type      (some-> (:base_type col-meta) name)
+                            :database_type  (some-> (:database_type col-meta) name)
+                            :root_type      (some-> (root-type (:base_type col-meta)) name)
+                            ;; replace nil values with values indicating how they behave in practice.
+                            ;; there may be better ways of doing this already, but i worry it's just implicit in QP
+                            :semantic_type  (some-> (effective-semantic-type-i-think col-meta) name)
+                            :effective_type (some-> (or (:effective_type col-meta) (:database_type col-meta)) name)
+                            ;; TODO get this passed through
+                            :field_id       (:id col-meta)})
+                         cols-meta)
+   :table_metadata {:table_id table-id}})
+
+(defn- maybe-fixup-v [col v]
+  ;; clickhouse returns bigdecimals for int64 values
+  (if (and (isa? (:base_type col) :type/Integer)
+           (or (instance? BigDecimal v)
+               (float? v)))
+    (bigint v)
+    v))
+
+(defn- write-table-data-to-file! [id temp-file cancel-chan]
+  (let [db-id           (t2/select-one-fn :db_id (t2/table-name :model/Table) :id id)
+        driver          (t2/select-one-fn :engine :model/Database db-id)
+        all-fields-meta (t2/select [:model/Field :id :name :base_type :effective_type :semantic_type :database_type :database_position :nfc_path :parent_id]
+                                   :table_id id
+                                   :active true
+                                   {:order-by [[:database_position :asc]]})
+        fields-meta (filter #(and (nil? (:parent_id %)) (nil? (:nfc_path %))) all-fields-meta)
+        query {:source-table id}
+        manifest (generate-manifest id fields-meta)]
+    (execute-mbql-query driver db-id query
+                        (fn [{cols-meta :cols} reducible-rows]
+                          (with-open [os (io/output-stream temp-file)]
+                            (let [filtered-col-meta (m/index-by :name fields-meta)
+                                  filtered-cols (filter #(contains? filtered-col-meta (:name %)) cols-meta)
+                                  col-name->index (into {} (map-indexed (fn [i col] [(:name col) i]) cols-meta))
+                                  filtered-indices (mapv (fn [col] [(col-name->index (:name col)) col]) filtered-cols)
+                                  filtered-rows (eduction (map (fn [row] (mapv (fn [[idx col]]
+                                                                                 (let [v (nth row idx)]
+                                                                                   (maybe-fixup-v col v))) filtered-indices)))
+                                                          reducible-rows)]
+                              (write-to-stream! os (mapv :name filtered-cols) filtered-rows))))
+                        cancel-chan)
+    manifest))
+
+(defn- maybe-with-endpoint-s3-client [^S3ClientBuilder builder endpoint]
+  (let [region (transforms.settings/python-storage-s-3-region)]
+    (case [(some? endpoint) (some? region)]
+      [true true]   (doto builder
+                      (.endpointOverride (URI/create endpoint))
+                      (.region (Region/of region)))
+      [true false]  (log/warn "Ignoring endpoint because region is not defined")
+      [false true]  (.region builder (Region/of region))
+      [false false] builder)))
+
+(defn- maybe-with-endpoint-s3-presigner [^S3Presigner$Builder builder endpoint]
+  (let [region (transforms.settings/python-storage-s-3-region)]
+    (case [(some? endpoint) (some? region)]
+      [true true]   (doto builder
+                      (.endpointOverride (URI/create endpoint))
+                      (.region (Region/of region)))
+      [true false]  (log/warn "Ignoring endpoint because region is not defined")
+      [false true]  (.region builder (Region/of region))
+      [false false] (.region builder (Region/US_EAST_1)))))
+
+(defn- create-s3-configuration
+  "Create S3Configuration with path-style access setting"
+  ^S3Configuration []
+  (-> (S3Configuration/builder)
+      (.pathStyleAccessEnabled (transforms.settings/python-storage-s-3-path-style-access))
+      (.build)))
+
+(defn- build-put-object-request
+  "Build a PutObjectRequest with bucket and key"
+  [^String bucket-name ^String key]
+  (-> (PutObjectRequest/builder)
+      (.bucket bucket-name)
+      (.key key)
+      (.build)))
+
+(defn- build-get-object-request
+  "Build a GetObjectRequest with bucket and key"
+  [^String bucket-name ^String key]
+  (-> (GetObjectRequest/builder)
+      (.bucket bucket-name)
+      (.key key)
+      (.build)))
+
+(defn- build-delete-object-request
+  "Build a DeleteObjectRequest with bucket and key"
+  [^String bucket-name ^String key]
+  (-> (DeleteObjectRequest/builder)
+      (.bucket bucket-name)
+      (.key key)
+      (.build)))
+
+(def ^:private ^Duration presigned-url-duration
+  "Default duration for presigned URLs"
+  (Duration/ofHours 1))
+
+(defn- maybe-with-credentials-s3-client [^S3ClientBuilder builder]
+  (let [access-key (transforms.settings/python-storage-s-3-access-key)
+        secret-key (transforms.settings/python-storage-s-3-secret-key)]
+    (if (or access-key secret-key)
+      (if-not (and access-key secret-key)
+        (do (log/warnf "Ignoring %s because %s is not defined"
+                       (if access-key "access-key" "secret-key")
+                       (if (not access-key) "access-key" "secret-key"))
+            (.credentialsProvider builder (DefaultCredentialsProvider/create)))
+        (.credentialsProvider builder
+                              (StaticCredentialsProvider/create
+                               (AwsBasicCredentials/create access-key secret-key))))
+      (.credentialsProvider builder (DefaultCredentialsProvider/create)))))
+
+(defn- maybe-with-credentials-s3-presigner [^S3Presigner$Builder builder]
+  (let [access-key (transforms.settings/python-storage-s-3-access-key)
+        secret-key (transforms.settings/python-storage-s-3-secret-key)]
+    (if (or access-key secret-key)
+      (if-not (and access-key secret-key)
+        (do (log/warnf "Ignoring %s because %s is not defined"
+                       (if access-key "access-key" "secret-key")
+                       (if (not access-key) "access-key" "secret-key"))
+            builder)
+        (.credentialsProvider builder
+                              (StaticCredentialsProvider/create
+                               (AwsBasicCredentials/create access-key secret-key))))
+      builder)))
+
+;; We just recreate the client every time, to keep things simple if config is changed.
+(defn- create-s3-client
+  "Create S3 client for host operations (uploads, reads)"
+  ^S3Client []
+  (let [^S3ClientBuilder builder (S3Client/builder)]
+    (doto builder
+      (maybe-with-endpoint-s3-client (transforms.settings/python-storage-s-3-endpoint))
+      maybe-with-credentials-s3-client
+      (.serviceConfiguration (create-s3-configuration)))
+    (.build builder)))
+
+(defn- create-s3-presigner-for-container
+  "Create S3 presigner for container operations (presigned URLs).
+  Uses container-endpoint if different from host endpoint."
+  ^S3Presigner []
+  (let [container-endpoint (transforms.settings/python-storage-s-3-container-endpoint)
+        builder (S3Presigner/builder)]
+    (doto builder
+      (maybe-with-endpoint-s3-presigner (or container-endpoint (transforms.settings/python-storage-s-3-endpoint)))
+      maybe-with-credentials-s3-presigner
+      (.serviceConfiguration (create-s3-configuration)))
+    (.build builder)))
+
+(defn- upload-file-to-s3
+  "Upload file using host client"
+  [^S3Client s3-client ^String bucket-name ^String key ^File file]
+  (let [^PutObjectRequest request (build-put-object-request bucket-name key)]
+    (.putObject s3-client request (RequestBody/fromFile file))))
+
+(defn- generate-presigned-get-url
+  "Generate GET URL using container presigner"
+  [^S3Presigner presigner ^String bucket-name ^String key]
+  (let [^GetObjectRequest get-request (build-get-object-request bucket-name key)
+        ^GetObjectPresignRequest presign-request (-> (GetObjectPresignRequest/builder)
+                                                     (.signatureDuration presigned-url-duration)
+                                                     (.getObjectRequest get-request)
+                                                     (.build))
+        presigned ^PresignedGetObjectRequest (.presignGetObject presigner presign-request)]
+    (.toString (.url presigned))))
+
+(defn- generate-presigned-put-url
+  "Generate PUT URL using container presigner"
+  [^S3Presigner presigner ^String bucket-name ^String key]
+  (let [^PutObjectRequest put-request (build-put-object-request bucket-name key)
+        ^PutObjectPresignRequest presign-request (-> (PutObjectPresignRequest/builder)
+                                                     (.signatureDuration presigned-url-duration)
+                                                     (.putObjectRequest put-request)
+                                                     (.build))
+        presigned ^PresignedPutObjectRequest (.presignPutObject presigner presign-request)]
+    (.toString (.url presigned))))
+
+(defn- delete-s3-object [^S3Client s3-client ^String bucket-name ^String key]
+  (try
+    (let [^DeleteObjectRequest request (build-delete-object-request bucket-name key)]
+      (.deleteObject s3-client request))
+    (catch Exception e
+      (log/debugf e "Error deleting s3 object %s" key)
+      ;; Ignore deletion errors - object might not exist, or we might not have permissions
+      ;; NOTE: we plan to put general retention on the bucket so that objects will eventually be deleted
+      nil)))
+
+(defn- cleanup-s3-objects [^S3Client s3-client bucket-name s3-keys]
+  (run! (partial delete-s3-object s3-client bucket-name) s3-keys))
+
+(defn- read-from-s3
+  ([s3-client bucket-name key] (read-from-s3 s3-client bucket-name key ::throw))
+  ([^S3Client s3-client ^String bucket-name ^String key not-found]
+   (try
+     (let [^GetObjectRequest request (build-get-object-request bucket-name key)
+           response                  (.getObject s3-client request)]
+       (slurp response))
+     (catch NoSuchKeyException e
+       (if (identical? ::throw not-found)
+         (throw e)
+         not-found)))))
+
+(defn get-logs
+  "Return the logs of the current running python process"
+  [run-id]
+  (let [server-url (transforms.settings/python-execution-server-url)]
+    (http/get (str server-url "/logs")
+              {:content-type     :json
+               :accept           :json
+               :throw-exceptions false
+               :as               :json
+               :query-params     {:request_id run-id}})))
+
+(defn- s3-shared-storage [table-name->id]
+  (let [prefix              (some-> (transforms.settings/python-storage-s-3-prefix) (str "/"))
+        work-dir-name       (str prefix "run-" (System/nanoTime) "-" (rand-int 10000))
+        container-presigner (create-s3-presigner-for-container)
+        bucket-name         (transforms.settings/python-storage-s-3-bucket)
+        loc
+        (fn [method relative-path]
+          ;; save a tracking root to a well known location
+          ;; later can ls these to collect those objects whose ttl has elapsed
+          ;; (OR setup a retention policy in s3?)
+          (let [path (str work-dir-name "/" relative-path)]
+            {:path   path
+             :method method
+             :url    (case method :put (generate-presigned-put-url container-presigner bucket-name path)
+                           :get (generate-presigned-get-url container-presigner bucket-name path))}))]
+    {:s3-client   (create-s3-client)                        ;; do not like mixing interactive things with descriptions, but its damn convenient to have it here for now
+     :bucket-name bucket-name
+     :objects
+     (into
+      {:output          (loc :put "output.csv")
+       :output-manifest (loc :put "output-manifest.json")
+       :events          (loc :put "events.jsonl")}
+      (for [[table-name id] table-name->id]
+        {[:table id :manifest] (loc :get (str "-table-" (name table-name) "-" id ".manifest.json"))
+         [:table id :data]     (loc :get (str "-table-" (name table-name) "-" id ".jsonl"))}))}))
+
+(defn open-s3-shared-storage!
+  "Returns a deref'able shared storage value, (.close) will optimistically delete any s3 objects named in storage (data files for tables, metadata files etc).
+  The intention is the bucket specifies a generic object retention policy to ensure objects are eventually deleted (e.g. because the process dies during writing and .close never gets called)"
+  ^Closeable [table-name->id]
+  (let [shared-storage (s3-shared-storage table-name->id)]
+    (reify IDeref
+      (deref [_] shared-storage)
+      Closeable
+      (close [_] (cleanup-s3-objects (:s3-client shared-storage) (:bucket-name shared-storage) (map :path (vals (:objects shared-storage))))))))
+
+(defn copy-tables-to-s3!
+  "Writes table content to their corresponding objects named in shared-storage, see (open-shared-storage!).
+  Blocks until all tables are fully written and committed to shared storage."
+  [{:keys [run-id
+           shared-storage
+           table-name->id
+           cancel-chan]}]
+  (doseq [id (vals table-name->id)
+          :let [{:keys [s3-client bucket-name objects]} shared-storage
+                {data-path :path}     (get objects [:table id :data])
+                {manifest-path :path} (get objects [:table id :manifest])]]
+    (let [temp-file       (File/createTempFile data-path "")
+          manifest-file   (File/createTempFile manifest-path "")]
+      (try
+        ;; Write table data to temporary file and get manifest
+        (let [manifest (transforms.instrumentation/with-stage-timing [run-id :data-transfer :dwh-to-file]
+                         (write-table-data-to-file! id temp-file cancel-chan))]
+          ;; Write manifest to file
+          (with-open [writer (io/writer manifest-file)]
+            (json/encode-to manifest writer {}))
+          (let [file-size (.length temp-file)
+                manifest-size (.length manifest-file)]
+            (transforms.instrumentation/record-data-transfer! run-id :dwh-to-file file-size nil)
+
+            ;; Upload both files to S3
+            (transforms.instrumentation/with-stage-timing [run-id :data-transfer :file-to-s3]
+              (upload-file-to-s3 s3-client bucket-name data-path temp-file)
+              (upload-file-to-s3 s3-client bucket-name manifest-path manifest-file))
+
+            (transforms.instrumentation/record-data-transfer! run-id :file-to-s3 (+ file-size manifest-size) nil)))
+        (finally
+          ;; Clean up temporary files
+          (safe-delete temp-file)
+          (safe-delete manifest-file))))))
+
+(defn execute-python-code-http-call!
+  "Calls the /execute endpoint of the python runner. Blocks until the run either succeeds or fails and returns
+  the response from the server."
+  [{:keys [server-url code run-id table-name->id shared-storage]}]
+  (let [{:keys [objects]} shared-storage
+        {:keys [output output-manifest events]} objects
+
+        table-name->url          (update-vals table-name->id (comp :url #(get objects [:table % :data])))
+        table-name->manifest-url (update-vals table-name->id (comp :url #(get objects [:table % :manifest])))
+
+        payload                  {:code                code
+                                  :timeout             30
+                                  :request_id          run-id
+                                  :output_url          (:url output)
+                                  :output_manifest_url (:url output-manifest)
+                                  :events_url          (:url events)
+                                  :table_mapping       table-name->url
+                                  :manifest_mapping    table-name->manifest-url}
+
+        response                 (transforms.instrumentation/with-python-api-timing [run-id]
+                                   (http/post (str server-url "/execute")
+                                              {:content-type     :json
+                                               :accept           :json
+                                               :body             (json/encode payload)
+                                               :throw-exceptions false
+                                               :as               :json}))]
+    ;; when a 500 is returned we observe a string in the body (despite the python returning json)
+    ;; always try to parse the returned string as json before yielding (could tighten this up at some point)
+    (update response :body (fn [string-if-error]
+                             (if (string? string-if-error)
+                               (try
+                                 (json/decode+kw string-if-error)
+                                 (catch Exception _
+                                   {:error string-if-error}))
+                               string-if-error)))))
+
+(defn open-cancellation-process!
+  "Starts a core.async process that optimistically sends a cancellation request to the python executor if cancel-chan receives a value.
+  Returns a channel that will receive either the async http call j.u.c.FutureTask in the case of cancellation, or nil when the cancel-chan is closed."
+  [server-url run-id cancel-chan]
+  (a/go (when (a/<! cancel-chan)
+          (http/post (str server-url "/cancel")
+                     {:content-type :json
+                      :body         (json/encode {:request_id run-id})
+                      :async?       true}
+                     identity identity))))
+
+;; temporary, we should not need to realize data/events files into memory longer term
+(defn read-output-objects
+  "Temporary function that strings/jsons stuff in S3 and returns it for compatibility."
+  [{:keys [s3-client bucket-name objects]}]
+  (let [{:keys [output output-manifest events]} objects
+        output-content          (read-from-s3 s3-client bucket-name (:path output) nil)
+        output-manifest-content (read-from-s3 s3-client bucket-name (:path output-manifest) "{}")
+        events-content          (read-from-s3 s3-client bucket-name (:path events))]
+    {:output output-content
+     :output-manifest (json/decode+kw output-manifest-content)
+     :events (mapv json/decode+kw (str/split-lines events-content))}))

enterprise/backend/src/metabase_enterprise/transforms/schedule.clj

@@ -101,6 +101,7 @@
         trigger (build-trigger job-id schedule)]
     (task/schedule-task! job trigger)))
 
+;; TODO: should this be part of transformjob hooks?
 (defn update-job!
   "Update the trigger for a transform job."
   [job-id schedule]

enterprise/backend/src/metabase_enterprise/transforms/settings.clj

@@ -1,17 +1,119 @@
 (ns metabase-enterprise.transforms.settings
   (:require
+   [metabase.config.core :as config]
    [metabase.settings.core :as setting]
    [metabase.util.i18n :refer [deferred-tru]]))
 
 (set! *warn-on-reflection* true)
 
+(def ^:private is-not-prod? (or config/is-dev? config/is-test?))
+
 (setting/defsetting transform-timeout
   (deferred-tru "The timeout for a transform job, in minutes.")
-  :type       :integer
+  :type :integer
   :visibility :internal
-  :default    (* 4 60)
-  :feature    :transforms
-  :doc        false
-  :export?    false
+  :default (* 4 60)
+  :feature :transforms
+  :doc false
+  :export? false
   :encryption :no
-  :audit      :getter)
+  :audit :getter)
+
+(setting/defsetting python-execution-server-url
+  (deferred-tru "URL for the Python execution server that runs transform functions.")
+  :type :string
+  :visibility :admin
+  :default (when is-not-prod? "http://localhost:5001")
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-endpoint
+  (deferred-tru "S3 endpoint URL for storing Python execution artifacts.")
+  :type :string
+  :visibility :admin
+  :default (when is-not-prod? "http://localhost:4566")
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-region
+  (deferred-tru "AWS region for S3 storage.")
+  :type :string
+  :visibility :admin
+  :default (when is-not-prod? "us-east-1")
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-bucket
+  (deferred-tru "S3 bucket name for storing Python execution artifacts.")
+  :type :string
+  :visibility :admin
+  :default (if is-not-prod? "metabase-python-runner" "metabase-cloud-python-runner-staging")
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-prefix
+  (deferred-tru "Prefix to use for S3 objects. Necessary in prod where we only have access to a particular prefix.")
+  :type :string
+  :visibility :admin
+  :feature :transforms
+  :default (if is-not-prod? "test-prefix" "hosting-4b9351bb22b84791")
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-access-key
+  (deferred-tru "AWS access key ID for S3 authentication.")
+  :type :string
+  :visibility :admin
+  :feature :transforms
+  :default (when is-not-prod? "test")
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-secret-key
+  (deferred-tru "AWS secret access key for S3 authentication.")
+  :type :string
+  :visibility :admin
+  :feature :transforms
+  :default (when is-not-prod? "test")
+  :doc false
+  :export? false
+  :encryption :when-encryption-key-set
+  :audit :never)
+
+(setting/defsetting python-storage-s-3-container-endpoint
+  (deferred-tru "Alternative S3 endpoint accessible from containers. Leave empty if same as main endpoint.")
+  :type :string
+  :visibility :admin
+  :default (when is-not-prod? "http://localstack:4566")
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-path-style-access
+  (deferred-tru "Use path-style access for S3 requests (required for LocalStack and some S3-compatible services).")
+  :type :boolean
+  :visibility :admin
+  :default (when is-not-prod? true)
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)

enterprise/backend/src/metabase_enterprise/transforms/util.clj

@@ -1,12 +1,16 @@
 (ns metabase-enterprise.transforms.util
   (:require
+   [clojure.string :as str]
    [java-time.api :as t]
    [metabase.driver :as driver]
+   [metabase.lib.schema.common :as lib.schema.common]
    [metabase.query-processor.compile :as qp.compile]
    [metabase.sync.core :as sync]
    [metabase.util :as u]
    [metabase.util.date-2 :as u.date]
    [metabase.util.log :as log]
+   [metabase.util.malli :as mu]
+   [metabase.util.malli.registry :as mr]
    [toucan2.core :as t2])
   (:import
    (java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)
@@ -21,10 +25,22 @@
     (keyword schema name)
     (keyword name)))
 
+(defn query-transform?
+  "Check if this is a query transform: native query / mbql query."
+  [transform]
+  (= :query (-> transform :source :type keyword)))
+
+(defn python-transform?
+  "Check if this is a Python transform."
+  [transform]
+  (= :python (-> transform :source :type keyword)))
+
 (defn target-table-exists?
   "Test if the target table of a transform already exists."
   [{:keys [source target] :as _transform}]
-  (let [db-id (-> source :query :database)
+  (let [db-id (or (-> source :query :database)
+                  ;; python transform target
+                  (-> target :database))
         {driver :engine :as database} (t2/select-one :model/Database db-id)]
     (driver/table-exists? driver database target)))
 
@@ -66,7 +82,9 @@
   [{:keys [id target source], :as _transform}]
   (when target
     (let [target (update target :type keyword)
-          database-id (-> source :query :database)
+          database-id (or (-> source :query :database)
+                          ;; python transform target
+                          (-> target :database))
           {driver :engine :as database} (t2/select-one :model/Database database-id)]
       (driver/drop-transform-target! driver database target)
       (log/info "Deactivating  target " (pr-str target) "for transform" id)
@@ -122,3 +140,60 @@
   (-> run
       (u/update-some :start_time utc-timestamp-string)
       (u/update-some :end_time   utc-timestamp-string)))
+
+(mr/def ::column-definition
+  [:map
+   [:name :string]
+   [:type ::lib.schema.common/base-type]
+   [:nullable? {:optional true} :boolean]])
+
+(mr/def ::table-definition
+  [:map
+   [:name :keyword]
+   [:columns [:sequential ::column-definition]]
+   [:primary-key {:optional true} [:sequential :string]]])
+
+(defn dtype->base-type
+  "Maps pandas dtype strings directly to Metabase base types in the type hierarchy."
+  [dtype-str]
+  (cond
+    (str/starts-with? dtype-str "int") :type/Integer
+    (str/starts-with? dtype-str "float") :type/Float
+    (str/starts-with? dtype-str "bool") :type/Boolean
+    ;; datetime64[ns, timezone] indicates timezone-aware datetime
+    (str/starts-with? dtype-str "datetime64[ns, ") :type/DateTimeWithTZ
+    (str/starts-with? dtype-str "datetime") :type/DateTime
+    ;; this is not a real dtype, pandas uses 'object', but we override it if there's source or custom field metadata
+    (str/starts-with? dtype-str "date") :type/Date
+    :else :type/Text))
+
+(mu/defn create-table-from-schema!
+  "Create a table from a table-schema"
+  [driver :- :keyword
+   database-id :- pos-int?
+   table-schema :- ::table-definition]
+  (let [{:keys [columns] table-name :name} table-schema
+        column-definitions (into {} (map (fn [{:keys [name type database-type]}]
+                                           (let [database-type (when-not (and (= driver :bigquery-cloud-sdk)
+                                                                              (#{"ARRAY" "RECORD"} database-type))
+                                                                 database-type)
+                                                 db-type (if database-type
+                                                           [[:raw database-type]]
+                                                           (driver/type->database-type driver type))]
+                                             [name db-type])))
+                                 columns)
+        primary-key-opts (select-keys table-schema [:primary-key])]
+    (log/infof "Creating table %s with %d columns" table-name (count columns))
+    (driver/create-table! driver database-id table-name column-definitions primary-key-opts)))
+
+(defn rename-table!
+  "kename a table in the database."
+  [driver database-id old-table-name new-table-name]
+  (log/infof "Renaming table %s to %s" old-table-name new-table-name)
+  (driver/rename-table! driver database-id old-table-name new-table-name))
+
+(defn drop-table!
+  "Drop a table in the database."
+  [driver database-id table-name]
+  (log/infof "Dropping table %s" table-name)
+  (driver/drop-table! driver database-id table-name))

enterprise/backend/test/metabase_enterprise/transforms/api_test.clj

@@ -8,8 +8,9 @@
    [metabase-enterprise.transforms.models.transform-transform-tag]
    [metabase-enterprise.transforms.query-test-util :as query-test-util]
    [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
-   [metabase-enterprise.transforms.test-util :refer [parse-instant with-transform-cleanup! utc-timestamp]]
+   [metabase-enterprise.transforms.test-util :as transforms.tu :refer [parse-instant with-transform-cleanup! utc-timestamp]]
    [metabase-enterprise.transforms.util :as transforms.util]
+   [metabase.driver :as driver]
    [metabase.legacy-mbql.normalize :as mbql.normalize]
    [metabase.lib.convert :as lib.convert]
    [metabase.lib.core :as lib]
@@ -72,6 +73,26 @@
                                             :schema schema
                                             :name   table-name}})))))))
 
+(deftest create-python-transform-test
+  (mt/with-premium-features #{:transforms}
+    (mt/dataset transforms-dataset/transforms-test
+      (with-transform-cleanup! [table-name "gadget_products"]
+        (let [schema    (get-test-schema)
+              transform-payload {:name   "My beautiful python runner"
+                                 :source {:type            "python"
+                                          :body            "print('hello world')"
+                                          :source-tables   {}}
+                                 :target {:type     "table"
+                                          :schema   schema
+                                          :name     table-name
+                                          :database (mt/id)}}
+              transform (mt/user-http-request :crowberto :post 200 "ee/transform"
+                                              transform-payload)]
+          (is (= "print('hello chris')"
+                 (-> (mt/user-http-request :crowberto :put 200 (format "ee/transform/%s" (:id transform))
+                                           (assoc-in transform-payload [:source :body] "print('hello chris')"))
+                     :source :body))))))))
+
 (deftest list-transforms-test
   (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
     (testing "Can list without query parameters"
@@ -319,6 +340,32 @@
                 (is (true? (transforms.util/target-table-exists? updated)))
                 (check-query-results table2-name [2 3 4 13] "Doohickey")))))))))
 
+(deftest execute-python-transform-test
+  (testing "transform execution with :transforms/table target"
+    (mt/test-drivers #{:h2 :postgres}
+      (mt/with-premium-features #{:transforms}
+        (mt/dataset transforms-dataset/transforms-test
+          (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
+            (with-transform-cleanup! [{table-name :name :as target} {:type   "table"
+                                                                     :schema schema
+                                                                     :name   "target_table"}]
+              (let [original           {:name   "Gadget Products"
+                                        :source {:type  "python"
+                                                 :source-database (mt/id)
+                                                 :source-tables {"transforms_customers" (mt/id :transforms_customers)}
+                                                 :body  (str "import pandas as pd\n"
+                                                             "\n"
+                                                             "def transform():\n"
+                                                             "    return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}
+                                        :target  (assoc target :database (mt/id))}
+                    {transform-id :id} (mt/user-http-request :crowberto :post 200 "ee/transform"
+                                                             original)]
+                (test-run transform-id)
+                (wait-for-table table-name 5000)
+                (is (true? (driver/table-exists? driver/*driver* (mt/db) target)))
+                (is (= [["Alice" 25] ["Bob" 30]]
+                       (transforms.tu/table-rows table-name)))))))))))
+
 (deftest get-runs-filter-by-single-transform-id-test
   (testing "GET /api/ee/transform/run - filter by single transform ID"
     (mt/with-premium-features #{:transforms}

enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj

@@ -5,7 +5,8 @@
    [metabase-enterprise.transforms.execute :as transforms.execute]
    [metabase-enterprise.transforms.query-test-util :as query-test-util]
    [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
-   [metabase-enterprise.transforms.test-util :refer [with-transform-cleanup!]]
+   [metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]
+   [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.driver :as driver]
    [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
    [metabase.lib.core :as lib]
@@ -13,6 +14,7 @@
    [metabase.query-processor :as qp]
    [metabase.sync.core :as sync]
    [metabase.test :as mt]
+   [metabase.test.data.sql :as sql.tx]
    [toucan2.core :as t2]
    [toucan2.util :as u]))
 
@@ -130,6 +132,24 @@
                           ["2024-01-23T00:00:00Z" 14.99]]
                          query-result)))))))))))
 
+(deftest create-table-from-schema!-test
+  (mt/test-drivers (mt/normal-drivers-with-feature :uploads)
+    (let [driver       driver/*driver*
+          db-id        (mt/id)
+          table-name   (mt/random-name)
+          schema-name  (sql.tx/session-schema driver)
+          table-schema {:name    (if schema-name
+                                   (keyword schema-name table-name)
+                                   (keyword table-name))
+                        :columns [{:name "id" :type :type/Integer :nullable? false}
+                                  {:name "name" :type :type/Text :nullable? true}]}]
+      (mt/as-admin
+        (testing "create-table-from-schema! should create the table successfully"
+          (transforms.util/create-table-from-schema! driver db-id table-schema)
+          (let [table-exists? (driver/table-exists? driver db-id {:schema schema-name :name table-name})]
+            (is (some? table-exists?) "Table should exist in the database schema")
+            (driver/drop-table! driver db-id (:name table-schema))))))))
+
 (deftest transform-schema-created-if-needed-test
   (mt/test-drivers (mt/normal-driver-select {:+features [:transforms/table :schemas]})
     (mt/dataset transforms-dataset/transforms-test
@@ -224,3 +244,98 @@
             (driver/execute-raw-queries! driver/*driver* spec
                                          [[(format "DROP OWNED BY %s;" no-schema-user)]
                                           [(format "DROP ROLE IF EXISTS %s;" no-schema-user)]])))))))
+
+(deftest atomic-python-transform-swap-test
+  (testing "Python transform execution with atomic table swap"
+    (mt/test-drivers #{:h2 :postgres}
+      (mt/with-premium-features #{:transforms}
+        (mt/dataset transforms-dataset/transforms-test
+          (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
+            (with-transform-cleanup! [{table-name :name :as target} {:type   "table"
+                                                                     :schema schema
+                                                                     :name   "atomic_python_swap"}]
+              (let [initial-transform {:name   "Python Transform Initial"
+                                       :source {:type  "python"
+                                                :source-tables {}
+                                                :body  (str "import pandas as pd\n"
+                                                            "\n"
+                                                            "def transform():\n"
+                                                            "    return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}
+                                       :target (assoc target :database (mt/id))}]
+                (mt/with-temp [:model/Transform transform initial-transform]
+                  (transforms.execute/execute-python-transform! transform {:run-method :manual})
+                  (wait-for-table table-name 10000)
+
+                  (let [initial-rows (transforms.tu/table-rows table-name)]
+                    (is (= [["Alice" 25] ["Bob" 30]] initial-rows) "Initial data should be Alice and Bob")
+
+                    (t2/update! :model/Transform (:id transform)
+                                {:source {:type "python"
+                                          :source-tables {}
+                                          :body (str "import pandas as pd\n"
+                                                     "\n"
+                                                     "def transform():\n"
+                                                     "    return pd.DataFrame({'name': ['Charlie', 'Diana', 'Eve'], 'age': [35, 40, 45]})")}}))
+
+                  ;; Test atomic swap by intercepting drop operation
+                  (let [drop-latch (java.util.concurrent.CountDownLatch. 1)
+                        original-drop-table! transforms.util/drop-table!]
+                    (with-redefs [transforms.util/drop-table! (fn [driver db-id table-name]
+                                                                ;; Wait for latch before proceeding with drop
+                                                                (.await drop-latch)
+                                                                (original-drop-table! driver db-id table-name))]
+                      ;; Start transform in background
+                      (let [transform-future (future
+                                               (transforms.execute/execute-python-transform!
+                                                (t2/select-one :model/Transform (:id transform))
+                                                {:run-method :manual}))]
+                        ;; Give transform time to create temp table and reach the drop operation
+                        (Thread/sleep 1000)
+                        ;; Verify old table is still accessible during transform
+                        (is (= [["Alice" 25] ["Bob" 30]]
+                               (transforms.tu/table-rows table-name))
+                            "Original data should still be accessible during transform")
+                        ;; Release the latch to allow drop and complete transform
+                        (.countDown drop-latch)
+                        ;; Wait for transform to complete
+                        @transform-future
+                        ;; Verify final state
+                        (is (= [["Charlie" 35] ["Diana" 40] ["Eve" 45]]
+                               (transforms.tu/table-rows table-name))
+                            "Table should now contain Charlie, Diana, and Eve")))))))))))))
+
+(deftest python-transform-temp-table-cleanup-test
+  (testing "Python transform cleans up temp tables on success"
+    (mt/test-drivers #{:h2 :postgres}
+      (mt/with-premium-features #{:transforms}
+        (mt/dataset transforms-dataset/transforms-test
+          (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
+            (with-transform-cleanup! [{table-name :name :as target} {:type   "table"
+                                                                     :schema schema
+                                                                     :name   "python_cleanup_test"}]
+              (let [transform-def {:name   "Python Transform Cleanup"
+                                   :source {:type  "python"
+                                            :source-tables {}
+                                            :body  (str "import pandas as pd\n"
+                                                        "\n"
+                                                        "def transform():\n"
+                                                        "    return pd.DataFrame({'col1': [1, 2, 3], 'col2': ['a', 'b', 'c']})")}
+                                   :target (assoc target :database (mt/id))}]
+                (mt/with-temp [:model/Transform transform transform-def]
+                  ;; Run initial transform
+                  (transforms.execute/execute-python-transform! transform {:run-method :manual})
+                  (wait-for-table table-name 10000)
+
+                  ;; Run again to trigger temp table creation and swap
+                  (transforms.execute/execute-python-transform! transform {:run-method :manual})
+
+                  ;; Check that no temp tables remain
+                  (let [db-id (mt/id)
+                        tables (t2/select :model/Table :db_id db-id :active true)
+                        temp-table-pattern (re-pattern (str ".*" table-name "_temp_.*"))]
+                    (is (not-any? #(re-matches temp-table-pattern (:name %)) tables)
+                        "No temp tables should remain after successful Python transform")
+
+                    ;; Verify the final data is correct
+                    (is (= [[1 "a"] [2 "b"] [3 "c"]] (transforms.tu/table-rows table-name))
+                        "Table should contain the expected data after swap")))))))))))

enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj

@@ -21,6 +21,20 @@
               :name name
               :type :table}}))
 
+(defn- make-python-transform [source-tables & [name schema target-db]]
+  (let [name (or name (mt/random-name))
+        default-schema (some-> driver/*driver* driver.sql/default-schema)
+        schema (or schema default-schema "public")
+        target-db (or target-db (mt/id))]
+    {:source {:type "python"
+              :source-tables source-tables
+              :body "# Python code here\nresult = df"}
+     :name (str "python_transform_" name)
+     :target {:database target-db
+              :schema schema
+              :name name
+              :type "table"}}))
+
 (deftest basic-ordering-test
   (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
     (mt/with-temp [:model/Transform {t1 :id} (make-transform
@@ -354,4 +368,105 @@
                                                 "table_1")]
         (is (= {:cycle-str "transform_table_2 -> transform_table_1 -> transform_table_3",
                 :cycle [t1 t3 t2]}
-               (ordering/get-transform-cycle (t2/select-one :model/Transform :id t1))))))))
+               (ordering/get-transform-cycle (t2/select-one :model/Transform :id t1)))))))
+
+  (deftest python-transform-basic-dependencies-test
+    (testing "Python transforms with source-tables dependencies are extracted correctly"
+      (mt/with-temp [:model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)
+                                                                       "products" (mt/id :products)})]
+        (is (= #{{:table (mt/id :orders)}
+                 {:table (mt/id :products)}}
+               (transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))
+
+  (deftest python-transform-ordering-test
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+      (testing "Python transforms are ordered correctly based on source-tables dependencies"
+        (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                  :name "output_1"}
+                       :model/Field _ {:table_id table1
+                                       :name "foo"}
+                       :model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)} "output_1")
+                       :model/Transform {t2 :id} (make-python-transform {"output_1" table1} "output_2")]
+          (is (= {t1 #{}
+                  t2 #{t1}}
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2]]))))))))
+
+  (deftest python-transform-multiple-dependencies-test
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+      (testing "Python transforms with multiple table dependencies"
+        (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                  :name "output_1"}
+                       :model/Field _ {:table_id table1
+                                       :name "foo"}
+                       :model/Table {table2 :id} {:schema "public"
+                                                  :name "output_2"}
+                       :model/Field _ {:table_id table2
+                                       :name "bar"}
+                       :model/Transform {t1 :id} (make-python-transform {"orders" (mt/id :orders)} "output_1")
+                       :model/Transform {t2 :id} (make-python-transform {"products" (mt/id :products)} "output_2")
+                       :model/Transform {t3 :id} (make-python-transform {"output_1" table1
+                                                                         "output_2" table2} "final_output")]
+          (is (= {t1 #{}
+                  t2 #{}
+                  t3 #{t1 t2}}
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]]))))))))
+
+  (deftest mixed-transform-ordering-test
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+      (testing "Python and query transforms are ordered together correctly"
+        (mt/with-temp [:model/Table {table1 :id} {:schema "public"
+                                                  :name "sql_output"}
+                       :model/Field _ {:table_id table1
+                                       :name "foo"}
+                       :model/Table {table2 :id} {:schema "public"
+                                                  :name "python_output"}
+                       :model/Field _ {:table_id table2
+                                       :name "bar"}
+                       ;; SQL transform that depends on orders table
+                       :model/Transform {t1 :id} (make-transform
+                                                  {:database (mt/id),
+                                                   :type "query",
+                                                   :query {:source-table (mt/id :orders)}}
+                                                  "sql_output")
+                       ;; Python transform that depends on the SQL transform's output
+                       :model/Transform {t2 :id} (make-python-transform {"sql_output" table1} "python_output")
+                       ;; Another SQL transform that depends on the Python transform's output
+                       :model/Transform {t3 :id} (make-transform
+                                                  {:database (mt/id)
+                                                   :type "query"
+                                                   :query {:source-table table2}}
+                                                  "final_output")]
+          (is (= {t1 #{}
+                  t2 #{t1}
+                  t3 #{t2}}
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]])))))))))
+
+#_(deftest mixed-database-transform-ordering-test
+    (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+      (testing "Python and query transforms are ordered together correctly, across multiple databases"
+      ;; orders (db-1) -> sql_output (db-1) -> python_output (db-2) -> final_output (db-2)
+        (mt/with-temp [:model/Database db-2    {}
+                       :model/Table    table-1 {:db_id (mt/id),    :schema "public", :name "sql_output"}
+                       :model/Table    table-2 {:db_id (:id db-2), :schema "public", :name "python_output"}
+                       :model/Field    _       {:table_id (:id table-1), :name "foo"}
+                       :model/Field    _       {:table_id (:id table-2), :name "bar"}
+
+                     ;; SQL transform that depends on orders table
+                       :model/Transform {t1 :id} (make-transform
+                                                  {:database (mt/id)
+                                                   :type     "query"
+                                                   :query    {:source-table (mt/id :orders)}}
+                                                  (:name table-1))
+                     ;; Python transform that depends on the SQL transform's output
+                       :model/Transform {t2 :id} (make-python-transform {"sql_output" (:id table-1)} (:name table-2))
+                     ;; Another SQL transform that depends on the Python transform's output
+                       :model/Transform {t3 :id} (make-transform
+                                                  {:database (:id db-2)
+                                                   :type     "query"
+                                                   :query    {:source-table (:id table-2)}}
+                                                  "final_output")]
+          (is (= {t1 #{}
+                  t2 #{t1}
+                  t3 #{t2}}
+
+                 (ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]]))))))))

enterprise/backend/test/metabase_enterprise/transforms/python_runner_test.clj

@@ -0,0 +1,424 @@
+(ns ^:mb/driver-tests metabase-enterprise.transforms.python-runner-test
+  (:require
+   [clojure.core.async :as a]
+   [clojure.data.csv :as csv]
+   [clojure.java.jdbc :as jdbc]
+   [clojure.string :as str]
+   [clojure.test :refer :all]
+   [java-time.api :as t]
+   [metabase-enterprise.transforms.python-runner :as python-runner]
+   [metabase-enterprise.transforms.schedule :as transforms.schedule]
+   [metabase-enterprise.transforms.settings :as transforms.settings]
+   [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
+   [metabase-enterprise.transforms.test-util :as transforms.tu]
+   [metabase-enterprise.transforms.util :as transforms.util]
+   [metabase.driver :as driver]
+   [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
+   [metabase.sync.core :as sync]
+   [metabase.task.core :as task]
+   [metabase.test :as mt]
+   [metabase.test.data.interface :as tx]
+   [metabase.test.data.sql :as sql.tx]
+   [metabase.util :as u]
+   [metabase.util.json :as json]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(defn template->regex
+  "Convert a template string with $var$ placeholders to a regex pattern.
+   Example: template->regex 'File ___PATH___/script.py, line ___LINE___'
+   will match 'File /tmp/unique-path/script.py, line 123'"
+  [template]
+  (-> template
+      (str/replace #"(?m)^\d{13}\s*" "")
+      (str/replace "\\" "\\\\")
+      (str/replace "." "\\.")
+      (str/replace "*" "\\*")
+      (str/replace "+" "\\+")
+      (str/replace "?" "\\?")
+      (str/replace "^" "\\^")
+      (str/replace "$" "\\$")
+      (str/replace "{" "\\{")
+      (str/replace "}" "\\}")
+      (str/replace "(" "\\(")
+      (str/replace ")" "\\)")
+      (str/replace "|" "\\|")
+      (str/replace "[" "\\[")
+      (str/replace "]" "\\]")
+      (str/replace #"___PATH___" "(/[\\\\w-]+)+")
+      (str/replace #"___LINE___" "\\\\d+")
+      re-pattern))
+
+(def ^:private test-id 1)
+
+(defn- execute [{:keys [code tables]}]
+  (with-open [shared-storage-ref (python-runner/open-s3-shared-storage! (or tables {}))]
+    (let [server-url     (transforms.settings/python-execution-server-url)
+          cancel-chan    (a/promise-chan)
+          table-name->id (or tables {})
+          _              (python-runner/copy-tables-to-s3! {:run-id         test-id
+                                                            :shared-storage @shared-storage-ref
+                                                            :table-name->id table-name->id
+                                                            :cancel-chan    cancel-chan})
+          response       (python-runner/execute-python-code-http-call! {:server-url     server-url
+                                                                        :code           code
+                                                                        :run-id         test-id
+                                                                        :table-name->id table-name->id
+                                                                        :shared-storage @shared-storage-ref})
+          {:keys [output output-manifest events]} (python-runner/read-output-objects @shared-storage-ref)]
+      ;; not sure about munging this all together but its what tests expect for now
+      (merge (:body response)
+             {:output          output
+              :output-manifest output-manifest
+              :stdout          (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
+              :stderr          (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))}))))
+
+(deftest ^:parallel transform-function-basic-test
+  (testing "executes transform function and returns CSV output"
+    (let [transform-code (str "import pandas as pd\n"
+                              "\n"
+                              "def transform():\n"
+                              "    return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")
+          result         (execute {:code transform-code})]
+      (is (=? {:output "name,age\nAlice,25\nBob,30\n"
+               :stdout "Successfully saved 2 rows to S3\nSuccessfully saved output manifest with 2 fields"
+               :stderr ""}
+              result)))))
+
+(deftest ^:parallel transform-function-missing-test
+  (testing "handles missing transform function"
+    (let [result (execute {:code (str "import pandas as pd\n"
+                                      "\n"
+                                      "# No transform function defined")})]
+      (is (=? {:exit_code 1
+               :stderr    "ERROR: User script must define a 'transform()' function"
+               :stdout    ""}
+              result)))))
+
+(deftest ^:parallel transform-function-wrong-return-type-test
+  (testing "handles transform function returning non-DataFrame"
+    (let [result (execute {:code (str "def transform():\n"
+                                      "    return 'not a dataframe'")})]
+      (is (=? {:exit_code 1
+               :stderr    "ERROR: Transform function must return a pandas DataFrame, got <class 'str'>"
+               :stdout    ""}
+              result)))))
+
+(deftest ^:parallel transform-function-error-test
+  (testing "handles transform function with error"
+    (let [result            (execute {:code (str "def transform():\n"
+                                                 "    raise ValueError('Something went wrong')")})
+          expected-template (str "ERROR: Transform function failed: Something went wrong\n"
+                                 "Traceback (most recent call last):\n"
+                                 "  File \"/app/external/transform_runner.py\", line ___LINE___, in main\n"
+                                 "    result = script.transform()\n"
+                                 "             ^^^^^^^^^^^^^^^^^^\n"
+                                 "  File \"___PATH___/script.py\", line 2, in transform\n"
+                                 "    raise ValueError('Something went wrong')\n"
+                                 "ValueError: Something went wrong")
+          stderr-pattern    (template->regex expected-template)]
+      (is (=? {:exit_code 1
+               :stderr    #(re-matches stderr-pattern %)
+               :stdout    ""
+               :timeout   false}
+              result)))))
+
+(deftest ^:parallel transform-function-complex-dataframe-test
+  (testing "can create complex DataFrames with transform"
+    (let [transform-code (str "import pandas as pd\n"
+                              "\n"
+                              "def transform():\n"
+                              "    data = {'x': [1, 2, 3], 'y': [10, 20, 30], 'z': ['a', 'b', 'c']}\n"
+                              "    return pd.DataFrame(data)")
+          result         (execute {:code transform-code})]
+      (is (=? {:output "x,y,z\n1,10,a\n2,20,b\n3,30,c\n"
+               :stdout "Successfully saved 3 rows to S3\nSuccessfully saved output manifest with 3 fields"
+               :stderr ""}
+              result)))))
+
+(deftest ^:parallel transform-function-with-db-parameter-test
+  (mt/test-drivers #{:postgres}
+    (testing "transform function can accept db parameter for forward compatibility"
+      (let [transform-code (str "import pandas as pd\n"
+                                "\n"
+                                "def transform():\n"
+                                "    data = {'name': ['Charlie', 'Dana'], 'score': [85, 92]}\n"
+                                "    return pd.DataFrame(data)")
+            result         (execute {:code transform-code})]
+        (is (=? {:output "name,score\nCharlie,85\nDana,92\n"
+                 :stdout "Successfully saved 2 rows to S3\nSuccessfully saved output manifest with 2 fields"
+                 :stderr ""}
+                result))))))
+
+(deftest transform-function-with-pass-thru
+  (testing "transform function successfully connects to PostgreSQL database and reads data"
+    (mt/test-drivers #{:postgres}
+      (mt/with-empty-db
+        (let [db-spec        (sql-jdbc.conn/db->pooled-connection-spec (mt/db))
+              _              (jdbc/execute! db-spec ["DROP TABLE IF EXISTS students"])
+              _              (jdbc/execute! db-spec ["CREATE TABLE students (id INTEGER PRIMARY KEY, name VARCHAR(100), score INTEGER)"])
+              _              (jdbc/execute! db-spec ["INSERT INTO students (id, name, score) VALUES (1, 'Alice', 85), (2, 'Bob', 92), (3, 'Charlie', 88), (4, 'Dana', 90)"])
+
+              _              (sync/sync-database! (mt/db))
+
+              transform-code (str "import pandas as pd\n"
+                                  "\n"
+                                  "def transform(students):\n"
+                                  "    return students")
+              result         (execute {:code   transform-code
+                                       :tables {"students" (mt/id :students)}})]
+
+          (is (=? {:output   "id,name,score\n1,Alice,85\n2,Bob,92\n3,Charlie,88\n4,Dana,90\n"
+                   :output-manifest {:fields [{:base_type "Integer",
+                                               :database_type "int4",
+                                               :dtype "int64",
+                                               :effective_type "Integer",
+                                               :name "id",
+                                               :root_type "Integer",
+                                               :semantic_type "PK"}
+                                              {:base_type "Text",
+                                               :database_type "varchar",
+                                               :dtype "object",
+                                               :effective_type "Text",
+                                               :name "name",
+                                               :root_type "Text",
+                                               :semantic_type "Name"}
+                                              {:base_type "Integer",
+                                               :database_type "int4",
+                                               :dtype "int64",
+                                               :effective_type "Integer",
+                                               :name "score",
+                                               :root_type "Integer",
+                                               :semantic_type "Score"}],
+                                     :source_metadata {:fields [{:base_type "Integer",
+                                                                 :database_type "int4",
+                                                                 :dtype "int64",
+                                                                 :effective_type "Integer",
+                                                                 :name "id",
+                                                                 :root_type "Integer",
+                                                                 :semantic_type "PK"}
+                                                                {:base_type "Text",
+                                                                 :database_type "varchar",
+                                                                 :dtype "object",
+                                                                 :effective_type "Text",
+                                                                 :name "name",
+                                                                 :root_type "Text",
+                                                                 :semantic_type "Name"}
+                                                                {:base_type "Integer",
+                                                                 :database_type "int4",
+                                                                 :dtype "int64",
+                                                                 :effective_type "Integer",
+                                                                 :name "score",
+                                                                 :root_type "Integer",
+                                                                 :semantic_type "Score"}],
+                                                       :table_metadata {:table_id (mt/malli=? int?)},
+                                                       :version "0.1.0"},
+                                     :version "0.1.0"}
+                   :stdout   (str "Successfully saved 4 rows to S3\n"
+                                  "Successfully saved output manifest with 3 fields")
+                   :stderr   ""}
+                  result)))))))
+
+(deftest transform-function-with-working-database-test
+  (testing "transform function successfully connects to PostgreSQL database and reads data"
+    (mt/test-drivers #{:postgres}
+      (mt/with-empty-db
+        (let [db-spec        (sql-jdbc.conn/db->pooled-connection-spec (mt/db))
+              _              (jdbc/execute! db-spec ["DROP TABLE IF EXISTS students"])
+              _              (jdbc/execute! db-spec ["CREATE TABLE students (id INTEGER PRIMARY KEY, name VARCHAR(100), score INTEGER)"])
+              _              (jdbc/execute! db-spec ["INSERT INTO students (id, name, score) VALUES (1, 'Alice', 85), (2, 'Bob', 92), (3, 'Charlie', 88), (4, 'Dana', 90)"])
+
+              _              (sync/sync-database! (mt/db))
+
+              transform-code (str "import pandas as pd\n"
+                                  "\n"
+                                  "def transform(students):\n"
+                                  "    # Calculate average score\n"
+                                  "    avg_score = students['score'].mean()\n"
+                                  "    result = pd.DataFrame({\n"
+                                  "        'student_count': [len(students)],\n"
+                                  "        'average_score': [round(avg_score, 2)]\n"
+                                  "    })\n"
+                                  "    return result")
+              result         (execute {:code   transform-code
+                                       :tables {"students" (mt/id :students)}})]
+
+          (is (=? {:output   "student_count,average_score\n4,88.75\n"
+                   :output-manifest {:version "0.1.0",
+                                     :fields  [{:name "student_count", :dtype "int64"}
+                                               {:name "average_score", :dtype "float64"}]}
+                   :stdout   (str "Successfully saved 1 rows to S3\n"
+                                  "Successfully saved output manifest with 2 fields")
+                   :stderr   ""}
+                  result)))))))
+
+(defn- datetime-equal?
+  [expected-iso-str actual-pandas-str]
+  (let [formatter   (t/formatter "yyyy-MM-dd HH:mm:ssXXX")
+        expected-dt (t/zoned-date-time expected-iso-str)
+        actual-dt   (t/offset-date-time formatter actual-pandas-str)]
+    (= (t/to-millis-from-epoch expected-dt)
+       (t/to-millis-from-epoch actual-dt))))
+
+(deftest transform-type-test
+  (mt/test-drivers #{:postgres}
+    (mt/with-empty-db
+      (let [db-spec (sql-jdbc.conn/db->pooled-connection-spec (mt/db))
+            _ (jdbc/execute! db-spec ["DROP TABLE IF EXISTS sample_table"])
+            _ (jdbc/execute! db-spec ["CREATE TABLE sample_table (
+                                                        id BIGSERIAL PRIMARY KEY,
+                                                        name VARCHAR(255),
+                                                        description TEXT,
+                                                        count BIGINT,
+                                                        price FLOAT,
+                                                        is_active BOOLEAN,
+                                                        created_date DATE,
+                                                        updated_at TIMESTAMP,
+                                                        scheduled_for TIMESTAMP WITH TIME ZONE
+                                                    )"])
+            _ (jdbc/execute! db-spec ["INSERT INTO sample_table
+                                                        (name, description, count, price, is_active, created_date, updated_at, scheduled_for)
+                                                      VALUES
+                                                        ('Product A', 'A sample product description', 100, 29.99, true, '2024-01-15', '2024-01-15 10:30:00', '2024-01-16 14:00:00+00'),
+                                                        ('Product B', 'Another product with longer description text', 50, 15.50, false, '2024-02-01', '2024-02-01 09:15:30', '2024-02-02 16:30:00-05'),
+                                                        ('Product C', NULL, 0, 0.0, true, '2024-03-10', '2024-03-10 18:45:15', '2024-03-11 08:00:00+02')"])
+
+            _ (sync/sync-database! (mt/db))
+
+            transform-code (str "import pandas as pd\n"
+                                "\n"
+                                "def transform(sample_table):\n"
+                                "    df = sample_table.copy()\n"
+                                "    return df")
+            result (execute {:code transform-code
+                             :tables {"sample_table" (mt/id :sample_table)}})
+            csv-data (csv/read-csv (:output result))
+            headers (first csv-data)
+            rows (rest csv-data)
+            [row1 row2 row3] rows
+            header-to-index (zipmap headers (range))
+            get-col (fn [row col-name] (nth row (header-to-index col-name)))
+            metadata (:output-manifest result)]
+
+        (is (= (set ["id" "name" "description" "count" "price" "is_active" "created_date" "updated_at" "scheduled_for"])
+               (set headers)))
+
+        (is (= "1" (get-col row1 "id")))
+        (is (= "Product A" (get-col row1 "name")))
+        (is (datetime-equal? "2024-01-16T14:00:00Z" (get-col row1 "scheduled_for")))
+
+        (is (= "2" (get-col row2 "id")))
+        (is (= "Product B" (get-col row2 "name")))
+        (is (datetime-equal? "2024-02-02T21:30:00Z" (get-col row2 "scheduled_for")))
+
+        (is (= "3" (get-col row3 "id")))
+        (is (= "Product C" (get-col row3 "name")))
+        (is (datetime-equal? "2024-03-11T06:00:00Z" (get-col row3 "scheduled_for")))
+        (testing "dtypes are preserved correctly"
+          (is (= {"id"            :type/Integer
+                  "name"          :type/Text
+                  "description"   :type/Text
+                  "count"         :type/Integer
+                  "price"         :type/Float
+                  "is_active"     :type/Boolean
+                  ;; Our hack works
+                  "created_date"  :type/Date
+                  "updated_at"    :type/DateTime
+                  "scheduled_for" :type/DateTimeWithTZ}
+                 (u/for-map [{:keys [name dtype]} (:fields metadata)]
+                   [name (transforms.util/dtype->base-type dtype)]))))))))
+
+(deftest python-transform-scheduled-job-test
+  (mt/test-helpers-set-global-values!
+    (mt/with-temp-scheduler!
+      (task/init! ::transforms.schedule/RunTransform)
+      (mt/test-drivers #{:h2 :postgres}
+        (mt/with-premium-features #{:transforms}
+          (mt/dataset transforms-dataset/transforms-test
+            (transforms.tu/with-transform-cleanup! [target {:type   "table"
+                                                            :schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))
+                                                            :name   "target_table"}]
+              (mt/with-temp
+                [:model/TransformTag {tag-id :id}       {:name "every second"}
+                 :model/Transform    {transform-id :id} {:name   "Gadget Products"
+                                                         :source {:type  "python"
+                                                                  :source-database (mt/id)
+                                                                  :source-tables {"transforms_customers" (mt/id :transforms_customers)}
+                                                                  :body  (str "import pandas as pd\n"
+                                                                              "\n"
+                                                                              "def transform():\n"
+                                                                              "    return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}
+                                                         :target  (assoc target :database (mt/id))}
+                 :model/TransformTransformTag   _  {:transform_id transform-id :tag_id tag-id :position 0}
+                 :model/TransformJob {job-id :id :as job} {:schedule "* * * * * ? *"}
+                 :model/TransformJobTransformTag _ {:job_id job-id :tag_id tag-id :position 0}]
+                (transforms.schedule/initialize-job! job)
+                (transforms.schedule/update-job! job-id "* * * * * ? *")
+                (is (true? (u/poll {:thunk   (fn [] (driver/table-exists? driver/*driver* (mt/db) target))
+                                    :done?   true?
+                                    :timeout 30000})))
+                (is (true? (t2/exists? :model/Table :name (:name target))))))))))))
+
+(deftest transform-type-roundtrip-test
+  (mt/test-drivers #{:postgres :h2 :mysql :bigquery-cloud-sdk :redshift :snowflake :sqlserver :clickhouse}
+    (mt/with-empty-db
+      (let [driver       driver/*driver*
+            db-id        (mt/id)
+            table-name   (if (= driver :redshift)
+                           (tx/db-qualified-table-name (get-in (mt/db) [:settings :database-source-dataset-name]) (mt/random-name))
+                           (mt/random-name))
+            schema-name  (sql.tx/session-schema driver)
+            qualified-table-name (if schema-name
+                                   (keyword schema-name table-name)
+                                   (keyword table-name))
+            table-schema {:name    qualified-table-name
+                          :columns [{:name "id"            :type :type/Integer  :nullable? false}
+                                    {:name "price"         :type :type/Float    :nullable? true}
+                                    {:name "active"        :type :type/Boolean  :nullable? true}
+                                    {:name "created_tz"    :type :type/DateTimeWithTZ :nullable? true}
+                                    {:name "created_at"    :type :type/DateTime :nullable? true}
+                                    {:name "created_date"  :type :type/Date     :nullable? true}
+                                    {:name "description"   :type :type/Text     :nullable? true}]}
+            _ (mt/as-admin
+                (transforms.util/create-table-from-schema! driver db-id table-schema))
+
+            row-values   [[1
+                           19.99
+                           (if (= driver/*driver* :sqlserver) 1 true)
+                           (if (= driver/*driver* :mysql) "2024-01-01 12:00:00" "2024-01-01T12:00:00Z")
+                           (if (= driver/*driver* :mysql) "2024-01-01 12:00:00" "2024-01-01T12:00:00")
+                           "2024-01-01"
+                           "Sample product"]]
+            _ (driver/insert-from-source! driver db-id qualified-table-name
+                                          (mapv :name (:columns table-schema))
+                                          {:type :rows :data row-values})
+
+            _ (sync/sync-database! (mt/db) {:scan :schema})
+
+            transform-code (str "import pandas as pd\n"
+                                "\n"
+                                "def transform(" table-name "):\n"
+                                "    return " table-name)
+            result (execute {:code transform-code
+                             :tables {table-name (mt/id qualified-table-name)}})
+
+            metadata (:output-manifest result)]
+
+        (testing "All expected columns are present"
+          (is (= #{"id" "price" "active" "created_tz" "created_at" "created_date" "description"}
+                 (set (map :name (:fields metadata))))))
+
+        (testing "dtypes are preserved correctly"
+          (is (= {"id"           :type/Integer
+                  "price"        :type/Float
+                  "active"       :type/Boolean
+                  "created_tz"   :type/DateTimeWithTZ
+                  "created_at"   :type/DateTime
+                  "created_date" :type/Date
+                  "description"  :type/Text}
+                 (u/for-map [{:keys [name dtype]} (:fields metadata)]
+                   [name (transforms.util/dtype->base-type dtype)]))))
+
+        ;; cleanup
+        (driver/drop-table! driver db-id qualified-table-name)))))

enterprise/backend/test/metabase_enterprise/transforms/test_util.clj

@@ -52,6 +52,12 @@
     `(mt/with-model-cleanup [:model/Transform]
        ~@body)))
 
+(defn table-rows
+  [table-name]
+  (mt/rows (mt/process-query {:database (mt/id)
+                              :query    {:source-table (t2/select-one-pk :model/Table :name table-name)}
+                              :type     :query})))
+
 (defn parse-timestamp
   "Parse a local datetime and convert it to a ZonedDateTime in the default timezone."
   ^ZonedDateTime [timestamp-string]

enterprise/frontend/src/metabase-enterprise/api/python-transform-library.ts

@@ -0,0 +1,35 @@
+import { Api } from "metabase/api";
+
+export type PythonLibrary = {
+  source: string;
+};
+
+export type GetPythonLibraryRequest = {
+  name: string;
+};
+
+export type UpdatePythonLibraryRequest = {
+  name: string;
+  source: string;
+};
+
+export const pythonLibraryApi = Api.injectEndpoints({
+  endpoints: (builder) => ({
+    getPythonLibrary: builder.query<PythonLibrary, GetPythonLibraryRequest>({
+      query: ({ name }) => ({
+        url: `/api/ee/python-transform/user-modules/${name}`,
+        method: "GET",
+      }),
+    }),
+    updatePythonLibrary: builder.mutation<void, UpdatePythonLibraryRequest>({
+      query: ({ name, source }) => ({
+        url: `/api/ee/python-transform/user-modules/${name}`,
+        method: "PUT",
+        body: { source },
+      }),
+    }),
+  }),
+});
+
+export const { useGetPythonLibraryQuery, useUpdatePythonLibraryMutation } =
+  pythonLibraryApi;

enterprise/frontend/src/metabase-enterprise/api/transform.ts

@@ -108,6 +108,14 @@ export const transformApi = EnterpriseApi.injectEndpoints({
       invalidatesTags: (_, error) =>
         invalidateTags(error, [listTag("transform")]),
     }),
+    cancelTransform: builder.mutation<void, TransformId>({
+      query: (id) => ({
+        method: "POST",
+        url: `/api/ee/transform/${id}/cancel`,
+      }),
+      invalidatesTags: (_, error, id) =>
+        invalidateTags(error, [idTag("transform", id)]),
+    }),
     deleteTransformTarget: builder.mutation<void, TransformId>({
       query: (id) => ({
         method: "DELETE",
@@ -126,6 +134,7 @@ export const {
   useGetTransformQuery,
   useLazyGetTransformQuery,
   useRunTransformMutation,
+  useCancelTransformMutation,
   useCreateTransformMutation,
   useUpdateTransformMutation,
   useDeleteTransformMutation,

enterprise/frontend/src/metabase-enterprise/transforms/analytics.ts

@@ -34,7 +34,7 @@ export function trackTransformCreate({
   creationType,
 }: {
   triggeredFrom: "transform-page-create-menu";
-  creationType: "query" | "native" | "saved-question";
+  creationType: "query" | "native" | "python" | "saved-question";
 }) {
   trackSimpleEvent({
     event: "transform_create",

enterprise/frontend/src/metabase-enterprise/transforms/api/python-runner.ts

@@ -0,0 +1,68 @@
+import { Api } from "metabase/api";
+
+export interface ExecutePythonRequest {
+  code: string;
+  tables: Record<string, number>;
+}
+
+export interface ExecutePythonResponse {
+  output?: string;
+  stdout?: string;
+  stderr?: string;
+  error?: string;
+  exit_code?: number;
+  timeout?: boolean;
+}
+
+export const pythonRunnerApi = Api.injectEndpoints({
+  endpoints: (builder) => ({
+    executePython: builder.mutation<
+      ExecutePythonResponse,
+      ExecutePythonRequest
+    >({
+      query: ({ code, tables }) => ({
+        url: "/api/ee/transform/test-python",
+        method: "POST",
+        body: {
+          code,
+          tables,
+        },
+      }),
+      transformResponse: (response: any) => {
+        // Extract the nested result from the backend response
+        if (response?.result?.body) {
+          return {
+            output: response.result.body.output,
+            stdout: response.result.body.stdout,
+            stderr: response.result.body.stderr,
+            error: response.result.body.error,
+            exit_code: response.result.body.exit_code,
+            timeout: response.result.body.timeout,
+          };
+        }
+        return response;
+      },
+      transformErrorResponse: (response: any) => {
+        return {
+          status: response?.status || 500,
+          data: {
+            error: response?.data?.error || "Python execution failed",
+            stdout: response?.data?.stdout || "",
+            stderr: response?.data?.stderr || "",
+            exit_code: response?.data?.exit_code,
+          },
+        };
+      },
+    }),
+    cancelPython: builder.mutation<void, void>({
+      query: () => ({
+        url: "/api/ee/transform/test-python/cancel",
+        method: "POST",
+      }),
+    }),
+  }),
+  overrideExisting: true,
+});
+
+export const { useExecutePythonMutation, useCancelPythonMutation } =
+  pythonRunnerApi;

enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/LogOutput.styled.tsx

@@ -0,0 +1,67 @@
+import { css } from "@emotion/react";
+import styled from "@emotion/styled";
+
+import { color } from "metabase/lib/colors";
+
+const LOG_PREFIX = ".react-ansi-style";
+
+const LOG_COLORS = {
+  black: color("text-dark"),
+  white: color("text-white"),
+  gray: color("text-medium"),
+  red: color("saturated-red"),
+  green: color("saturated-green"),
+  yellow: color("saturated-yellow"),
+  blue: color("saturated-blue"),
+  magenta: color("saturated-purple"),
+  cyan: "cyan",
+};
+
+export const LogOutputContainer = styled.div`
+  border: 1px solid var(--mb-color-border);
+  border-radius: 0.5rem;
+  background-color: var(--mb-color-bg-light);
+  font-family: Monaco, Menlo, "Ubuntu Mono", "Lucida Console", monospace;
+  font-size: 13px;
+  white-space: pre;
+  padding: 1rem;
+  overflow: auto;
+  line-height: 1.4;
+
+  ${LOG_PREFIX}-bold {
+    font-weight: bold;
+  }
+
+  ${LOG_PREFIX}-dim {
+    font-weight: lighter;
+  }
+
+  ${LOG_PREFIX}-italic {
+    font-style: italic;
+  }
+
+  ${LOG_PREFIX}-underline {
+    text-decoration: underline;
+  }
+
+  ${LOG_PREFIX}-inverse {
+    color: var(--mb-color-text-dark);
+    background-color: var(--mb-color-bg-white);
+  }
+
+  ${LOG_PREFIX}-hidden {
+    color: transparent;
+  }
+
+  ${LOG_PREFIX}-strikethrough {
+    text-decoration: line-through;
+  }
+
+  ${Object.entries(LOG_COLORS).map(
+    ([key, color]) => css`
+      ${LOG_PREFIX}-${key} {
+        color: ${color};
+      }
+    `,
+  )};
+`;

enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/LogOutput.tsx

@@ -0,0 +1,85 @@
+import * as React from "react";
+import { useEffect, useMemo, useRef, useState } from "react";
+import reactAnsiStyle from "react-ansi-style";
+
+import { LogOutputContainer } from "./LogOutput.styled";
+
+interface LogOutputProps {
+  content?: string;
+  showLineNumbers?: boolean;
+  maxLines?: number;
+}
+
+export function LogOutput({
+  content,
+  showLineNumbers = true,
+  maxLines = 20,
+}: LogOutputProps) {
+  const containerRef = useRef<HTMLDivElement>(null);
+  const [isUserScrolled, setIsUserScrolled] = useState(false);
+  const [shouldAutoScroll, setShouldAutoScroll] = useState(true);
+
+  const { displayContent, actualLines } = useMemo(() => {
+    if (!content || content.trim().length === 0) {
+      return { displayContent: "No output", actualLines: 1 };
+    }
+
+    const lines = content.split("\n");
+    let processedContent = content;
+
+    if (showLineNumbers) {
+      const maxLineLength = lines.length.toString().length;
+      processedContent = lines
+        .map(
+          (line, index) =>
+            `${(index + 1).toString().padStart(maxLineLength, " ")} | ${line}`,
+        )
+        .join("\n");
+    }
+
+    return {
+      displayContent: reactAnsiStyle(React, processedContent),
+      actualLines: lines.length,
+    };
+  }, [content, showLineNumbers]);
+
+  // Calculate the height based on actual lines (up to maxLines)
+  const linesToShow = Math.min(actualLines, maxLines);
+  const containerHeight = `${linesToShow * 1.4 + 2}em`; // 1.4 is line-height + padding
+  const maxContainerHeight = `${maxLines * 1.4 + 2}em`;
+
+  // Auto-scroll to bottom when content changes
+  useEffect(() => {
+    if (containerRef.current && shouldAutoScroll && !isUserScrolled) {
+      containerRef.current.scrollTop = containerRef.current.scrollHeight;
+    }
+  }, [displayContent, shouldAutoScroll, isUserScrolled]);
+
+  // Handle scroll events to detect user scrolling
+  const handleScroll = () => {
+    if (!containerRef.current) {
+      return;
+    }
+
+    const { scrollTop, scrollHeight, clientHeight } = containerRef.current;
+    const isAtBottom = Math.abs(scrollHeight - clientHeight - scrollTop) < 5;
+
+    if (isAtBottom) {
+      setIsUserScrolled(false);
+      setShouldAutoScroll(true);
+    } else {
+      setIsUserScrolled(true);
+      setShouldAutoScroll(false);
+    }
+  };
+
+  return (
+    <LogOutputContainer
+      ref={containerRef}
+      style={{ height: containerHeight, maxHeight: maxContainerHeight }}
+      onScroll={handleScroll}
+    >
+      {displayContent}
+    </LogOutputContainer>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/index.ts

@@ -0,0 +1 @@
+export { LogOutput } from "./LogOutput";

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonDataPicker/PythonDataPicker.module.css

@@ -0,0 +1,3 @@
+.dataPicker {
+  border-right: 1px solid var(--mb-color-border);
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonDataPicker/PythonDataPicker.tsx

@@ -0,0 +1,365 @@
+import { useState } from "react";
+import { t } from "ttag";
+
+import { useListDatabasesQuery, useListTablesQuery } from "metabase/api";
+import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
+import {
+  ActionIcon,
+  Box,
+  Button,
+  Group,
+  Icon,
+  Select,
+  Stack,
+  Text,
+  TextInput,
+} from "metabase/ui";
+import type { Database, Table } from "metabase-types/api";
+
+import S from "./PythonDataPicker.module.css";
+
+type TableSelection = {
+  id: string; // unique ID for this selection row
+  tableId: number | undefined;
+  alias: string;
+  aliasManuallySet?: boolean; // Track if user manually edited the alias
+};
+
+type PythonDataPickerProps = {
+  database?: number;
+  tables?: Record<string, number>; // alias -> table-id mapping
+  onChange: (
+    database: number,
+    tables: Record<string, { id: number; name: string }>,
+  ) => void;
+};
+
+// Helper function to slugify table names
+function slugify(text: string): string {
+  return text
+    .toLowerCase()
+    .replace(/[^\w\s-]/g, "") // Remove non-word chars
+    .replace(/[\s_-]+/g, "_") // Replace spaces, underscores, hyphens with single underscore
+    .replace(/^_+|_+$/g, ""); // Remove leading/trailing underscores
+}
+
+export function PythonDataPicker({
+  database,
+  tables,
+  onChange,
+}: PythonDataPickerProps) {
+  const [selectedDatabaseId, setSelectedDatabaseId] = useState<
+    number | undefined
+  >(database);
+
+  // Initialize table selections from props
+  const initializeTableSelections = (): TableSelection[] => {
+    if (tables && Object.keys(tables).length > 0) {
+      return Object.entries(tables).map(([alias, tableId], index) => ({
+        id: `table-${index}-${Date.now()}`,
+        tableId,
+        alias,
+        aliasManuallySet: true, // Existing aliases are considered manually set
+      }));
+    }
+    // Start with one empty row by default
+    return [
+      {
+        id: `table-0-${Date.now()}`,
+        tableId: undefined,
+        alias: "",
+        aliasManuallySet: false,
+      },
+    ];
+  };
+
+  const [tableSelections, setTableSelections] = useState<TableSelection[]>(
+    initializeTableSelections,
+  );
+
+  const {
+    data: databasesResponse,
+    isLoading: isLoadingDatabases,
+    error: databaseError,
+  } = useListDatabasesQuery();
+
+  const {
+    data: tablesData,
+    isLoading: isLoadingTables,
+    error: tablesError,
+  } = useListTablesQuery(
+    selectedDatabaseId
+      ? {
+          dbId: selectedDatabaseId,
+          include_hidden: false,
+          include_editable_data_model: true,
+        }
+      : undefined,
+    {
+      skip: !selectedDatabaseId,
+    },
+  );
+
+  const notifyParentOfChange = (
+    dbId: number | undefined,
+    selections: TableSelection[],
+    tableIdToName?: Map<number, string>,
+  ) => {
+    if (dbId) {
+      const tablesMap: Record<string, { id: number; name: string }> = {};
+
+      selections.forEach((selection) => {
+        if (selection.tableId && selection.alias) {
+          const tableName =
+            tableIdToName?.get(selection.tableId) ||
+            `Table ${selection.tableId}`;
+          tablesMap[selection.alias] = {
+            id: selection.tableId,
+            name: tableName,
+          };
+        }
+      });
+      onChange(dbId, tablesMap);
+    }
+  };
+
+  if (databaseError || tablesError) {
+    return <LoadingAndErrorWrapper error={databaseError || tablesError} />;
+  }
+
+  if (isLoadingDatabases) {
+    return <LoadingAndErrorWrapper loading />;
+  }
+
+  const databases = databasesResponse?.data || [];
+  const databaseOptions = databases.map((db: Database) => ({
+    value: db.id.toString(),
+    label: db.name,
+  }));
+
+  const availableTables = (tablesData || [])
+    .filter((tbl: Table) => tbl.db_id === selectedDatabaseId && tbl.active)
+    .map((tbl: Table) => ({
+      value: tbl.id.toString(),
+      label: tbl.display_name || tbl.name,
+      table: tbl,
+    }));
+
+  const tableIdToNameMap = new Map<number, string>(
+    availableTables.map((t) => [parseInt(t.value), t.table.name]),
+  );
+
+  const handleDatabaseChange = (value: string | null) => {
+    const dbId = value ? parseInt(value) : undefined;
+    setSelectedDatabaseId(dbId);
+    const newSelections = [
+      {
+        id: `table-0-${Date.now()}`,
+        tableId: undefined,
+        alias: "",
+        aliasManuallySet: false,
+      },
+    ];
+    setTableSelections(newSelections);
+    notifyParentOfChange(dbId, newSelections);
+  };
+
+  const handleTableChange = (selectionId: string, value: string | null) => {
+    const newSelections = tableSelections.map((selection) => {
+      if (selection.id === selectionId) {
+        const tableId = value ? parseInt(value) : undefined;
+        const table = availableTables.find((t) => t.value === value)?.table;
+
+        const newAlias = table ? slugify(table.name) : "";
+
+        return {
+          ...selection,
+          tableId,
+          alias: selection.aliasManuallySet ? selection.alias : newAlias,
+          aliasManuallySet: selection.aliasManuallySet || false,
+        };
+      }
+      return selection;
+    });
+    setTableSelections(newSelections);
+    notifyParentOfChange(selectedDatabaseId, newSelections, tableIdToNameMap);
+  };
+
+  const handleAliasChange = (selectionId: string, alias: string) => {
+    // Only allow valid Python identifier characters (letters, numbers, underscores)
+    // and ensure it doesn't start with a number
+    const validAlias = alias
+      .replace(/[^a-zA-Z0-9_]/g, "")
+      .replace(/^(\d)/, "_$1");
+
+    const newSelections = tableSelections.map((selection) =>
+      selection.id === selectionId
+        ? { ...selection, alias: validAlias, aliasManuallySet: true }
+        : selection,
+    );
+    setTableSelections(newSelections);
+    notifyParentOfChange(selectedDatabaseId, newSelections, tableIdToNameMap);
+  };
+
+  const handleResetAlias = (selectionId: string) => {
+    const newSelections = tableSelections.map((selection) => {
+      if (selection.id === selectionId && selection.tableId) {
+        const table = availableTables.find(
+          (t) => t.value === selection.tableId!.toString(),
+        )?.table;
+        if (table) {
+          return {
+            ...selection,
+            alias: slugify(table.name),
+            aliasManuallySet: false,
+          };
+        }
+      }
+      return selection;
+    });
+    setTableSelections(newSelections);
+    notifyParentOfChange(selectedDatabaseId, newSelections, tableIdToNameMap);
+  };
+
+  const handleAddTable = () => {
+    const newSelections = [
+      ...tableSelections,
+      {
+        id: `table-${tableSelections.length}-${Date.now()}`,
+        tableId: undefined,
+        alias: "",
+        aliasManuallySet: false,
+      },
+    ];
+    setTableSelections(newSelections);
+    // Don't notify parent for adding empty row
+  };
+
+  const handleRemoveTable = (selectionId: string) => {
+    const newSelections = tableSelections.filter(
+      (selection) => selection.id !== selectionId,
+    );
+    setTableSelections(newSelections);
+    notifyParentOfChange(selectedDatabaseId, newSelections, tableIdToNameMap);
+  };
+
+  const selectedTableIds = new Set(
+    tableSelections
+      .filter((s) => s.tableId !== undefined)
+      .map((s) => s.tableId?.toString()),
+  );
+
+  return (
+    <Stack p="md" gap="md" h="100%" className={S.dataPicker}>
+      <Box>
+        <Text size="sm" fw="bold" mb="xs">
+          {t`Source Database`}
+        </Text>
+        <Text size="xs" c="dimmed" mb="sm">
+          {t`Select the database that contains your source data`}
+        </Text>
+        <Select
+          data={databaseOptions}
+          value={selectedDatabaseId?.toString() || null}
+          onChange={handleDatabaseChange}
+          placeholder={t`Select a database`}
+          clearable
+        />
+      </Box>
+      {selectedDatabaseId && (
+        <Box>
+          <Text size="sm" fw="bold" mb="xs">
+            {t`Source Tables`}
+          </Text>
+          <Text size="xs" c="dimmed" mb="sm">
+            {t`Select tables to use as data sources and provide aliases for each`}
+          </Text>
+          <Stack gap="sm">
+            {tableSelections.map((selection, index) => {
+              // Filter available tables to exclude already selected ones (except current selection)
+              const filteredTables = availableTables.filter(
+                (table) =>
+                  !selectedTableIds.has(table.value) ||
+                  table.value === selection.tableId?.toString(),
+              );
+
+              const table = availableTables.find(
+                (t) => t.value === selection.tableId?.toString(),
+              )?.table;
+              const defaultAlias = table ? slugify(table.name) : "";
+              const showReset =
+                selection.aliasManuallySet &&
+                selection.alias !== defaultAlias &&
+                table;
+
+              return (
+                <Group key={selection.id} gap="sm" align="flex-end">
+                  <Select
+                    style={{ flex: 1 }}
+                    data={filteredTables}
+                    value={selection.tableId?.toString() || null}
+                    onChange={(value) => handleTableChange(selection.id, value)}
+                    placeholder={t`Select a table`}
+                    clearable
+                    disabled={isLoadingTables}
+                  />
+                  <TextInput
+                    style={{ flex: 0.7 }}
+                    value={selection.alias}
+                    onChange={(e) =>
+                      handleAliasChange(selection.id, e.target.value)
+                    }
+                    placeholder={t`Enter alias`}
+                    label={index === 0 ? t`Alias` : undefined}
+                    styles={{
+                      input: {
+                        color: selection.aliasManuallySet
+                          ? undefined
+                          : "var(--mb-color-text-light)",
+                      },
+                    }}
+                    rightSection={
+                      showReset ? (
+                        <ActionIcon
+                          onClick={() => handleResetAlias(selection.id)}
+                          aria-label={t`Reset alias to default`}
+                          color="gray"
+                          variant="subtle"
+                          size="xs"
+                        >
+                          <Icon name="refresh" size={12} />
+                        </ActionIcon>
+                      ) : null
+                    }
+                  />
+                  {tableSelections.length > 1 && (
+                    <ActionIcon
+                      onClick={() => handleRemoveTable(selection.id)}
+                      aria-label={t`Remove table`}
+                      color="gray"
+                      variant="subtle"
+                    >
+                      <Icon name="trash" />
+                    </ActionIcon>
+                  )}
+                </Group>
+              );
+            })}
+            <Button
+              leftSection={<Icon name="add" />}
+              variant="subtle"
+              onClick={handleAddTable}
+              disabled={
+                !tableSelections[tableSelections.length - 1]?.tableId ||
+                availableTables.length === 0 ||
+                selectedTableIds.size >= availableTables.length
+              }
+            >
+              {t`Add another table`}
+            </Button>
+          </Stack>
+        </Box>
+      )}
+    </Stack>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonDataPicker/index.ts

@@ -0,0 +1 @@
+export * from "./PythonDataPicker";

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonEditor/PythonEditor.module.css

@@ -0,0 +1,4 @@
+.editor {
+  width: 100%;
+  height: 100%;
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonEditor/PythonEditor.tsx

@@ -0,0 +1,28 @@
+import cx from "classnames";
+
+import { CodeEditor } from "metabase/common/components/CodeEditor";
+
+import S from "./PythonEditor.module.css";
+import { completion } from "./utils";
+
+export function PythonEditor({
+  value,
+  onChange,
+  withPandasCompletions,
+  className,
+}: {
+  value: string;
+  onChange: (value: string) => void;
+  withPandasCompletions?: boolean;
+  className?: string;
+}) {
+  return (
+    <CodeEditor
+      className={cx(S.editor, className)}
+      value={value}
+      onChange={onChange}
+      language="python"
+      extensions={withPandasCompletions ? completion : undefined}
+    />
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonEditor/index.ts

@@ -0,0 +1 @@
+export * from "./PythonEditor";

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonEditor/utils.ts

@@ -0,0 +1,675 @@
+import {
+  type Completion,
+  type CompletionContext,
+  autocompletion,
+} from "@codemirror/autocomplete";
+import { pythonLanguage } from "@codemirror/lang-python";
+
+// This is the closest you can get to "loading pandas definitions"
+
+const pandasDefinitions: Record<string, Completion[]> = {
+  pd: [
+    {
+      label: "DataFrame",
+      type: "class",
+      info: "Two-dimensional, size-mutable, potentially heterogeneous tabular data",
+    },
+    {
+      label: "Series",
+      type: "class",
+      info: "One-dimensional ndarray with axis labels",
+    },
+    {
+      label: "read_csv",
+      type: "function",
+      info: "Read a comma-separated values (csv) file into DataFrame",
+    },
+    {
+      label: "read_excel",
+      type: "function",
+      info: "Read an Excel file into a pandas DataFrame",
+    },
+    {
+      label: "read_json",
+      type: "function",
+      info: "Convert a JSON string to pandas object",
+    },
+    {
+      label: "read_sql",
+      type: "function",
+      info: "Read SQL query or database table into a DataFrame",
+    },
+    {
+      label: "read_parquet",
+      type: "function",
+      info: "Load a parquet object from file path",
+    },
+    {
+      label: "concat",
+      type: "function",
+      info: "Concatenate pandas objects along a particular axis",
+    },
+    {
+      label: "merge",
+      type: "function",
+      info: "Merge DataFrame or named Series objects with a database-style join",
+    },
+    {
+      label: "pivot_table",
+      type: "function",
+      info: "Create a spreadsheet-style pivot table as a DataFrame",
+    },
+    {
+      label: "crosstab",
+      type: "function",
+      info: "Compute a simple cross tabulation of two (or more) factors",
+    },
+    {
+      label: "get_dummies",
+      type: "function",
+      info: "Convert categorical variable(s) into dummy/indicator variables",
+    },
+    {
+      label: "date_range",
+      type: "function",
+      info: "Return a fixed frequency DatetimeIndex",
+    },
+    {
+      label: "to_datetime",
+      type: "function",
+      info: "Convert argument to datetime",
+    },
+    {
+      label: "to_numeric",
+      type: "function",
+      info: "Convert argument to a numeric type",
+    },
+    {
+      label: "cut",
+      type: "function",
+      info: "Bin values into discrete intervals",
+    },
+    {
+      label: "qcut",
+      type: "function",
+      info: "Quantile-based discretization function",
+    },
+    {
+      label: "isna",
+      type: "function",
+      info: "Detect missing values for an array-like object",
+    },
+    {
+      label: "isnull",
+      type: "function",
+      info: "Detect missing values for an array-like object",
+    },
+    {
+      label: "notna",
+      type: "function",
+      info: "Detect non-missing values for an array-like object",
+    },
+    {
+      label: "notnull",
+      type: "function",
+      info: "Detect non-missing values for an array-like object",
+    },
+  ],
+
+  // DataFrame methods and properties
+  DataFrame: [
+    // Viewing/Inspection
+    { label: "head", type: "method", info: "Return the first n rows" },
+    { label: "tail", type: "method", info: "Return the last n rows" },
+    {
+      label: "info",
+      type: "method",
+      info: "Print a concise summary of a DataFrame",
+    },
+    {
+      label: "describe",
+      type: "method",
+      info: "Generate descriptive statistics",
+    },
+    {
+      label: "shape",
+      type: "property",
+      info: "Return a tuple representing the dimensionality",
+    },
+    {
+      label: "size",
+      type: "property",
+      info: "Return an int representing the number of elements",
+    },
+    {
+      label: "columns",
+      type: "property",
+      info: "The column labels of the DataFrame",
+    },
+    {
+      label: "index",
+      type: "property",
+      info: "The index (row labels) of the DataFrame",
+    },
+    {
+      label: "dtypes",
+      type: "property",
+      info: "Return the dtypes in the DataFrame",
+    },
+    {
+      label: "values",
+      type: "property",
+      info: "Return a Numpy representation of the DataFrame",
+    },
+
+    // Selection/Indexing
+    {
+      label: "loc",
+      type: "property",
+      info: "Access a group of rows and columns by label(s)",
+    },
+    {
+      label: "iloc",
+      type: "property",
+      info: "Purely integer-location based indexing",
+    },
+    {
+      label: "at",
+      type: "property",
+      info: "Access a single value for a row/column label pair",
+    },
+    {
+      label: "iat",
+      type: "property",
+      info: "Access a single value for a row/column pair by integer position",
+    },
+    {
+      label: "query",
+      type: "method",
+      info: "Query the columns of a DataFrame with a boolean expression",
+    },
+    {
+      label: "filter",
+      type: "method",
+      info: "Subset the dataframe rows or columns according to labels",
+    },
+    {
+      label: "where",
+      type: "method",
+      info: "Replace values where the condition is False",
+    },
+    {
+      label: "mask",
+      type: "method",
+      info: "Replace values where the condition is True",
+    },
+
+    // Data Cleaning
+    {
+      label: "drop",
+      type: "method",
+      info: "Drop specified labels from rows or columns",
+    },
+    {
+      label: "drop_duplicates",
+      type: "method",
+      info: "Return DataFrame with duplicate rows removed",
+    },
+    { label: "dropna", type: "method", info: "Remove missing values" },
+    {
+      label: "fillna",
+      type: "method",
+      info: "Fill NA/NaN values using the specified method",
+    },
+    {
+      label: "replace",
+      type: "method",
+      info: "Replace values given in to_replace with value",
+    },
+    {
+      label: "interpolate",
+      type: "method",
+      info: "Fill NaN values using an interpolation method",
+    },
+    {
+      label: "duplicated",
+      type: "method",
+      info: "Return boolean Series denoting duplicate rows",
+    },
+    { label: "isnull", type: "method", info: "Detect missing values" },
+    { label: "isna", type: "method", info: "Detect missing values" },
+    {
+      label: "notnull",
+      type: "method",
+      info: "Detect existing (non-missing) values",
+    },
+    {
+      label: "notna",
+      type: "method",
+      info: "Detect existing (non-missing) values",
+    },
+
+    // Transformations
+    {
+      label: "sort_values",
+      type: "method",
+      info: "Sort by the values along either axis",
+    },
+    {
+      label: "sort_index",
+      type: "method",
+      info: "Sort object by labels (along an axis)",
+    },
+    {
+      label: "reset_index",
+      type: "method",
+      info: "Reset the index, or a level of it",
+    },
+    {
+      label: "set_index",
+      type: "method",
+      info: "Set the DataFrame index using existing columns",
+    },
+    {
+      label: "reindex",
+      type: "method",
+      info: "Conform Series/DataFrame to new index",
+    },
+    { label: "rename", type: "method", info: "Alter axes labels" },
+    {
+      label: "assign",
+      type: "method",
+      info: "Assign new columns to a DataFrame",
+    },
+    {
+      label: "pipe",
+      type: "method",
+      info: "Apply chainable functions that expect Series or DataFrames",
+    },
+
+    // Aggregation/GroupBy
+    {
+      label: "groupby",
+      type: "method",
+      info: "Group DataFrame using a mapper or by a Series of columns",
+    },
+    {
+      label: "agg",
+      type: "method",
+      info: "Aggregate using one or more operations",
+    },
+    {
+      label: "aggregate",
+      type: "method",
+      info: "Aggregate using one or more operations",
+    },
+    {
+      label: "apply",
+      type: "method",
+      info: "Apply a function along an axis of the DataFrame",
+    },
+    {
+      label: "applymap",
+      type: "method",
+      info: "Apply a function to a Dataframe elementwise",
+    },
+    {
+      label: "transform",
+      type: "method",
+      info: "Call func on self producing a DataFrame with transformed values",
+    },
+    {
+      label: "rolling",
+      type: "method",
+      info: "Provide rolling window calculations",
+    },
+    {
+      label: "expanding",
+      type: "method",
+      info: "Provide expanding transformations",
+    },
+    { label: "resample", type: "method", info: "Resample time-series data" },
+
+    // Statistical Methods
+    {
+      label: "sum",
+      type: "method",
+      info: "Return the sum of the values for the requested axis",
+    },
+    {
+      label: "mean",
+      type: "method",
+      info: "Return the mean of the values for the requested axis",
+    },
+    {
+      label: "median",
+      type: "method",
+      info: "Return the median of the values for the requested axis",
+    },
+    {
+      label: "mode",
+      type: "method",
+      info: "Get the mode(s) of each element along the selected axis",
+    },
+    {
+      label: "std",
+      type: "method",
+      info: "Return sample standard deviation over requested axis",
+    },
+    {
+      label: "var",
+      type: "method",
+      info: "Return unbiased variance over requested axis",
+    },
+    {
+      label: "min",
+      type: "method",
+      info: "Return the minimum of the values for the requested axis",
+    },
+    {
+      label: "max",
+      type: "method",
+      info: "Return the maximum of the values for the requested axis",
+    },
+    {
+      label: "count",
+      type: "method",
+      info: "Count non-NA cells for each column or row",
+    },
+    {
+      label: "nunique",
+      type: "method",
+      info: "Count number of distinct elements in specified axis",
+    },
+    {
+      label: "value_counts",
+      type: "method",
+      info: "Return a Series containing counts of unique values",
+    },
+    {
+      label: "corr",
+      type: "method",
+      info: "Compute pairwise correlation of columns",
+    },
+    {
+      label: "cov",
+      type: "method",
+      info: "Compute pairwise covariance of columns",
+    },
+    {
+      label: "rank",
+      type: "method",
+      info: "Compute numerical data ranks (1 through n) along axis",
+    },
+    {
+      label: "quantile",
+      type: "method",
+      info: "Return values at the given quantile over requested axis",
+    },
+    {
+      label: "clip",
+      type: "method",
+      info: "Trim values at input threshold(s)",
+    },
+
+    // Merging/Joining
+    {
+      label: "merge",
+      type: "method",
+      info: "Merge DataFrame or named Series objects",
+    },
+    {
+      label: "join",
+      type: "method",
+      info: "Join columns of another DataFrame",
+    },
+    {
+      label: "concat",
+      type: "method",
+      info: "Concatenate pandas objects along a particular axis",
+    },
+    {
+      label: "append",
+      type: "method",
+      info: "Append rows of other to the end of caller",
+    },
+    {
+      label: "update",
+      type: "method",
+      info: "Modify in place using non-NA values from another DataFrame",
+    },
+
+    // Reshaping
+    {
+      label: "pivot",
+      type: "method",
+      info: "Return reshaped DataFrame organized by given index / column values",
+    },
+    {
+      label: "pivot_table",
+      type: "method",
+      info: "Create a spreadsheet-style pivot table",
+    },
+    {
+      label: "melt",
+      type: "method",
+      info: "Unpivot a DataFrame from wide to long format",
+    },
+    {
+      label: "stack",
+      type: "method",
+      info: "Stack the prescribed level(s) from columns to index",
+    },
+    {
+      label: "unstack",
+      type: "method",
+      info: "Pivot a level of the (necessarily hierarchical) index labels",
+    },
+    { label: "transpose", type: "method", info: "Transpose index and columns" },
+    { label: "T", type: "property", info: "Transpose index and columns" },
+
+    // I/O
+    {
+      label: "to_csv",
+      type: "method",
+      info: "Write object to a comma-separated values (csv) file",
+    },
+    {
+      label: "to_excel",
+      type: "method",
+      info: "Write object to an Excel sheet",
+    },
+    {
+      label: "to_json",
+      type: "method",
+      info: "Convert the object to a JSON string",
+    },
+    {
+      label: "to_sql",
+      type: "method",
+      info: "Write records stored in a DataFrame to a SQL database",
+    },
+    {
+      label: "to_parquet",
+      type: "method",
+      info: "Write a DataFrame to the binary parquet format",
+    },
+    {
+      label: "to_dict",
+      type: "method",
+      info: "Convert the DataFrame to a dictionary",
+    },
+    {
+      label: "to_html",
+      type: "method",
+      info: "Render a DataFrame as an HTML table",
+    },
+    {
+      label: "to_string",
+      type: "method",
+      info: "Render a DataFrame to a console-friendly tabular output",
+    },
+    {
+      label: "to_numpy",
+      type: "method",
+      info: "Convert the DataFrame to a NumPy array",
+    },
+
+    // Time Series
+    {
+      label: "shift",
+      type: "method",
+      info: "Shift index by desired number of periods",
+    },
+    { label: "tshift", type: "method", info: "Shift the time index" },
+    {
+      label: "diff",
+      type: "method",
+      info: "First discrete difference of element",
+    },
+    {
+      label: "pct_change",
+      type: "method",
+      info: "Percentage change between the current and a prior element",
+    },
+
+    // Plotting (if matplotlib available)
+    {
+      label: "plot",
+      type: "method",
+      info: "Make plots of Series or DataFrame",
+    },
+    {
+      label: "hist",
+      type: "method",
+      info: "Make a histogram of the DataFrame's columns",
+    },
+    {
+      label: "boxplot",
+      type: "method",
+      info: "Make a box plot of the DataFrame columns",
+    },
+  ],
+
+  // Series methods (subset of DataFrame methods that make sense for Series)
+  Series: [
+    { label: "head", type: "method", info: "Return the first n values" },
+    { label: "tail", type: "method", info: "Return the last n values" },
+    {
+      label: "describe",
+      type: "method",
+      info: "Generate descriptive statistics",
+    },
+    {
+      label: "value_counts",
+      type: "method",
+      info: "Return a Series containing counts of unique values",
+    },
+    {
+      label: "unique",
+      type: "method",
+      info: "Return unique values of Series object",
+    },
+    {
+      label: "nunique",
+      type: "method",
+      info: "Return number of unique elements in the object",
+    },
+    { label: "sort_values", type: "method", info: "Sort by the values" },
+    {
+      label: "sort_index",
+      type: "method",
+      info: "Sort Series by index labels",
+    },
+    {
+      label: "reset_index",
+      type: "method",
+      info: "Generate a new DataFrame or Series with the index reset",
+    },
+    {
+      label: "dropna",
+      type: "method",
+      info: "Return a new Series with missing values removed",
+    },
+    {
+      label: "fillna",
+      type: "method",
+      info: "Fill NA/NaN values using the specified method",
+    },
+    {
+      label: "apply",
+      type: "method",
+      info: "Invoke function on values of Series",
+    },
+    {
+      label: "map",
+      type: "method",
+      info: "Map values of Series according to input correspondence",
+    },
+    {
+      label: "replace",
+      type: "method",
+      info: "Replace values given in to_replace with value",
+    },
+    {
+      label: "str",
+      type: "property",
+      info: "Vectorized string functions for Series and Index",
+    },
+    {
+      label: "dt",
+      type: "property",
+      info: "Accessor object for datetimelike properties",
+    },
+    {
+      label: "cat",
+      type: "property",
+      info: "Accessor object for categorical properties",
+    },
+  ],
+};
+
+function genericPandasCompletions(context: CompletionContext) {
+  const word = context.matchBefore(/[\w.]*/);
+  if (!word || (word.from === word.to && !context.explicit)) {
+    return null;
+  }
+
+  const text = word.text;
+
+  // Match patterns like "pd.", "df.", etc.
+  if (text.includes(".")) {
+    const parts = text.split(".");
+    const obj = parts.at(-2);
+    const partial = parts.at(-1) ?? "";
+
+    if (!obj || !pandasDefinitions[obj]) {
+      return null;
+    }
+
+    const options = pandasDefinitions[obj]
+      .filter(
+        (item) =>
+          !partial ||
+          item.label.toLowerCase().startsWith(partial.toLowerCase()),
+      )
+      .map((comp) => ({
+        ...comp,
+        info: undefined,
+        boost: comp.type === "method" ? 2 : comp.type === "function" ? 1 : 0,
+      }));
+
+    return {
+      filter: false,
+      from: word.from + text.length - partial.length,
+      options,
+    };
+  }
+}
+
+export const completion = [
+  pythonLanguage.data.of({
+    autocomplete: genericPandasCompletions,
+  }),
+  autocompletion(),
+];

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonResultsTable/PythonResultsTable.tsx

@@ -0,0 +1,181 @@
+import { useMemo } from "react";
+import { t } from "ttag";
+
+import { colors } from "metabase/lib/colors/colors";
+import { Alert, Box, Text } from "metabase/ui";
+
+interface PythonResultsTableProps {
+  csvData?: string;
+  error?: string;
+  stdout?: string;
+  stderr?: string;
+  isLoading?: boolean;
+}
+
+function parseCSV(csv: string): { headers: string[]; rows: string[][] } {
+  if (!csv || !csv.trim()) {
+    return { headers: [], rows: [] };
+  }
+
+  // Split by newlines to get rows
+  const lines = csv.trim().split("\n");
+  if (lines.length === 0) {
+    return { headers: [], rows: [] };
+  }
+
+  // Parse headers from first line
+  const headers = lines[0].split(",").map((h) => h.trim());
+
+  // Parse data rows
+  const rows = lines.slice(1).map((line) => {
+    // Simple CSV parsing - handles basic comma-separated values
+    // For more complex CSV (with quotes, escapes), we'd need a proper parser
+    return line.split(",").map((cell) => cell.trim());
+  });
+
+  return { headers, rows };
+}
+
+export function PythonResultsTable({
+  csvData,
+  error,
+  stdout,
+  stderr,
+  isLoading = false,
+}: PythonResultsTableProps) {
+  const { headers, rows } = useMemo(() => {
+    if (!csvData) {
+      return { headers: [], rows: [] };
+    }
+    const parsed = parseCSV(csvData);
+    return parsed;
+  }, [csvData]);
+
+  if (isLoading) {
+    return (
+      <Box p="lg">
+        <Text c="text-medium">{t`Running Python script...`}</Text>
+      </Box>
+    );
+  }
+
+  if (error) {
+    return (
+      <Box p="lg">
+        <Alert color="red" title={t`Execution Error`} mb="md">
+          {error}
+        </Alert>
+        {stdout && (
+          <Box mb="md">
+            <Text fw="bold" mb="xs">{t`Standard Output:`}</Text>
+            <Box
+              p="sm"
+              bg="bg-light"
+              style={{ fontFamily: "monospace", whiteSpace: "pre-wrap" }}
+            >
+              {stdout}
+            </Box>
+          </Box>
+        )}
+        {stderr && (
+          <Box>
+            <Text fw="bold" mb="xs">{t`Standard Error:`}</Text>
+            <Box
+              p="sm"
+              bg="bg-light"
+              style={{ fontFamily: "monospace", whiteSpace: "pre-wrap" }}
+            >
+              {stderr}
+            </Box>
+          </Box>
+        )}
+      </Box>
+    );
+  }
+
+  if (!csvData || headers.length === 0) {
+    return (
+      <Box p="lg">
+        <Text c="text-medium">{t`No results to display. Click "Run" to execute the script.`}</Text>
+        {stdout && (
+          <Box mt="md">
+            <Text fw="bold" mb="xs">{t`Standard Output:`}</Text>
+            <Box
+              p="sm"
+              bg="bg-light"
+              style={{ fontFamily: "monospace", whiteSpace: "pre-wrap" }}
+            >
+              {stdout}
+            </Box>
+          </Box>
+        )}
+      </Box>
+    );
+  }
+
+  return (
+    <Box
+      style={{
+        overflow: "auto",
+        padding: "1rem",
+        background: colors["bg-light"],
+        border: `1px solid ${colors["border"]}`,
+      }}
+    >
+      {stdout && (
+        <Box p="md" pb={0}>
+          <Text fw="bold" mb="xs">{t`Standard Output:`}</Text>
+          <Box
+            p="sm"
+            bg="bg-light"
+            style={{ fontFamily: "monospace", whiteSpace: "pre-wrap" }}
+            mb="md"
+          >
+            {stdout}
+          </Box>
+        </Box>
+      )}
+      <Box style={{ background: "white", padding: "1rem" }}>
+        <Text fw="bold" mb="md">{t`Results Table:`}</Text>
+        <table
+          style={{
+            width: "100%",
+            borderCollapse: "collapse",
+            border: `1px solid ${colors["border"]}`,
+          }}
+        >
+          <thead>
+            <tr style={{ backgroundColor: colors["bg-medium"] }}>
+              {headers.map((header, index) => (
+                <th
+                  key={index}
+                  style={{
+                    padding: "8px",
+                    textAlign: "left",
+                    borderBottom: `2px solid ${colors["border"]}`,
+                  }}
+                >
+                  {header}
+                </th>
+              ))}
+            </tr>
+          </thead>
+          <tbody>
+            {rows.map((row, rowIndex) => (
+              <tr
+                key={rowIndex}
+                style={{ borderBottom: `1px solid ${colors["bg-medium"]}` }}
+              >
+                {row.map((cell, cellIndex) => (
+                  <td key={cellIndex} style={{ padding: "8px" }}>
+                    {cell}
+                  </td>
+                ))}
+              </tr>
+            ))}
+          </tbody>
+        </table>
+      </Box>
+    </Box>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonResultsTable/index.ts

@@ -0,0 +1 @@
+export { PythonResultsTable } from "./PythonResultsTable";

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonTransformEditor/PythonTransformEditor.tsx

@@ -0,0 +1,235 @@
+import { useEffect, useState } from "react";
+
+import { Flex, Stack } from "metabase/ui";
+import type { PythonTransformSource } from "metabase-types/api";
+
+import { PythonDataPicker } from "../PythonDataPicker";
+import { EditorHeader } from "../QueryEditor/EditorHeader";
+import { PythonEditor } from "../QueryEditor/PythonEditor";
+
+type PythonTransformEditorProps = {
+  initialSource: PythonTransformSource;
+  isNew?: boolean;
+  isSaving?: boolean;
+  onSave: (newSource: PythonTransformSource) => void;
+  onCancel: () => void;
+  onSourceChange?: (newSource: PythonTransformSource) => void;
+};
+
+export function PythonTransformEditor({
+  initialSource,
+  isNew = true,
+  isSaving = false,
+  onSave,
+  onCancel,
+  onSourceChange,
+}: PythonTransformEditorProps) {
+  const [source, setSource] = useState(initialSource);
+  const [isSourceDirty, setIsSourceDirty] = useState(false);
+
+  useEffect(() => {
+    if (onSourceChange) {
+      onSourceChange(source);
+    }
+  }, [source, onSourceChange]);
+
+  const handleScriptChange = (body: string) => {
+    const newSource = {
+      ...source,
+      body,
+    };
+    setSource(newSource);
+    setIsSourceDirty(true);
+  };
+
+  const updateTransformSignature = (
+    script: string,
+    tables: Record<string, { id: number; name: string }>,
+  ): string => {
+    const tableAliases = Object.keys(tables);
+
+    const transformRegex = /^def\s+transform\s*\([^)]*\)\s*:\s*\n(\s*)/m;
+
+    const signatureOneLine = `def transform(${tableAliases.join(", ")}):`;
+    const newSignature =
+      signatureOneLine.length > 80 && tableAliases.length > 0
+        ? `def transform(\n${tableAliases.map((alias) => `    ${alias}`).join(",\n")},\n):`
+        : signatureOneLine;
+
+    if (transformRegex.test(script)) {
+      // Capture the existing indentation after the colon
+      const match = script.match(transformRegex);
+      const originalIndent = match ? match[1] : "    ";
+
+      let updatedScript = script.replace(
+        transformRegex,
+        newSignature + "\n" + originalIndent,
+      );
+
+      const argsRegex =
+        /(\s+"""[\s\S]*?)\s*Args:\s*\n([\s\S]*?)\s*(\n\s+Returns:|\n\s+""")/;
+      const argsMatch = updatedScript.match(argsRegex);
+
+      if (tableAliases.length > 0) {
+        const maxAliasLength = Math.max(...tableAliases.map((a) => a.length));
+        const newArgsSection = tableAliases
+          .map((alias) => {
+            const padding = " ".repeat(maxAliasLength - alias.length);
+            const tableName = tables[alias].name;
+            if (alias === tableName) {
+              return `        ${alias}:${padding} DataFrame containing the data from the corresponding table`;
+            }
+            return `        ${alias}:${padding} DataFrame containing the data from table "${tableName}"`;
+          })
+          .join("\n");
+
+        if (argsMatch) {
+          // Replace existing Args section
+          updatedScript = updatedScript.replace(
+            argsRegex,
+            `    $1
+
+    Args:
+${newArgsSection}
+$3`,
+          );
+        } else {
+          // No Args section exists, look for the pattern before Returns and insert Args
+          // Also remove the instructional text about selecting tables
+          const insertRegex =
+            /(\s+"""[\s\S]*?)\s*Select tables above to add them as function parameters\.\s*(\n\s+Returns:|\n\s+""")/;
+          const insertMatch = updatedScript.match(insertRegex);
+          if (insertMatch) {
+            updatedScript = updatedScript.replace(
+              insertRegex,
+              `$1
+
+    Args:
+${newArgsSection}
+$2`,
+            );
+          } else {
+            // Fallback: just look for Returns section without the specific text
+            // Remove any extra whitespace before Returns and ensure single blank line
+            const fallbackRegex = /(\s+"""[\s\S]*?)\s*(\n\s+Returns:|\n\s+""")/;
+            const fallbackMatch = updatedScript.match(fallbackRegex);
+            if (fallbackMatch) {
+              updatedScript = updatedScript.replace(
+                fallbackRegex,
+                `$1
+
+    Args:
+${newArgsSection}
+$2`,
+              );
+            }
+          }
+        }
+      } else if (argsMatch) {
+        // Remove existing Args section when no tables
+        updatedScript = updatedScript.replace(argsRegex, "$1$3");
+      }
+
+      return updatedScript;
+    }
+
+    const maxAliasLength =
+      tableAliases.length > 0
+        ? Math.max(...tableAliases.map((a) => a.length))
+        : 0;
+    const functionTemplate = `
+${newSignature}
+    """
+    Transform function that processes the input data.
+    ${
+      tableAliases.length > 0
+        ? `
+
+    Args:
+${tableAliases
+  .map((alias) => {
+    const padding = " ".repeat(maxAliasLength - alias.length);
+    const tableName = tables[alias].name;
+    if (alias === tableName) {
+      return `        ${alias}:${padding} DataFrame containing the data from the corresponding table`;
+    }
+    return `        ${alias}:${padding} DataFrame containing the data from table "${tableName}"`;
+  })
+  .join("\n")}`
+        : ""
+    }
+
+    Returns:
+        DataFrame: The transformed data
+    """
+    import pandas as pd
+    return pd.DataFrame()
+`;
+
+    return script + functionTemplate;
+  };
+
+  const handleDataChange = (
+    database: number,
+    tables: Record<string, { id: number; name: string }>,
+  ) => {
+    const updatedScript = updateTransformSignature(source.body, tables);
+
+    const sourceTables: Record<string, number> = {};
+    Object.entries(tables).forEach(([alias, tableInfo]) => {
+      sourceTables[alias] = tableInfo.id;
+    });
+
+    const newSource = {
+      ...source,
+      body: updatedScript,
+      "source-database": database,
+      "source-tables": sourceTables,
+    };
+    setSource(newSource);
+    setIsSourceDirty(true);
+  };
+
+  const handleSave = () => {
+    onSave(source);
+  };
+
+  const canSave = Boolean(
+    source.body.trim() &&
+      source["source-database"] &&
+      source["source-tables"] &&
+      Object.keys(source["source-tables"]).length > 0,
+  );
+
+  return (
+    <Stack
+      w="100%"
+      h="100%"
+      bg="bg-white"
+      data-testid="python-transform-editor"
+      gap={0}
+    >
+      <EditorHeader
+        isNew={isNew}
+        isSaving={isSaving}
+        canSave={canSave && (isNew || isSourceDirty)}
+        onSave={handleSave}
+        onCancel={onCancel}
+      />
+      <Flex h="100%" w="100%">
+        <PythonDataPicker
+          database={source["source-database"]}
+          tables={source["source-tables"]}
+          onChange={handleDataChange}
+        />
+
+        <PythonEditor
+          script={source.body}
+          isRunnable={true}
+          onChange={handleScriptChange}
+          tables={source["source-tables"]}
+        />
+      </Flex>
+    </Stack>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonTransformEditor/index.ts

@@ -0,0 +1 @@
+export { PythonTransformEditor } from "./PythonTransformEditor";

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/EditorBody/EditorBody.tsx

@@ -11,7 +11,13 @@ import { doesDatabaseSupportTransforms } from "metabase-enterprise/transforms/ut
 import type Question from "metabase-lib/v1/Question";
 import type Database from "metabase-lib/v1/metadata/Database";
 import type NativeQuery from "metabase-lib/v1/queries/NativeQuery";
-import type { Database as ApiDatabase, RecentItem } from "metabase-types/api";
+import type {
+  Database as ApiDatabase,
+  PythonTransformSource,
+  RecentItem,
+} from "metabase-types/api";
+
+import { PythonEditor } from "../PythonEditor";
 
 import S from "./EditorBody.module.css";
 import { ResizableBoxHandle } from "./ResizableBoxHandle";
@@ -21,10 +27,13 @@ const EDITOR_HEIGHT = 400;
 type EditorBodyProps = {
   question: Question;
   isNative: boolean;
+  isPython?: boolean;
+  pythonSource?: PythonTransformSource;
   isRunnable: boolean;
   isRunning: boolean;
   isResultDirty: boolean;
   onChange: (newQuestion: Question) => void;
+  onPythonChange?: (script: string) => void;
   onRunQuery: () => Promise<void>;
   onCancelQuery: () => void;
   databases: ApiDatabase[];
@@ -33,10 +42,13 @@ type EditorBodyProps = {
 export function EditorBody({
   question,
   isNative,
+  isPython = false,
+  pythonSource,
   isRunnable,
   isRunning,
   isResultDirty,
   onChange,
+  onPythonChange,
   onRunQuery,
   onCancelQuery,
   databases,
@@ -70,6 +82,24 @@ export function EditorBody({
 
   const dataPickerOptions = useDataPickerOptions({ databases });
 
+  const handlePythonChange = (script: string) => {
+    onPythonChange?.(script);
+  };
+
+  if (isPython) {
+    return (
+      <PythonEditor
+        script={pythonSource?.body || ""}
+        isRunnable={isRunnable}
+        isRunning={isRunning}
+        isResultDirty={isResultDirty}
+        onChange={handlePythonChange}
+        onRunScript={onRunQuery}
+        onCancelScript={onCancelQuery}
+      />
+    );
+  }
+
   return isNative ? (
     <NativeQueryEditor
       question={question}

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/EditorHeader/EditorHeader.tsx

@@ -23,6 +23,7 @@ export function EditorHeader({
       title={getTitle(isNew)}
       admin
       buttons={[
+        <Button key="cancel" small onClick={onCancel}>{t`Cancel`}</Button>,
         <Button
           key="save"
           onClick={onSave}
@@ -32,7 +33,6 @@ export function EditorHeader({
         >
           {getSaveButtonLabel(isNew, isSaving)}
         </Button>,
-        <Button key="cancel" small onClick={onCancel}>{t`Cancel`}</Button>,
       ]}
     />
   );

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/PythonEditor/ExecutionOutputTable.tsx

@@ -0,0 +1,25 @@
+import { useMemo } from "react";
+
+import { DataGrid, useDataGridInstance } from "metabase/data-grid";
+
+import { parseCSV } from "./utils";
+
+export function ExecutionOutputTable({ output }: { output?: string }) {
+  const { headers, rows } = useMemo(() => parseCSV(output || ""), [output]);
+
+  const tableProps = useDataGridInstance<string[], unknown>({
+    data: rows,
+    columnsOptions: headers.map((header, index) => ({
+      id: header,
+      name: header,
+      accessorFn: (row) => row[index],
+    })),
+    minGridWidth: 100,
+  });
+
+  if (!output || headers.length === 0) {
+    return null;
+  }
+
+  return <DataGrid {...tableProps} />;
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/PythonEditor/PythonEditor.module.css

@@ -0,0 +1,68 @@
+.root {
+  border-bottom: 1px solid var(--mb-color-border);
+}
+
+.editor {
+  width: 100%;
+  height: 100%;
+}
+
+.logs {
+  white-space: pre;
+  border: 1px solid var(--mb-color-border);
+  font-family: var(--mb-default-monospace-font-family);
+}
+
+.visualization {
+  flex: 1 0 auto;
+}
+
+.results {
+  width: 100%;
+  border-collapse: collapse;
+  border: 1px solid var(--mb-color-border);
+  border-radius: var(--mantine-radius-md);
+}
+
+.tableHeader {
+  background-color: var(--mb-color-bg-light);
+  padding: var(--mantine-spacing-sm);
+}
+
+.tableCell {
+  border-top: 1px solid var(--mb-color-border);
+  padding: var(--mantine-spacing-sm);
+}
+
+.error {
+  color: var(--mb-color-error);
+  background-color: var(--mb-color-bg-error);
+  border-bottom: 1px solid var(--mb-color-border);
+}
+
+.info {
+  color: var(--mb-color-text-dark);
+  border: 1px solid var(--mb-color-bg-medium);
+  background-color: var(--mb-color-bg-light);
+}
+
+.success {
+  color: var(--mb-color-success);
+  background-color: var(--mb-color-bg-success);
+  border-bottom: 1px solid var(--mb-color-border);
+}
+
+.loading {
+  position: absolute;
+  top: 0;
+  left: 0;
+  bottom: 0;
+  right: 0;
+  background-color: rgba(255, 255, 255, 0.5);
+}
+
+.libraryActions {
+  position: absolute;
+  top: 0;
+  right: 0;
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/PythonEditor/PythonEditor.tsx

@@ -0,0 +1,247 @@
+import type { ReactNode } from "react";
+import { ResizableBox } from "react-resizable";
+import { c, t } from "ttag";
+
+import EmptyCodeResult from "assets/img/empty-states/code.svg";
+import DebouncedFrame from "metabase/common/components/DebouncedFrame";
+import Link from "metabase/common/components/Link";
+import { LoadingSpinner } from "metabase/common/components/MetadataInfo/MetadataInfo.styled";
+import { isMac } from "metabase/lib/browser";
+import RunButtonWithTooltip from "metabase/query_builder/components/RunButtonWithTooltip";
+import { Box, Checkbox, Flex, Icon, Stack, Text, Title } from "metabase/ui";
+import { getPythonLibraryUrl } from "metabase-enterprise/transforms/urls";
+
+import { SHARED_LIB_IMPORT_NAME } from "../../../constants";
+import { PythonEditor as PythonCodeEditor } from "../../PythonEditor";
+import { ResizableBoxHandle } from "../EditorBody/ResizableBoxHandle";
+
+import { ExecutionOutputTable } from "./ExecutionOutputTable";
+import S from "./PythonEditor.module.css";
+import {
+  type ExecutionResult,
+  hasImport,
+  insertImport,
+  removeImport,
+  useTestPythonScript,
+} from "./utils";
+
+type PythonEditorProps = {
+  script: string;
+  isRunnable: boolean;
+  onChange: (script: string) => void;
+  onRunScript?: () => Promise<void>;
+  onCancelScript?: () => void;
+  tables?: Record<string, number>;
+};
+
+const EDITOR_HEIGHT = 400;
+
+export function PythonEditor({
+  script,
+  isRunnable,
+  onChange,
+  tables = {},
+}: PythonEditorProps) {
+  const { isRunning, isDirty, cancel, run, executionResult } =
+    useTestPythonScript(script, tables);
+
+  const hasSharedLib = hasImport(script, SHARED_LIB_IMPORT_NAME);
+  function handleToggleSharedLib() {
+    if (hasImport(script, SHARED_LIB_IMPORT_NAME)) {
+      onChange(removeImport(script, SHARED_LIB_IMPORT_NAME));
+    } else {
+      onChange(insertImport(script, SHARED_LIB_IMPORT_NAME));
+    }
+  }
+
+  return (
+    <Stack h="100%" w="100%" gap={0}>
+      <ResizableBox
+        className={S.root}
+        axis="y"
+        height={EDITOR_HEIGHT}
+        handle={<ResizableBoxHandle />}
+        resizeHandles={["s"]}
+      >
+        <Flex h="100%" align="end" bg="bg-light">
+          <PythonCodeEditor
+            value={script}
+            onChange={onChange}
+            withPandasCompletions
+          />
+
+          <Box p="md">
+            <RunButtonWithTooltip
+              disabled={!isRunnable}
+              isRunning={isRunning}
+              isDirty={isDirty}
+              onRun={run}
+              onCancel={cancel}
+              getTooltip={() => t`Run Python script`}
+            />
+          </Box>
+        </Flex>
+        <Stack className={S.libraryActions} p="md" gap="sm">
+          <Checkbox
+            label={t`Import shared library`}
+            checked={hasSharedLib}
+            onChange={handleToggleSharedLib}
+            size="sm"
+          />
+          <Flex
+            component={Link}
+            target="_blank"
+            to={getPythonLibraryUrl({ name: SHARED_LIB_IMPORT_NAME })}
+            gap="sm"
+          >
+            <Icon name="pencil" />
+            {t`Edit shared library`}
+          </Flex>
+        </Stack>
+      </ResizableBox>
+
+      <DebouncedFrame className={S.visualization}>
+        {executionResult ? (
+          <ExecutionResult executionResult={executionResult} />
+        ) : (
+          <EmptyState />
+        )}
+        {isRunning && <LoadingState />}
+      </DebouncedFrame>
+    </Stack>
+  );
+}
+
+function getRunQueryShortcut() {
+  return isMac() ? t`⌘ + return` : t`Ctrl + enter`;
+}
+
+function LoadingState() {
+  return (
+    <Flex p="md" className={S.loading}>
+      <LoadingSpinner />
+    </Flex>
+  );
+}
+
+function EmptyState() {
+  const keyboardShortcut = getRunQueryShortcut();
+
+  return (
+    <Flex h="100%" align="center" justify="center">
+      <Stack maw="25rem" gap={0} ta="center" align="center">
+        <Box maw="3rem" mb="0.75rem">
+          <img src={EmptyCodeResult} alt="Code prompt icon" />
+        </Box>
+        <Text c="text-medium">
+          {c("{0} refers to the keyboard shortcut")
+            .jt`To run your code, click on the Run button or type ${(
+            <b key="shortcut">({keyboardShortcut})</b>
+          )}`}
+        </Text>
+      </Stack>
+    </Flex>
+  );
+}
+
+function ExecutionResult({
+  executionResult,
+}: {
+  executionResult: ExecutionResult | null;
+}) {
+  if (!executionResult) {
+    return null;
+  }
+
+  const message = getMessageForExecutionResult(executionResult);
+
+  return (
+    <Stack gap={0}>
+      {message}
+
+      <ExecutionLogs
+        label={t`Standard output:`}
+        content={executionResult.stdout}
+      />
+      <ExecutionLogs
+        label={t`Standard error:`}
+        content={executionResult.stderr}
+      />
+
+      <ExecutionOutputTable output={executionResult.output} />
+    </Stack>
+  );
+}
+
+function getMessageForExecutionResult(executionResult: ExecutionResult) {
+  if (executionResult.error) {
+    return (
+      <Box className={S.error} p="sm">
+        <Flex gap="sm">
+          <Icon name="warning" />
+          <Stack gap={0}>
+            <Title order={5} mb="xs">
+              {t`An error occurred while executing your Python script`}
+            </Title>
+            {executionResult.error}
+          </Stack>
+        </Flex>
+      </Box>
+    );
+  }
+
+  if (!executionResult.output) {
+    return (
+      <Flex className={S.info} p="sm" bdrs="xs" gap="sm" align="center">
+        <Icon name="info" />
+        {t`No results to display.`}
+      </Flex>
+    );
+  }
+
+  return (
+    <Flex className={S.success} p="sm" gap="sm" align="center">
+      <Icon name="check" />
+      {t`Script executed successfully.`}
+    </Flex>
+  );
+}
+
+function ExecutionLogs({
+  label,
+  content,
+}: {
+  label: string;
+  content?: string | null;
+}) {
+  if (!content) {
+    return null;
+  }
+
+  return (
+    <Section title={label}>
+      <Box p="sm" bg="bg-light" mah="150px" bdrs="xs" className={S.logs}>
+        {content}
+      </Box>
+    </Section>
+  );
+}
+
+function Section({
+  title,
+  children,
+}: {
+  title: ReactNode;
+  children: ReactNode;
+}) {
+  return (
+    <Box>
+      {title && (
+        <Title order={5} mb="xs">
+          {title}
+        </Title>
+      )}
+      {children}
+    </Box>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/PythonEditor/index.ts

@@ -0,0 +1 @@
+export { PythonEditor } from "./PythonEditor";

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/PythonEditor/utils.ts

@@ -0,0 +1,131 @@
+import { useRef, useState } from "react";
+import { t } from "ttag";
+
+import {
+  type ExecutePythonResponse,
+  useExecutePythonMutation,
+} from "metabase-enterprise/transforms/api/python-runner";
+
+export function parseCSV(csv: string): { headers: string[]; rows: string[][] } {
+  if (!csv || !csv.trim()) {
+    return { headers: [], rows: [] };
+  }
+
+  const lines = csv.trim().split("\n");
+  if (lines.length === 0) {
+    return { headers: [], rows: [] };
+  }
+
+  const headers = lines[0].split(",").map((h) => h.trim());
+  const rows = lines.slice(1).map((line) => {
+    return line.split(",").map((cell) => cell.trim());
+  });
+
+  return { headers, rows };
+}
+
+export type ExecutionResult = {
+  output?: string;
+  stdout?: string;
+  stderr?: string;
+  error?: string;
+};
+
+type TestPythonScriptState = {
+  isRunning: boolean;
+  isDirty: boolean;
+  executionResult: ExecutionResult | null;
+  run: () => void;
+  cancel: () => void;
+};
+
+export function useTestPythonScript(
+  code: string,
+  tables: Record<string, number>,
+): TestPythonScriptState {
+  const [executePython, { isLoading: isRunning, originalArgs }] =
+    useExecutePythonMutation();
+  const abort = useRef<(() => void) | null>(null);
+  const [executionResult, setData] = useState<ExecutePythonResponse | null>(
+    null,
+  );
+
+  const isDirty = originalArgs?.code !== code;
+
+  const run = async () => {
+    const request = executePython({ code, tables });
+    abort.current = () => request.abort();
+
+    try {
+      const data = await request.unwrap();
+      setData(data);
+    } catch (error) {
+      if (typeof error === "object" && error !== null) {
+        if ("name" in error && error.name === "AbortError") {
+          setData({ error: t`Python script execution was canceled` });
+          return;
+        }
+
+        if ("message" in error && typeof error.message === "string") {
+          setData({ error: error.message });
+          return;
+        }
+
+        if (
+          "data" in error &&
+          typeof error.data === "object" &&
+          error.data !== null &&
+          "error" in error.data &&
+          typeof error.data.error === "string"
+        ) {
+          setData({ error: error?.data?.error });
+          return;
+        }
+      }
+
+      setData({ error: t`An unknown error occurred` });
+    }
+  };
+
+  const cancel = () => {
+    abort.current?.();
+  };
+
+  return {
+    isRunning,
+    isDirty,
+    cancel,
+    run,
+    executionResult,
+  };
+}
+
+export function insertImport(source: string, lib: string) {
+  const lines = source.split("\n");
+
+  // Find the first line that is not a comment
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    if (line.trim().startsWith("#")) {
+      continue;
+    }
+    lines.splice(i, 0, `import ${lib}`);
+    break;
+  }
+
+  return lines.join("\n");
+}
+
+function libImportRegex(lib: string) {
+  return new RegExp(`import ${lib}`, "g");
+}
+
+export function removeImport(source: string, lib: string) {
+  const lines = source.split("\n");
+  return lines.filter((line) => !libImportRegex(lib).test(line)).join("\n");
+}
+
+export function hasImport(source: string, lib: string) {
+  const regex = libImportRegex(lib);
+  return regex.test(source);
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/RunButton/RunButton.tsx

@@ -2,7 +2,7 @@ import { type ReactNode, type Ref, forwardRef, useState } from "react";
 import { useUpdateEffect } from "react-use";
 import { t } from "ttag";
 
-import { Button, Icon, Loader } from "metabase/ui";
+import { Button, Group, Icon, Loader } from "metabase/ui";
 import type { TransformRun } from "metabase-types/api";
 
 const RECENT_TIMEOUT = 5000;
@@ -12,6 +12,8 @@ type RunButtonProps = {
   isLoading: boolean;
   isDisabled?: boolean;
   onRun: () => void;
+  onCancel?: () => void;
+  isCanceling?: boolean;
 };
 
 export const RunButton = forwardRef(function RunButton(
@@ -20,6 +22,8 @@ export const RunButton = forwardRef(function RunButton(
     isLoading,
     isDisabled: isExternallyDisabled = false,
     onRun,
+    onCancel,
+    isCanceling = false,
   }: RunButtonProps,
   ref: Ref<HTMLButtonElement>,
 ) {
@@ -37,6 +41,35 @@ export const RunButton = forwardRef(function RunButton(
     return () => clearTimeout(timeoutId);
   }, [run]);
 
+  const showCancelButton = (run?.status === "started" || isLoading) && onCancel;
+
+  if (showCancelButton) {
+    return (
+      <Group gap="sm">
+        <Button
+          ref={ref}
+          variant="filled"
+          color={color}
+          leftSection={leftSection}
+          disabled={isDisabled}
+          data-testid="run-button"
+          onClick={onRun}
+        >
+          {label}
+        </Button>
+        <Button
+          variant="outline"
+          color="error"
+          disabled={isCanceling}
+          data-testid="cancel-button"
+          onClick={onCancel}
+        >
+          {isCanceling ? t`Canceling…` : t`Cancel`}
+        </Button>
+      </Group>
+    );
+  }
+
   return (
     <Button
       ref={ref}

enterprise/frontend/src/metabase-enterprise/transforms/constants.ts

@@ -6,3 +6,5 @@ export const POLLING_INTERVAL = isCypressActive ? 100 : 3000;
 
 export const FILTER_WIDGET_MIN_WIDTH = 300;
 export const FILTER_WIDGET_MAX_HEIGHT = 400;
+
+export const SHARED_LIB_IMPORT_NAME = "lib";

enterprise/frontend/src/metabase-enterprise/transforms/index.tsx

@@ -9,7 +9,9 @@ import { hasPremiumFeature } from "metabase-enterprise/settings";
 import { JobListPage } from "./pages/JobListPage";
 import { JobPage } from "./pages/JobPage";
 import { NewJobPage } from "./pages/NewJobPage";
+import { NewPythonTransformPage } from "./pages/NewPythonTransformPage";
 import { NewTransformQueryPage } from "./pages/NewTransformQueryPage";
+import { PythonLibraryEditorPage } from "./pages/PythonLibraryEditorPage";
 import { RunListPage } from "./pages/RunListPage";
 import { TransformListPage } from "./pages/TransformListPage";
 import { TransformPage } from "./pages/TransformPage";
@@ -32,7 +34,21 @@ if (hasPremiumFeature("transforms")) {
           <Route path="runs" component={RunListPage} />
           <Route path=":transformId" component={TransformPage} />
         </Route>
-        <Route path="new/:type" component={NewTransformQueryPage} />
+        <Route
+          component={(props) => <TransformPageLayout {...props} fullWidth />}
+        >
+          <Route path="library/:name" component={PythonLibraryEditorPage} />
+        </Route>
+        <Route
+          path="new/:type"
+          component={(props: any) =>
+            props.params.type === "python" ? (
+              <NewPythonTransformPage />
+            ) : (
+              <NewTransformQueryPage params={props.params} />
+            )
+          }
+        />
         <Route path="new/card/:cardId" component={NewTransformQueryPage} />
         <Route path=":transformId/query" component={TransformQueryPage} />
       </Route>

enterprise/frontend/src/metabase-enterprise/transforms/pages/NewPythonTransformPage/CreatePythonTransformModal/CreatePythonTransformModal.tsx

@@ -0,0 +1,154 @@
+import { Field, Form, Formik } from "formik";
+import { t } from "ttag";
+import * as Yup from "yup";
+
+import { Button, Group, Modal, Stack, TextInput } from "metabase/ui";
+import { useCreateTransformMutation } from "metabase-enterprise/api";
+import type { PythonTransformSource, Transform } from "metabase-types/api";
+
+type PythonTransformApiSource = {
+  type: "python";
+  body: string;
+  "source-database": number;
+  "source-tables": Record<string, number>;
+};
+
+type CreatePythonTransformModalProps = {
+  source: PythonTransformSource;
+  onCreate: (transform: Transform) => void;
+  onClose: () => void;
+};
+
+type FormValues = {
+  name: string;
+  description: string | null;
+  targetSchema: string | null;
+  targetName: string;
+};
+
+const getValidationSchema = () =>
+  Yup.object({
+    name: Yup.string().required(t`Name is required`),
+    description: Yup.string().nullable(),
+    targetSchema: Yup.string().nullable(),
+    targetName: Yup.string().required(t`Table name is required`),
+  });
+
+export function CreatePythonTransformModal({
+  source,
+  onCreate,
+  onClose,
+}: CreatePythonTransformModalProps) {
+  const [createTransform] = useCreateTransformMutation();
+
+  const initialValues: FormValues = {
+    name: "",
+    description: null,
+    targetSchema: "public",
+    targetName: "",
+  };
+
+  const handleSubmit = async (values: FormValues) => {
+    try {
+      const transformedSource: PythonTransformApiSource = {
+        type: "python",
+        body: source.body,
+        "source-database": source["source-database"],
+        "source-tables": source["source-tables"] || {},
+      };
+
+      const transform = await createTransform({
+        name: values.name,
+        description: values.description,
+        source: transformedSource as any,
+        target: {
+          type: "table",
+          schema: values.targetSchema?.trim() || null,
+          name: values.targetName,
+          database: source["source-database"], // for now the same as source
+        },
+      }).unwrap();
+
+      onCreate(transform);
+    } catch (error) {
+      console.error("Error creating Python transform:", error);
+      // TODO: Show error toast
+    }
+  };
+
+  return (
+    <Modal
+      opened
+      onClose={onClose}
+      title={t`Create Python Transform`}
+      size="md"
+    >
+      <Formik
+        initialValues={initialValues}
+        validationSchema={getValidationSchema()}
+        onSubmit={handleSubmit}
+      >
+        {({ isSubmitting, isValid }) => (
+          <Form>
+            <Stack gap="md">
+              <Field name="name">
+                {({ field, meta }: any) => (
+                  <TextInput
+                    {...field}
+                    label={t`Transform name`}
+                    placeholder={t`My Python Transform`}
+                    error={meta.touched && meta.error}
+                  />
+                )}
+              </Field>
+
+              <Field name="description">
+                {({ field }: any) => (
+                  <TextInput
+                    {...field}
+                    label={t`Description (optional)`}
+                    placeholder={t`What does this transform do?`}
+                  />
+                )}
+              </Field>
+
+              <Field name="targetName">
+                {({ field, meta }: any) => (
+                  <TextInput
+                    {...field}
+                    label={t`Output table name`}
+                    placeholder={t`my_python_transform`}
+                    error={meta.touched && meta.error}
+                  />
+                )}
+              </Field>
+
+              <Field name="targetSchema">
+                {({ field }: any) => (
+                  <TextInput
+                    {...field}
+                    label={t`Schema (optional)`}
+                    placeholder={t`public`}
+                  />
+                )}
+              </Field>
+
+              <Group justify="flex-end" mt="md">
+                <Button variant="subtle" onClick={onClose}>
+                  {t`Cancel`}
+                </Button>
+                <Button
+                  type="submit"
+                  loading={isSubmitting}
+                  disabled={!isValid}
+                >
+                  {t`Create transform`}
+                </Button>
+              </Group>
+            </Stack>
+          </Form>
+        )}
+      </Formik>
+    </Modal>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/pages/NewPythonTransformPage/CreatePythonTransformModal/index.ts

@@ -0,0 +1 @@
+export { CreatePythonTransformModal } from "./CreatePythonTransformModal";

enterprise/frontend/src/metabase-enterprise/transforms/pages/NewPythonTransformPage/NewPythonTransformPage.tsx

@@ -0,0 +1,71 @@
+import { useDisclosure } from "@mantine/hooks";
+import { useState } from "react";
+import { push } from "react-router-redux";
+
+import { useDispatch } from "metabase/lib/redux";
+import type { PythonTransformSource, Transform } from "metabase-types/api";
+
+import { PythonTransformEditor } from "../../components/PythonTransformEditor";
+import { getTransformListUrl, getTransformUrl } from "../../urls";
+
+import { CreatePythonTransformModal } from "./CreatePythonTransformModal";
+
+const DEFAULT_PYTHON_SOURCE: PythonTransformSource = {
+  type: "python",
+  "source-database": 1, // Default to first database, will be updated by user
+  "source-tables": {}, // Will be populated when user selects tables
+  body: `# Write your Python transformation script here
+import pandas as pd
+
+def transform():
+    """
+    Your transformation function.
+
+    Select tables above to add them as function parameters.
+
+    Returns:
+        DataFrame to write to the destination table
+    """
+    # Your transformation logic here
+    return pd.DataFrame([{"message": "Hello from Python transform!"}])`,
+};
+
+export function NewPythonTransformPage() {
+  const [source, setSource] = useState<PythonTransformSource>(
+    DEFAULT_PYTHON_SOURCE,
+  );
+  const [isModalOpened, { open: openModal, close: closeModal }] =
+    useDisclosure();
+  const dispatch = useDispatch();
+
+  const handleCreate = (transform: Transform) => {
+    dispatch(push(getTransformUrl(transform.id)));
+  };
+
+  const handleSaveClick = (newSource: PythonTransformSource) => {
+    setSource(newSource);
+    openModal();
+  };
+
+  const handleCancelClick = () => {
+    dispatch(push(getTransformListUrl()));
+  };
+
+  return (
+    <>
+      <PythonTransformEditor
+        initialSource={DEFAULT_PYTHON_SOURCE}
+        isNew
+        onSave={handleSaveClick}
+        onCancel={handleCancelClick}
+      />
+      {isModalOpened && (
+        <CreatePythonTransformModal
+          source={source}
+          onCreate={handleCreate}
+          onClose={closeModal}
+        />
+      )}
+    </>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/pages/NewPythonTransformPage/index.ts

@@ -0,0 +1 @@
+export { NewPythonTransformPage } from "./NewPythonTransformPage";

enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/CreateTransformModal/CreateTransformModal.tsx

@@ -107,7 +107,10 @@ function CreateTransformForm({
   }
 
   const handleSubmit = async (values: NewTransformValues) => {
-    const request = getCreateRequest(query, values);
+    if (!databaseId) {
+      throw new Error("Database ID is required");
+    }
+    const request = getCreateRequest(query, values, databaseId);
     const transform = await createTransform(request).unwrap();
 
     trackTransformCreated({ transformId: transform.id });
@@ -170,6 +173,7 @@ function getInitialValues(schemas: string[]): NewTransformValues {
 function getCreateRequest(
   query: DatasetQuery,
   { name, description, targetName, targetSchema }: NewTransformValues,
+  databaseId: number,
 ): CreateTransformRequest {
   return {
     name: name,
@@ -182,6 +186,7 @@ function getCreateRequest(
       type: "table",
       name: targetName,
       schema: targetSchema,
+      database: databaseId,
     },
   };
 }

enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/NewTransformQueryPage.tsx

@@ -90,7 +90,7 @@ function getParsedParams({
   cardId,
 }: NewTransformQueryPageParams): NewTransformQueryPageParsedParams {
   return {
-    type: type === "native" ? "native" : "query",
+    type: type === "native" ? "native" : type === "python" ? "query" : "query",
     cardId: cardId != null ? Urls.extractEntityId(cardId) : undefined,
   };
 }

enterprise/frontend/src/metabase-enterprise/transforms/pages/PythonLibraryEditorPage/PythonLibraryEditorPage.module.css

@@ -0,0 +1,5 @@
+.editor {
+  /* HACK: 133px is the height of the headers */
+  height: calc(100vh - 133px);
+  flex-grow: 1;
+}

enterprise/frontend/src/metabase-enterprise/transforms/pages/PythonLibraryEditorPage/PythonLibraryEditorPage.tsx

@@ -0,0 +1,119 @@
+import { useEffect, useState } from "react";
+import { t } from "ttag";
+
+import Button from "metabase/common/components/Button";
+import EditBar from "metabase/common/components/EditBar";
+import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
+import { useMetadataToasts } from "metabase/metadata/hooks";
+import { Box, Flex } from "metabase/ui";
+import {
+  useGetPythonLibraryQuery,
+  useUpdatePythonLibraryMutation,
+} from "metabase-enterprise/api/python-transform-library";
+import { PythonEditor } from "metabase-enterprise/transforms/components/PythonEditor";
+import type { PythonLibraryEditorPageParams } from "metabase-enterprise/transforms/types";
+
+import S from "./PythonLibraryEditorPage.module.css";
+
+type PythonLibraryEditorPageProps = {
+  params: PythonLibraryEditorPageParams;
+};
+
+export function PythonLibraryEditorPage({
+  params,
+}: PythonLibraryEditorPageProps) {
+  const { name } = params;
+  const [source, setSource] = useState("");
+
+  const {
+    data: library,
+    isLoading,
+    error,
+  } = useGetPythonLibraryQuery({ name });
+  const [updatePythonLibrary, { isLoading: isSaving }] =
+    useUpdatePythonLibraryMutation();
+  const { sendSuccessToast, sendErrorToast } = useMetadataToasts();
+
+  useEffect(() => {
+    setSource(library?.source || "");
+  }, [library?.source]);
+
+  const isDirty = source !== library?.source;
+
+  function handleRevert() {
+    setSource(library?.source || "");
+  }
+
+  async function handleSave() {
+    try {
+      await updatePythonLibrary({ name, source }).unwrap();
+      sendSuccessToast(t`Python library saved`);
+    } catch (error) {
+      sendErrorToast(t`Python library could not be saved`);
+    }
+  }
+
+  if (isLoading || error) {
+    return (
+      <Box p="md">
+        <LoadingAndErrorWrapper loading={isLoading} error={error} />
+      </Box>
+    );
+  }
+
+  return (
+    <Flex h="100%" w="100%" bg="bg-light" gap={0} direction="column">
+      <LibraryEditorHeader
+        onSave={handleSave}
+        onRevert={handleRevert}
+        isDirty={isDirty}
+        isSaving={isSaving}
+      />
+      <PythonEditor
+        value={source}
+        onChange={setSource}
+        withPandasCompletions
+        className={S.editor}
+      />
+    </Flex>
+  );
+}
+
+export function LibraryEditorHeader({
+  isDirty,
+  isSaving,
+  onSave,
+  onRevert,
+}: {
+  isDirty?: boolean;
+  isSaving?: boolean;
+  onSave: () => void;
+  onRevert: () => void;
+}) {
+  return (
+    <EditBar
+      title={t`You are editing the shared Python library`}
+      admin
+      buttons={[
+        <Button
+          key="save"
+          onClick={onRevert}
+          primary
+          small
+          disabled={!isDirty || isSaving}
+        >
+          {t`Revert`}
+        </Button>,
+        <Button
+          key="save"
+          onClick={onSave}
+          primary
+          small
+          disabled={!isDirty || isSaving}
+        >
+          {t`Save`}
+        </Button>,
+      ]}
+    />
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/pages/PythonLibraryEditorPage/index.ts

@@ -0,0 +1 @@
+export * from "./PythonLibraryEditorPage";

enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformListPage/CreateTransformMenu/CreateTransformMenu.tsx

@@ -85,6 +85,19 @@ export function CreateTransformMenu() {
               >
                 {t`SQL query`}
               </Menu.Item>
+              <Menu.Item
+                component={ForwardRefLink}
+                to={getNewTransformFromTypeUrl("python")}
+                leftSection={<Icon name="code_block" />}
+                onClick={() => {
+                  trackTransformCreate({
+                    triggeredFrom: "transform-page-create-menu",
+                    creationType: "python",
+                  });
+                }}
+              >
+                {t`Python script`}
+              </Menu.Item>
               <Menu.Item
                 leftSection={<Icon name="copy" />}
                 onClick={handleSavedQuestionClick}

enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/ManageSection/ManageSection.tsx

@@ -19,9 +19,12 @@ type ManageSectionProps = {
 };
 
 export function ManageSection({ transform }: ManageSectionProps) {
+  const sectionLabel =
+    transform.source.type === "python" ? t`Python Script` : t`Query`;
+
   return (
     <TitleSection
-      label={t`Query`}
+      label={sectionLabel}
       rightSection={
         <Group>
           <EditQueryButton transform={transform} />
@@ -30,7 +33,14 @@ export function ManageSection({ transform }: ManageSectionProps) {
       }
     >
       <Card p={0} shadow="none" withBorder>
-        <QueryView query={transform.source.query} />
+        {transform.source.type === "query" && (
+          <QueryView query={transform.source.query} />
+        )}
+        {transform.source.type === "python" && (
+          <div style={{ padding: "1rem" }}>
+            <pre>{transform.source.body}</pre>
+          </div>
+        )}
       </Card>
     </TitleSection>
   );
@@ -41,13 +51,16 @@ type EditQueryButtonProps = {
 };
 
 function EditQueryButton({ transform }: EditQueryButtonProps) {
+  const buttonText =
+    transform.source.type === "python" ? t`Edit script` : t`Edit query`;
+
   return (
     <Button
       component={Link}
       to={getTransformQueryUrl(transform.id)}
       leftSection={<Icon name="pencil_lines" aria-hidden />}
     >
-      {t`Edit query`}
+      {buttonText}
     </Button>
   );
 }

enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/RunSection/RunSection.tsx

@@ -5,6 +5,7 @@ import { useSetting } from "metabase/common/hooks";
 import { useMetadataToasts } from "metabase/metadata/hooks";
 import { Anchor, Box, Divider, Group, Icon, Stack } from "metabase/ui";
 import {
+  useCancelTransformMutation,
   useLazyGetTransformQuery,
   useRunTransformMutation,
   useUpdateTransformMutation,
@@ -12,6 +13,7 @@ import {
 import { trackTranformTriggerManualRun } from "metabase-enterprise/transforms/analytics";
 import type { Transform, TransformTagId } from "metabase-types/api";
 
+import { LogOutput } from "../../../components/LogOutput";
 import { RunButton } from "../../../components/RunButton";
 import { RunErrorInfo } from "../../../components/RunErrorInfo";
 import { SplitSection } from "../../../components/SplitSection";
@@ -29,10 +31,16 @@ export function RunSection({ transform }: RunSectionProps) {
       label={t`Run this transform`}
       description={t`This transform will be run whenever the jobs it belongs to are scheduled.`}
     >
-      <Group p="lg" justify="space-between">
-        <RunStatusSection transform={transform} />
-        <RunButtonSection transform={transform} />
-      </Group>
+      <Stack>
+        <Group p="lg" justify="space-between">
+          <RunStatusSection transform={transform} />
+          <RunButtonSection transform={transform} />
+        </Group>
+        {transform?.last_run?.message &&
+          ["started", "succeeded"].some(
+            (s) => transform?.last_run?.status === s,
+          ) && <LogOutput content={transform.last_run.message} />}
+      </Stack>
       <Divider />
       <Group p="lg" gap="lg">
         <Stack gap="sm">
@@ -143,7 +151,9 @@ type RunButtonSectionProps = {
 function RunButtonSection({ transform }: RunButtonSectionProps) {
   const [fetchTransform, { isFetching }] = useLazyGetTransformQuery();
   const [runTransform, { isLoading: isRunning }] = useRunTransformMutation();
-  const { sendErrorToast } = useMetadataToasts();
+  const [cancelTransform, { isLoading: isCanceling }] =
+    useCancelTransformMutation();
+  const { sendErrorToast, sendSuccessToast } = useMetadataToasts();
 
   const handleRun = async () => {
     trackTranformTriggerManualRun({
@@ -161,11 +171,24 @@ function RunButtonSection({ transform }: RunButtonSectionProps) {
     return { error };
   };
 
+  const handleCancel = async () => {
+    const { error } = await cancelTransform(transform.id);
+    if (error) {
+      sendErrorToast(t`Failed to cancel transform`);
+    } else {
+      sendSuccessToast(t`Transform cancellation requested`);
+      // fetch the transform to get the updated status
+      fetchTransform(transform.id);
+    }
+  };
+
   return (
     <RunButton
       run={transform.last_run}
       isLoading={isFetching || isRunning}
       onRun={handleRun}
+      onCancel={handleCancel}
+      isCanceling={isCanceling}
     />
   );
 }

enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/SourceSection/SourceSection.tsx

@@ -0,0 +1,190 @@
+import { t } from "ttag";
+
+import { skipToken, useGetDatabaseQuery, useGetTableQuery } from "metabase/api";
+import Link from "metabase/common/components/Link";
+import CS from "metabase/css/core/index.css";
+import {
+  Divider,
+  Group,
+  Icon,
+  type IconName,
+  Loader,
+  Stack,
+  Text,
+} from "metabase/ui";
+import type { Transform } from "metabase-types/api";
+
+import { SplitSection } from "../../../components/SplitSection";
+import {
+  getBrowseDatabaseUrl,
+  getBrowseSchemaUrl,
+  getQueryBuilderUrl,
+} from "../../../urls";
+
+type SourceSectionProps = {
+  transform: Transform;
+};
+
+export function SourceSection({ transform }: SourceSectionProps) {
+  if (transform.source.type !== "python") {
+    return null;
+  }
+
+  return (
+    <SplitSection
+      label={t`Transform source`}
+      description={t`The data sources for this Python transform.`}
+    >
+      <Group p="lg">
+        <SourceInfo transform={transform} />
+      </Group>
+    </SplitSection>
+  );
+}
+
+type SourceInfoProps = {
+  transform: Transform;
+};
+
+function SourceInfo({ transform }: SourceInfoProps) {
+  const source = transform.source;
+  const sourceDatabaseId =
+    source.type === "python" ? source["source-database"] : undefined;
+  const sourceTables =
+    source.type === "python" ? source["source-tables"] : undefined;
+
+  const { data: database, isLoading: isDatabaseLoading } = useGetDatabaseQuery(
+    sourceDatabaseId ? { id: sourceDatabaseId } : skipToken,
+  );
+
+  if (!database) {
+    return null;
+  }
+
+  const hasMultipleTables =
+    sourceTables && Object.keys(sourceTables).length > 0;
+
+  if (isDatabaseLoading) {
+    return <Loader size="sm" />;
+  }
+
+  // Display for multiple tables
+  if (hasMultipleTables) {
+    return (
+      <Stack gap="sm">
+        {database && (
+          <Group gap="sm">
+            <SourceItemLink
+              label={database.name}
+              icon="database"
+              to={getBrowseDatabaseUrl(database.id)}
+              data-testid="source-database-link"
+            />
+          </Group>
+        )}
+        <Divider />
+        <Stack gap="xs">
+          {Object.entries(sourceTables).map(([alias, tableId]) => (
+            <TableWithAlias
+              key={alias}
+              alias={alias}
+              tableId={tableId as number}
+              databaseId={sourceDatabaseId as number}
+            />
+          ))}
+        </Stack>
+      </Stack>
+    );
+  }
+
+  // Just show database if no tables selected
+  return (
+    <Group gap="sm">
+      {database != null && (
+        <SourceItemLink
+          label={database.name}
+          icon="database"
+          to={getBrowseDatabaseUrl(database.id)}
+          data-testid="source-database-link"
+        />
+      )}
+    </Group>
+  );
+}
+
+type TableWithAliasProps = {
+  alias: string;
+  tableId: number;
+  databaseId: number;
+};
+
+function TableWithAlias({ alias, tableId, databaseId }: TableWithAliasProps) {
+  const { data: table, isLoading } = useGetTableQuery(
+    tableId ? { id: tableId } : skipToken,
+  );
+
+  if (isLoading) {
+    return <Loader size="xs" />;
+  }
+
+  return (
+    <Group gap="sm" align="flex-start">
+      <Group gap="xs">
+        {table?.schema && (
+          <>
+            <SourceItemLink
+              label={table.schema}
+              icon="folder"
+              to={getBrowseSchemaUrl(databaseId, table.schema)}
+              small
+            />
+            <SourceItemDivider />
+          </>
+        )}
+        <SourceItemLink
+          label={table?.display_name || table?.name || `Table ${tableId}`}
+          icon="table2"
+          to={table ? getQueryBuilderUrl(table.id, table.db_id) : undefined}
+          data-testid={`source-table-${alias}-link`}
+          small
+        />
+      </Group>
+    </Group>
+  );
+}
+
+type SourceItemLinkProps = {
+  label: string;
+  icon: IconName;
+  to?: string;
+  "data-testid"?: string;
+  small?: boolean;
+};
+
+function SourceItemLink({
+  label,
+  icon,
+  to,
+  "data-testid": dataTestId,
+  small = false,
+}: SourceItemLinkProps) {
+  return (
+    <Link
+      className={CS.link}
+      to={to ?? ""}
+      disabled={to == null}
+      data-testid={dataTestId}
+    >
+      <Group gap="xs">
+        <Icon name={icon} size={small ? 14 : 16} />
+        <Text c="inherit" size={small ? "sm" : "md"}>
+          {label}
+        </Text>
+      </Group>
+    </Link>
+  );
+}
+
+function SourceItemDivider() {
+  return <Icon name="chevronright" size={8} />;
+}

enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/SourceSection/index.ts

@@ -0,0 +1 @@
+export * from "./SourceSection";

enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TargetSection/TargetSection.tsx

@@ -58,7 +58,13 @@ type TargetInfoProps = {
 
 function TargetInfo({ transform }: TargetInfoProps) {
   const { source, target, table } = transform;
-  const { database: databaseId } = source.query;
+
+  const databaseId =
+    source.type === "query"
+      ? source.query.database
+      : source.type === "python"
+        ? source["source-database"]
+        : undefined;
 
   const { data: databaseFromApi, isLoading: isDatabaseLoading } =
     useGetDatabaseQuery(

enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TargetSection/UpdateTargetModal/UpdateTargetModal.tsx

@@ -83,7 +83,12 @@ function UpdateTargetForm({
   onClose,
 }: UpdateTargetFormProps) {
   const { source, target, table } = transform;
-  const { database: databaseId } = source.query;
+  const databaseId =
+    source.type === "query"
+      ? source.query.database
+      : source.type === "python"
+        ? source["source-database"]
+        : undefined;
   const [updateTransform] = useUpdateTransformMutation();
   const [deleteTransformTarget] = useDeleteTransformTargetMutation();
   const initialValues = useMemo(() => getInitialValues(transform), [transform]);
@@ -113,10 +118,15 @@ function UpdateTargetForm({
   }
 
   const handleSubmit = async (values: EditTransformValues) => {
+    if (!databaseId) {
+      throw new Error("Database ID is required");
+    }
     if (shouldDeleteTarget) {
       await deleteTransformTarget(transform.id).unwrap();
     }
-    await updateTransform(getUpdateRequest(transform, values)).unwrap();
+    await updateTransform(
+      getUpdateRequest(transform, values, databaseId),
+    ).unwrap();
     onUpdate();
   };
 
@@ -199,6 +209,7 @@ function getSubmitButtonColor(shouldDeleteTarget: boolean) {
 function getUpdateRequest(
   { id }: Transform,
   { name, schema }: EditTransformValues,
+  databaseId: number,
 ): UpdateTransformRequest {
   return {
     id,
@@ -206,6 +217,7 @@ function getUpdateRequest(
       type: "table",
       name,
       schema,
+      database: databaseId,
     },
   };
 }

enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TransformPage.tsx

@@ -15,6 +15,7 @@ import { HeaderSection } from "./HeaderSection";
 import { ManageSection } from "./ManageSection";
 import { NameSection } from "./NameSection";
 import { RunSection } from "./RunSection";
+import { SourceSection } from "./SourceSection";
 import { TargetSection } from "./TargetSection";
 
 type TransformPageParams = {
@@ -59,6 +60,7 @@ export function TransformPage({ params }: TransformPageProps) {
         <NameSection transform={transform} />
       </Stack>
       <RunSection transform={transform} />
+      <SourceSection transform={transform} />
       <TargetSection transform={transform} />
       <ManageSection transform={transform} />
       <DependenciesSection transform={transform} />

enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPageLayout/TransformPageLayout.tsx

@@ -8,8 +8,14 @@ import {
 import { AdminSettingsLayout } from "metabase/common/components/AdminLayout/AdminSettingsLayout";
 import { useSelector } from "metabase/lib/redux";
 import { getLocation } from "metabase/selectors/routing";
+import { SHARED_LIB_IMPORT_NAME } from "metabase-enterprise/transforms/constants";
 
-import { getJobListUrl, getRunListUrl, getTransformListUrl } from "../../urls";
+import {
+  getJobListUrl,
+  getPythonLibraryUrl,
+  getRunListUrl,
+  getTransformListUrl,
+} from "../../urls";
 
 type TransformPageLayoutParams = {
   transformId?: string;
@@ -18,17 +24,20 @@ type TransformPageLayoutParams = {
 
 type TransformPageLayoutProps = {
   params: TransformPageLayoutParams;
+  fullWidth?: boolean;
   children?: ReactNode;
 };
 
 export function TransformPageLayout({
   params,
   children,
+  fullWidth,
 }: TransformPageLayoutProps) {
   return (
     <AdminSettingsLayout
       sidebar={<TransformSidebar params={params} />}
       maw="60rem"
+      fullWidth={fullWidth}
     >
       {children}
     </AdminSettingsLayout>
@@ -61,6 +70,11 @@ function TransformSidebar({ params }: TransformSidebarProps) {
         active={pathname?.startsWith(jobListUrl)}
       />
       <AdminNavItem label={t`Runs`} path={getRunListUrl()} icon="list" />
+      <AdminNavItem
+        label={t`Python library`}
+        path={getPythonLibraryUrl({ name: SHARED_LIB_IMPORT_NAME })}
+        icon="code_block"
+      />
     </AdminNavWrapper>
   );
 }

enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformQueryPage/TransformQueryPage.tsx

@@ -10,7 +10,12 @@ import {
   useGetTransformQuery,
   useUpdateTransformMutation,
 } from "metabase-enterprise/api";
-import type { DatasetQuery, Transform } from "metabase-types/api";
+import { PythonTransformEditor } from "metabase-enterprise/transforms/components/PythonTransformEditor";
+import type {
+  DatasetQuery,
+  PythonTransformSource,
+  Transform,
+} from "metabase-types/api";
 
 import { QueryEditor } from "../../components/QueryEditor";
 import { getTransformUrl } from "../../urls";
@@ -53,7 +58,7 @@ export function TransformQueryPageBody({
   const dispatch = useDispatch();
   const { sendErrorToast, sendSuccessToast } = useMetadataToasts();
 
-  const handleSave = async (query: DatasetQuery) => {
+  const handleDatasetSave = async (query: DatasetQuery) => {
     const { error } = await updateTransform({
       id: transform.id,
       source: {
@@ -70,16 +75,41 @@ export function TransformQueryPageBody({
     }
   };
 
+  const handlePythonSave = async (source: PythonTransformSource) => {
+    const { error } = await updateTransform({
+      id: transform.id,
+      source: source,
+    });
+
+    if (error) {
+      sendErrorToast(t`Failed to update transform query`);
+    } else {
+      sendSuccessToast(t`Transform query updated`);
+      dispatch(push(getTransformUrl(transform.id)));
+    }
+  };
+
   const handleCancel = () => {
     dispatch(push(getTransformUrl(transform.id)));
   };
 
+  if (transform.source.type === "python") {
+    return (
+      <PythonTransformEditor
+        initialSource={transform.source}
+        isNew={false}
+        isSaving={isLoading}
+        onSave={handlePythonSave}
+        onCancel={handleCancel}
+      />
+    );
+  }
   return (
     <QueryEditor
       initialQuery={transform.source.query}
       isNew={false}
       isSaving={isLoading}
-      onSave={handleSave}
+      onSave={handleDatasetSave}
       onCancel={handleCancel}
     />
   );

enterprise/frontend/src/metabase-enterprise/transforms/types.ts

@@ -20,3 +20,7 @@ export type JobListParams = {
   nextRunStartTime?: string;
   transformTagIds?: TransformTagId[];
 };
+
+export type PythonLibraryEditorPageParams = {
+  name: string;
+};

enterprise/frontend/src/metabase-enterprise/transforms/urls.ts

@@ -7,7 +7,11 @@ import type {
   TransformJobId,
 } from "metabase-types/api";
 
-import type { JobListParams, RunListParams } from "./types";
+import type {
+  JobListParams,
+  PythonLibraryEditorPageParams,
+  RunListParams,
+} from "./types";
 
 export const ROOT_URL = "/admin/transforms";
 
@@ -15,7 +19,9 @@ export function getTransformListUrl() {
   return ROOT_URL;
 }
 
-export function getNewTransformFromTypeUrl(type: DatasetQuery["type"]) {
+export function getNewTransformFromTypeUrl(
+  type: DatasetQuery["type"] | "python",
+) {
   return `${ROOT_URL}/new/${type}`;
 }
 
@@ -121,3 +127,7 @@ export function getTableMetadataUrl(
 ) {
   return `/admin/datamodel/database/${databaseId}/schema/${databaseId}:${encodeURIComponent(schema ?? "")}/table/${tableId}`;
 }
+
+export function getPythonLibraryUrl({ name }: PythonLibraryEditorPageParams) {
+  return `/admin/transforms/library/${name}`;
+}

frontend/src/metabase-types/analytics/event.ts

@@ -294,7 +294,7 @@ export type TransformJobTriggerManualRunEvent = ValidateEvent<{
 export type TransformCreateEvent = ValidateEvent<{
   event: "transform_create";
   triggered_from: "transform-page-create-menu";
-  event_detail: "query" | "native" | "saved-question";
+  event_detail: "query" | "native" | "python" | "saved-question";
 }>;
 
 export type TransformCreatedEvent = ValidateEvent<{

frontend/src/metabase-types/api/mocks/transform.ts

@@ -9,13 +9,19 @@ import type {
 
 import { createMockStructuredDatasetQuery } from "./query";
 
-export function createMockTransformSource(
-  opts?: Partial<TransformSource>,
-): TransformSource {
+export function createMockTransformSource(): TransformSource {
   return {
     type: "query",
     query: createMockStructuredDatasetQuery(),
-    ...opts,
+  };
+}
+
+export function createMockPythonTransformSource(): TransformSource {
+  return {
+    type: "python",
+    body: "# Python script\nprint('Hello, world!')",
+    "source-database": 1,
+    "source-tables": {},
   };
 }
 
@@ -26,6 +32,7 @@ export function createMockTransformTarget(
     type: "table",
     name: "Table",
     schema: null,
+    database: 1,
     ...opts,
   };
 }

frontend/src/metabase-types/api/settings.ts

@@ -669,6 +669,14 @@ export interface EnterpriseSettings extends Settings {
   "saml-group-mappings": Record<string, GroupId[]> | null;
   "database-replication-enabled": boolean | null;
   "database-replication-connections"?: DatabaseReplicationConnections | null;
+  "python-execution-server-url"?: string | null;
+  "python-storage-s-3-endpoint"?: string | null;
+  "python-storage-s-3-region"?: string | null;
+  "python-storage-s-3-bucket"?: string | null;
+  "python-storage-s-3-access-key"?: string | null;
+  "python-storage-s-3-secret-key"?: string | null;
+  "python-storage-s-3-container-endpoint"?: string | null;
+  "python-storage-s-3-path-style-access"?: boolean | null;
   /**
    * @deprecated
    */

frontend/src/metabase-types/api/transform.ts

@@ -22,17 +22,26 @@ export type Transform = {
   last_run?: TransformRun | null;
 };
 
-export type TransformSource = {
+export type PythonTransformSource = {
+  type: "python";
+  body: string;
+  "source-database": number;
+  "source-tables": Record<string, number>; // alias -> table-id mapping
+};
+export type QueryTransformSource = {
   type: "query";
   query: DatasetQuery;
 };
 
+export type TransformSource = QueryTransformSource | PythonTransformSource;
+
 export type TransformTargetType = "table";
 
 export type TransformTarget = {
   type: TransformTargetType;
   name: string;
   schema: string | null;
+  database: number;
 };
 
 export type TransformRun = {

frontend/src/metabase/admin/settings/components/SettingsNav/SettingsNav.tsx

@@ -24,6 +24,7 @@ export function SettingsNav() {
   const hasSaml = useHasTokenFeature("sso_saml");
   const hasJwt = useHasTokenFeature("sso_jwt");
   const hasScim = useHasTokenFeature("scim");
+  const hasTransforms = useHasTokenFeature("transforms");
 
   return (
     <AdminNavWrapper>
@@ -88,6 +89,13 @@ export function SettingsNav() {
       </SettingsNavItem>
       <NavDivider />
       <SettingsNavItem path="uploads" label={t`Uploads`} icon="upload" />
+      {hasTransforms && (
+        <SettingsNavItem
+          path="python-runner"
+          label={t`Python Runner`}
+          icon="snippet"
+        />
+      )}
       <SettingsNavItem
         path="public-sharing"
         label={t`Public sharing`}

frontend/src/metabase/admin/settings/components/SettingsPages/PythonRunnerSettingsPage.tsx

@@ -0,0 +1,91 @@
+import { t } from "ttag";
+
+import {
+  SettingsPageWrapper,
+  SettingsSection,
+} from "metabase/admin/components/SettingsSection";
+import { useGetSettingsQuery } from "metabase/api";
+import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
+import { useHasTokenFeature } from "metabase/common/hooks";
+
+import { AdminSettingInput } from "../widgets/AdminSettingInput";
+
+export function PythonRunnerSettingsPage() {
+  // it seems we don't need to wire in _settingValues to the inputs - what is this magic??
+  const { data: _settingValues, isLoading } = useGetSettingsQuery();
+  const hasTransformsFeature = useHasTokenFeature("transforms");
+
+  if (isLoading) {
+    return <LoadingAndErrorWrapper loading />;
+  }
+
+  if (!hasTransformsFeature) {
+    return (
+      <SettingsPageWrapper title={t`Python Runner`}>
+        <SettingsSection>
+          <div className="text-medium">
+            {t`Python Runner configuration requires the transforms feature.`}
+          </div>
+        </SettingsSection>
+      </SettingsPageWrapper>
+    );
+  }
+
+  return (
+    <SettingsPageWrapper title={t`Python Runner`}>
+      <SettingsSection title={t`Service Configuration`}>
+        <AdminSettingInput
+          name="python-execution-server-url"
+          title={t`Python Execution Server URL`}
+          description={t`URL for the Python execution server that runs transform functions.`}
+          placeholder="http://localhost:5001"
+          inputType="text"
+        />
+      </SettingsSection>
+      <SettingsSection title={t`S3 Storage Configuration`}>
+        <AdminSettingInput
+          name="python-storage-s-3-endpoint"
+          title={t`S3 Endpoint`}
+          description={t`S3 endpoint URL for storing Python execution artifacts.`}
+          inputType="text"
+        />
+        <AdminSettingInput
+          name="python-storage-s-3-region"
+          title={t`S3 Region`}
+          description={t`AWS region for S3 storage.`}
+          inputType="text"
+        />
+        <AdminSettingInput
+          name="python-storage-s-3-bucket"
+          title={t`S3 Bucket`}
+          description={t`S3 bucket name for storing Python execution artifacts.`}
+          inputType="text"
+        />
+        <AdminSettingInput
+          name="python-storage-s-3-access-key"
+          title={t`S3 Access Key ID`}
+          description={t`AWS access key ID for S3 authentication.`}
+          inputType="password"
+        />
+        <AdminSettingInput
+          name="python-storage-s-3-secret-key"
+          title={t`S3 Secret Access Key`}
+          description={t`AWS secret access key for S3 authentication.`}
+          inputType="password"
+        />
+        <AdminSettingInput
+          name="python-storage-s-3-container-endpoint"
+          title={t`S3 Container Endpoint (Optional)`}
+          description={t`Alternative S3 endpoint accessible from containers. Leave empty if same as main endpoint.`}
+          inputType="text"
+        />
+        <AdminSettingInput
+          name="python-storage-s-3-path-style-access"
+          title={t`Use Path-Style Access`}
+          description={t`Use path-style access for S3 requests (required for LocalStack and some S3-compatible services).`}
+          inputType="boolean"
+        />
+      </SettingsSection>
+    </SettingsPageWrapper>
+  );
+}

frontend/src/metabase/admin/settingsRoutes.tsx

@@ -22,6 +22,7 @@ import { LocalizationSettingsPage } from "./settings/components/SettingsPages/Lo
 import { MapsSettingsPage } from "./settings/components/SettingsPages/MapsSettingsPage";
 import { NotificationSettingsPage } from "./settings/components/SettingsPages/NotificationSettingsPage";
 import { PublicSharingSettingsPage } from "./settings/components/SettingsPages/PublicSharingSettingsPage";
+import { PythonRunnerSettingsPage } from "./settings/components/SettingsPages/PythonRunnerSettingsPage";
 import { UpdatesSettingsPage } from "./settings/components/SettingsPages/UpdatesSettingsPage";
 import { UploadSettingsPage } from "./settings/components/SettingsPages/UploadSettingsPage";
 
@@ -63,6 +64,7 @@ export const getSettingsRoutes = () => (
     <Route path="maps" component={MapsSettingsPage} />
     <Route path="localization" component={LocalizationSettingsPage} />
     <Route path="uploads" component={UploadSettingsPage} />
+    <Route path="python-runner" component={PythonRunnerSettingsPage} />
     <Route path="public-sharing" component={PublicSharingSettingsPage} />
     <Route
       path="embedding-in-other-applications"

frontend/src/metabase/common/components/AdminLayout/AdminSettingsLayout.module.css

@@ -22,7 +22,6 @@
   display: flex;
   justify-content: center;
   width: 100%;
-  padding: 2rem;
   position: relative;
   background-color: var(--mb-color-bg-light);
 }

frontend/src/metabase/common/components/AdminLayout/AdminSettingsLayout.tsx

@@ -10,11 +10,13 @@ import S from "./AdminSettingsLayout.module.css";
 export const AdminSettingsLayout = ({
   sidebar,
   children,
+  fullWidth = false,
   maw = "50rem",
 }: {
   sidebar?: React.ReactNode;
   children?: React.ReactNode;
   maw?: string;
+  fullWidth?: boolean;
 }) => {
   return (
     <Box className={S.Wrapper}>
@@ -24,9 +26,13 @@ export const AdminSettingsLayout = ({
             {sidebar}
           </Box>
         )}
-        <Box className={S.Content} data-testid="admin-layout-content">
-          <Box maw={maw} w="100%">
-            <Box pb="2rem">
+        <Box
+          className={S.Content}
+          data-testid="admin-layout-content"
+          p={fullWidth ? 0 : "2rem"}
+        >
+          <Box maw={fullWidth ? undefined : maw} w="100%">
+            <Box pb={fullWidth ? 0 : "2rem"}>
               <ErrorBoundary>{children ?? <NotFound />}</ErrorBoundary>
             </Box>
           </Box>

frontend/src/metabase/common/components/CodeEditor/CodeEditor.module.css

@@ -1,8 +0,0 @@
-.codeEditor {
-  :global {
-    .cm-gutter {
-      background-color: var(--mb-color-brand-light);
-      color: var(--mb-color-brand);
-    }
-  }
-}

frontend/src/metabase/common/components/CodeEditor/CodeEditor.tsx

@@ -1,8 +1,7 @@
-import cx from "classnames";
+import type { Extension } from "@uiw/react-codemirror";
 
 import { CodeMirror } from "metabase/common/components/CodeMirror";
 
-import S from "./CodeEditor.module.css";
 import type { CodeLanguage } from "./types";
 import { useExtensions } from "./utils";
 
@@ -15,6 +14,7 @@ type Props = {
   readOnly?: boolean;
   value: string;
   onChange?: (value: string) => void;
+  extensions?: Extension[];
 };
 
 export function CodeEditor({
@@ -26,8 +26,12 @@ export function CodeEditor({
   readOnly,
   value,
   onChange,
+  extensions: externalExtensions,
 }: Props) {
-  const extensions = useExtensions({ language });
+  const extensions = useExtensions({
+    language,
+    extensions: externalExtensions,
+  });
 
   return (
     <CodeMirror
@@ -37,7 +41,7 @@ export function CodeEditor({
         highlightActiveLine: false,
         highlightActiveLineGutter: false,
       }}
-      className={cx(S.codeEditor, className)}
+      className={className}
       extensions={extensions}
       id={id}
       readOnly={readOnly}

frontend/src/metabase/common/components/CodeEditor/utils.ts

@@ -2,7 +2,7 @@ import { html } from "@codemirror/lang-html";
 import { javascript } from "@codemirror/lang-javascript";
 import { json } from "@codemirror/lang-json";
 import { python } from "@codemirror/lang-python";
-import { StreamLanguage } from "@codemirror/language";
+import { StreamLanguage, indentUnit } from "@codemirror/language";
 import { clojure } from "@codemirror/legacy-modes/mode/clojure";
 import { pug } from "@codemirror/legacy-modes/mode/pug";
 import { ruby } from "@codemirror/legacy-modes/mode/ruby";
@@ -12,10 +12,19 @@ import { useMemo } from "react";
 
 import type { CodeLanguage } from "./types";
 
-export function useExtensions({ language }: { language?: CodeLanguage }) {
+export function useExtensions({
+  language,
+  extensions,
+}: {
+  language?: CodeLanguage;
+  extensions?: Extension[];
+}) {
   return useMemo(
-    () => [...(language ? [getLanguageExtension(language)] : [])],
-    [language],
+    () => [
+      ...(extensions ?? []),
+      ...(language ? [getLanguageExtension(language)] : []),
+    ],
+    [language, extensions],
   );
 }
 
@@ -28,7 +37,7 @@ export function getLanguageExtension(language: CodeLanguage): Extension {
     case "json":
       return json();
     case "python":
-      return python();
+      return [python(), indentUnit.of("    ")];
     case "mustache":
       return handlebars;
     case "pug":

modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj

@@ -12,11 +12,13 @@
    [metabase.driver.bigquery-cloud-sdk.query-processor :as bigquery.qp]
    [metabase.driver.common.table-rows-sample :as table-rows-sample]
    [metabase.driver.sql :as driver.sql]
+   [metabase.driver.sql-jdbc :as driver.sql-jdbc]
    [metabase.driver.sql-jdbc.sync.describe-database :as sql-jdbc.describe-database]
    [metabase.driver.sql.query-processor :as sql.qp]
    [metabase.driver.sql.util :as sql.u]
    [metabase.driver.sync :as driver.s]
    [metabase.util :as u]
+   [metabase.util.date-2 :as u.date]
    [metabase.util.i18n :refer [tru]]
    [metabase.util.log :as log]
    [metabase.util.malli :as mu]
@@ -38,6 +40,8 @@
     Field$Mode
     FieldValue
     FieldValueList
+    InsertAllRequest
+    InsertAllResponse
     QueryJobConfiguration
     Schema
     Table
@@ -45,6 +49,7 @@
     TableId
     TableResult)
    (com.google.common.collect ImmutableMap)
+   (com.google.gson JsonParser)
    (java.util Iterator)))
 
 (set! *warn-on-reflection* true)
@@ -230,6 +235,33 @@
     "BIGNUMERIC" :type/Decimal
     :type/*))
 
+(defmulti ^:private type->database-type
+  "Internal type->database-type multimethod for BigQuery that dispatches on type."
+  {:arglists '([type])}
+  identity)
+
+;; we can't recover the parameterized types
+(defmethod type->database-type :type/Array [_] [[:raw "JSON"]])
+(defmethod type->database-type :type/Dictionary [_] [[:raw "JSON"]])
+
+(defmethod type->database-type :type/Boolean [_] [[:raw "BOOL"]])
+(defmethod type->database-type :type/Float [_] [[:raw "FLOAT64"]])
+(defmethod type->database-type :type/Integer [_] [[:raw "INT"]])
+(defmethod type->database-type :type/Number [_] [[:raw "INT"]])
+(defmethod type->database-type :type/Text [_] [[:raw "STRING"]])
+(defmethod type->database-type :type/TextLike [_] [[:raw "STRING"]])
+(defmethod type->database-type :type/Date [_] [[:raw "DATE"]])
+(defmethod type->database-type :type/DateTime [_] [[:raw "DATETIME"]])
+(defmethod type->database-type :type/DateTimeWithTZ [_] [[:raw "TIMESTAMP"]])
+(defmethod type->database-type :type/Time [_] [[:raw "TIME"]])
+(defmethod type->database-type :type/JSON [_] [[:raw "JSON"]])
+(defmethod type->database-type :type/SerializedJSON [_] [[:raw "JSON"]])
+(defmethod type->database-type :type/Decimal [_] [[:raw "BIGDECIMAL"]])
+
+(defmethod driver/type->database-type :bigquery-cloud-sdk
+  [_driver base-type]
+  (type->database-type base-type))
+
 (defn- field->database+base-type
   "Returns a normalized `database-type` and its `base-type` for a type from BigQuery Field type.
 
@@ -836,6 +868,110 @@
   (let [table-str (get-table-str table)]
     [(str "DROP TABLE IF EXISTS " table-str)]))
 
+(defmethod driver/create-table! :bigquery-cloud-sdk
+  [driver database-id table-name column-definitions & {:keys [primary-key]}]
+  (let [sql (#'driver.sql-jdbc/create-table!-sql driver table-name column-definitions :primary-key primary-key)]
+    (driver/execute-raw-queries! driver (t2/select-one :model/Database database-id) [sql])))
+
+(defmethod driver/rename-table! :bigquery-cloud-sdk
+  [driver db-id old-table-name new-table-name]
+  (let [database (t2/select-one :model/Database db-id)
+        old-table-str (get-table-str old-table-name)
+        new-table-str (name new-table-name)]
+    (try
+      (let [sql (format "ALTER TABLE %s RENAME TO %s" old-table-str new-table-str)]
+        (driver/execute-raw-queries! driver database [sql]))
+      (catch Exception e
+        (if (and (instance? com.google.cloud.bigquery.BigQueryException e)
+                 (str/includes? (.getMessage e) "streaming data"))
+          ;; rename failed, we must create + drop
+          (let [create-sql (driver/compile-transform driver {:query (format "SELECT * FROM %s" old-table-str)
+                                                             :output-table new-table-name})
+                drop-sql (driver/compile-drop-table driver old-table-name)]
+            (driver/execute-raw-queries! driver database (concat create-sql drop-sql)))
+          (throw e))))))
+
+(defmethod driver/drop-table! :bigquery-cloud-sdk
+  [driver database-id table-name]
+  (let [sql (driver/compile-drop-table driver table-name)]
+    (driver/execute-raw-queries! driver (t2/select-one :model/Database database-id) [sql])))
+
+(defn- convert-value-for-insertion
+  [base-type value]
+  (if-not (string? value)
+    value
+    (case base-type
+      :type/JSON
+      (.toString (JsonParser/parseString value))
+
+      (:type/Dictionary :type/Array)
+      (JsonParser/parseString value)
+
+      :type/Integer
+      (parse-long value)
+
+      :type/Float
+      (parse-double value)
+
+      :type/Boolean
+      (parse-boolean value)
+
+      :type/Numeric
+      (bigdec value)
+
+      :type/Decimal
+      (bigdec value)
+
+      (:type/Date :type/DateTime :type/DateTimeWithLocalTZ)
+      (u.date/format (u.date/parse value))
+
+      value)))
+
+(defn- prepare-row-with-field-types
+  "Convert a row using field type information to properly type values."
+  [field-types column-names row-values]
+  (into {}
+        (map (fn [col-name value]
+               (let [base-type (get field-types col-name :type/Text)]
+                 [col-name (convert-value-for-insertion base-type value)]))
+             column-names
+             row-values)))
+
+(defmethod driver/insert-into! :bigquery-cloud-sdk
+  [_driver db-id table-name column-names values]
+  (let [database (t2/select-one :model/Database db-id)
+        details (:details database)
+        client (database-details->client details)
+        project-id (get-project-id details)
+
+        [dataset-id actual-table-name] [(namespace table-name) (name table-name)]
+
+        ^Table bq-table (if dataset-id
+                          (get-table client project-id dataset-id actual-table-name)
+                          (throw (ex-info "Table name must include dataset" {:table-name table-name})))
+
+        schema (.getSchema (.getDefinition bq-table))
+        table-id (.getTableId bq-table)
+
+        field-types (into {}
+                          (map (fn [^Field field]
+                                 (let [[_db-type base-type] (field->database+base-type field)]
+                                   [(.getName field) base-type])))
+                          (.getFields schema))
+
+        prepared-rows (map #(prepare-row-with-field-types field-types column-names %) values)]
+
+    (doseq [chunk (partition-all 1000 prepared-rows)]
+      (let [insert-request-builder (InsertAllRequest/newBuilder table-id)]
+        (doseq [row chunk]
+          (.addRow insert-request-builder ^java.util.Map row))
+        (let [insert-request (.build insert-request-builder)
+              ^InsertAllResponse response (.insertAll client insert-request)]
+          (when (.hasErrors response)
+            (let [errors (.getInsertErrors response)]
+              (throw (ex-info "BigQuery insert failed"
+                              {:errors (into [] (map str errors))})))))))))
+
 (defmethod driver/execute-raw-queries! :bigquery-cloud-sdk
   [_driver connection-details queries]
   ;; connection-details is either database details directly (from transforms)

modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj

@@ -482,3 +482,11 @@
 
   (execute! "select name from `%s`.metabase_test_tracking.datasets order by accessed_at" (project-id))
   (database-exists?! (tx/get-dataset-definition (data.impl/resolve-dataset-definition *ns* 'test-data))))
+
+(defn ^:private get-test-data-name
+  []
+  (test-dataset-id
+   (tx/get-dataset-definition (or data.impl/*dbdef-used-to-create-db*
+                                  (tx/default-dataset :bigquery-cloud-sdk)))))
+
+(defmethod sql.tx/session-schema :bigquery-cloud-sdk [_driver] (get-test-data-name))

modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj

@@ -2,6 +2,7 @@
   "Driver for ClickHouse databases"
   (:require
    [clojure.core.memoize :as memoize]
+   [clojure.java.jdbc :as jdbc]
    [clojure.string :as str]
    [metabase.driver :as driver]
    [metabase.driver-api.core :as driver-api]
@@ -212,6 +213,27 @@
     :metabase.upload/datetime                 "Nullable(DateTime64(3))"
     :metabase.upload/offset-datetime          nil))
 
+(defmulti ^:private type->database-type
+  "Internal type->database-type multimethod for ClickHouse that dispatches on type."
+  {:arglists '([type])}
+  identity)
+
+(defmethod type->database-type :type/Boolean [_] [[:raw "Nullable(Boolean)"]])
+(defmethod type->database-type :type/Float [_] [[:raw "Nullable(Float64)"]])
+(defmethod type->database-type :type/Integer [_] [[:raw "Nullable(Int64)"]])
+(defmethod type->database-type :type/Number [_] [[:raw "Nullable(Int64)"]])
+(defmethod type->database-type :type/Text [_] [[:raw "Nullable(String)"]])
+(defmethod type->database-type :type/TextLike [_] [[:raw "Nullable(String)"]])
+(defmethod type->database-type :type/Date [_] [[:raw "Nullable(Date32)"]])
+(defmethod type->database-type :type/DateTime [_] [[:raw "Nullable(DateTime64(3))"]])
+(defmethod type->database-type :type/DateTimeWithTZ [_]
+  ;; ?
+  [[:raw "Nullable(DateTime64(3))"]])
+
+(defmethod driver/type->database-type :clickhouse
+  [_driver base-type]
+  (type->database-type base-type))
+
 (defmethod driver/table-name-length-limit :clickhouse
   [_driver]
   ;; FIXME: This is a lie because you're really limited by a filesystems' limits, because Clickhouse uses
@@ -241,7 +263,14 @@
    {:write? true}
    (fn [^java.sql.Connection conn]
      (with-open [stmt (.createStatement conn)]
-       (.execute stmt (create-table!-sql driver table-name column-definitions :primary-key primary-key))))))
+       (let [sql (create-table!-sql driver table-name column-definitions :primary-key primary-key)]
+         (.execute stmt sql))))))
+
+(defmethod driver/rename-table! :clickhouse
+  [_driver db-id old-table-name new-table-name]
+  (let [sql [(format "RENAME TABLE %s TO %s" (quote-name old-table-name) (quote-name new-table-name))]]
+    (jdbc/with-db-transaction [conn (sql-jdbc.conn/db->pooled-connection-spec db-id)]
+      (jdbc/execute! conn sql))))
 
 (defmethod driver/insert-into! :clickhouse
   [driver db-id table-name column-names values]

modules/drivers/mongo/src/metabase/driver/mongo.clj

@@ -1,8 +1,11 @@
 (ns metabase.driver.mongo
   "MongoDB Driver."
   (:require
+   [clojure.data.csv :as csv]
+   [clojure.java.io :as io]
    [clojure.set :as set]
    [clojure.string :as str]
+   [medley.core :as m]
    [metabase.driver :as driver]
    [metabase.driver-api.core :as driver-api]
    [metabase.driver.common.table-rows-sample :as table-rows-sample]
@@ -17,6 +20,7 @@
    [metabase.driver.settings :as driver.settings]
    [metabase.driver.util :as driver.u]
    [metabase.util :as u]
+   [metabase.util.date-2 :as u.date]
    [metabase.util.json :as json]
    [metabase.util.log :as log]
    [taoensso.nippy :as nippy])
@@ -553,6 +557,127 @@
       (log/debugf e "Query:\n%s" native-form)
       native-form)))
 
+(defmethod driver/create-table! :mongo
+  [_driver database-id table-name _column-definitions & {:keys [primary-key]}]
+  ;; MongoDB collections are created implicitly when first document is inserted
+  ;; We can create an empty collection explicitly if needed
+  (mongo.connection/with-mongo-database [^MongoDatabase db database-id]
+    (.createCollection db (name table-name))
+    ;; Create indexes for any primary key fields
+    (when primary-key
+      (doseq [pk-field primary-key]
+        (mongo.util/create-index
+         (mongo.util/collection db (name table-name))
+         {pk-field 1})))))
+
+(defmethod driver/drop-table! :mongo
+  [_driver db-id table-name]
+  (mongo.connection/with-mongo-database [^MongoDatabase db db-id]
+    (some-> (mongo.util/collection db (name table-name))
+            .drop)))
+
+(defmethod driver/rename-table! :mongo
+  [_driver db-id old-table-name new-table-name]
+  (mongo.connection/with-mongo-database [^MongoDatabase db db-id]
+    (let [old-collection (mongo.util/collection db (name old-table-name))]
+      (.renameCollection old-collection
+                         (com.mongodb.MongoNamespace.
+                          (.getName db)
+                          (name new-table-name))))))
+
+;; Insert documents into a collection
+(defmethod driver/insert-into! :mongo
+  [_driver db-id table-name column-names values]
+  (mongo.connection/with-mongo-database [^MongoDatabase db db-id]
+    (let [collection (mongo.util/collection db (name table-name))
+          documents (map (fn [row]
+                           (zipmap (map keyword column-names) row))
+                         values)]
+      (if (> (count documents) 1)
+        (mongo.util/insert-many collection documents)
+        (mongo.util/insert-one collection (first documents))))))
+
+(defmethod driver/drop-transform-target! [:mongo :table]
+  [driver database target]
+  (driver/drop-table! driver (:id database) (:name target)))
+
+(defmethod driver/connection-spec :mongo
+  [_driver database]
+  (:details database))
+
+(defmethod driver/type->database-type :mongo
+  [_driver base-type]
+  (case base-type
+    :type/TextLike "string"
+    :type/Text "string"
+    :type/Number "long"
+    :type/Integer "int"
+    :type/BigInteger "long"
+    :type/Float "double"
+    :type/Decimal "decimal"
+    :type/Boolean "bool"
+    :type/Date "date"
+    :type/DateTime "date"
+    :type/DateTimeWithTZ "date"
+    :type/Time "date"
+    :type/TimeWithTZ "date"
+    :type/Instant "date"
+    :type/UUID "uuid"
+    :type/JSON "object"
+    :type/SerializedJSON "string"
+    :type/Array "array"
+    :type/Dictionary "object"
+    :type/MongoBSONID "objectId"
+    :type/MongoBinData "binData"
+    :type/IPAddress "string"
+    ;; Default fallback
+    "object"))
+
+(defn- convert-value-for-insertion
+  [base-type value]
+  (if-not (string? value)
+    value
+    (case base-type
+      (:type/JSON :type/Dictionary :type/Array)
+      (json/decode value)
+
+      :type/Integer
+      (parse-long value)
+
+      (:type/Number :type/BigInteger)
+      (bigint value)
+
+      :type/Float
+      (parse-double value)
+
+      :type/Decimal
+      (bigdec value)
+
+      :type/Boolean
+      (parse-boolean value)
+
+      (:type/Date :type/DateTime :type/DateTimeWithTZ :type/Time :type/TimeWithTZ :type/Instant)
+      (u.date/parse value)
+
+      :type/UUID
+      (parse-uuid value)
+
+      value)))
+
+(defmethod driver/insert-from-source! [:mongo :csv-file]
+  [driver db-id table-name column-names {:keys [file table-schema]}]
+  (let [csv-rows (csv/read-csv (io/reader file))
+        header-rows (first csv-rows)
+        meta (m/index-by :name (:columns table-schema))
+        convert-value (fn [name v]
+                        (let [ty (:type (get meta name))]
+                          (convert-value-for-insertion ty v)))
+        data-rows (map (fn [rows]
+                         (let [m (zipmap header-rows rows)]
+                           (mapv #(convert-value % (get m %)) column-names)))
+                       (rest csv-rows))]
+    (driver/insert-from-source! driver db-id table-name column-names {:type :rows :data data-rows})))
+
 ;; Following code is using monger. Leaving it here for a reference as it could be transformed when there is need
 ;; for ssl experiments.
 #_(comment

modules/drivers/mongo/src/metabase/driver/mongo/execute.clj

@@ -213,9 +213,10 @@
         db-name (mongo.db/db-name database)
         client-database (mongo.util/database mongo.connection/*mongo-client* db-name)]
     (with-open [session ^ClientSession (mongo.util/start-session! mongo.connection/*mongo-client*)]
-      (a/go
-        (when (a/<! (driver-api/canceled-chan))
-          (mongo.util/kill-session! client-database session)))
+      (when-let [cancel-chan (driver-api/canceled-chan)]
+        (a/go
+          (when (a/<! cancel-chan)
+            (mongo.util/kill-session! client-database session))))
       (let [aggregate ^AggregateIterable (*aggregate* client-database
                                                       collection-name
                                                       session

modules/drivers/redshift/src/metabase/driver/redshift.clj

@@ -539,6 +539,25 @@
     :metabase.upload/datetime                 [:timestamp]
     :metabase.upload/offset-datetime          [:timestamp-with-time-zone]))
 
+(defmulti ^:private type->database-type
+  "Internal type->database-type multimethod for Redshift that dispatches on type."
+  {:arglists '([type])}
+  identity)
+
+(defmethod type->database-type :type/TextLike [_] [[:varchar 65535]])
+(defmethod type->database-type :type/Text [_] [[:varchar 65535]])
+(defmethod type->database-type :type/Number [_] [:bigint])
+(defmethod type->database-type :type/Float [_] [(keyword "double precision")])
+(defmethod type->database-type :type/Boolean [_] [:boolean])
+(defmethod type->database-type :type/Date [_] [:date])
+(defmethod type->database-type :type/DateTime [_] [:timestamp])
+(defmethod type->database-type :type/DateTimeWithTZ [_] [:timestamp-with-time-zone])
+(defmethod type->database-type :type/Time [_] [:time])
+
+(defmethod driver/type->database-type :redshift
+  [_driver base-type]
+  (type->database-type base-type))
+
 (defmethod driver/allowed-promotions :redshift [_] {})
 
 (defmethod driver/table-name-length-limit :redshift

modules/drivers/snowflake/src/metabase/driver/snowflake.clj

@@ -262,7 +262,7 @@
     :FLOAT4                     :type/Float
     :FLOAT8                     :type/Float
     :DOUBLE                     :type/Float
-    (keyword "DOUBLE PRECISON") :type/Float
+    (keyword "DOUBLE PRECISION") :type/Float
     :REAL                       :type/Float
     :VARCHAR                    :type/Text
     :CHAR                       :type/Text
@@ -290,6 +290,27 @@
     :OBJECT                     :type/Dictionary
     :ARRAY                      :type/*} base-type))
 
+(defmulti ^:private type->database-type
+  "Internal type->database-type multimethod for Snowflake that dispatches on type."
+  {:arglists '([type])}
+  identity)
+
+(defmethod type->database-type :type/Array [_] [:ARRAY])
+(defmethod type->database-type :type/Boolean [_] [:BOOLEAN])
+(defmethod type->database-type :type/Date [_] [:DATE])
+(defmethod type->database-type :type/DateTime [_] [:DATETIME])
+(defmethod type->database-type :type/DateTimeWithLocalTZ [_] [:TIMESTAMPTZ])
+(defmethod type->database-type :type/DateTimeWithTZ [_] [:TIMESTAMPLTZ])
+(defmethod type->database-type :type/Decimal [_] [:DECIMAL])
+(defmethod type->database-type :type/Float [_] [:DOUBLE])
+(defmethod type->database-type :type/Number [_] [:BIGINT])
+(defmethod type->database-type :type/Text [_] [:TEXT])
+(defmethod type->database-type :type/Time [_] [:TIME])
+
+(defmethod driver/type->database-type :snowflake
+  [_driver base-type]
+  (type->database-type base-type))
+
 (defmethod sql.qp/unix-timestamp->honeysql [:snowflake :seconds]      [_ _ expr] [:to_timestamp_tz expr])
 (defmethod sql.qp/unix-timestamp->honeysql [:snowflake :milliseconds] [_ _ expr] [:to_timestamp_tz expr 3])
 (defmethod sql.qp/unix-timestamp->honeysql [:snowflake :microseconds] [_ _ expr] [:to_timestamp_tz expr 6])

modules/drivers/snowflake/test/metabase/test/data/snowflake.clj

@@ -379,3 +379,5 @@
          order by created"]
        (jdbc/reducible-query (no-db-connection-spec))
        (into [] (map (juxt :database_name :created)))))
+
+(defmethod sql.tx/session-schema :snowflake [_driver] "PUBLIC")

modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj

@@ -3,6 +3,8 @@
   (:require
    [clojure.data.xml :as xml]
    [clojure.java.io :as io]
+
+   [clojure.java.jdbc :as jdbc]
    [clojure.string :as str]
    [clojure.walk :as walk]
    [honey.sql :as sql]
@@ -119,6 +121,27 @@
     :xml              :type/*
     (keyword "int identity") :type/Integer} column-type)) ; auto-incrementing integer (ie pk) field
 
+(defmulti ^:private type->database-type
+  "Internal type->database-type multimethod for SQL Server that dispatches on type."
+  {:arglists '([type])}
+  identity)
+
+(defmethod type->database-type :type/Boolean [_] [:bit])
+(defmethod type->database-type :type/Date [_] [:date])
+(defmethod type->database-type :type/DateTime [_] [:datetime2])
+(defmethod type->database-type :type/DateTimeWithTZ [_] [:datetimeoffset])
+(defmethod type->database-type :type/Decimal [_] [:decimal])
+(defmethod type->database-type :type/Float [_] [:float])
+(defmethod type->database-type :type/Integer [_] [:int])
+(defmethod type->database-type :type/Number [_] [:bigint])
+(defmethod type->database-type :type/Text [_] [:text])
+(defmethod type->database-type :type/Time [_] [:time])
+(defmethod type->database-type :type/UUID [_] [:uniqueidentifier])
+
+(defmethod driver/type->database-type :sqlserver
+  [_driver base-type]
+  (type->database-type base-type))
+
 (defmethod sql-jdbc.conn/connection-details->spec :sqlserver
   [_ {:keys [user password db host port instance domain ssl]
       :or   {user "dbuser", password "dbpassword", db "", host "localhost"}
@@ -969,3 +992,9 @@
   [driver conn-spec schema]
   (let [sql [[(format "IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '%s') EXEC('CREATE SCHEMA [%s];');" schema schema)]]]
     (driver/execute-raw-queries! driver conn-spec sql)))
+
+(defmethod driver/rename-table! :sqlserver
+  [_driver db-id old-table-name new-table-name]
+  (let [sql (format "EXEC sp_rename '%s', '%s'" (name old-table-name) (name new-table-name))]
+    (jdbc/with-db-transaction [conn (sql-jdbc.conn/db->pooled-connection-spec db-id)]
+      (jdbc/execute! conn sql))))

src/metabase/analytics/prometheus.clj

@@ -398,7 +398,56 @@
    (prometheus/counter :metabase-gsheets/connection-deleted
                        {:description "How many times the instance has deleted their Google Sheets connection."})
    (prometheus/counter :metabase-gsheets/connection-manually-synced
-                       {:description "How many times the instance has manually sync'ed their Google Sheets connection."})])
+                       {:description "How many times the instance has manually sync'ed their Google Sheets connection."})
+
+   ;; transform metrics
+   (prometheus/counter :metabase-transforms/job-runs-total
+                       {:description "Total number of transform job runs started."
+                        :labels [:job-id :run-method]})
+   (prometheus/counter :metabase-transforms/job-runs-completed
+                       {:description "Number of transform job runs that completed successfully."
+                        :labels [:job-id :run-method]})
+   (prometheus/counter :metabase-transforms/job-runs-failed
+                       {:description "Number of transform job runs that failed."
+                        :labels [:job-id :run-method]})
+   (prometheus/histogram :metabase-transforms/job-run-duration-ms
+                         {:description "Duration in milliseconds of transform job runs."
+                          :labels [:job-id :run-method]
+                          ;; 100ms -> 30 minutes
+                          :buckets [100 500 1000 5000 10000 30000 60000 300000 1800000]})
+   (prometheus/counter :metabase-transforms/stage-started
+                       {:description "Number of transform stages started."
+                        :labels [:job-run-id :stage-type :stage-label]})
+   (prometheus/counter :metabase-transforms/stage-completed
+                       {:description "Number of transform stages completed successfully."
+                        :labels [:job-run-id :stage-type :stage-label]})
+   (prometheus/counter :metabase-transforms/stage-failed
+                       {:description "Number of transform stages that failed."
+                        :labels [:job-run-id :stage-type :stage-label]})
+   (prometheus/histogram :metabase-transforms/stage-duration-ms
+                         {:description "Duration in milliseconds of individual transform stages."
+                          :labels [:job-run-id :stage-type :stage-label]
+                          ;; 10ms -> 10 minutes
+                          :buckets [10 100 500 1000 5000 10000 30000 60000 300000 600000]})
+   (prometheus/histogram :metabase-transforms/data-transfer-bytes
+                         {:description "Size in bytes of data transferred during transform stages."
+                          :labels [:job-run-id :stage-label]
+                          ;; 1KB -> 10GB
+                          :buckets [1000 10000 100000 1000000 10000000 100000000 1000000000 10000000000]})
+   (prometheus/histogram :metabase-transforms/data-transfer-rows
+                         {:description "Number of rows transferred during transform stages."
+                          :labels [:job-run-id :stage-label]
+                          ;; 10 -> 10M rows
+                          :buckets [10 100 1000 10000 100000 1000000 10000000]})
+   ;; Python-transform specific metrics
+   (prometheus/histogram :metabase-transforms/python-api-call-duration-ms
+                         {:description "Duration of Python runner API calls."
+                          :labels [:job-run-id]
+                          ;; 100ms -> 30 minutes
+                          :buckets [100 500 1000 5000 10000 30000 60000 300000 1800000]})
+   (prometheus/counter :metabase-transforms/python-api-calls-total
+                       {:description "Total number of Python runner API calls."
+                        :labels [:job-run-id :status]})])
 
 (defn- quartz-collectors
   []
@@ -575,7 +624,7 @@
   (spit "metrics" (iapetos.export/text-format (:registry system)))
 
   ;; See all metrics that match a given prefix:
-  ; (filter #(.startsWith % "metabase_search_") (clojure.string/split-lines (iapetos.export/text-format (:registry system))))
+                                        ; (filter #(.startsWith % "metabase_search_") (clojure.string/split-lines (iapetos.export/text-format (:registry system))))
 
   ;; need to restart the server to see the metrics? use:
   (shutdown!)

src/metabase/driver.clj

@@ -9,6 +9,8 @@
    [[metabase.driver.sql-jdbc]] for more details."
   #_{:clj-kondo/ignore [:metabase/modules]}
   (:require
+   [clojure.data.csv :as csv]
+   [clojure.java.io :as io]
    [clojure.set :as set]
    [clojure.string :as str]
    [metabase.auth-provider.core :as auth-provider]
@@ -1268,6 +1270,14 @@
   dispatch-on-initialized-driver
   :hierarchy #'hierarchy)
 
+(defmulti rename-table!
+  "Rename a table from `old-table-name` to `new-table-name`. Table names may be qualified by schema e.g.
+
+    schema.table"
+  {:added "0.57.0", :arglists '([driver db-id old-table-name new-table-name])}
+  dispatch-on-initialized-driver
+  :hierarchy #'hierarchy)
+
 (defmulti truncate!
   "Delete the current contents of `table-name`.
   If something like a SQL TRUNCATE statement is supported, we use that, but may otherwise fall back to explicitly
@@ -1294,6 +1304,31 @@
   dispatch-on-initialized-driver
   :hierarchy #'hierarchy)
 
+(defmulti insert-from-source!
+  "Inserts data from a data source into an existing table.
+   This abstracts away the conversion and insertion process, allowing drivers
+   to optimize based on the data source type. Returns number of rows inserted. "
+  {:added "0.57.0", :arglists '([driver database-id table-name column-names data-source])}
+  (fn [driver _ _ _ data-source]
+    [(dispatch-on-initialized-driver driver) (:type data-source)])
+  :hierarchy #'hierarchy)
+
+(defmethod insert-from-source! [::driver :rows]
+  [driver db-id table-name column-names {:keys [data]}]
+  (when (seq data)
+    (insert-into! driver db-id table-name column-names data))
+  (count data))
+
+(defmethod insert-from-source! [::driver :csv-file]
+  [driver db-id table-name column-names {:keys [file]}]
+  (let [csv-rows (csv/read-csv (io/reader file))
+        header-rows (first csv-rows)
+        data-rows (map (fn [rows]
+                         (let [m (zipmap header-rows rows)]
+                           (mapv #(get m %) column-names)))
+                       (rest csv-rows))]
+    (insert-from-source! driver db-id table-name column-names {:type :rows :data data-rows})))
+
 (defmulti add-columns!
   "Add columns given by `column-definitions` to a table named `table-name`. If the table doesn't exist it will throw an error.
   `args` is an optional map with an optional key `primary-key`. The `primary-key` value is a vector of column names
@@ -1361,6 +1396,12 @@
   dispatch-on-initialized-driver
   :hierarchy #'hierarchy)
 
+(defmulti type->database-type
+  "Returns the database type for a given Metabase type (from the type hierarchy) as a HoneySQL spec."
+  {:added "0.57.0", :arglists '([driver base-type])}
+  (fn [driver _base-type] driver)
+  :hierarchy #'hierarchy)
+
 (defmulti allowed-promotions
   "Returns a mapping of which types a column can be implicitly relaxed to, based on the content of appended values.
   In the context of uploads, this permits certain appends or replacements of an existing csv table

src/metabase/driver/h2.clj

@@ -606,6 +606,28 @@
     :metabase.upload/datetime                 [:timestamp]
     :metabase.upload/offset-datetime          [:timestamp-with-time-zone]))
 
+(defmulti ^:private type->database-type
+  "Internal type->database-type multimethod for H2 that dispatches on type."
+  {:arglists '([type])}
+  identity)
+
+(defmethod type->database-type :type/TextLike [_] [:varchar])
+(defmethod type->database-type :type/Text [_] [:varchar])
+(defmethod type->database-type :type/Number [_] [:bigint])
+(defmethod type->database-type :type/Float [_] [(keyword "DOUBLE PRECISION")])
+(defmethod type->database-type :type/Decimal [_] [:decimal])
+(defmethod type->database-type :type/Boolean [_] [:boolean])
+(defmethod type->database-type :type/Date [_] [:date])
+(defmethod type->database-type :type/DateTime [_] [:timestamp])
+(defmethod type->database-type :type/DateTimeWithTZ [_] [:timestamp-with-time-zone])
+(defmethod type->database-type :type/Time [_] [:time])
+(defmethod type->database-type :type/TimeWithTZ [_] [:time-with-time-zone])
+(defmethod type->database-type :type/UUID [_] [:uuid])
+
+(defmethod driver/type->database-type :h2
+  [_driver base-type]
+  (type->database-type base-type))
+
 (defmethod driver/create-auto-pk-with-append-csv? :h2 [_driver] true)
 
 (defmethod driver/table-name-length-limit :h2