Use DataGrid for displaying results

For me 50 was exceeded today and I could not start transforms. I am
guessing these are starter numbers and we'll need to figure out values
anyway

.clj-kondo/config.edn

@@ -1,7 +1,7 @@
 ;; -*- comment-column: 100; -*-
 {:config-paths ["macros" "src" "config"]
  :linters
- { ;;
+ {;;
   ;; enabled optional linters (these are disabled by default)
   ;;
 
@@ -186,7 +186,7 @@
   :metabase/disallow-hardcoded-driver-names-in-tests
   ;; don't include `:h2` since that will probably lead to a mountain of tests where we are specifically testing
   ;; middleware or compilation behavior with just H2 since it's the default.
-  {:drivers #{ ;; core drivers
+  {:drivers #{;; core drivers
               #_:h2
               :postgres
               :mysql
@@ -418,6 +418,7 @@
   metabase-enterprise.transforms.api-test/with-transform-tags                           clojure.core/let
   metabase-enterprise.transforms.api-test/with-worker-runs                              clojure.core/let
   metabase-enterprise.transforms.test-util/with-transform-cleanup!                      clojure.core/let
+  metabase-enterprise.transforms.execute/with-transform-lifecycle                       clojure.core/let
   metabase.actions.test-util/with-actions                                               clojure.core/let
   metabase.api.common/let-404                                                           clojure.core/let
   metabase.app-db.custom-migrations.util/with-temp-scheduler!                           clojure.core/fn
@@ -734,7 +735,9 @@
   ;; `metabase-enterprise.sandbox.query-processor*-test`
   ;;
   {:pattern "^metabase(?:((?:-enterprise\\.sandbox)?\\.query-processor)|(?:\\.driver\\.sql(?!server)(?!ite))).*-test$"
-   :name    general-qp-and-driver-test-namespaces}]
+   :name    general-qp-and-driver-test-namespaces}
+  {:pattern "^metabase\\.python-runner\\..*"
+   :name    python-runner-namespaces}]
 
  :config-in-ns
  {test-namespaces

.clj-kondo/config/modules/config.edn

@@ -886,6 +886,19 @@
            enterprise/advanced-permissions
            enterprise/sandbox}}
 
+  python-runner
+  {:team "Workflows"
+   :api  #{metabase.python-runner.api
+           metabase.python-runner.core
+           metabase.python-runner.init}
+   :uses #{api
+           driver
+           models
+           query-processor
+           request
+           util
+           enterprise/transforms}}
+
   queries
   {:team "Querying"
    :api  #{metabase.queries.api
@@ -1947,8 +1960,11 @@
   {:team "Querying"
    :api  #{metabase-enterprise.transforms.api
            metabase-enterprise.transforms.core
-           metabase-enterprise.transforms.init}
-   :uses #{api
+           metabase-enterprise.transforms.init
+           ;; TODO: Remove this from API once we remove the python-runner callback to metabase
+           metabase-enterprise.transforms.settings}
+   :uses #{analytics
+           api
            app-db
            config
            driver
@@ -1964,6 +1980,8 @@
            sync
            task
            task-history
+           ;; TODO: Remove this once we're using a better file format than CSV
+           upload
            util}}
 
   enterprise/upload-management

.github/workflows/backend.yml

@@ -195,6 +195,47 @@ jobs:
       - name: Run cljfmt
         run: ./bin/mage cljfmt-all --force-check
 
+  be-python-runner-tests:
+    if: ${{ !inputs.skip }}
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    name: Python Runner Tests
+    env:
+      CI: 'true'
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      - name: Prepare back-end environment
+        uses: ./.github/actions/prepare-backend
+        with:
+          m2-cache-key: be-python-runner-tests
+      - name: Start python-runner using Makefile
+        run: |
+          echo "🚀 Starting python-runner using make run..."
+          cd python-runner
+
+          # Use the official make run command
+          make run
+
+          echo "✅ Python runner started successfully!"
+
+      - name: Run Python Runner tests
+        run: >-
+          clojure -X:dev:ee:ee-dev:driver:test:test/python-transform
+      - name: Upload Test Results
+        uses: ./.github/actions/upload-test-results
+        if: always()
+        with:
+          input-path: ./target/junit/
+          output-name: be-python-runner-tests
+          bucket: ${{ vars.AWS_S3_TEST_RESULTS_BUCKET }}
+          aws-access-key-id: ${{ secrets.AWS_TEST_RESULTS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.AWS_REGION }}
+          trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
+
   be-tests-result:
     needs:
       - be-linter-eastwood
@@ -202,6 +243,7 @@ jobs:
       - be-cljfmt
       - be-check
       - be-tests
+      - be-python-runner-tests
     if: always() && !cancelled()
     runs-on: ubuntu-latest
     timeout-minutes: 5

.gitignore

@@ -163,3 +163,4 @@ CLAUDE.local.md
 
 /logs
 /.clojure-mcp
+__pycache__/

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "python-runner"]
+	path = python-runner
+	url = https://github.com/metabase/python-runner-container.git
+	branch = prototype

deps.edn

@@ -36,7 +36,7 @@
   com.clearspring.analytics/stream          {:mvn/version "2.9.8"               ; Various sketching algorithms
                                              :exclusions  [it.unimi.dsi/fastutil
                                                            org.slf4j/slf4j-api]}
-  com.clojure-goes-fast/jvm-alloc-rate-meter{:mvn/version "0.1.4"}              ; Monitor heap allocation rate.
+  com.clojure-goes-fast/jvm-alloc-rate-meter {:mvn/version "0.1.4"}              ; Monitor heap allocation rate.
   com.clojure-goes-fast/jvm-hiccup-meter    {:mvn/version "0.1.1"}              ; Monitor GC pauses and other system-induced pauses.
   com.draines/postal                        {:mvn/version "2.0.5"               ; SMTP library
                                              :exclusions [com.sun.mail/jakarta.mail]}
@@ -45,7 +45,7 @@
   com.github.albfernandez/juniversalchardet {:mvn/version "2.5.0"}
   com.github.blagerweij/liquibase-sessionlock {:mvn/version "1.6.9"}            ; session lock for liquibase migrations
   com.github.jknack/handlebars              ^:antq/exclude                      ; 4.4.0 requires java 17+
-                                            {:mvn/version "4.3.1"}              ; templating engine
+  {:mvn/version "4.3.1"}              ; templating engine
   com.github.oliyh/martian-clj-http         {:mvn/version "0.1.31"}             ; openapi client
   com.github.pangloss/transducers           {:git/sha "c91d39046c8c64dc31b84ebb27154a8084fcda4c" ; like medley for transducers
                                              :git/url "https://github.com/pangloss/transducers"}
@@ -55,14 +55,14 @@
   com.github.vertical-blank/sql-formatter   {:mvn/version "2.0.5"}              ; Java SQL formatting library https://github.com/vertical-blank/sql-formatter
   com.google.guava/guava                    {:mvn/version "33.4.8-jre"}         ; dep for BigQuery, Spark, and GA. Require here rather than letting different dep versions stomp on each other — see comments on #9697
   com.h2database/h2                         ^:antq/exclude                      ; https://metaboat.slack.com/archives/CKZEMT1MJ/p1727191010259979
-                                            {:mvn/version "2.1.214"}            ; embedded SQL database
+  {:mvn/version "2.1.214"}            ; embedded SQL database
   com.ibm.icu/icu4j                         {:mvn/version "77.1"}               ; Used to transliterate Unicode characters into Latin characters
   com.knuddels/jtokkit                      {:mvn/version "1.1.0"}              ; Java tokenizer library for OpenAI models, used for token counting in semantic search
   com.mchange/c3p0                          {:mvn/version "0.10.1"}             ; DB connection pool
   com.snowplowanalytics/snowplow-java-tracker ^:antq/exclude                    ; 2+ depend on apache httpclient 5.x, but saml and clj-http depend on 4.x
-                                            {:mvn/version "1.0.1"}              ; Snowplow analytics
+  {:mvn/version "1.0.1"}              ; Snowplow analytics
   com.sun.mail/jakarta.mail                 ^:antq/exclude                      ; 2+ changes imports from javax.mail to jakarta.mail
-                                            {:mvn/version "1.6.8"}              ; SMTP lib, required by postal
+  {:mvn/version "1.6.8"}              ; SMTP lib, required by postal
   com.taoensso/nippy                        {:mvn/version "3.6.0"}              ; Fast serialization (i.e., GZIP) library for Clojure
   com.vladsch.flexmark/flexmark             {:mvn/version "0.64.8"}             ; Markdown parsing
   com.vladsch.flexmark/flexmark-ext-autolink
@@ -92,7 +92,7 @@
   ;; TODO: replace back to normal artifact
   io.github.metabase/macaw                  {:mvn/version "0.2.27"}             ; Parse native SQL queries
   io.github.resilience4j/resilience4j-retry ^:antq/exclude                      ; 2.x do not support Java 8
-                                            {:mvn/version "1.7.1"}              ; Support for retrying operations
+  {:mvn/version "1.7.1"}              ; Support for retrying operations
   io.prometheus/simpleclient_hotspot        {:mvn/version "0.16.0"}             ; prometheus jvm collector
   junegunn/grouper                          {:mvn/version "0.1.1"}              ; Batch processing helper
   kixi/stats                                {:mvn/version "0.5.7"               ; Various statistic measures implemented as transducers
@@ -198,6 +198,8 @@
                                                            org.eclipse.jetty.ee9/jetty-ee9-servlet
                                                            org.eclipse.jetty.ee9.websocket/jetty-ee9-websocket-jetty-server]}
   slingshot/slingshot                       {:mvn/version "0.12.2"}             ; enhanced throw/catch, used by other deps
+  software.amazon.awssdk/s3                 {:mvn/version "2.33.0"}
+  software.amazon.awssdk/sts                {:mvn/version "2.33.0"}           ; AWS SDK v2 for S3
   stencil/stencil                           {:mvn/version "0.5.0"}              ; Mustache templates for Clojure
   user-agent/user-agent                     {:mvn/version "0.1.1"}              ; User-Agent string parser, for Login History page & elsewhere
   weavejester/dependency                    {:mvn/version "1.0.0"}              ; Dependency graphs and topological sorting
@@ -836,8 +838,14 @@
   :test/lib
   {:exec-args {:only ["test/metabase/lib"
                       "test/metabase/lib_be"]}}
+  ;; test the python transforms
+  ;;    clj -X:dev:ee:ee-dev:driver:test:test/python-transform
+  :test/python-transform
+  {:exec-args {:only [metabase-enterprise.transforms.api-test
+                      metabase.driver.common-test
+                      "enterprise/backend/test/metabase_enterprise/transforms"]}}
 
-  ;; test the Query Processor
+;; test the Query Processor
   ;;
   ;;    clj -X:dev:ee:ee-dev:test:test/qp
   :test/qp

enterprise/backend/src/metabase_enterprise/transforms/api.clj

@@ -1,7 +1,9 @@
 (ns metabase-enterprise.transforms.api
   (:require
+   [clojure.core.async :as a]
    [metabase-enterprise.transforms.api.transform-job]
    [metabase-enterprise.transforms.api.transform-tag]
+   [metabase-enterprise.transforms.canceling :as trasnforms.canceling]
    [metabase-enterprise.transforms.execute :as transforms.execute]
    [metabase-enterprise.transforms.models.transform :as transform.model]
    [metabase-enterprise.transforms.models.transform-run :as transform-run]
@@ -29,12 +31,21 @@
 (set! *warn-on-reflection* true)
 
 (mr/def ::transform-source
-  [:map
-   [:type [:= "query"]]
-   [:query [:map [:database :int]]]])
+  [:multi {:dispatch (comp keyword :type)}
+   [:query
+    [:map
+     [:type [:= "query"]]
+     [:query [:map [:database :int]]]]]
+   [:python
+    [:map {:closed true}
+     [:source-database {:optional true} :int]
+     [:source-tables   [:map-of :string :int]]
+     [:type [:= "python"]]
+     [:body :string]]]])
 
 (mr/def ::transform-target
   [:map
+   [:database :int]
    [:type [:enum "table"]]
    [:schema {:optional true} [:or ms/NonBlankString :nil]]
    [:name :string]])
@@ -57,19 +68,22 @@
 
 (defn- source-database-id
   [transform]
-  (-> transform :source :query :database))
+  (if (transforms.util/python-transform? transform)
+    (-> transform :source :source-database)
+    (-> transform :source :query :database)))
 
 (defn- check-database-feature
   [transform]
-  (let [database (api/check-400 (t2/select-one :model/Database (source-database-id transform))
-                                (deferred-tru "The source database cannot be found."))
-        feature (transforms.util/required-database-feature transform)]
-    (api/check-400 (not (:is_sample database))
-                   (deferred-tru "Cannot run transforms on the sample database."))
-    (api/check-400 (not (:is_audit database))
-                   (deferred-tru "Cannot run transforms on audit databases."))
-    (api/check-400 (driver.u/supports? (:engine database) feature database)
-                   (deferred-tru "The database does not support the requested transform target type."))))
+  (when (transforms.util/query-transform? transform)
+    (let [database (api/check-400 (t2/select-one :model/Database (source-database-id transform))
+                                  (deferred-tru "The source database cannot be found."))
+          feature (transforms.util/required-database-feature transform)]
+      (api/check-400 (not (:is_sample database))
+                     (deferred-tru "Cannot run transforms on the sample database."))
+      (api/check-400 (not (:is_audit database))
+                     (deferred-tru "Cannot run transforms on audit databases."))
+      (api/check-400 (driver.u/supports? (:engine database) feature database)
+                     (deferred-tru "The database does not support the requested transform target type.")))))
 
 (defn- localize-timestamps
   [run]
@@ -178,9 +192,10 @@
           new (merge old body)
           target-fields #(-> % :target (select-keys [:schema :name]))]
       (check-database-feature new)
-      (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]
-        (throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)
-                        {:status-code 400})))
+      (when (transforms.util/query-transform? old)
+        (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]
+          (throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)
+                          {:status-code 400}))))
       (api/check (not (and (not= (target-fields old) (target-fields new))
                            (transforms.util/target-table-exists? new)))
                  403
@@ -213,11 +228,14 @@
 (api.macros/defendpoint :post "/:id/cancel"
   "Cancel the current run for a given transform."
   [{:keys [id]} :- [:map
-                    [:id :string]]]
+                    [:id ms/PositiveInt]]]
   (log/info "canceling transform " id)
   (api/check-superuser)
-  (let [run (api/check-404 (transform-run/running-run-for-run-id id))]
-    (transform-run-cancelation/mark-cancel-started-run! (:id run)))
+  (let [transform (api/check-404 (t2/select-one :model/Transform id))
+        run (api/check-404 (transform-run/running-run-for-transform-id id))]
+    (transform-run-cancelation/mark-cancel-started-run! (:id run))
+    (when (transforms.util/python-transform? transform)
+      (trasnforms.canceling/cancel-run! (:id run))))
   nil)
 
 (api.macros/defendpoint :post "/:id/run"
@@ -228,9 +246,16 @@
   (api/check-superuser)
   (let [transform (api/check-404 (t2/select-one :model/Transform id))
         start-promise (promise)]
-    (u.jvm/in-virtual-thread*
-     (transforms.execute/run-mbql-transform! transform {:start-promise start-promise
-                                                        :run-method :manual}))
+    (if (transforms.util/python-transform? transform)
+      (u.jvm/in-virtual-thread*
+       (transforms.execute/execute-python-transform! transform {:start-promise start-promise
+                                                                :message-log   (atom {:pre-python  []
+                                                                                      :python      nil
+                                                                                      :post-python []})
+                                                                :run-method :manual}))
+      (u.jvm/in-virtual-thread*
+       (transforms.execute/run-mbql-transform! transform {:start-promise start-promise
+                                                          :run-method :manual})))
     (when (instance? Throwable @start-promise)
       (throw @start-promise))
     (let [result @start-promise
@@ -240,6 +265,52 @@
                               :run_id run-id})
           (assoc :status 202)))))
 
+;; hack
+(def ^:private python-test-run-id Integer/MAX_VALUE)
+
+(api.macros/defendpoint :post "/test-python"
+  "Test Python code execution without creating a transform."
+  [_route-params
+   _query-params
+   body :- [:map
+            [:code :string]
+            [:tables [:map-of :string :int]]]]
+  (log/info "test python code execution")
+  (api/check-superuser)
+  (let [run-id python-test-run-id
+        cancel-chan (a/promise-chan)]
+    (trasnforms.canceling/chan-start-run! run-id cancel-chan)
+    (try
+      (let [{:keys [body status]} (transforms.execute/call-python-runner-api! (:code body) (:tables body) run-id cancel-chan)]
+        (if (= status 200)
+          (do
+            (log/info "Python test execution succeeded")
+            (-> (response/response {:message (deferred-tru "Python code executed successfully")
+                                    :result  {:body body}})
+                (assoc :status 200)))
+          (do
+            (log/error "Error executing Python test code")
+            (-> (response/response {:message   (deferred-tru "Python code execution failed")
+                                    :error     (:error body)
+                                    :stdout    (:stdout body)
+                                    :stderr    (:stderr body)
+                                    :exit_code (:exit-code body)})
+                (assoc :status status)))))
+      (finally
+        (trasnforms.canceling/chan-end-run! run-id)))))
+
+(api.macros/defendpoint :post "/test-python/cancel"
+  "Cancel the current test-python execution."
+  [_route-params
+   _query-params]
+  (log/info "canceling test python execution")
+  (api/check-superuser)
+  (if (trasnforms.canceling/chan-signal-cancel! python-test-run-id)
+    (-> (response/response {:message (deferred-tru "Python test canceled")})
+        (assoc :status 200))
+    (-> (response/response {:message (deferred-tru "No running Python test to cancel")})
+        (assoc :status 404))))
+
 (def ^{:arglists '([request respond raise])} routes
   "`/api/ee/transform` routes."
   (handlers/routes

enterprise/backend/src/metabase_enterprise/transforms/canceling.clj

@@ -49,7 +49,9 @@
    (chan-signal-cancel! run-id)
    (transform-run/timeout-run! run-id)))
 
-(defn- cancel-run! [run-id]
+(defn cancel-run!
+  "Cancel run"
+  [run-id]
   (when (chan-signal-cancel! run-id)
     (transform-run/cancel-run! run-id)))
 

enterprise/backend/src/metabase_enterprise/transforms/execute.clj

@@ -1,17 +1,28 @@
 (ns metabase-enterprise.transforms.execute
   (:require
    [clojure.core.async :as a]
+   [clojure.java.io :as io]
+   [clojure.string :as str]
    [metabase-enterprise.transforms.canceling :as canceling]
+   [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
    [metabase-enterprise.transforms.models.transform-run :as transform-run]
+   [metabase-enterprise.transforms.python-runner :as python-runner]
    [metabase-enterprise.transforms.settings :as transforms.settings]
    [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.driver :as driver]
    [metabase.driver.util :as driver.u]
    [metabase.lib.schema.common :as schema.common]
    [metabase.query-processor.pipeline :as qp.pipeline]
+   [metabase.util :as u]
+   [metabase.util.json :as json]
+   [metabase.util.jvm :as u.jvm]
    [metabase.util.log :as log]
    [metabase.util.malli.registry :as mr]
-   [toucan2.core :as t2]))
+   [toucan2.core :as t2])
+  (:import
+   (clojure.lang IDeref)
+   (java.io Closeable File)
+   (java.time Duration)))
 
 (set! *warn-on-reflection* true)
 
@@ -26,6 +37,24 @@
   [:map
    [:overwrite? :boolean]])
 
+(defmacro with-transform-lifecycle
+  "Macro to manage transform run lifecycle. Starts a run, executes body,
+   and handles success/failure appropriately.
+
+   Usage:
+   (with-transform-lifecycle [run-id transform-id opts]
+     ;; body that returns result on success or throws on failure
+     )"
+  [[run-id-sym [transform-id opts]] & body]
+  `(let [{~run-id-sym :id} (transform-run/start-run! ~transform-id ~opts)]
+     (try
+       (let [result# (do ~@body)]
+         (transform-run/succeed-started-run! ~run-id-sym)
+         result#)
+       (catch Throwable t#
+         (transform-run/fail-started-run! ~run-id-sym {:message (.getMessage t#)})
+         (throw t#)))))
+
 (defn- sync-target!
   ([transform-id run-id]
    (let [{:keys [source target]} (t2/select-one :model/Transform transform-id)
@@ -37,22 +66,35 @@
    (log/info "Syncing target" (pr-str target) "for transform")
    (transforms.util/activate-table-and-mark-computed! database target)))
 
-(defn run-transform!
+(defn run-cancelable-transform!
   "Run a compiled transform"
-  [run-id driver transform-details opts]
-  ;; local run is responsible for status
+  [run-id run-transform!]
+  ;; local run is responsible for status, using canceling lifecycle
   (try
     (canceling/chan-start-timeout-vthread! run-id (transforms.settings/transform-timeout))
-    (binding [qp.pipeline/*canceled-chan* (a/promise-chan)]
-      (canceling/chan-start-run! run-id qp.pipeline/*canceled-chan*)
-      (driver/run-transform! driver transform-details opts))
-    (transform-run/succeed-started-run! run-id)
+    (let [cancel-chan (a/promise-chan)
+          ret (binding [qp.pipeline/*canceled-chan* cancel-chan]
+                (canceling/chan-start-run! run-id cancel-chan)
+                (run-transform! cancel-chan))]
+      (transform-run/succeed-started-run! run-id)
+      ret)
     (catch Throwable t
       (transform-run/fail-started-run! run-id {:message (.getMessage t)})
       (throw t))
     (finally
       (canceling/chan-end-run! run-id))))
 
+(defn- try-start-unless-already-running [id run-method]
+  (try
+    (transform-run/start-run! id {:run_method run-method})
+    (catch java.sql.SQLException e
+      (if (= (.getSQLState e) "23505")
+        (throw (ex-info "Transform is already running"
+                        {:error :already-running
+                         :transform-id id}
+                        e))
+        (throw e)))))
+
 (defn run-mbql-transform!
   "Run `transform` and sync its target table.
 
@@ -74,20 +116,13 @@
          (throw (ex-info "The database does not support the requested transform target type."
                          {:driver driver, :database database, :feature feature})))
        ;; mark the execution as started and notify any observers
-       (let [{run-id :id} (try
-                            (transform-run/start-run! id {:run_method run-method})
-                            (catch java.sql.SQLException e
-                              (if (= (.getSQLState e) "23505")
-                                (throw (ex-info "Transform is already running"
-                                                {:error :already-running
-                                                 :transform-id id}
-                                                e))
-                                (throw e))))]
+       (let [{run-id :id} (try-start-unless-already-running id run-method)]
          (when start-promise
            (deliver start-promise [:started run-id]))
          (log/info "Executing transform" id "with target" (pr-str target))
-         (run-transform! run-id driver transform-details opts)
-         (sync-target! target database run-id)))
+         (run-cancelable-transform! run-id (fn [_cancel-chan] (driver/run-transform! driver transform-details opts)))
+         (transforms.instrumentation/with-stage-timing [run-id :sync :table-sync]
+           (sync-target! target database run-id))))
      (catch Throwable t
        (log/error t "Error executing transform")
        (when start-promise
@@ -95,3 +130,177 @@
          ;; but we assume nobody would catch the exception anyway
          (deliver start-promise t))
        (throw t)))))
+
+(defn- log! [message-log s]
+  (when message-log
+    (swap! message-log (fn [m] (update m (if (:python m) :post-python :pre-python) conj s))))
+  nil)
+
+(defn- replace-python-logs! [message-log events]
+  (when message-log (swap! message-log assoc :python events))
+  nil)
+
+(defn- save-log-as-message! [run-id message-log]
+  (when message-log
+    (let [{:keys [pre-python python post-python]} @message-log]
+      (t2/update! :model/TransformRun
+                  :id run-id
+                  {:message (str/join "\n" (concat pre-python
+                                                   (for [{:keys [stream message]} python]
+                                                     (if (= "stdout" stream)
+                                                       (str "\033[32m" message "\033[0m")
+                                                       (str "\033[31m" message "\033[0m")))
+                                                   post-python))}))))
+
+(defn- log-loop! [run-id message-log]
+  (let [poll #(python-runner/get-logs run-id)]
+    (try
+      (loop []
+        (if (.isInterrupted (Thread/currentThread))
+          (log/debug "Message update loop interrupted")
+          (do (Thread/sleep 1000)
+              (let [{:keys [status body]} (poll)]
+                (cond
+                  (<= 200 status 299)
+                  (let [{:keys [execution_id events]} body]
+                    (if-not (= run-id execution_id)
+                      (do (log/debugf "Run id did not match expected: %s actual: %s" run-id execution_id)
+                          (recur))
+                      (do
+                        (replace-python-logs! message-log events)
+                        (save-log-as-message! run-id message-log)
+                        (recur))))
+                  (= 404 status)
+                  (do
+                    (log/debugf "No logs yet (or run finished), run-id: %s" run-id)
+                    (recur))
+                  :else
+                  (do
+                    (log/warnf "Unexpected status polling for logs %s %s, run-id: %s" status body run-id)
+                    (log/debug "Exiting due to poll error")))))))
+      (catch InterruptedException _)
+      (catch Throwable e
+        (log/errorf e "An exception was caught during msg update loop, run-id: %s" run-id)))))
+
+(defn- open-log-thread! ^Closeable [run-id message-log]
+  (let [log-thread-promise (promise)
+        cleanup (fn []
+                  (let [^Thread log-thread (deref log-thread-promise 1000 nil)]
+                    (when-not log-thread (log/fatalf "Log thread reference not bound, run-id: %s" run-id))
+                    (when (.isAlive log-thread)
+                      (.interrupt log-thread)
+                      (when-not (.join log-thread (Duration/ofSeconds 10))
+                        (log/fatalf "Log thread could not be interrupted, run-id: %s" run-id)))))]
+    (u.jvm/in-virtual-thread*
+     (deliver log-thread-promise (Thread/currentThread))
+     (log-loop! run-id message-log))
+    (when-not (deref log-thread-promise 1000 nil) (log/fatalf "Log thread reference not bound, run-id: %s" run-id))
+    (reify
+      IDeref
+      (deref [_] @log-thread-promise)
+      Closeable
+      (close [_] (cleanup)))))
+
+(defn call-python-runner-api!
+  "Call the Python runner API endpoint to execute Python code.
+   Returns the result map or throws on error."
+  [code table-name->id run-id cancel-chan]
+  ;; TODO probably don't need this hack anymore, double check
+  (transforms.instrumentation/with-python-api-timing [run-id]
+    (update (python-runner/execute-python-code run-id code table-name->id cancel-chan)
+            :body #(if (string? %) json/decode+kw %))))
+
+(defn- debug-info-str [{:keys [exit-code events]}]
+  (str/join "\n"
+            ;; todo this was temporary before the log stuff, needs a rethink at some point
+            ["stdout"
+             "======"
+             (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
+             "stderr"
+             "======"
+             (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))
+             "======"
+             (format "exit code %d" exit-code)]))
+
+(defn- transfer-file-to-db [driver db {:keys [target] :as transform} metadata temp-file]
+  (let [table-name (transforms.util/qualified-table-name driver target)
+        table-schema {:name table-name
+                      :columns (mapv (fn [{:keys [name dtype]}]
+                                       {:name name
+                                        :type (transforms.util/dtype->base-type dtype)
+                                        :nullable? true})
+                                     (:fields metadata))}
+        data-source {:type :csv-file
+                     :file temp-file}]
+    ;; TODO: should be transactional, perharps go through driver/run-transform!
+    (transforms.util/delete-target-table! transform)
+    (transforms.util/create-table-from-schema! driver (:id db) table-schema)
+    (driver/insert-from-source! driver (:id db) table-name (mapv :name (:columns table-schema)) data-source)))
+
+(defn- run-python-transform! [{:keys [source] :as transform} db run-id cancel-chan message-log]
+  (with-open [log-thread-ref (open-log-thread! run-id message-log)]
+    (let [driver                           (:engine db)
+          {:keys [source-tables body]}     source
+          {:keys [body status] :as result} (call-python-runner-api! body source-tables run-id cancel-chan)
+          {:keys [events]}   body]
+      (.close log-thread-ref)           ; early close to force any writes to flush
+      (when (seq events)
+        (replace-python-logs! message-log events))
+      (if (not= 200 status)
+        (throw (ex-info (debug-info-str body)
+                        {:status-code     400
+                         :api-status-code status
+                         :body            body
+                         :error           (:error body)}))
+        (try
+          (let [temp-file (File/createTempFile "transform-output-" ".csv")
+                csv-data  (:output body)
+                metadata  (-> body :metadata json/decode+kw)]
+            (when-not (seq (:fields metadata))
+              (throw (ex-info "No fields in metadata"
+                              {:metadata metadata
+                               :raw-body body})))
+            (try
+              (with-open [writer (io/writer temp-file)]
+                (.write writer ^String csv-data))
+              (let [file-size (.length temp-file)]
+                (transforms.instrumentation/with-stage-timing [run-id :data-transfer :file-to-dwh]
+                  (transfer-file-to-db driver db transform metadata temp-file))
+                (transforms.instrumentation/record-data-transfer! run-id :file-to-dwh file-size nil))
+              (finally
+                (.delete temp-file))))
+          result
+          (catch Exception e
+            (log/error e "Failed to to create resulting table")
+            (throw (ex-info "Failed to create the resulting table" {:error (.getMessage e)}))))))))
+
+(defn execute-python-transform!
+  "Execute a Python transform by calling the python runner.
+
+  This is executing synchronously, but supports being kicked off in the background
+  by delivering the `start-promise` just before the start when the beginning of the execution has been booked
+  in the database."
+  [transform {:keys [run-method start-promise message-log]}]
+  (when (transforms.util/python-transform? transform)
+    (try
+      (let [message-log (or message-log (atom {:pre-python  []
+                                               :python      nil
+                                               :post-python []}))
+            {:keys [target] transform-id :id} transform
+            db (t2/select-one :model/Database (:database target))
+            {run-id :id} (try-start-unless-already-running transform-id run-method)]
+        (some-> start-promise (deliver [:started run-id]))
+        (log! message-log "Executing Python transform")
+        (log/info "Executing Python transform" transform-id "with target" (pr-str target))
+        (let [start-ms (u/start-timer)
+              result   (run-cancelable-transform! run-id (fn [cancel-chan] (run-python-transform! transform db run-id cancel-chan message-log)))]
+          (transforms.instrumentation/with-stage-timing [run-id :sync :table-sync]
+            (sync-target! target db run-id))
+          (log! message-log (format "Python execution finished in %s" (Duration/ofMillis (u/since-ms start-ms))))
+          (save-log-as-message! run-id message-log)
+          {:run_id run-id
+           :result result}))
+      (catch Throwable t
+        (log/error t "Error executing Python transform")
+        (log! message-log "Error executing python transform")
+        (throw t)))))

enterprise/backend/src/metabase_enterprise/transforms/instrumentation.clj

@@ -0,0 +1,181 @@
+(ns metabase-enterprise.transforms.instrumentation
+  (:require
+   [metabase.analytics.prometheus :as prometheus]
+   [metabase.util :as u]
+   [metabase.util.log :as log]
+   [metabase.util.malli :as mu]
+   [metabase.util.malli.registry :as mr]))
+
+(set! *warn-on-reflection* true)
+
+(mr/def ::stage-type
+  [:enum :data-transfer :computation :sync])
+
+(mr/def ::stage-label
+  [:enum
+   :dwh-to-file
+   :file-to-s3
+   :file-to-dwh
+
+   :python-execution
+   :mbql-query
+   :table-sync])
+
+(mu/defn record-stage-start!
+  "Record the start of a transform stage."
+  [job-run-id :- pos-int?
+   stage-type :- ::stage-type
+   stage-label :- ::stage-label]
+  (log/infof "Transform stage started: run-id=%d type=%s label=%s" job-run-id (name stage-type) (name stage-label))
+  (prometheus/inc! :metabase-transforms/stage-started
+                   {:job-run-id (str job-run-id)
+                    :stage-type (name stage-type)
+                    :stage-label (name stage-label)}))
+
+(mu/defn record-stage-completion!
+  "Record the successful completion of a transform stage."
+  [job-run-id :- pos-int?
+   stage-type :- ::stage-type
+   stage-label :- ::stage-label
+   duration-ms :- int?]
+  (log/infof "Transform stage completed: run-id=%d type=%s label=%s duration=%dms" job-run-id (name stage-type) (name stage-label) duration-ms)
+  (let [labels {:job-run-id (str job-run-id)
+                :stage-type (name stage-type)
+                :stage-label (name stage-label)}]
+    (prometheus/inc! :metabase-transforms/stage-completed labels)
+    (prometheus/observe! :metabase-transforms/stage-duration-ms labels duration-ms)))
+
+(mu/defn record-stage-failure!
+  "Record the failure of a transform stage."
+  [job-run-id :- pos-int?
+   stage-type :- ::stage-type
+   stage-label :- ::stage-label
+   duration-ms :- [:maybe int?]]
+  (log/warnf "Transform stage failed: run-id=%d type=%s label=%s duration=%s" job-run-id (name stage-type) (name stage-label) (if duration-ms (str duration-ms "ms") "unknown"))
+  (let [labels {:job-run-id (str job-run-id)
+                :stage-type (name stage-type)
+                :stage-label (name stage-label)}]
+    (prometheus/inc! :metabase-transforms/stage-failed labels)
+    (when duration-ms
+      (prometheus/observe! :metabase-transforms/stage-duration-ms labels duration-ms))))
+
+(defn with-timing
+  "Generic timing function that executes body while recording metrics.
+
+   Takes a map of optional functions:
+   - :start-fn - function to call when starting
+   - :success-fn - function to call on success (receives duration-ms as last arg)
+   - :error-fn - function to call on error (receives duration-ms as last arg)
+
+   Usage:
+   (with-timing {:start-fn #(record-stage-start! %1 %2 %3)
+                 :success-fn #(record-stage-completion! %1 %2 %3 %4)
+                 :error-fn #(record-stage-failure! %1 %2 %3 %4)}
+                [job-run-id stage-type stage-label]
+                #(do-work)) "
+  [callbacks args body-fn]
+  (let [start-time (u/start-timer)]
+    (when-let [start-fn (:start-fn callbacks)]
+      (apply start-fn args))
+    (try
+      (let [result (body-fn)]
+        (when-let [success-fn (:success-fn callbacks)]
+          (apply success-fn (conj (vec args) (long (u/since-ms start-time)))))
+        result)
+      (catch Throwable t
+        (when-let [error-fn (:error-fn callbacks)]
+          (apply error-fn (conj (vec args) (long (u/since-ms start-time)))))
+        (throw t)))))
+
+(defmacro with-stage-timing
+  "Execute body while timing a transform stage. Automatically records start, completion or failure.
+
+   Usage:
+   (with-stage-timing [job-run-id :data-transfer :dwh-to-file]
+     ;; stage implementation
+     result)"
+  [args & body]
+  `(with-timing {:start-fn record-stage-start!
+                 :success-fn record-stage-completion!
+                 :error-fn record-stage-failure!}
+     ~args
+     (^:once fn* [] ~@body)))
+
+(mu/defn record-data-transfer!
+  "Record metrics about data transfer (size and rows)."
+  [job-run-id :- pos-int?
+   stage-label :- ::stage-label
+   bytes :- [:maybe int?]
+   rows :- [:maybe int?]]
+  (log/infof "Data transfer recorded: run-id=%d stage=%s%s%s"
+             job-run-id (name stage-label)
+             (if bytes (str " bytes=" bytes) "")
+             (if rows (str " rows=" rows) ""))
+  (let [labels {:job-run-id (str job-run-id)
+                :stage-label (name stage-label)}]
+    (when bytes
+      (prometheus/observe! :metabase-transforms/data-transfer-bytes labels bytes))
+    (when rows
+      (prometheus/observe! :metabase-transforms/data-transfer-rows labels rows))))
+
+(defn record-job-start!
+  "Record the start of a transform job run."
+  [job-id run-method]
+  (log/infof "Transform job started: job-id=%d run-method=%s" job-id (name run-method))
+  (prometheus/inc! :metabase-transforms/job-runs-total
+                   {:job-id (str job-id)
+                    :run-method (name run-method)}))
+
+(defn record-job-completion!
+  "Record the successful completion of a transform job run."
+  [job-id run-method duration-ms]
+  (log/infof "Transform job completed: job-id=%d run-method=%s duration=%dms" job-id (name run-method) duration-ms)
+  (let [labels {:job-id (str job-id)
+                :run-method (name run-method)}]
+    (prometheus/inc! :metabase-transforms/job-runs-completed labels)
+    (prometheus/observe! :metabase-transforms/job-run-duration-ms labels duration-ms)))
+
+(defn record-job-failure!
+  "Record the failure of a transform job run."
+  [job-id run-method duration-ms]
+  (log/warnf "Transform job failed: job-id=%d run-method=%s duration=%s" job-id (name run-method) (if duration-ms (str duration-ms "ms") "unknown"))
+  (let [labels {:job-id (str job-id)
+                :run-method (name run-method)}]
+    (prometheus/inc! :metabase-transforms/job-runs-failed labels)
+    (when duration-ms
+      (prometheus/observe! :metabase-transforms/job-run-duration-ms labels duration-ms))))
+
+(defmacro with-job-timing
+  "Execute body while timing a transform job run. Automatically records start, completion or failure.
+
+   Usage:
+   (with-job-timing [job-id run-method]
+     ;; job implementation
+     result)"
+  [[job-id run-method] & body]
+  `(with-timing {:start-fn record-job-start!
+                 :success-fn record-job-completion!
+                 :error-fn record-job-failure!}
+     [~job-id ~run-method]
+     (^:once fn* [] ~@body)))
+
+(mu/defn record-python-api-call!
+  "Record metrics about Python API calls."
+  [job-run-id :- pos-int?
+   duration-ms :- int?
+   status :- [:enum :success :error :timeout]]
+  (log/infof "Python API call %s: run-id=%d duration=%dms" (name status) job-run-id duration-ms)
+  (let [labels {:job-run-id (str job-run-id)}]
+    (prometheus/inc! :metabase-transforms/python-api-calls-total
+                     (assoc labels :status (name status)))
+    (prometheus/observe! :metabase-transforms/python-api-call-duration-ms labels duration-ms)))
+
+(defmacro with-python-api-timing
+  "Execute body while timing a Python API call."
+  [[job-run-id] & body]
+  `(with-timing {:success-fn (fn [job-run-id# duration-ms#]
+                               (record-python-api-call! job-run-id# duration-ms# :success))
+                 :error-fn (fn [job-run-id# duration-ms#]
+                             (record-python-api-call! job-run-id# duration-ms# :error))}
+     [~job-run-id]
+     (^:once fn* [] ~@body)))

enterprise/backend/src/metabase_enterprise/transforms/jobs.clj

@@ -6,10 +6,12 @@
    [clojurewerkz.quartzite.triggers :as triggers]
    [flatland.ordered.set :as ordered-set]
    [metabase-enterprise.transforms.execute :as transforms.execute]
+   [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
    [metabase-enterprise.transforms.models.job-run :as transforms.job-run]
    [metabase-enterprise.transforms.models.transform-run :as transform-run]
    [metabase-enterprise.transforms.ordering :as transforms.ordering]
    [metabase-enterprise.transforms.settings :as transforms.settings]
+   [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.task.core :as task]
    [metabase.util.log :as log]
    [toucan2.core :as t2]))
@@ -64,14 +66,21 @@
         (map transforms-by-id complete)))))
 
 (defn- run-transform! [run-id run-method {transform-id :id :as transform}]
-  (when (transform-run/running-run-for-run-id transform-id)
+  (when
+   (transform-run/running-run-for-transform-id transform-id)
     (log/warn "Transform" (pr-str transform-id) "already running, waiting")
     (loop []
       (Thread/sleep 2000)
-      (when (transform-run/running-run-for-run-id transform-id)
+      (when (transform-run/running-run-for-transform-id transform-id)
         (recur))))
+
   (log/info "Executing job transform" (pr-str transform-id))
-  (transforms.execute/run-mbql-transform! transform {:run-method run-method})
+  (if (transforms.util/python-transform? transform)
+    (transforms.instrumentation/with-stage-timing [run-id :computation :python-execution]
+      (transforms.execute/execute-python-transform! transform {:run-method run-method}))
+    (transforms.instrumentation/with-stage-timing [run-id :computation :mbql-query]
+      (transforms.execute/run-mbql-transform! transform {:run-method run-method})))
+
   (transforms.job-run/add-run-activity! run-id))
 
 (defn run-transforms!
@@ -110,12 +119,13 @@
     (let [transforms (job-transform-ids job-id)]
       (log/info "Executing transform job" (pr-str job-id) "with transforms" (pr-str transforms))
       (let [{run-id :id} (transforms.job-run/start-run! job-id run-method)]
-        (try
-          (run-transforms! run-id transforms opts)
-          (transforms.job-run/succeed-started-run! run-id)
-          (catch Throwable t
-            (transforms.job-run/fail-started-run! run-id {:message (.getMessage t)})
-            (throw t)))))))
+        (transforms.instrumentation/with-job-timing [job-id run-method]
+          (try
+            (run-transforms! run-id transforms opts)
+            (transforms.job-run/succeed-started-run! run-id)
+            (catch Throwable t
+              (transforms.job-run/fail-started-run! run-id {:message (.getMessage t)})
+              (throw t))))))))
 
 (def ^:private job-key "metabase-enterprise.transforms.jobs.timeout-job")
 

enterprise/backend/src/metabase_enterprise/transforms/models/transform_run.clj

@@ -147,11 +147,11 @@
                         :message   "Canceled by user but could not guarantee run stopped."})
     (cancel/delete-old-canceling-runs!)))
 
-(defn running-run-for-run-id
+(defn running-run-for-transform-id
   "Return a single active transform run or nil."
-  [id]
+  [transform-id]
   (t2/select-one :model/TransformRun
-                 :id id
+                 :transform_id transform-id
                  :is_active true))
 
 (defn- timestamp-constraint

enterprise/backend/src/metabase_enterprise/transforms/models/transform_run_cancelation.clj

@@ -14,14 +14,10 @@
   ([run-id]
    (mark-cancel-started-run! run-id {}))
   ([run-id properties]
-   (t2/insert! :model/TransformRunCancelation
-               (assoc properties
-                      :run_id run-id)
-               {:where [:exists {:select [1]
-                                 :from   [:transform_run]
-                                 :where  [:and
-                                          [:= :transform_run.id run-id]
-                                          :transform_run.is_active]}]})))
+   (when (t2/exists? :transform_run :id run-id :is_active true)
+     (t2/insert! :model/TransformRunCancelation
+                 (assoc properties
+                        :run_id run-id)))))
 
 (defn reducible-canceled-local-runs
   "Return a reducible sequence of local canceled runs."

enterprise/backend/src/metabase_enterprise/transforms/ordering.clj

@@ -56,20 +56,28 @@
 
   The result is a map of transform id -> #{transform ids the transform depends on}. Dependencies are limited to just
   the transforms in the original list -- if a transform depends on some transform not in the list, the 'extra'
-  dependency is ignored."
+  dependency is ignored. Python transforms have no dependencies and run sequentially."
   [transforms]
-  (let [transforms-by-db (->> transforms
-                              (map (fn [transform]
-                                     {(get-in transform [:source :query :database]) [transform]}))
-                              (apply merge-with into))
+  (let [{python-transforms true
+         sql-transforms false} (group-by transforms.util/python-transform? transforms)
 
-        {:keys [output-tables dependencies]} (->> transforms-by-db
+        ;; TODO: figure out python transforms deps
+        python-deps (into {} (map (fn [transform] [(:id transform) #{}])) python-transforms)
+
+        sql-transforms-by-db (->> sql-transforms
+                                  (map (fn [transform]
+                                         {(get-in transform [:source :query :database]) [transform]}))
+                                  (apply merge-with into))
+
+        {:keys [output-tables dependencies]} (->> sql-transforms-by-db
                                                   (map (fn [[db-id db-transforms]]
                                                          (qp.store/with-metadata-provider db-id
                                                            {:output-tables (output-table-map db-transforms)
                                                             :dependencies (dependency-map db-transforms)})))
-                                                  (apply merge-with merge))]
-    (update-vals dependencies #(into #{} (keep output-tables) %))))
+                                                  (apply merge-with merge))
+
+        sql-deps (update-vals dependencies #(into #{} (keep output-tables) %))]
+    (merge python-deps sql-deps)))
 
 (defn find-cycle
   "Finds a path containing a cycle in the directed graph `node->children`.

enterprise/backend/src/metabase_enterprise/transforms/python_runner.clj

@@ -0,0 +1,414 @@
+(ns metabase-enterprise.transforms.python-runner
+  (:require
+   [clj-http.client :as http]
+   [clojure.core.async :as a]
+   [clojure.java.io :as io]
+   [clojure.string :as str]
+   [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
+   [metabase-enterprise.transforms.settings :as transforms.settings]
+   [metabase.driver :as driver]
+   [metabase.query-processor.compile :as qp.compile]
+   ^{:clj-kondo/ignore [:discouraged-namespace]}
+   [metabase.query-processor.store :as qp.store]
+   [metabase.util.json :as json]
+   [metabase.util.log :as log]
+   [toucan2.core :as t2])
+  (:import
+   (java.io BufferedWriter File OutputStream OutputStreamWriter)
+   (java.net URI)
+   (java.nio.charset StandardCharsets)
+   (java.time Duration)
+   (java.util.concurrent CancellationException)
+   (software.amazon.awssdk.auth.credentials AwsBasicCredentials StaticCredentialsProvider)
+   (software.amazon.awssdk.auth.credentials DefaultCredentialsProvider)
+   (software.amazon.awssdk.core.sync RequestBody)
+   (software.amazon.awssdk.regions Region)
+   (software.amazon.awssdk.services.s3 S3Client S3ClientBuilder S3Configuration)
+   (software.amazon.awssdk.services.s3.model DeleteObjectRequest GetObjectRequest NoSuchKeyException PutObjectRequest)
+   (software.amazon.awssdk.services.s3.presigner S3Presigner S3Presigner$Builder)
+   (software.amazon.awssdk.services.s3.presigner.model GetObjectPresignRequest PresignedGetObjectRequest PresignedPutObjectRequest PutObjectPresignRequest)))
+
+(set! *warn-on-reflection* true)
+
+(defn- safe-delete
+  "Safely delete a file."
+  [^File file]
+  (try (.delete file) (catch Exception _)))
+
+(defn- write-to-stream! [^OutputStream os col-names reducible-rows]
+  (let [writer (-> os
+                   (OutputStreamWriter. StandardCharsets/UTF_8)
+                   (BufferedWriter.))]
+
+    (run! (fn [row]
+            (let [row-map (zipmap col-names row)]
+              (json/encode-to row-map writer {})
+              (.newLine writer)))
+          reducible-rows)
+
+    (doto writer
+      (.flush)
+      (.close))))
+
+(defn- execute-mbql-query
+  [driver db-id query respond cancel-chan]
+  (driver/with-driver driver
+    (let [native (qp.compile/compile {:type :query, :database db-id :query query})
+
+          query  {:database db-id
+                  :type     :native
+                  :native   native}]
+      (qp.store/with-metadata-provider db-id
+        (driver/execute-reducible-query driver query {:canceled-chan cancel-chan} respond)))))
+
+(defn- root-type
+  [base-type]
+  (when base-type
+    (some #(when (isa? base-type %) %)
+          [:type/Float
+           :type/Integer
+           :type/Boolean
+           :type/DateTimeWithTZ
+           :type/DateTime
+           :type/Date
+           :type/Text])))
+
+(defn- closest-ancestor [t pred]
+  (loop [remaining (conj clojure.lang.PersistentQueue/EMPTY [t])]
+    (when-let [t (first remaining)]
+      (if (pred t)
+        t
+        (recur (into (pop t) (parents t)))))))
+
+(defn- effective-semantic-type-i-think
+  "Kinda sketchy but maybe reasonable way to infer the effective-semantic-type"
+  [{:keys [base_type effective_type semantic_type]}]
+  (or semantic_type (closest-ancestor (or effective_type base_type) #(isa? % :Semantic/*))))
+
+(defn- generate-manifest
+  "Generate a metadata manifest for the table columns."
+  [table-id cols-meta]
+  {:version        "0.1.0"
+   :fields         (mapv (fn [col-meta]
+                           {:name           (:name col-meta)
+                            :base_type      (some-> (:base_type col-meta) name)
+                            :database_type  (some-> (:database_type col-meta) name)
+                            :root_type      (some-> (root-type (:base_type col-meta)) name)
+                            ;; replace nil values with values indicating how they behave in practice.
+                            ;; there may be better ways of doing this already, but i worry it's just implicit in QP
+                            :semantic_type  (some-> (effective-semantic-type-i-think col-meta) name)
+                            :effective_type (some-> (or (:effective_type col-meta) (:database_type col-meta)) name)
+                            ;; TODO get this passed through
+                            #_#_:field_id   (:id col-meta)})
+                         cols-meta)
+   :table_metadata {:table_id table-id}})
+
+(defn- write-table-data-to-file! [id temp-file cancel-chan]
+  (let [db-id         (t2/select-one-fn :db_id (t2/table-name :model/Table) :id id)
+        driver        (t2/select-one-fn :engine :model/Database db-id)
+        ;; TODO: limit
+        query         {:source-table id}
+        manifest-atom (atom nil)]
+    (execute-mbql-query driver db-id query
+                        (fn [{cols-meta :cols} reducible-rows]
+                          (reset! manifest-atom (generate-manifest id cols-meta))
+                          (with-open [os (io/output-stream temp-file)]
+                            (write-to-stream! os (mapv :name cols-meta) reducible-rows)))
+                        cancel-chan)
+    @manifest-atom))
+
+(defn- maybe-with-endpoint-s3-client [^S3ClientBuilder builder endpoint]
+  (let [region (transforms.settings/python-storage-s-3-region)]
+    (case [(some? endpoint) (some? region)]
+      [true true]   (doto builder
+                      (.endpointOverride (URI/create endpoint))
+                      (.region (Region/of region)))
+      [true false]  (log/warn "Ignoring endpoint because region is not defined")
+      [false true]  (.region builder (Region/of region))
+      [false false] builder)))
+
+(defn- maybe-with-endpoint-s3-presigner [^S3Presigner$Builder builder endpoint]
+  (let [region (transforms.settings/python-storage-s-3-region)]
+    (case [(some? endpoint) (some? region)]
+      [true true]   (doto builder
+                      (.endpointOverride (URI/create endpoint))
+                      (.region (Region/of region)))
+      [true false]  (log/warn "Ignoring endpoint because region is not defined")
+      [false true]  (.region builder (Region/of region))
+      [false false] (.region builder (Region/US_EAST_1)))))
+
+(defn- create-s3-configuration
+  "Create S3Configuration with path-style access setting"
+  ^S3Configuration []
+  (-> (S3Configuration/builder)
+      (.pathStyleAccessEnabled (transforms.settings/python-storage-s-3-path-style-access))
+      (.build)))
+
+(defn- build-put-object-request
+  "Build a PutObjectRequest with bucket and key"
+  [^String bucket-name ^String key]
+  (-> (PutObjectRequest/builder)
+      (.bucket bucket-name)
+      (.key key)
+      (.build)))
+
+(defn- build-get-object-request
+  "Build a GetObjectRequest with bucket and key"
+  [^String bucket-name ^String key]
+  (-> (GetObjectRequest/builder)
+      (.bucket bucket-name)
+      (.key key)
+      (.build)))
+
+(defn- build-delete-object-request
+  "Build a DeleteObjectRequest with bucket and key"
+  [^String bucket-name ^String key]
+  (-> (DeleteObjectRequest/builder)
+      (.bucket bucket-name)
+      (.key key)
+      (.build)))
+
+(def ^:private ^Duration presigned-url-duration
+  "Default duration for presigned URLs"
+  (Duration/ofHours 1))
+
+(defn- maybe-with-credentials-s3-client [^S3ClientBuilder builder]
+  (let [access-key (transforms.settings/python-storage-s-3-access-key)
+        secret-key (transforms.settings/python-storage-s-3-secret-key)]
+    (if (or access-key secret-key)
+      (if-not (and access-key secret-key)
+        (do (log/warnf "Ignoring %s because %s is not defined"
+                       (if access-key "access-key" "secret-key")
+                       (if (not access-key) "access-key" "secret-key"))
+            (.credentialsProvider builder (DefaultCredentialsProvider/create)))
+        (.credentialsProvider builder
+                              (StaticCredentialsProvider/create
+                               (AwsBasicCredentials/create access-key secret-key))))
+      (.credentialsProvider builder (DefaultCredentialsProvider/create)))))
+
+(defn- maybe-with-credentials-s3-presigner [^S3Presigner$Builder builder]
+  (let [access-key (transforms.settings/python-storage-s-3-access-key)
+        secret-key (transforms.settings/python-storage-s-3-secret-key)]
+    (if (or access-key secret-key)
+      (if-not (and access-key secret-key)
+        (do (log/warnf "Ignoring %s because %s is not defined"
+                       (if access-key "access-key" "secret-key")
+                       (if (not access-key) "access-key" "secret-key"))
+            builder)
+        (.credentialsProvider builder
+                              (StaticCredentialsProvider/create
+                               (AwsBasicCredentials/create access-key secret-key))))
+      builder)))
+
+;; We just recreate the client every time, to keep things simple if config is changed.
+(defn- create-s3-client
+  "Create S3 client for host operations (uploads, reads)"
+  ^S3Client []
+  (let [^S3ClientBuilder builder (S3Client/builder)]
+    (doto builder
+      (maybe-with-endpoint-s3-client (transforms.settings/python-storage-s-3-endpoint))
+      maybe-with-credentials-s3-client
+      (.serviceConfiguration (create-s3-configuration)))
+    (.build builder)))
+
+(defn- create-s3-presigner-for-container
+  "Create S3 presigner for container operations (presigned URLs).
+  Uses container-endpoint if different from host endpoint."
+  ^S3Presigner []
+  (let [container-endpoint (transforms.settings/python-storage-s-3-container-endpoint)
+        builder (S3Presigner/builder)]
+    (doto builder
+      (maybe-with-endpoint-s3-presigner (or container-endpoint (transforms.settings/python-storage-s-3-endpoint)))
+      maybe-with-credentials-s3-presigner
+      (.serviceConfiguration (create-s3-configuration)))
+    (.build builder)))
+
+(defn- upload-file-to-s3
+  "Upload file using host client"
+  [^S3Client s3-client ^String bucket-name ^String key ^File file]
+  (let [^PutObjectRequest request (build-put-object-request bucket-name key)]
+    (.putObject s3-client request (RequestBody/fromFile file))))
+
+(defn- generate-presigned-get-url
+  "Generate GET URL using container presigner"
+  [^S3Presigner presigner ^String bucket-name ^String key]
+  (let [^GetObjectRequest get-request (build-get-object-request bucket-name key)
+        ^GetObjectPresignRequest presign-request (-> (GetObjectPresignRequest/builder)
+                                                     (.signatureDuration presigned-url-duration)
+                                                     (.getObjectRequest get-request)
+                                                     (.build))
+        presigned ^PresignedGetObjectRequest (.presignGetObject presigner presign-request)]
+    (.toString (.url presigned))))
+
+(defn- generate-presigned-put-url
+  "Generate PUT URL using container presigner"
+  [^S3Presigner presigner ^String bucket-name ^String key]
+  (let [^PutObjectRequest put-request (build-put-object-request bucket-name key)
+        ^PutObjectPresignRequest presign-request (-> (PutObjectPresignRequest/builder)
+                                                     (.signatureDuration presigned-url-duration)
+                                                     (.putObjectRequest put-request)
+                                                     (.build))
+        presigned ^PresignedPutObjectRequest (.presignPutObject presigner presign-request)]
+    (.toString (.url presigned))))
+
+(defn- delete-s3-object [^S3Client s3-client ^String bucket-name ^String key]
+  (try
+    (let [^DeleteObjectRequest request (build-delete-object-request bucket-name key)]
+      (.deleteObject s3-client request))
+    (catch Exception _
+      ;; Ignore deletion errors - object might not exist or we might not have permissions
+      nil)))
+
+(defn- cleanup-s3-objects [^S3Client s3-client bucket-name s3-keys]
+  (run! (partial delete-s3-object s3-client bucket-name) s3-keys))
+
+(defn- read-from-s3 [^S3Client s3-client ^String bucket-name ^String key & [fallback-content]]
+  (try
+    (let [^GetObjectRequest request (build-get-object-request bucket-name key)
+          response                  (.getObject s3-client request)]
+      (slurp response))
+    (catch NoSuchKeyException e
+      (if fallback-content
+        fallback-content
+        (throw e)))))
+
+(defn get-logs
+  "Return the logs of the current running python process"
+  ;; TODO: we should be given an id for the expected job, se we don't return unrelated logs
+  ;;       if the job has already finished, we could fethc the logs from the db instead
+  [run-id]
+  (let [server-url (transforms.settings/python-execution-server-url)]
+    (http/get (str server-url "/logs")
+              {:content-type     :json
+               :accept           :json
+               :throw-exceptions false
+               :as               :json
+               :query-params     {:request_id run-id}})))
+
+(defn execute-python-code
+  "Execute Python code using the Python execution server."
+  [run-id code table-name->id cancel-chan]
+  (let [prefix        (some-> (transforms.settings/python-storage-s-3-prefix) (str "/"))
+        work-dir-name (str prefix "run-" (System/nanoTime) "-" (rand-int 10000))]
+
+    (try
+      (let [server-url               (transforms.settings/python-execution-server-url)
+            bucket-name              (transforms.settings/python-storage-s-3-bucket)
+            s3-client                (create-s3-client)
+            container-presigner      (create-s3-presigner-for-container)
+
+            ;; Generate S3 keys for output files
+            output-key               (str work-dir-name "/output.csv")
+            output-manifest-key      (str work-dir-name "/output.manifest.json")
+            events-key               (str work-dir-name "/events.jsonl")
+            ;; Generate presigned URLs for writing (using container client)
+            output-url               (generate-presigned-put-url container-presigner bucket-name output-key)
+            output-manifest-url      (generate-presigned-put-url container-presigner bucket-name output-manifest-key)
+            events-url               (generate-presigned-put-url container-presigner bucket-name events-key)
+            ;; Upload input table data (write to disk first, then upload to S3)
+            table-results            (for [[table-name id] table-name->id]
+                                       (let [temp-file       (File/createTempFile
+                                                              (str work-dir-name "-table-" (name table-name) "-" id)
+                                                              ".jsonl")
+                                             manifest-file   (File/createTempFile
+                                                              (str work-dir-name "-table-" (name table-name) "-" id)
+                                                              ".manifest.json")
+                                             s3-key          (str work-dir-name "/" (.getName temp-file))
+                                             manifest-s3-key (str work-dir-name "/" (.getName manifest-file))]
+                                         (try
+                                           ;; Write table data to temporary file and get manifest
+                                           (let [manifest (transforms.instrumentation/with-stage-timing [run-id :data-transfer :dwh-to-file]
+                                                            (write-table-data-to-file! id temp-file cancel-chan))]
+                                             ;; Write manifest to file
+                                             (with-open [writer (io/writer manifest-file)]
+                                               (json/encode-to manifest writer {}))
+                                             (let [file-size (.length temp-file)
+                                                   manifest-size (.length manifest-file)]
+                                               (transforms.instrumentation/record-data-transfer! run-id :dwh-to-file file-size nil)
+
+                                               ;; Upload both files to S3
+                                               (transforms.instrumentation/with-stage-timing [run-id :data-transfer :file-to-s3]
+                                                 (upload-file-to-s3 s3-client bucket-name s3-key temp-file)
+                                                 (upload-file-to-s3 s3-client bucket-name manifest-s3-key manifest-file))
+
+                                               (let [data-url     (generate-presigned-get-url container-presigner bucket-name s3-key)
+                                                     manifest-url (generate-presigned-get-url container-presigner bucket-name manifest-s3-key)]
+                                                 (transforms.instrumentation/record-data-transfer! run-id :file-to-s3 (+ file-size manifest-size) nil)
+                                                 {:table-name      (name table-name)
+                                                  :url             data-url
+                                                  :manifest-url    manifest-url
+                                                  :s3-key          s3-key
+                                                  :manifest-s3-key manifest-s3-key})))
+                                           (finally
+                                             ;; Clean up temporary files
+                                             (safe-delete temp-file)
+                                             (safe-delete manifest-file)))))
+            table-name->url          (into {} (map (juxt :table-name :url) table-results))
+            table-name->manifest-url (into {} (map (juxt :table-name :manifest-url) table-results))
+            all-s3-keys              (concat [output-key output-manifest-key events-key]
+                                             (map :s3-key table-results)
+                                             (map :manifest-s3-key table-results))
+            canc                     (a/go (when (a/<! cancel-chan)
+                                             (http/post (str server-url "/cancel")
+                                                        {:content-type :json
+                                                         :body         (json/encode {:request_id run-id})
+                                                         :async?       true}
+                                                        identity identity)))
+            payload                  {:code                code
+                                      :timeout             30
+                                      :request_id          run-id
+                                      :output_url          output-url
+                                      :output_manifest_url output-manifest-url
+                                      :events_url          events-url
+                                      :table_mapping       table-name->url
+                                      :manifest_mapping    table-name->manifest-url}
+            response                 (http/post (str server-url "/execute")
+                                                {:content-type     :json
+                                                 :accept           :json
+                                                 :body             (json/encode payload)
+                                                 :throw-exceptions false
+                                                 :as               :json})
+            _                        (a/close! canc)
+
+            result                   (:body response)
+            ;; TODO look into why some tests return json and others strings
+            result                   (if (string? result) (json/decode result keyword) result)]
+
+        (try
+          (if (and (= 200 (:status response))
+                   (zero? (:exit_code result)))
+           ;; Success - read the output from S3
+            (let [output-content  (read-from-s3 s3-client bucket-name output-key)
+                  output-manifest (read-from-s3 s3-client bucket-name output-manifest-key "{}")
+                  events-content  (read-from-s3 s3-client bucket-name events-key "[]")]
+              (if (not (str/blank? output-content))
+                {:status 200
+                 :body   {:output output-content
+                          :metadata output-manifest
+                          :events (mapv json/decode+kw (str/split-lines events-content))}}
+                {:status 500
+                 :body   {:error  "Transform did not produce output CSV"
+                          :events (mapv json/decode+kw (str/split-lines events-content))}}))
+           ;; Error from execution server - read events .jsonl (including stderr/stdout) from S3
+            (let [events-content (read-from-s3 s3-client bucket-name events-key "[]")]
+              {:status 500
+               :body
+               {:error       (or (:error result) "Execution failed")
+                :exit-code   (:exit_code result)
+                :status-code (:status response)
+                :timeout     (:timeout result)
+                :events      (mapv json/decode+kw (str/split-lines events-content))}}))
+          (finally
+            ;; Clean up S3 objects
+            (try
+              (cleanup-s3-objects s3-client bucket-name all-s3-keys)
+              (catch Exception _)))))
+
+      (catch CancellationException _
+        {:status 408
+         :body   {:error "Interrupted"}})
+
+      (catch Exception e
+        (.printStackTrace e)
+        {:status 500
+         :body   {:error (str "Failed to connect to Python execution server: " (.getMessage e))}}))))

enterprise/backend/src/metabase_enterprise/transforms/schedule.clj

@@ -96,6 +96,7 @@
         trigger (build-trigger job-id schedule)]
     (task/schedule-task! job trigger)))
 
+;; TODO: should this be part of transformjob hooks?
 (defn update-job!
   "Update the trigger for a transform job."
   [job-id schedule]

enterprise/backend/src/metabase_enterprise/transforms/settings.clj

@@ -1,17 +1,119 @@
 (ns metabase-enterprise.transforms.settings
   (:require
+   [metabase.config.core :as config]
    [metabase.settings.core :as setting]
    [metabase.util.i18n :refer [deferred-tru]]))
 
 (set! *warn-on-reflection* true)
 
+(def ^:privatg is-not-prod? (or config/is-dev? config/is-test?))
+
 (setting/defsetting transform-timeout
   (deferred-tru "The timeout for a transform job, in minutes.")
-  :type       :integer
+  :type :integer
   :visibility :internal
-  :default    (* 4 60)
-  :feature    :transforms
-  :doc        false
-  :export?    false
+  :default (* 4 60)
+  :feature :transforms
+  :doc false
+  :export? false
   :encryption :no
-  :audit      :getter)
+  :audit :getter)
+
+(setting/defsetting python-execution-server-url
+  (deferred-tru "URL for the Python execution server that runs transform functions.")
+  :type :string
+  :visibility :admin
+  :default (when is-not-prod? "http://localhost:5001")
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-endpoint
+  (deferred-tru "S3 endpoint URL for storing Python execution artifacts.")
+  :type :string
+  :visibility :admin
+  :default (when is-not-prod? "http://localhost:4566")
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-region
+  (deferred-tru "AWS region for S3 storage.")
+  :type :string
+  :visibility :admin
+  :default (when is-not-prod? "us-east-1")
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-bucket
+  (deferred-tru "S3 bucket name for storing Python execution artifacts.")
+  :type :string
+  :visibility :admin
+  :default (if is-not-prod? "metabase-python-runner" "metabase-cloud-python-runner-staging")
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-prefix
+  (deferred-tru "Prefix to use for S3 objects. Necessary in prod where we only have access to a particular prefix.")
+  :type :string
+  :visibility :admin
+  :feature :transforms
+  :default (if is-not-prod? "test-prefix" "hosting-4b9351bb22b84791")
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-access-key
+  (deferred-tru "AWS access key ID for S3 authentication.")
+  :type :string
+  :visibility :admin
+  :feature :transforms
+  :default (when is-not-prod? "test")
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-secret-key
+  (deferred-tru "AWS secret access key for S3 authentication.")
+  :type :string
+  :visibility :admin
+  :feature :transforms
+  :default (when is-not-prod? "test")
+  :doc false
+  :export? false
+  :encryption :when-encryption-key-set
+  :audit :never)
+
+(setting/defsetting python-storage-s-3-container-endpoint
+  (deferred-tru "Alternative S3 endpoint accessible from containers. Leave empty if same as main endpoint.")
+  :type :string
+  :visibility :admin
+  :default (when is-not-prod? "http://localstack:4566")
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-path-style-access
+  (deferred-tru "Use path-style access for S3 requests (required for LocalStack and some S3-compatible services).")
+  :type :boolean
+  :visibility :admin
+  :default (when is-not-prod? true)
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)

enterprise/backend/src/metabase_enterprise/transforms/util.clj

@@ -1,10 +1,14 @@
 (ns metabase-enterprise.transforms.util
   (:require
+   [clojure.string :as str]
    [metabase.driver :as driver]
+   [metabase.lib.schema.common :as lib.schema.common]
    [metabase.query-processor.compile :as qp.compile]
    [metabase.sync.core :as sync]
    [metabase.util.date-2 :as u.date]
    [metabase.util.log :as log]
+   [metabase.util.malli :as mu]
+   [metabase.util.malli.registry :as mr]
    [toucan2.core :as t2])
   (:import
    (java.time Instant LocalDateTime OffsetDateTime ZonedDateTime ZoneId)
@@ -19,12 +23,23 @@
     (keyword schema name)
     (keyword name)))
 
+(defn query-transform?
+  "Check if this is a query transform: native query / mbql query."
+  [transform]
+  (= :query (-> transform :source :type keyword)))
+
+(defn python-transform?
+  "Check if this is a Python transform."
+  [transform]
+  (= :python (-> transform :source :type keyword)))
+
 (defn target-table-exists?
   "Test if the target table of a transform already exists."
-  [{:keys [source target] :as _transform}]
-  (let [db-id (-> source :query :database)
-        {driver :engine :as database} (t2/select-one :model/Database db-id)]
-    (driver/table-exists? driver database target)))
+  [{:keys [source target] :as transform}]
+  (when (query-transform? transform)
+    (let [db-id (-> source :query :database)
+          {driver :engine :as database} (t2/select-one :model/Database db-id)]
+      (driver/table-exists? driver database target))))
 
 (defn target-table
   "Load the `target` table of a transform from the database specified by `database-id`."
@@ -64,7 +79,9 @@
   [{:keys [id target source], :as _transform}]
   (when target
     (let [target (update target :type keyword)
-          database-id (-> source :query :database)
+          database-id (or (-> source :query :database)
+                          ;; python transform target
+                          (-> target :database))
           {driver :engine :as database} (t2/select-one :model/Database database-id)]
       (driver/drop-transform-target! driver database target)
       (log/info "Deactivating  target " (pr-str target) "for transform" id)
@@ -91,7 +108,6 @@
   [transform]
   (case (-> transform :target :type)
     "table"             :transforms/table))
-
 (defn local-timestamp
   "Convert the timestamp t to a ZonedDateTime instance in the system timezone."
   [t]
@@ -109,3 +125,41 @@
   "Convert the timestamp t to a string encoding the it in the system timezone."
   [t]
   (-> t local-timestamp str))
+
+(mr/def ::column-definition
+  [:map
+   [:name :string]
+   [:type ::lib.schema.common/base-type]
+   [:nullable? {:optional true} :boolean]])
+
+(mr/def ::table-definition
+  [:map
+   [:name :keyword]
+   [:columns [:sequential ::column-definition]]
+   [:primary-key {:optional true} [:sequential :string]]])
+
+(defn dtype->base-type
+  "Maps pandas dtype strings directly to Metabase base types in the type hierarchy."
+  [dtype-str]
+  (cond
+    (str/starts-with? dtype-str "int") :type/Integer
+    (str/starts-with? dtype-str "float") :type/Float
+    (str/starts-with? dtype-str "bool") :type/Boolean
+    ;; datetime64[ns, timezone] indicates timezone-aware datetime
+    (str/starts-with? dtype-str "datetime64[ns, ") :type/DateTimeWithTZ
+    (str/starts-with? dtype-str "datetime") :type/DateTime
+    ;; this is not a real dtype, pandas uses 'object', but we override it if there's source or custom field metadata
+    (str/starts-with? dtype-str "date") :type/Date
+    :else :type/Text))
+
+(mu/defn create-table-from-schema!
+  "Create a table from a table-schema"
+  [driver :- :keyword
+   database-id :- pos-int?
+   table-schema :- ::table-definition]
+  (let [{:keys [columns] table-name :name} table-schema
+        column-definitions (into {} (map (fn [{:keys [name type]}] [name (driver/type->database-type driver type)]))
+                                 columns)
+        primary-key-opts (select-keys table-schema [:primary-key])]
+    (log/infof "Creating table %s with %d columns" table-name (count columns))
+    (driver/create-table! driver database-id table-name column-definitions primary-key-opts)))

enterprise/backend/test/metabase_enterprise/transforms/api_test.clj

@@ -8,14 +8,16 @@
    [metabase-enterprise.transforms.models.transform-transform-tag]
    [metabase-enterprise.transforms.query-test-util :as query-test-util]
    [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
-   [metabase-enterprise.transforms.test-util :refer [with-transform-cleanup!]]
+   [metabase-enterprise.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]
    [metabase-enterprise.transforms.util :as transforms.util]
+   [metabase.driver :as driver]
    [metabase.legacy-mbql.normalize :as mbql.normalize]
    [metabase.lib.convert :as lib.convert]
    [metabase.lib.core :as lib]
    [metabase.lib.metadata :as lib.metadata]
    [metabase.query-processor :as qp]
    [metabase.test :as mt]
+   [metabase.test.util :as tu]
    [metabase.util :as u]
    [toucan2.core :as t2])
   (:import
@@ -74,6 +76,26 @@
                                             :schema schema
                                             :name   table-name}})))))))
 
+(deftest create-python-transform-test
+  (mt/with-premium-features #{:transforms}
+    (mt/dataset transforms-dataset/transforms-test
+      (with-transform-cleanup! [table-name "gadget_products"]
+        (let [schema    (get-test-schema)
+              transform-payload {:name   "My beautiful python runner"
+                                 :source {:type            "python"
+                                          :body            "print('hello world')"
+                                          :source-tables   {}}
+                                 :target {:type     "table"
+                                          :schema   schema
+                                          :name     table-name
+                                          :database (mt/id)}}
+              transform (mt/user-http-request :crowberto :post 200 "ee/transform"
+                                              transform-payload)]
+          (is (= "print('hello chris')"
+                 (-> (mt/user-http-request :crowberto :put 200 (format "ee/transform/%s" (:id transform))
+                                           (assoc-in transform-payload [:source :body] "print('hello chris')"))
+                     :source :body))))))))
+
 (deftest list-transforms-test
   (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
     (testing "Can list without query parameters"
@@ -321,6 +343,32 @@
                 (is (true? (transforms.util/target-table-exists? updated)))
                 (check-query-results table2-name [2 3 4 13] "Doohickey")))))))))
 
+(deftest execute-python-transform-test
+  (testing "transform execution with :transforms/table target"
+    (mt/test-drivers #{:h2 :postgres}
+      (mt/with-premium-features #{:transforms}
+        (mt/dataset transforms-dataset/transforms-test
+          (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
+            (with-transform-cleanup! [{table-name :name :as target} {:type   "table"
+                                                                     :schema schema
+                                                                     :name   "target_table"}]
+              (let [original           {:name   "Gadget Products"
+                                        :source {:type  "python"
+                                                 :source-database (mt/id)
+                                                 :source-tables {"transforms_customers" (mt/id :transforms_customers)}
+                                                 :body  (str "import pandas as pd\n"
+                                                             "\n"
+                                                             "def transform():\n"
+                                                             "    return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}
+                                        :target  (assoc target :database (mt/id))}
+                    {transform-id :id} (mt/user-http-request :crowberto :post 200 "ee/transform"
+                                                             original)]
+                (do (test-run transform-id)
+                    (wait-for-table table-name 5000))
+                (is (true? (driver/table-exists? driver/*driver* (mt/db) target)))
+                (is (= [["Alice" 25] ["Bob" 30]]
+                       (transforms.tu/table-rows table-name)))))))))))
+
 (deftest get-runs-filter-by-single-transform-id-test
   (testing "GET /api/ee/transform/run - filter by single transform ID"
     (mt/with-premium-features #{:transforms}

enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj

@@ -6,10 +6,13 @@
    [metabase-enterprise.transforms.query-test-util :as query-test-util]
    [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
    [metabase-enterprise.transforms.test-util :refer [with-transform-cleanup!]]
+   [metabase-enterprise.transforms.util :as transforms.util]
+   [metabase.driver :as driver]
    [metabase.lib.core :as lib]
    [metabase.lib.metadata :as lib.metadata]
    [metabase.query-processor :as qp]
    [metabase.test :as mt]
+   [metabase.test.data.sql :as sql.tx]
    [toucan2.core :as t2]))
 
 (set! *warn-on-reflection* true)
@@ -125,3 +128,21 @@
                           ["2024-01-21T00:00:00Z" 19.99]
                           ["2024-01-23T00:00:00Z" 14.99]]
                          query-result)))))))))))
+
+(deftest create-table-from-schema!-test
+  (mt/test-drivers (mt/normal-drivers-with-feature :uploads)
+    (let [driver       driver/*driver*
+          db-id        (mt/id)
+          table-name   (mt/random-name)
+          schema-name  (sql.tx/session-schema driver)
+          table-schema {:name    (if schema-name
+                                   (keyword schema-name table-name)
+                                   (keyword table-name))
+                        :columns [{:name "id" :type :type/Integer :nullable? false}
+                                  {:name "name" :type :type/Text :nullable? true}]}]
+      (mt/as-admin
+        (testing "create-table-from-schema! should create the table successfully"
+          (transforms.util/create-table-from-schema! driver db-id table-schema)
+          (let [table-exists? (driver/table-exists? driver db-id {:schema schema-name :name table-name})]
+            (is (some? table-exists?) "Table should exist in the database schema")
+            (driver/drop-table! driver db-id (:name table-schema))))))))

enterprise/backend/test/metabase_enterprise/transforms/python_runner_test.clj

@@ -0,0 +1,408 @@
+(ns ^:mb/driver-tests metabase-enterprise.transforms.python-runner-test
+  (:require
+   [clojure.core.async :as a]
+   [clojure.data.csv :as csv]
+   [clojure.java.jdbc :as jdbc]
+   [clojure.string :as str]
+   [clojure.test :refer :all]
+   [java-time.api :as t]
+   [metabase-enterprise.transforms.python-runner :as python-runner]
+   [metabase-enterprise.transforms.schedule :as transforms.schedule]
+   [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
+   [metabase-enterprise.transforms.test-util :as transforms.tu]
+   [metabase-enterprise.transforms.util :as transforms.util]
+   [metabase.driver :as driver]
+   [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
+   [metabase.sync.core :as sync]
+   [metabase.task.core :as task]
+   [metabase.test :as mt]
+   [metabase.test.data.sql :as sql.tx]
+   [metabase.util :as u]
+   [metabase.util.json :as json]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(defn template->regex
+  "Convert a template string with $var$ placeholders to a regex pattern.
+   Example: template->regex 'File ___PATH___/script.py, line ___LINE___'
+   will match 'File /tmp/unique-path/script.py, line 123'"
+  [template]
+  (-> template
+      (str/replace #"(?m)^\d{13}\s*" "")
+      (str/replace "\\" "\\\\")
+      (str/replace "." "\\.")
+      (str/replace "*" "\\*")
+      (str/replace "+" "\\+")
+      (str/replace "?" "\\?")
+      (str/replace "^" "\\^")
+      (str/replace "$" "\\$")
+      (str/replace "{" "\\{")
+      (str/replace "}" "\\}")
+      (str/replace "(" "\\(")
+      (str/replace ")" "\\)")
+      (str/replace "|" "\\|")
+      (str/replace "[" "\\[")
+      (str/replace "]" "\\]")
+      (str/replace #"___PATH___" "(/[\\\\w-]+)+")
+      (str/replace #"___LINE___" "\\\\d+")
+      re-pattern))
+
+(def ^:private test-id 1)
+
+(defn- reconstruct-logs [{:keys [events] :as body}]
+  (assoc body :stdout (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
+         :stderr (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))))
+
+(defn- execute [{:keys [code tables]}]
+  (reconstruct-logs (:body (python-runner/execute-python-code test-id code (or tables {}) (a/promise-chan)))))
+
+(deftest ^:parallel transform-function-basic-test
+  (testing "executes transform function and returns CSV output"
+    (let [transform-code (str "import pandas as pd\n"
+                              "\n"
+                              "def transform():\n"
+                              "    return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")
+          result         (execute {:code transform-code})]
+      (is (=? {:output "name,age\nAlice,25\nBob,30\n"
+               :stdout "Successfully saved 2 rows to CSV\nSuccessfully saved output manifest with 2 fields"
+               :stderr ""}
+              result)))))
+
+(deftest ^:parallel transform-function-missing-test
+  (testing "handles missing transform function"
+    (let [result (execute {:code (str "import pandas as pd\n"
+                                      "\n"
+                                      "# No transform function defined")})]
+      (is (=? {:error     "Execution failed"
+               :exit-code 1
+               :stderr    "ERROR: User script must define a 'transform()' function"
+               :stdout    ""}
+              result)))))
+
+(deftest ^:parallel transform-function-wrong-return-type-test
+  (testing "handles transform function returning non-DataFrame"
+    (let [result (execute {:code (str "def transform():\n"
+                                      "    return 'not a dataframe'")})]
+      (is (=? {:error     "Execution failed"
+               :exit-code 1
+               :stderr    "ERROR: Transform function must return a pandas DataFrame, got <class 'str'>"
+               :stdout    ""}
+              result)))))
+
+(deftest ^:parallel transform-function-error-test
+  (testing "handles transform function with error"
+    (let [result            (execute {:code (str "def transform():\n"
+                                                 "    raise ValueError('Something went wrong')")})
+          expected-template (str "ERROR: Transform function failed: Something went wrong\n"
+                                 "Traceback (most recent call last):\n"
+                                 "  File \"/app/transform_runner.py\", line ___LINE___, in main\n"
+                                 "    result = script.transform()\n"
+                                 "             ^^^^^^^^^^^^^^^^^^\n"
+                                 "  File \"___PATH___/script.py\", line 2, in transform\n"
+                                 "    raise ValueError('Something went wrong')\n"
+                                 "ValueError: Something went wrong")
+          stderr-pattern    (template->regex expected-template)]
+      (is (=? {:error     "Execution failed"
+               :exit-code 1
+               :stderr    #(re-matches stderr-pattern %)
+               :stdout    ""
+               :timeout   false}
+              result)))))
+
+(deftest ^:parallel transform-function-complex-dataframe-test
+  (testing "can create complex DataFrames with transform"
+    (let [transform-code (str "import pandas as pd\n"
+                              "\n"
+                              "def transform():\n"
+                              "    data = {'x': [1, 2, 3], 'y': [10, 20, 30], 'z': ['a', 'b', 'c']}\n"
+                              "    return pd.DataFrame(data)")
+          result         (execute {:code transform-code})]
+      (is (=? {:output "x,y,z\n1,10,a\n2,20,b\n3,30,c\n"
+               :stdout "Successfully saved 3 rows to CSV\nSuccessfully saved output manifest with 3 fields"
+               :stderr ""}
+              result)))))
+
+(deftest ^:parallel transform-function-with-db-parameter-test
+  (mt/test-drivers #{:postgres}
+    (testing "transform function can accept db parameter for forward compatibility"
+      (let [transform-code (str "import pandas as pd\n"
+                                "\n"
+                                "def transform():\n"
+                                "    data = {'name': ['Charlie', 'Dana'], 'score': [85, 92]}\n"
+                                "    return pd.DataFrame(data)")
+            result         (execute {:code transform-code})]
+        (is (=? {:output "name,score\nCharlie,85\nDana,92\n"
+                 :stdout "Successfully saved 2 rows to CSV\nSuccessfully saved output manifest with 2 fields"
+                 :stderr ""}
+                result))))))
+
+(deftest transform-function-with-pass-thru
+  (testing "transform function successfully connects to PostgreSQL database and reads data"
+    (mt/test-drivers #{:postgres}
+      (mt/with-empty-db
+        (let [db-spec        (sql-jdbc.conn/db->pooled-connection-spec (mt/db))
+              _              (jdbc/execute! db-spec ["DROP TABLE IF EXISTS students"])
+              _              (jdbc/execute! db-spec ["CREATE TABLE students (id INTEGER PRIMARY KEY, name VARCHAR(100), score INTEGER)"])
+              _              (jdbc/execute! db-spec ["INSERT INTO students (id, name, score) VALUES (1, 'Alice', 85), (2, 'Bob', 92), (3, 'Charlie', 88), (4, 'Dana', 90)"])
+
+              _              (sync/sync-database! (mt/db))
+
+              transform-code (str "import pandas as pd\n"
+                                  "\n"
+                                  "def transform(students):\n"
+                                  "    return students")
+              result         (execute {:code   transform-code
+                                       :tables {"students" (mt/id :students)}})]
+
+          (is (=? {:output   "id,name,score\n1,Alice,85\n2,Bob,92\n3,Charlie,88\n4,Dana,90\n"
+                   :metadata #(= (json/decode+kw %)
+                                 {:version         "0.1.0"
+                                  :fields          [{:base_type      "Integer",
+                                                     :dtype          "int64",
+                                                     :database_type  "int4",
+                                                     :effective_type "int4",
+                                                     :name           "id",
+                                                     :semantic_type  nil,
+                                                     :root_type      "Integer"}
+                                                    {:base_type      "Text",
+                                                     :dtype          "object",
+                                                     :database_type  "varchar",
+                                                     :effective_type "varchar",
+                                                     :name           "name",
+                                                     :semantic_type  nil,
+                                                     :root_type      "Text"}
+                                                    {:base_type      "Integer",
+                                                     :dtype          "int64",
+                                                     :database_type  "int4",
+                                                     :effective_type "int4",
+                                                     :name           "score",
+                                                     :semantic_type  nil,
+                                                     :root_type      "Integer"}],
+                                  :source_metadata {:fields         [{:base_type      "Integer",
+                                                                      :dtype          "int64",
+                                                                      :database_type  "int4",
+                                                                      :effective_type "int4",
+                                                                      :name           "id",
+                                                                      :semantic_type  nil,
+                                                                      :root_type      "Integer"}
+                                                                     {:base_type      "Text",
+                                                                      :dtype          "object",
+                                                                      :database_type  "varchar",
+                                                                      :effective_type "varchar",
+                                                                      :name           "name",
+                                                                      :semantic_type  nil,
+                                                                      :root_type      "Text"}
+                                                                     {:base_type      "Integer",
+                                                                      :dtype          "int64",
+                                                                      :database_type  "int4",
+                                                                      :effective_type "int4",
+                                                                      :name           "score",
+                                                                      :semantic_type  nil,
+                                                                      :root_type      "Integer"}],
+                                                    :table_metadata {:table_id (mt/id :students)},
+                                                    :version        "0.1.0"}})
+                   :stdout   (str "Successfully saved 4 rows to CSV\n"
+                                  "Successfully saved output manifest with 3 fields")
+                   :stderr   ""}
+                  result)))))))
+
+(deftest transform-function-with-working-database-test
+  (testing "transform function successfully connects to PostgreSQL database and reads data"
+    (mt/test-drivers #{:postgres}
+      (mt/with-empty-db
+        (let [db-spec        (sql-jdbc.conn/db->pooled-connection-spec (mt/db))
+              _              (jdbc/execute! db-spec ["DROP TABLE IF EXISTS students"])
+              _              (jdbc/execute! db-spec ["CREATE TABLE students (id INTEGER PRIMARY KEY, name VARCHAR(100), score INTEGER)"])
+              _              (jdbc/execute! db-spec ["INSERT INTO students (id, name, score) VALUES (1, 'Alice', 85), (2, 'Bob', 92), (3, 'Charlie', 88), (4, 'Dana', 90)"])
+
+              _              (sync/sync-database! (mt/db))
+
+              transform-code (str "import pandas as pd\n"
+                                  "\n"
+                                  "def transform(students):\n"
+                                  "    # Calculate average score\n"
+                                  "    avg_score = students['score'].mean()\n"
+                                  "    result = pd.DataFrame({\n"
+                                  "        'student_count': [len(students)],\n"
+                                  "        'average_score': [round(avg_score, 2)]\n"
+                                  "    })\n"
+                                  "    return result")
+              result         (execute {:code   transform-code
+                                       :tables {"students" (mt/id :students)}})]
+
+          (is (=? {:output   "student_count,average_score\n4,88.75\n"
+                   :metadata #(= (json/decode+kw %)
+                                 {:version        "0.1.0",
+                                  :fields         [{:name "student_count", :dtype "int64"}
+                                                   {:name "average_score", :dtype "float64"}]})
+                   :stdout   (str "Successfully saved 1 rows to CSV\n"
+                                  "Successfully saved output manifest with 2 fields")
+                   :stderr   ""}
+                  result)))))))
+
+(defn- datetime-equal?
+  [expected-iso-str actual-pandas-str]
+  (let [formatter   (t/formatter "yyyy-MM-dd HH:mm:ssXXX")
+        expected-dt (t/zoned-date-time expected-iso-str)
+        actual-dt   (t/offset-date-time formatter actual-pandas-str)]
+    (= (t/to-millis-from-epoch expected-dt)
+       (t/to-millis-from-epoch actual-dt))))
+
+(deftest transform-type-test
+  (mt/test-drivers #{:postgres}
+    (mt/with-empty-db
+      (let [db-spec (sql-jdbc.conn/db->pooled-connection-spec (mt/db))
+            _ (jdbc/execute! db-spec ["DROP TABLE IF EXISTS sample_table"])
+            _ (jdbc/execute! db-spec ["CREATE TABLE sample_table (
+                                                        id BIGSERIAL PRIMARY KEY,
+                                                        name VARCHAR(255),
+                                                        description TEXT,
+                                                        count BIGINT,
+                                                        price FLOAT,
+                                                        is_active BOOLEAN,
+                                                        created_date DATE,
+                                                        updated_at TIMESTAMP,
+                                                        scheduled_for TIMESTAMP WITH TIME ZONE
+                                                    )"])
+            _ (jdbc/execute! db-spec ["INSERT INTO sample_table
+                                                        (name, description, count, price, is_active, created_date, updated_at, scheduled_for)
+                                                      VALUES
+                                                        ('Product A', 'A sample product description', 100, 29.99, true, '2024-01-15', '2024-01-15 10:30:00', '2024-01-16 14:00:00+00'),
+                                                        ('Product B', 'Another product with longer description text', 50, 15.50, false, '2024-02-01', '2024-02-01 09:15:30', '2024-02-02 16:30:00-05'),
+                                                        ('Product C', NULL, 0, 0.0, true, '2024-03-10', '2024-03-10 18:45:15', '2024-03-11 08:00:00+02')"])
+
+            _ (sync/sync-database! (mt/db))
+
+            transform-code (str "import pandas as pd\n"
+                                "\n"
+                                "def transform(sample_table):\n"
+                                "    df = sample_table.copy()\n"
+                                "    return df")
+            result (execute {:code transform-code
+                             :tables {"sample_table" (mt/id :sample_table)}})
+            csv-data (csv/read-csv (:output result))
+            headers (first csv-data)
+            rows (rest csv-data)
+            [row1 row2 row3] rows
+            header-to-index (zipmap headers (range))
+            get-col (fn [row col-name] (nth row (header-to-index col-name)))
+            metadata (some-> (:metadata result) json/decode+kw)]
+
+        (is (= (set ["id" "name" "description" "count" "price" "is_active" "created_date" "updated_at" "scheduled_for"])
+               (set headers)))
+
+        (is (= "1" (get-col row1 "id")))
+        (is (= "Product A" (get-col row1 "name")))
+        (is (datetime-equal? "2024-01-16T14:00:00Z" (get-col row1 "scheduled_for")))
+
+        (is (= "2" (get-col row2 "id")))
+        (is (= "Product B" (get-col row2 "name")))
+        (is (datetime-equal? "2024-02-02T21:30:00Z" (get-col row2 "scheduled_for")))
+
+        (is (= "3" (get-col row3 "id")))
+        (is (= "Product C" (get-col row3 "name")))
+        (is (datetime-equal? "2024-03-11T06:00:00Z" (get-col row3 "scheduled_for")))
+        (testing "dtypes are preserved correctly"
+          (is (= {"id"            :type/Integer
+                  "name"          :type/Text
+                  "description"   :type/Text
+                  "count"         :type/Integer
+                  "price"         :type/Float
+                  "is_active"     :type/Boolean
+                  ;; Our hack works
+                  "created_date"  :type/Date
+                  "updated_at"    :type/DateTime
+                  "scheduled_for" :type/DateTimeWithTZ}
+                 (u/for-map [{:keys [name dtype]} (:fields metadata)]
+                   [name (transforms.util/dtype->base-type dtype)]))))))))
+
+(deftest python-transform-scheduled-job-test
+  (mt/test-helpers-set-global-values!
+    (mt/with-temp-scheduler!
+      (task/init! ::transforms.schedule/RunTransform)
+      (mt/test-drivers #{:h2 :postgres}
+        (mt/with-premium-features #{:transforms}
+          (mt/dataset transforms-dataset/transforms-test
+            (transforms.tu/with-transform-cleanup! [target {:type   "table"
+                                                            :schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))
+                                                            :name   "target_table"}]
+              (mt/with-temp
+                [:model/TransformTag {tag-id :id}       {:name "every second"}
+                 :model/Transform    {transform-id :id} {:name   "Gadget Products"
+                                                         :source {:type  "python"
+                                                                  :source-database (mt/id)
+                                                                  :source-tables {"transforms_customers" (mt/id :transforms_customers)}
+                                                                  :body  (str "import pandas as pd\n"
+                                                                              "\n"
+                                                                              "def transform():\n"
+                                                                              "    return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}
+                                                         :target  (assoc target :database (mt/id))}
+                 :model/TransformTransformTag   _  {:transform_id transform-id :tag_id tag-id :position 0}
+                 :model/TransformJob {job-id :id :as job} {:schedule "* * * * * ? *"}
+                 :model/TransformJobTransformTag _ {:job_id job-id :tag_id tag-id :position 0}]
+                (transforms.schedule/initialize-job! job)
+                (transforms.schedule/update-job! job-id "* * * * * ? *")
+                (is (true? (u/poll {:thunk   (fn [] (driver/table-exists? driver/*driver* (mt/db) target))
+                                    :done?   true?
+                                    :timeout 30000})))
+                (is (true? (t2/exists? :model/Table :name (:name target))))))))))))
+
+(deftest transform-type-roundtrip-test
+  (mt/test-drivers #{:postgres :h2 :mysql :bigquery-cloud-sdk :redshift :snowflake :sqlserver :clickhouse}
+    (mt/with-empty-db
+      (let [driver       driver/*driver*
+            db-id        (mt/id)
+            table-name   (mt/random-name)
+            schema-name  (sql.tx/session-schema driver)
+            qualified-table-name (if schema-name
+                                   (keyword schema-name table-name)
+                                   (keyword table-name))
+            table-schema {:name    qualified-table-name
+                          :columns [{:name "id"            :type :type/Integer  :nullable? false}
+                                    {:name "price"         :type :type/Float    :nullable? true}
+                                    {:name "active"        :type :type/Boolean  :nullable? true}
+                                    {:name "created_tz"    :type :type/DateTimeWithTZ :nullable? true}
+                                    {:name "created_at"    :type :type/DateTime :nullable? true}
+                                    {:name "created_date"  :type :type/Date     :nullable? true}
+                                    {:name "description"   :type :type/Text     :nullable? true}]}
+            _ (mt/as-admin
+                (transforms.util/create-table-from-schema! driver db-id table-schema))
+
+            row-values   [[1
+                           19.99
+                           (if (= driver/*driver* :sqlserver) 1 true)
+                           (if (= driver/*driver* :mysql) "2024-01-01 12:00:00" "2024-01-01T12:00:00Z")
+                           (if (= driver/*driver* :mysql) "2024-01-01 12:00:00" "2024-01-01T12:00:00")
+                           "2024-01-01"
+                           "Sample product"]]
+            _ (driver/insert-from-source! driver db-id qualified-table-name
+                                          (mapv :name (:columns table-schema))
+                                          {:type :rows :data row-values})
+
+            _ (sync/sync-database! (mt/db))
+            transform-code (str "import pandas as pd\n"
+                                "\n"
+                                "def transform(" table-name "):\n"
+                                "    return " table-name)
+            result (execute {:code transform-code
+                             :tables {table-name (mt/id qualified-table-name)}})
+            metadata (some-> (:metadata result) json/decode+kw)]
+
+        (testing "All expected columns are present"
+          (is (= #{"id" "price" "active" "created_tz" "created_at" "created_date" "description"}
+                 (set (map :name (:fields metadata))))))
+
+        (testing "dtypes are preserved correctly"
+          (is (= {"id"           :type/Integer
+                  "price"        :type/Float
+                  "active"       :type/Boolean
+                  "created_tz"   (if (= driver/*driver* :mysql) :type/DateTime :type/DateTimeWithTZ)
+                  "created_at"   :type/DateTime
+                  "created_date" :type/Date
+                  "description"  :type/Text}
+                 (u/for-map [{:keys [name dtype]} (:fields metadata)]
+                   [name (transforms.util/dtype->base-type dtype)]))))
+
+        ;; cleanup
+        (driver/drop-table! driver db-id (:name table-schema))))))

enterprise/backend/test/metabase_enterprise/transforms/test_util.clj

@@ -47,3 +47,9 @@
            (drop-target! target#))))
     `(mt/with-model-cleanup [:model/Transform]
        ~@body)))
+
+(defn table-rows
+  [table-name]
+  (mt/rows (mt/process-query {:database (mt/id)
+                              :query    {:source-table (t2/select-one-pk :model/Table :name table-name)}
+                              :type     :query})))

enterprise/frontend/src/metabase-enterprise/api/transform.ts

@@ -108,6 +108,14 @@ export const transformApi = EnterpriseApi.injectEndpoints({
       invalidatesTags: (_, error) =>
         invalidateTags(error, [listTag("transform")]),
     }),
+    cancelTransform: builder.mutation<void, TransformId>({
+      query: (id) => ({
+        method: "POST",
+        url: `/api/ee/transform/${id}/cancel`,
+      }),
+      invalidatesTags: (_, error, id) =>
+        invalidateTags(error, [idTag("transform", id)]),
+    }),
     deleteTransformTarget: builder.mutation<void, TransformId>({
       query: (id) => ({
         method: "DELETE",
@@ -126,6 +134,7 @@ export const {
   useGetTransformQuery,
   useLazyGetTransformQuery,
   useRunTransformMutation,
+  useCancelTransformMutation,
   useCreateTransformMutation,
   useUpdateTransformMutation,
   useDeleteTransformMutation,

enterprise/frontend/src/metabase-enterprise/transforms/analytics.ts

@@ -34,7 +34,7 @@ export function trackTransformCreate({
   creationType,
 }: {
   triggeredFrom: "transform-page-create-menu";
-  creationType: "query" | "native" | "saved-question";
+  creationType: "query" | "native" | "python" | "saved-question";
 }) {
   trackSimpleEvent({
     event: "transform_create",

enterprise/frontend/src/metabase-enterprise/transforms/api/python-runner.ts

@@ -0,0 +1,68 @@
+import { Api } from "metabase/api";
+
+export interface ExecutePythonRequest {
+  code: string;
+  tables: Record<string, number>;
+}
+
+export interface ExecutePythonResponse {
+  output?: string;
+  stdout?: string;
+  stderr?: string;
+  error?: string;
+  exit_code?: number;
+  timeout?: boolean;
+}
+
+export const pythonRunnerApi = Api.injectEndpoints({
+  endpoints: (builder) => ({
+    executePython: builder.mutation<
+      ExecutePythonResponse,
+      ExecutePythonRequest
+    >({
+      query: ({ code, tables }) => ({
+        url: "/api/ee/transform/test-python",
+        method: "POST",
+        body: {
+          code,
+          tables,
+        },
+      }),
+      transformResponse: (response: any) => {
+        // Extract the nested result from the backend response
+        if (response?.result?.body) {
+          return {
+            output: response.result.body.output,
+            stdout: response.result.body.stdout,
+            stderr: response.result.body.stderr,
+            error: response.result.body.error,
+            exit_code: response.result.body.exit_code,
+            timeout: response.result.body.timeout,
+          };
+        }
+        return response;
+      },
+      transformErrorResponse: (response: any) => {
+        return {
+          status: response?.status || 500,
+          data: {
+            error: response?.data?.error || "Python execution failed",
+            stdout: response?.data?.stdout || "",
+            stderr: response?.data?.stderr || "",
+            exit_code: response?.data?.exit_code,
+          },
+        };
+      },
+    }),
+    cancelPython: builder.mutation<void, void>({
+      query: () => ({
+        url: "/api/ee/transform/test-python/cancel",
+        method: "POST",
+      }),
+    }),
+  }),
+  overrideExisting: true,
+});
+
+export const { useExecutePythonMutation, useCancelPythonMutation } =
+  pythonRunnerApi;

enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/LogOutput.styled.tsx

@@ -0,0 +1,67 @@
+import { css } from "@emotion/react";
+import styled from "@emotion/styled";
+
+import { color } from "metabase/lib/colors";
+
+const LOG_PREFIX = ".react-ansi-style";
+
+const LOG_COLORS = {
+  black: color("text-dark"),
+  white: color("text-white"),
+  gray: color("text-medium"),
+  red: color("saturated-red"),
+  green: color("saturated-green"),
+  yellow: color("saturated-yellow"),
+  blue: color("saturated-blue"),
+  magenta: color("saturated-purple"),
+  cyan: "cyan",
+};
+
+export const LogOutputContainer = styled.div`
+  border: 1px solid var(--mb-color-border);
+  border-radius: 0.5rem;
+  background-color: var(--mb-color-bg-light);
+  font-family: Monaco, Menlo, "Ubuntu Mono", "Lucida Console", monospace;
+  font-size: 13px;
+  white-space: pre;
+  padding: 1rem;
+  overflow: auto;
+  line-height: 1.4;
+
+  ${LOG_PREFIX}-bold {
+    font-weight: bold;
+  }
+
+  ${LOG_PREFIX}-dim {
+    font-weight: lighter;
+  }
+
+  ${LOG_PREFIX}-italic {
+    font-style: italic;
+  }
+
+  ${LOG_PREFIX}-underline {
+    text-decoration: underline;
+  }
+
+  ${LOG_PREFIX}-inverse {
+    color: var(--mb-color-text-dark);
+    background-color: var(--mb-color-bg-white);
+  }
+
+  ${LOG_PREFIX}-hidden {
+    color: transparent;
+  }
+
+  ${LOG_PREFIX}-strikethrough {
+    text-decoration: line-through;
+  }
+
+  ${Object.entries(LOG_COLORS).map(
+    ([key, color]) => css`
+      ${LOG_PREFIX}-${key} {
+        color: ${color};
+      }
+    `,
+  )};
+`;

enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/LogOutput.tsx

@@ -0,0 +1,85 @@
+import * as React from "react";
+import { useEffect, useMemo, useRef, useState } from "react";
+import reactAnsiStyle from "react-ansi-style";
+
+import { LogOutputContainer } from "./LogOutput.styled";
+
+interface LogOutputProps {
+  content?: string;
+  showLineNumbers?: boolean;
+  maxLines?: number;
+}
+
+export function LogOutput({
+  content,
+  showLineNumbers = true,
+  maxLines = 20,
+}: LogOutputProps) {
+  const containerRef = useRef<HTMLDivElement>(null);
+  const [isUserScrolled, setIsUserScrolled] = useState(false);
+  const [shouldAutoScroll, setShouldAutoScroll] = useState(true);
+
+  const { displayContent, actualLines } = useMemo(() => {
+    if (!content || content.trim().length === 0) {
+      return { displayContent: "No output", actualLines: 1 };
+    }
+
+    const lines = content.split("\n");
+    let processedContent = content;
+
+    if (showLineNumbers) {
+      const maxLineLength = lines.length.toString().length;
+      processedContent = lines
+        .map(
+          (line, index) =>
+            `${(index + 1).toString().padStart(maxLineLength, " ")} | ${line}`,
+        )
+        .join("\n");
+    }
+
+    return {
+      displayContent: reactAnsiStyle(React, processedContent),
+      actualLines: lines.length,
+    };
+  }, [content, showLineNumbers]);
+
+  // Calculate the height based on actual lines (up to maxLines)
+  const linesToShow = Math.min(actualLines, maxLines);
+  const containerHeight = `${linesToShow * 1.4 + 2}em`; // 1.4 is line-height + padding
+  const maxContainerHeight = `${maxLines * 1.4 + 2}em`;
+
+  // Auto-scroll to bottom when content changes
+  useEffect(() => {
+    if (containerRef.current && shouldAutoScroll && !isUserScrolled) {
+      containerRef.current.scrollTop = containerRef.current.scrollHeight;
+    }
+  }, [displayContent, shouldAutoScroll, isUserScrolled]);
+
+  // Handle scroll events to detect user scrolling
+  const handleScroll = () => {
+    if (!containerRef.current) {
+      return;
+    }
+
+    const { scrollTop, scrollHeight, clientHeight } = containerRef.current;
+    const isAtBottom = Math.abs(scrollHeight - clientHeight - scrollTop) < 5;
+
+    if (isAtBottom) {
+      setIsUserScrolled(false);
+      setShouldAutoScroll(true);
+    } else {
+      setIsUserScrolled(true);
+      setShouldAutoScroll(false);
+    }
+  };
+
+  return (
+    <LogOutputContainer
+      ref={containerRef}
+      style={{ height: containerHeight, maxHeight: maxContainerHeight }}
+      onScroll={handleScroll}
+    >
+      {displayContent}
+    </LogOutputContainer>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/LogOutput/index.ts

@@ -0,0 +1 @@
+export { LogOutput } from "./LogOutput";

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonDataPicker/PythonDataPicker.module.css

@@ -0,0 +1,3 @@
+.dataPicker {
+  border-right: 1px solid var(--mb-color-border);
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonDataPicker/PythonDataPicker.tsx

@@ -0,0 +1,365 @@
+import { useState } from "react";
+import { t } from "ttag";
+
+import { useListDatabasesQuery, useListTablesQuery } from "metabase/api";
+import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
+import {
+  ActionIcon,
+  Box,
+  Button,
+  Group,
+  Icon,
+  Select,
+  Stack,
+  Text,
+  TextInput,
+} from "metabase/ui";
+import type { Database, Table } from "metabase-types/api";
+
+import S from "./PythonDataPicker.module.css";
+
+type TableSelection = {
+  id: string; // unique ID for this selection row
+  tableId: number | undefined;
+  alias: string;
+  aliasManuallySet?: boolean; // Track if user manually edited the alias
+};
+
+type PythonDataPickerProps = {
+  database?: number;
+  tables?: Record<string, number>; // alias -> table-id mapping
+  onChange: (
+    database: number,
+    tables: Record<string, { id: number; name: string }>,
+  ) => void;
+};
+
+// Helper function to slugify table names
+function slugify(text: string): string {
+  return text
+    .toLowerCase()
+    .replace(/[^\w\s-]/g, "") // Remove non-word chars
+    .replace(/[\s_-]+/g, "_") // Replace spaces, underscores, hyphens with single underscore
+    .replace(/^_+|_+$/g, ""); // Remove leading/trailing underscores
+}
+
+export function PythonDataPicker({
+  database,
+  tables,
+  onChange,
+}: PythonDataPickerProps) {
+  const [selectedDatabaseId, setSelectedDatabaseId] = useState<
+    number | undefined
+  >(database);
+
+  // Initialize table selections from props
+  const initializeTableSelections = (): TableSelection[] => {
+    if (tables && Object.keys(tables).length > 0) {
+      return Object.entries(tables).map(([alias, tableId], index) => ({
+        id: `table-${index}-${Date.now()}`,
+        tableId,
+        alias,
+        aliasManuallySet: true, // Existing aliases are considered manually set
+      }));
+    }
+    // Start with one empty row by default
+    return [
+      {
+        id: `table-0-${Date.now()}`,
+        tableId: undefined,
+        alias: "",
+        aliasManuallySet: false,
+      },
+    ];
+  };
+
+  const [tableSelections, setTableSelections] = useState<TableSelection[]>(
+    initializeTableSelections,
+  );
+
+  const {
+    data: databasesResponse,
+    isLoading: isLoadingDatabases,
+    error: databaseError,
+  } = useListDatabasesQuery();
+
+  const {
+    data: tablesData,
+    isLoading: isLoadingTables,
+    error: tablesError,
+  } = useListTablesQuery(
+    selectedDatabaseId
+      ? {
+          dbId: selectedDatabaseId,
+          include_hidden: false,
+          include_editable_data_model: true,
+        }
+      : undefined,
+    {
+      skip: !selectedDatabaseId,
+    },
+  );
+
+  const notifyParentOfChange = (
+    dbId: number | undefined,
+    selections: TableSelection[],
+    tableIdToName?: Map<number, string>,
+  ) => {
+    if (dbId) {
+      const tablesMap: Record<string, { id: number; name: string }> = {};
+
+      selections.forEach((selection) => {
+        if (selection.tableId && selection.alias) {
+          const tableName =
+            tableIdToName?.get(selection.tableId) ||
+            `Table ${selection.tableId}`;
+          tablesMap[selection.alias] = {
+            id: selection.tableId,
+            name: tableName,
+          };
+        }
+      });
+      onChange(dbId, tablesMap);
+    }
+  };
+
+  if (databaseError || tablesError) {
+    return <LoadingAndErrorWrapper error={databaseError || tablesError} />;
+  }
+
+  if (isLoadingDatabases) {
+    return <LoadingAndErrorWrapper loading />;
+  }
+
+  const databases = databasesResponse?.data || [];
+  const databaseOptions = databases.map((db: Database) => ({
+    value: db.id.toString(),
+    label: db.name,
+  }));
+
+  const availableTables = (tablesData || [])
+    .filter((tbl: Table) => tbl.db_id === selectedDatabaseId && tbl.active)
+    .map((tbl: Table) => ({
+      value: tbl.id.toString(),
+      label: tbl.display_name || tbl.name,
+      table: tbl,
+    }));
+
+  const tableIdToNameMap = new Map<number, string>(
+    availableTables.map((t) => [parseInt(t.value), t.table.name]),
+  );
+
+  const handleDatabaseChange = (value: string | null) => {
+    const dbId = value ? parseInt(value) : undefined;
+    setSelectedDatabaseId(dbId);
+    const newSelections = [
+      {
+        id: `table-0-${Date.now()}`,
+        tableId: undefined,
+        alias: "",
+        aliasManuallySet: false,
+      },
+    ];
+    setTableSelections(newSelections);
+    notifyParentOfChange(dbId, newSelections);
+  };
+
+  const handleTableChange = (selectionId: string, value: string | null) => {
+    const newSelections = tableSelections.map((selection) => {
+      if (selection.id === selectionId) {
+        const tableId = value ? parseInt(value) : undefined;
+        const table = availableTables.find((t) => t.value === value)?.table;
+
+        const newAlias = table ? slugify(table.name) : "";
+
+        return {
+          ...selection,
+          tableId,
+          alias: selection.aliasManuallySet ? selection.alias : newAlias,
+          aliasManuallySet: selection.aliasManuallySet || false,
+        };
+      }
+      return selection;
+    });
+    setTableSelections(newSelections);
+    notifyParentOfChange(selectedDatabaseId, newSelections, tableIdToNameMap);
+  };
+
+  const handleAliasChange = (selectionId: string, alias: string) => {
+    // Only allow valid Python identifier characters (letters, numbers, underscores)
+    // and ensure it doesn't start with a number
+    const validAlias = alias
+      .replace(/[^a-zA-Z0-9_]/g, "")
+      .replace(/^(\d)/, "_$1");
+
+    const newSelections = tableSelections.map((selection) =>
+      selection.id === selectionId
+        ? { ...selection, alias: validAlias, aliasManuallySet: true }
+        : selection,
+    );
+    setTableSelections(newSelections);
+    notifyParentOfChange(selectedDatabaseId, newSelections, tableIdToNameMap);
+  };
+
+  const handleResetAlias = (selectionId: string) => {
+    const newSelections = tableSelections.map((selection) => {
+      if (selection.id === selectionId && selection.tableId) {
+        const table = availableTables.find(
+          (t) => t.value === selection.tableId!.toString(),
+        )?.table;
+        if (table) {
+          return {
+            ...selection,
+            alias: slugify(table.name),
+            aliasManuallySet: false,
+          };
+        }
+      }
+      return selection;
+    });
+    setTableSelections(newSelections);
+    notifyParentOfChange(selectedDatabaseId, newSelections, tableIdToNameMap);
+  };
+
+  const handleAddTable = () => {
+    const newSelections = [
+      ...tableSelections,
+      {
+        id: `table-${tableSelections.length}-${Date.now()}`,
+        tableId: undefined,
+        alias: "",
+        aliasManuallySet: false,
+      },
+    ];
+    setTableSelections(newSelections);
+    // Don't notify parent for adding empty row
+  };
+
+  const handleRemoveTable = (selectionId: string) => {
+    const newSelections = tableSelections.filter(
+      (selection) => selection.id !== selectionId,
+    );
+    setTableSelections(newSelections);
+    notifyParentOfChange(selectedDatabaseId, newSelections, tableIdToNameMap);
+  };
+
+  const selectedTableIds = new Set(
+    tableSelections
+      .filter((s) => s.tableId !== undefined)
+      .map((s) => s.tableId?.toString()),
+  );
+
+  return (
+    <Stack p="md" gap="md" h="100%" className={S.dataPicker}>
+      <Box>
+        <Text size="sm" fw="bold" mb="xs">
+          {t`Source Database`}
+        </Text>
+        <Text size="xs" c="dimmed" mb="sm">
+          {t`Select the database that contains your source data`}
+        </Text>
+        <Select
+          data={databaseOptions}
+          value={selectedDatabaseId?.toString() || null}
+          onChange={handleDatabaseChange}
+          placeholder={t`Select a database`}
+          clearable
+        />
+      </Box>
+      {selectedDatabaseId && (
+        <Box>
+          <Text size="sm" fw="bold" mb="xs">
+            {t`Source Tables`}
+          </Text>
+          <Text size="xs" c="dimmed" mb="sm">
+            {t`Select tables to use as data sources and provide aliases for each`}
+          </Text>
+          <Stack gap="sm">
+            {tableSelections.map((selection, index) => {
+              // Filter available tables to exclude already selected ones (except current selection)
+              const filteredTables = availableTables.filter(
+                (table) =>
+                  !selectedTableIds.has(table.value) ||
+                  table.value === selection.tableId?.toString(),
+              );
+
+              const table = availableTables.find(
+                (t) => t.value === selection.tableId?.toString(),
+              )?.table;
+              const defaultAlias = table ? slugify(table.name) : "";
+              const showReset =
+                selection.aliasManuallySet &&
+                selection.alias !== defaultAlias &&
+                table;
+
+              return (
+                <Group key={selection.id} gap="sm" align="flex-end">
+                  <Select
+                    style={{ flex: 1 }}
+                    data={filteredTables}
+                    value={selection.tableId?.toString() || null}
+                    onChange={(value) => handleTableChange(selection.id, value)}
+                    placeholder={t`Select a table`}
+                    clearable
+                    disabled={isLoadingTables}
+                  />
+                  <TextInput
+                    style={{ flex: 0.7 }}
+                    value={selection.alias}
+                    onChange={(e) =>
+                      handleAliasChange(selection.id, e.target.value)
+                    }
+                    placeholder={t`Enter alias`}
+                    label={index === 0 ? t`Alias` : undefined}
+                    styles={{
+                      input: {
+                        color: selection.aliasManuallySet
+                          ? undefined
+                          : "var(--mb-color-text-light)",
+                      },
+                    }}
+                    rightSection={
+                      showReset ? (
+                        <ActionIcon
+                          onClick={() => handleResetAlias(selection.id)}
+                          aria-label={t`Reset alias to default`}
+                          color="gray"
+                          variant="subtle"
+                          size="xs"
+                        >
+                          <Icon name="refresh" size={12} />
+                        </ActionIcon>
+                      ) : null
+                    }
+                  />
+                  {tableSelections.length > 1 && (
+                    <ActionIcon
+                      onClick={() => handleRemoveTable(selection.id)}
+                      aria-label={t`Remove table`}
+                      color="gray"
+                      variant="subtle"
+                    >
+                      <Icon name="trash" />
+                    </ActionIcon>
+                  )}
+                </Group>
+              );
+            })}
+            <Button
+              leftSection={<Icon name="add" />}
+              variant="subtle"
+              onClick={handleAddTable}
+              disabled={
+                !tableSelections[tableSelections.length - 1]?.tableId ||
+                availableTables.length === 0 ||
+                selectedTableIds.size >= availableTables.length
+              }
+            >
+              {t`Add another table`}
+            </Button>
+          </Stack>
+        </Box>
+      )}
+    </Stack>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonDataPicker/index.ts

@@ -0,0 +1 @@
+export * from "./PythonDataPicker";

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonResultsTable/PythonResultsTable.tsx

@@ -0,0 +1,181 @@
+import { useMemo } from "react";
+import { t } from "ttag";
+
+import { colors } from "metabase/lib/colors/colors";
+import { Alert, Box, Text } from "metabase/ui";
+
+interface PythonResultsTableProps {
+  csvData?: string;
+  error?: string;
+  stdout?: string;
+  stderr?: string;
+  isLoading?: boolean;
+}
+
+function parseCSV(csv: string): { headers: string[]; rows: string[][] } {
+  if (!csv || !csv.trim()) {
+    return { headers: [], rows: [] };
+  }
+
+  // Split by newlines to get rows
+  const lines = csv.trim().split("\n");
+  if (lines.length === 0) {
+    return { headers: [], rows: [] };
+  }
+
+  // Parse headers from first line
+  const headers = lines[0].split(",").map((h) => h.trim());
+
+  // Parse data rows
+  const rows = lines.slice(1).map((line) => {
+    // Simple CSV parsing - handles basic comma-separated values
+    // For more complex CSV (with quotes, escapes), we'd need a proper parser
+    return line.split(",").map((cell) => cell.trim());
+  });
+
+  return { headers, rows };
+}
+
+export function PythonResultsTable({
+  csvData,
+  error,
+  stdout,
+  stderr,
+  isLoading = false,
+}: PythonResultsTableProps) {
+  const { headers, rows } = useMemo(() => {
+    if (!csvData) {
+      return { headers: [], rows: [] };
+    }
+    const parsed = parseCSV(csvData);
+    return parsed;
+  }, [csvData]);
+
+  if (isLoading) {
+    return (
+      <Box p="lg">
+        <Text c="text-medium">{t`Running Python script...`}</Text>
+      </Box>
+    );
+  }
+
+  if (error) {
+    return (
+      <Box p="lg">
+        <Alert color="red" title={t`Execution Error`} mb="md">
+          {error}
+        </Alert>
+        {stdout && (
+          <Box mb="md">
+            <Text fw="bold" mb="xs">{t`Standard Output:`}</Text>
+            <Box
+              p="sm"
+              bg="bg-light"
+              style={{ fontFamily: "monospace", whiteSpace: "pre-wrap" }}
+            >
+              {stdout}
+            </Box>
+          </Box>
+        )}
+        {stderr && (
+          <Box>
+            <Text fw="bold" mb="xs">{t`Standard Error:`}</Text>
+            <Box
+              p="sm"
+              bg="bg-light"
+              style={{ fontFamily: "monospace", whiteSpace: "pre-wrap" }}
+            >
+              {stderr}
+            </Box>
+          </Box>
+        )}
+      </Box>
+    );
+  }
+
+  if (!csvData || headers.length === 0) {
+    return (
+      <Box p="lg">
+        <Text c="text-medium">{t`No results to display. Click "Run" to execute the script.`}</Text>
+        {stdout && (
+          <Box mt="md">
+            <Text fw="bold" mb="xs">{t`Standard Output:`}</Text>
+            <Box
+              p="sm"
+              bg="bg-light"
+              style={{ fontFamily: "monospace", whiteSpace: "pre-wrap" }}
+            >
+              {stdout}
+            </Box>
+          </Box>
+        )}
+      </Box>
+    );
+  }
+
+  return (
+    <Box
+      style={{
+        overflow: "auto",
+        padding: "1rem",
+        background: colors["bg-light"],
+        border: `1px solid ${colors["border"]}`,
+      }}
+    >
+      {stdout && (
+        <Box p="md" pb={0}>
+          <Text fw="bold" mb="xs">{t`Standard Output:`}</Text>
+          <Box
+            p="sm"
+            bg="bg-light"
+            style={{ fontFamily: "monospace", whiteSpace: "pre-wrap" }}
+            mb="md"
+          >
+            {stdout}
+          </Box>
+        </Box>
+      )}
+      <Box style={{ background: "white", padding: "1rem" }}>
+        <Text fw="bold" mb="md">{t`Results Table:`}</Text>
+        <table
+          style={{
+            width: "100%",
+            borderCollapse: "collapse",
+            border: `1px solid ${colors["border"]}`,
+          }}
+        >
+          <thead>
+            <tr style={{ backgroundColor: colors["bg-medium"] }}>
+              {headers.map((header, index) => (
+                <th
+                  key={index}
+                  style={{
+                    padding: "8px",
+                    textAlign: "left",
+                    borderBottom: `2px solid ${colors["border"]}`,
+                  }}
+                >
+                  {header}
+                </th>
+              ))}
+            </tr>
+          </thead>
+          <tbody>
+            {rows.map((row, rowIndex) => (
+              <tr
+                key={rowIndex}
+                style={{ borderBottom: `1px solid ${colors["bg-medium"]}` }}
+              >
+                {row.map((cell, cellIndex) => (
+                  <td key={cellIndex} style={{ padding: "8px" }}>
+                    {cell}
+                  </td>
+                ))}
+              </tr>
+            ))}
+          </tbody>
+        </table>
+      </Box>
+    </Box>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonResultsTable/index.ts

@@ -0,0 +1 @@
+export { PythonResultsTable } from "./PythonResultsTable";

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonTransformEditor/PythonTransformEditor.tsx

@@ -0,0 +1,235 @@
+import { useEffect, useState } from "react";
+
+import { Flex, Stack } from "metabase/ui";
+import type { PythonTransformSource } from "metabase-types/api";
+
+import { PythonDataPicker } from "../PythonDataPicker";
+import { EditorHeader } from "../QueryEditor/EditorHeader";
+import { PythonEditor } from "../QueryEditor/PythonEditor";
+
+type PythonTransformEditorProps = {
+  initialSource: PythonTransformSource;
+  isNew?: boolean;
+  isSaving?: boolean;
+  onSave: (newSource: PythonTransformSource) => void;
+  onCancel: () => void;
+  onSourceChange?: (newSource: PythonTransformSource) => void;
+};
+
+export function PythonTransformEditor({
+  initialSource,
+  isNew = true,
+  isSaving = false,
+  onSave,
+  onCancel,
+  onSourceChange,
+}: PythonTransformEditorProps) {
+  const [source, setSource] = useState(initialSource);
+  const [isSourceDirty, setIsSourceDirty] = useState(false);
+
+  useEffect(() => {
+    if (onSourceChange) {
+      onSourceChange(source);
+    }
+  }, [source, onSourceChange]);
+
+  const handleScriptChange = (body: string) => {
+    const newSource = {
+      ...source,
+      body,
+    };
+    setSource(newSource);
+    setIsSourceDirty(true);
+  };
+
+  const updateTransformSignature = (
+    script: string,
+    tables: Record<string, { id: number; name: string }>,
+  ): string => {
+    const tableAliases = Object.keys(tables);
+
+    const transformRegex = /^def\s+transform\s*\([^)]*\)\s*:\s*\n(\s*)/m;
+
+    const signatureOneLine = `def transform(${tableAliases.join(", ")}):`;
+    const newSignature =
+      signatureOneLine.length > 80 && tableAliases.length > 0
+        ? `def transform(\n${tableAliases.map((alias) => `    ${alias}`).join(",\n")},\n):`
+        : signatureOneLine;
+
+    if (transformRegex.test(script)) {
+      // Capture the existing indentation after the colon
+      const match = script.match(transformRegex);
+      const originalIndent = match ? match[1] : "    ";
+
+      let updatedScript = script.replace(
+        transformRegex,
+        newSignature + "\n" + originalIndent,
+      );
+
+      const argsRegex =
+        /(\s+"""[\s\S]*?)\s*Args:\s*\n([\s\S]*?)\s*(\n\s+Returns:|\n\s+""")/;
+      const argsMatch = updatedScript.match(argsRegex);
+
+      if (tableAliases.length > 0) {
+        const maxAliasLength = Math.max(...tableAliases.map((a) => a.length));
+        const newArgsSection = tableAliases
+          .map((alias) => {
+            const padding = " ".repeat(maxAliasLength - alias.length);
+            const tableName = tables[alias].name;
+            if (alias === tableName) {
+              return `        ${alias}:${padding} DataFrame containing the data from the corresponding table`;
+            }
+            return `        ${alias}:${padding} DataFrame containing the data from table "${tableName}"`;
+          })
+          .join("\n");
+
+        if (argsMatch) {
+          // Replace existing Args section
+          updatedScript = updatedScript.replace(
+            argsRegex,
+            `    $1
+
+    Args:
+${newArgsSection}
+$3`,
+          );
+        } else {
+          // No Args section exists, look for the pattern before Returns and insert Args
+          // Also remove the instructional text about selecting tables
+          const insertRegex =
+            /(\s+"""[\s\S]*?)\s*Select tables above to add them as function parameters\.\s*(\n\s+Returns:|\n\s+""")/;
+          const insertMatch = updatedScript.match(insertRegex);
+          if (insertMatch) {
+            updatedScript = updatedScript.replace(
+              insertRegex,
+              `$1
+
+    Args:
+${newArgsSection}
+$2`,
+            );
+          } else {
+            // Fallback: just look for Returns section without the specific text
+            // Remove any extra whitespace before Returns and ensure single blank line
+            const fallbackRegex = /(\s+"""[\s\S]*?)\s*(\n\s+Returns:|\n\s+""")/;
+            const fallbackMatch = updatedScript.match(fallbackRegex);
+            if (fallbackMatch) {
+              updatedScript = updatedScript.replace(
+                fallbackRegex,
+                `$1
+
+    Args:
+${newArgsSection}
+$2`,
+              );
+            }
+          }
+        }
+      } else if (argsMatch) {
+        // Remove existing Args section when no tables
+        updatedScript = updatedScript.replace(argsRegex, "$1$3");
+      }
+
+      return updatedScript;
+    }
+
+    const maxAliasLength =
+      tableAliases.length > 0
+        ? Math.max(...tableAliases.map((a) => a.length))
+        : 0;
+    const functionTemplate = `
+${newSignature}
+    """
+    Transform function that processes the input data.
+    ${
+      tableAliases.length > 0
+        ? `
+
+    Args:
+${tableAliases
+  .map((alias) => {
+    const padding = " ".repeat(maxAliasLength - alias.length);
+    const tableName = tables[alias].name;
+    if (alias === tableName) {
+      return `        ${alias}:${padding} DataFrame containing the data from the corresponding table`;
+    }
+    return `        ${alias}:${padding} DataFrame containing the data from table "${tableName}"`;
+  })
+  .join("\n")}`
+        : ""
+    }
+
+    Returns:
+        DataFrame: The transformed data
+    """
+    import pandas as pd
+    return pd.DataFrame()
+`;
+
+    return script + functionTemplate;
+  };
+
+  const handleDataChange = (
+    database: number,
+    tables: Record<string, { id: number; name: string }>,
+  ) => {
+    const updatedScript = updateTransformSignature(source.body, tables);
+
+    const sourceTables: Record<string, number> = {};
+    Object.entries(tables).forEach(([alias, tableInfo]) => {
+      sourceTables[alias] = tableInfo.id;
+    });
+
+    const newSource = {
+      ...source,
+      body: updatedScript,
+      "source-database": database,
+      "source-tables": sourceTables,
+    };
+    setSource(newSource);
+    setIsSourceDirty(true);
+  };
+
+  const handleSave = () => {
+    onSave(source);
+  };
+
+  const canSave = Boolean(
+    source.body.trim() &&
+      source["source-database"] &&
+      source["source-tables"] &&
+      Object.keys(source["source-tables"]).length > 0,
+  );
+
+  return (
+    <Stack
+      w="100%"
+      h="100%"
+      bg="bg-white"
+      data-testid="python-transform-editor"
+      gap={0}
+    >
+      <EditorHeader
+        isNew={isNew}
+        isSaving={isSaving}
+        canSave={canSave && (isNew || isSourceDirty)}
+        onSave={handleSave}
+        onCancel={onCancel}
+      />
+      <Flex h="100%" w="100%">
+        <PythonDataPicker
+          database={source["source-database"]}
+          tables={source["source-tables"]}
+          onChange={handleDataChange}
+        />
+
+        <PythonEditor
+          script={source.body}
+          isRunnable={true}
+          onChange={handleScriptChange}
+          tables={source["source-tables"]}
+        />
+      </Flex>
+    </Stack>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/PythonTransformEditor/index.ts

@@ -0,0 +1 @@
+export { PythonTransformEditor } from "./PythonTransformEditor";

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/EditorBody/EditorBody.tsx

@@ -11,7 +11,13 @@ import { doesDatabaseSupportTransforms } from "metabase-enterprise/transforms/ut
 import type Question from "metabase-lib/v1/Question";
 import type Database from "metabase-lib/v1/metadata/Database";
 import type NativeQuery from "metabase-lib/v1/queries/NativeQuery";
-import type { Database as ApiDatabase, RecentItem } from "metabase-types/api";
+import type {
+  Database as ApiDatabase,
+  PythonTransformSource,
+  RecentItem,
+} from "metabase-types/api";
+
+import { PythonEditor } from "../PythonEditor";
 
 import S from "./EditorBody.module.css";
 import { ResizableBoxHandle } from "./ResizableBoxHandle";
@@ -21,10 +27,13 @@ const EDITOR_HEIGHT = 400;
 type EditorBodyProps = {
   question: Question;
   isNative: boolean;
+  isPython?: boolean;
+  pythonSource?: PythonTransformSource;
   isRunnable: boolean;
   isRunning: boolean;
   isResultDirty: boolean;
   onChange: (newQuestion: Question) => void;
+  onPythonChange?: (script: string) => void;
   onRunQuery: () => Promise<void>;
   onCancelQuery: () => void;
   databases: ApiDatabase[];
@@ -33,10 +42,13 @@ type EditorBodyProps = {
 export function EditorBody({
   question,
   isNative,
+  isPython = false,
+  pythonSource,
   isRunnable,
   isRunning,
   isResultDirty,
   onChange,
+  onPythonChange,
   onRunQuery,
   onCancelQuery,
   databases,
@@ -70,6 +82,24 @@ export function EditorBody({
 
   const dataPickerOptions = useDataPickerOptions({ databases });
 
+  const handlePythonChange = (script: string) => {
+    onPythonChange?.(script);
+  };
+
+  if (isPython) {
+    return (
+      <PythonEditor
+        script={pythonSource?.body || ""}
+        isRunnable={isRunnable}
+        isRunning={isRunning}
+        isResultDirty={isResultDirty}
+        onChange={handlePythonChange}
+        onRunScript={onRunQuery}
+        onCancelScript={onCancelQuery}
+      />
+    );
+  }
+
   return isNative ? (
     <NativeQueryEditor
       question={question}

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/EditorHeader/EditorHeader.tsx

@@ -23,6 +23,7 @@ export function EditorHeader({
       title={getTitle(isNew)}
       admin
       buttons={[
+        <Button key="cancel" small onClick={onCancel}>{t`Cancel`}</Button>,
         <Button
           key="save"
           onClick={onSave}
@@ -32,7 +33,6 @@ export function EditorHeader({
         >
           {getSaveButtonLabel(isNew, isSaving)}
         </Button>,
-        <Button key="cancel" small onClick={onCancel}>{t`Cancel`}</Button>,
       ]}
     />
   );

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/PythonEditor/ExecutionOutputTable.tsx

@@ -0,0 +1,25 @@
+import { useMemo } from "react";
+
+import { DataGrid, useDataGridInstance } from "metabase/data-grid";
+
+import { parseCSV } from "./utils";
+
+export function ExecutionOutputTable({ output }: { output?: string }) {
+  const { headers, rows } = useMemo(() => parseCSV(output || ""), [output]);
+
+  const tableProps = useDataGridInstance<string[], unknown>({
+    data: rows,
+    columnsOptions: headers.map((header, index) => ({
+      id: header,
+      name: header,
+      accessorFn: (row) => row[index],
+    })),
+    minGridWidth: 100,
+  });
+
+  if (!output || headers.length === 0) {
+    return null;
+  }
+
+  return <DataGrid {...tableProps} />;
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/PythonEditor/PythonEditor.module.css

@@ -0,0 +1,62 @@
+.root {
+  border-bottom: 1px solid var(--mb-color-border);
+}
+
+.editor {
+  width: 100%;
+  height: 100%;
+}
+
+.logs {
+  white-space: pre;
+  border: 1px solid var(--mb-color-border);
+  font-family: var(--mb-default-monospace-font-family);
+}
+
+.visualization {
+  flex: 1 0 auto;
+}
+
+.results {
+  width: 100%;
+  border-collapse: collapse;
+  border: 1px solid var(--mb-color-border);
+  border-radius: var(--mantine-radius-md);
+}
+
+.tableHeader {
+  background-color: var(--mb-color-bg-light);
+  padding: var(--mantine-spacing-sm);
+}
+
+.tableCell {
+  border-top: 1px solid var(--mb-color-border);
+  padding: var(--mantine-spacing-sm);
+}
+
+.error {
+  color: var(--mb-color-error);
+  background-color: var(--mb-color-bg-error);
+  border-bottom: 1px solid var(--mb-color-border);
+}
+
+.info {
+  color: var(--mb-color-text-dark);
+  border: 1px solid var(--mb-color-bg-medium);
+  background-color: var(--mb-color-bg-light);
+}
+
+.success {
+  color: var(--mb-color-success);
+  background-color: var(--mb-color-bg-success);
+  border-bottom: 1px solid var(--mb-color-border);
+}
+
+.loading {
+  position: absolute;
+  top: 0;
+  left: 0;
+  bottom: 0;
+  right: 0;
+  background-color: rgba(255, 255, 255, 0.5);
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/PythonEditor/PythonEditor.tsx

@@ -0,0 +1,214 @@
+import type { ReactNode } from "react";
+import { ResizableBox } from "react-resizable";
+import { c, t } from "ttag";
+
+import EmptyCodeResult from "assets/img/empty-states/code.svg";
+import { CodeEditor } from "metabase/common/components/CodeEditor";
+import DebouncedFrame from "metabase/common/components/DebouncedFrame";
+import { LoadingSpinner } from "metabase/common/components/MetadataInfo/MetadataInfo.styled";
+import { isMac } from "metabase/lib/browser";
+import RunButtonWithTooltip from "metabase/query_builder/components/RunButtonWithTooltip";
+import { Box, Flex, Icon, Stack, Text, Title } from "metabase/ui";
+
+import { ResizableBoxHandle } from "../EditorBody/ResizableBoxHandle";
+
+import { ExecutionOutputTable } from "./ExecutionOutputTable";
+import S from "./PythonEditor.module.css";
+import { type ExecutionResult, completion, useTestPythonScript } from "./utils";
+
+type PythonEditorProps = {
+  script: string;
+  isRunnable: boolean;
+  onChange: (script: string) => void;
+  onRunScript?: () => Promise<void>;
+  onCancelScript?: () => void;
+  tables?: Record<string, number>;
+};
+
+const EDITOR_HEIGHT = 400;
+
+export function PythonEditor({
+  script,
+  isRunnable,
+  onChange,
+  tables = {},
+}: PythonEditorProps) {
+  const { isRunning, isDirty, cancel, run, executionResult } =
+    useTestPythonScript(script, tables);
+
+  return (
+    <Stack h="100%" w="100%" gap={0}>
+      <ResizableBox
+        className={S.root}
+        axis="y"
+        height={EDITOR_HEIGHT}
+        handle={<ResizableBoxHandle />}
+        resizeHandles={["s"]}
+      >
+        <Flex h="100%" align="end" bg="bg-light">
+          <CodeEditor
+            className={S.editor}
+            value={script}
+            onChange={onChange}
+            language="python"
+            extensions={completion}
+          />
+
+          <Box p="md">
+            <RunButtonWithTooltip
+              disabled={!isRunnable}
+              isRunning={isRunning}
+              isDirty={isDirty}
+              onRun={run}
+              onCancel={cancel}
+              getTooltip={() => t`Run Python script`}
+            />
+          </Box>
+        </Flex>
+      </ResizableBox>
+
+      <DebouncedFrame className={S.visualization}>
+        {executionResult ? (
+          <ExecutionResult executionResult={executionResult} />
+        ) : (
+          <EmptyState />
+        )}
+        {isRunning && <LoadingState />}
+      </DebouncedFrame>
+    </Stack>
+  );
+}
+
+function getRunQueryShortcut() {
+  return isMac() ? t`⌘ + return` : t`Ctrl + enter`;
+}
+
+function LoadingState() {
+  return (
+    <Flex p="md" className={S.loading}>
+      <LoadingSpinner />
+    </Flex>
+  );
+}
+
+function EmptyState() {
+  const keyboardShortcut = getRunQueryShortcut();
+
+  return (
+    <Flex h="100%" align="center" justify="center">
+      <Stack maw="25rem" gap={0} ta="center" align="center">
+        <Box maw="3rem" mb="0.75rem">
+          <img src={EmptyCodeResult} alt="Code prompt icon" />
+        </Box>
+        <Text c="text-medium">
+          {c("{0} refers to the keyboard shortcut")
+            .jt`To run your code, click on the Run button or type ${(
+            <b key="shortcut">({keyboardShortcut})</b>
+          )}`}
+        </Text>
+      </Stack>
+    </Flex>
+  );
+}
+
+function ExecutionResult({
+  executionResult,
+}: {
+  executionResult: ExecutionResult | null;
+}) {
+  if (!executionResult) {
+    return null;
+  }
+
+  const message = getMessageForExecutionResult(executionResult);
+
+  return (
+    <Stack gap={0}>
+      {message}
+
+      <ExecutionLogs
+        label={t`Standard output:`}
+        content={executionResult.stdout}
+      />
+      <ExecutionLogs
+        label={t`Standard error:`}
+        content={executionResult.stderr}
+      />
+
+      <ExecutionOutputTable output={executionResult.output} />
+    </Stack>
+  );
+}
+
+function getMessageForExecutionResult(executionResult: ExecutionResult) {
+  if (executionResult.error) {
+    return (
+      <Box className={S.error} p="sm">
+        <Flex gap="sm">
+          <Icon name="warning" />
+          <Stack gap={0}>
+            <Title order={5} mb="xs">
+              {t`An error occurred while executing your Python script`}
+            </Title>
+            {executionResult.error}
+          </Stack>
+        </Flex>
+      </Box>
+    );
+  }
+
+  if (!executionResult.output) {
+    return (
+      <Flex className={S.info} p="sm" bdrs="xs" gap="sm" align="center">
+        <Icon name="info" />
+        {t`No results to display.`}
+      </Flex>
+    );
+  }
+
+  return (
+    <Flex className={S.success} p="sm" gap="sm" align="center">
+      <Icon name="check" />
+      {t`Script executed successfully.`}
+    </Flex>
+  );
+}
+
+function ExecutionLogs({
+  label,
+  content,
+}: {
+  label: string;
+  content?: string | null;
+}) {
+  if (!content) {
+    return null;
+  }
+
+  return (
+    <Section title={label}>
+      <Box p="sm" bg="bg-light" mah="150px" bdrs="xs" className={S.logs}>
+        {content}
+      </Box>
+    </Section>
+  );
+}
+
+function Section({
+  title,
+  children,
+}: {
+  title: ReactNode;
+  children: ReactNode;
+}) {
+  return (
+    <Box>
+      {title && (
+        <Title order={5} mb="xs">
+          {title}
+        </Title>
+      )}
+      {children}
+    </Box>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/PythonEditor/index.ts

@@ -0,0 +1 @@
+export { PythonEditor } from "./PythonEditor";

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/PythonEditor/utils.ts

@@ -0,0 +1,777 @@
+import {
+  type Completion,
+  type CompletionContext,
+  autocompletion,
+} from "@codemirror/autocomplete";
+import { pythonLanguage } from "@codemirror/lang-python";
+import { useRef, useState } from "react";
+import { t } from "ttag";
+
+import {
+  type ExecutePythonResponse,
+  useExecutePythonMutation,
+} from "metabase-enterprise/transforms/api/python-runner";
+
+export function parseCSV(csv: string): { headers: string[]; rows: string[][] } {
+  if (!csv || !csv.trim()) {
+    return { headers: [], rows: [] };
+  }
+
+  const lines = csv.trim().split("\n");
+  if (lines.length === 0) {
+    return { headers: [], rows: [] };
+  }
+
+  const headers = lines[0].split(",").map((h) => h.trim());
+  const rows = lines.slice(1).map((line) => {
+    return line.split(",").map((cell) => cell.trim());
+  });
+
+  return { headers, rows };
+}
+
+export type ExecutionResult = {
+  output?: string;
+  stdout?: string;
+  stderr?: string;
+  error?: string;
+};
+
+type TestPythonScriptState = {
+  isRunning: boolean;
+  isDirty: boolean;
+  executionResult: ExecutionResult | null;
+  run: () => void;
+  cancel: () => void;
+};
+
+export function useTestPythonScript(
+  code: string,
+  tables: Record<string, number>,
+): TestPythonScriptState {
+  const [executePython, { isLoading: isRunning, originalArgs }] =
+    useExecutePythonMutation();
+  const abort = useRef<(() => void) | null>(null);
+  const [executionResult, setData] = useState<ExecutePythonResponse | null>(
+    null,
+  );
+
+  const isDirty = originalArgs?.code !== code;
+
+  const run = async () => {
+    const request = executePython({ code, tables });
+    abort.current = () => request.abort();
+
+    try {
+      const data = await request.unwrap();
+      setData(data);
+    } catch (error) {
+      if (typeof error === "object" && error !== null) {
+        if ("name" in error && error.name === "AbortError") {
+          setData({ error: t`Python script execution was canceled` });
+          return;
+        }
+
+        if ("message" in error && typeof error.message === "string") {
+          setData({ error: error.message });
+          return;
+        }
+
+        if (
+          "data" in error &&
+          typeof error.data === "object" &&
+          error.data !== null &&
+          "error" in error.data &&
+          typeof error.data.error === "string"
+        ) {
+          setData({ error: error?.data?.error });
+          return;
+        }
+      }
+
+      setData({ error: t`An unknown error occurred` });
+    }
+  };
+
+  const cancel = () => {
+    abort.current?.();
+  };
+
+  return {
+    isRunning,
+    isDirty,
+    cancel,
+    run,
+    executionResult,
+  };
+}
+
+// Static Pandas Definitions for CodeMirror Autocomplete
+// This is the closest you can get to "loading pandas definitions"
+
+const pandasDefinitions: Record<string, Completion[]> = {
+  pd: [
+    {
+      label: "DataFrame",
+      type: "class",
+      info: "Two-dimensional, size-mutable, potentially heterogeneous tabular data",
+    },
+    {
+      label: "Series",
+      type: "class",
+      info: "One-dimensional ndarray with axis labels",
+    },
+    {
+      label: "read_csv",
+      type: "function",
+      info: "Read a comma-separated values (csv) file into DataFrame",
+    },
+    {
+      label: "read_excel",
+      type: "function",
+      info: "Read an Excel file into a pandas DataFrame",
+    },
+    {
+      label: "read_json",
+      type: "function",
+      info: "Convert a JSON string to pandas object",
+    },
+    {
+      label: "read_sql",
+      type: "function",
+      info: "Read SQL query or database table into a DataFrame",
+    },
+    {
+      label: "read_parquet",
+      type: "function",
+      info: "Load a parquet object from file path",
+    },
+    {
+      label: "concat",
+      type: "function",
+      info: "Concatenate pandas objects along a particular axis",
+    },
+    {
+      label: "merge",
+      type: "function",
+      info: "Merge DataFrame or named Series objects with a database-style join",
+    },
+    {
+      label: "pivot_table",
+      type: "function",
+      info: "Create a spreadsheet-style pivot table as a DataFrame",
+    },
+    {
+      label: "crosstab",
+      type: "function",
+      info: "Compute a simple cross tabulation of two (or more) factors",
+    },
+    {
+      label: "get_dummies",
+      type: "function",
+      info: "Convert categorical variable(s) into dummy/indicator variables",
+    },
+    {
+      label: "date_range",
+      type: "function",
+      info: "Return a fixed frequency DatetimeIndex",
+    },
+    {
+      label: "to_datetime",
+      type: "function",
+      info: "Convert argument to datetime",
+    },
+    {
+      label: "to_numeric",
+      type: "function",
+      info: "Convert argument to a numeric type",
+    },
+    {
+      label: "cut",
+      type: "function",
+      info: "Bin values into discrete intervals",
+    },
+    {
+      label: "qcut",
+      type: "function",
+      info: "Quantile-based discretization function",
+    },
+    {
+      label: "isna",
+      type: "function",
+      info: "Detect missing values for an array-like object",
+    },
+    {
+      label: "isnull",
+      type: "function",
+      info: "Detect missing values for an array-like object",
+    },
+    {
+      label: "notna",
+      type: "function",
+      info: "Detect non-missing values for an array-like object",
+    },
+    {
+      label: "notnull",
+      type: "function",
+      info: "Detect non-missing values for an array-like object",
+    },
+  ],
+
+  // DataFrame methods and properties
+  DataFrame: [
+    // Viewing/Inspection
+    { label: "head", type: "method", info: "Return the first n rows" },
+    { label: "tail", type: "method", info: "Return the last n rows" },
+    {
+      label: "info",
+      type: "method",
+      info: "Print a concise summary of a DataFrame",
+    },
+    {
+      label: "describe",
+      type: "method",
+      info: "Generate descriptive statistics",
+    },
+    {
+      label: "shape",
+      type: "property",
+      info: "Return a tuple representing the dimensionality",
+    },
+    {
+      label: "size",
+      type: "property",
+      info: "Return an int representing the number of elements",
+    },
+    {
+      label: "columns",
+      type: "property",
+      info: "The column labels of the DataFrame",
+    },
+    {
+      label: "index",
+      type: "property",
+      info: "The index (row labels) of the DataFrame",
+    },
+    {
+      label: "dtypes",
+      type: "property",
+      info: "Return the dtypes in the DataFrame",
+    },
+    {
+      label: "values",
+      type: "property",
+      info: "Return a Numpy representation of the DataFrame",
+    },
+
+    // Selection/Indexing
+    {
+      label: "loc",
+      type: "property",
+      info: "Access a group of rows and columns by label(s)",
+    },
+    {
+      label: "iloc",
+      type: "property",
+      info: "Purely integer-location based indexing",
+    },
+    {
+      label: "at",
+      type: "property",
+      info: "Access a single value for a row/column label pair",
+    },
+    {
+      label: "iat",
+      type: "property",
+      info: "Access a single value for a row/column pair by integer position",
+    },
+    {
+      label: "query",
+      type: "method",
+      info: "Query the columns of a DataFrame with a boolean expression",
+    },
+    {
+      label: "filter",
+      type: "method",
+      info: "Subset the dataframe rows or columns according to labels",
+    },
+    {
+      label: "where",
+      type: "method",
+      info: "Replace values where the condition is False",
+    },
+    {
+      label: "mask",
+      type: "method",
+      info: "Replace values where the condition is True",
+    },
+
+    // Data Cleaning
+    {
+      label: "drop",
+      type: "method",
+      info: "Drop specified labels from rows or columns",
+    },
+    {
+      label: "drop_duplicates",
+      type: "method",
+      info: "Return DataFrame with duplicate rows removed",
+    },
+    { label: "dropna", type: "method", info: "Remove missing values" },
+    {
+      label: "fillna",
+      type: "method",
+      info: "Fill NA/NaN values using the specified method",
+    },
+    {
+      label: "replace",
+      type: "method",
+      info: "Replace values given in to_replace with value",
+    },
+    {
+      label: "interpolate",
+      type: "method",
+      info: "Fill NaN values using an interpolation method",
+    },
+    {
+      label: "duplicated",
+      type: "method",
+      info: "Return boolean Series denoting duplicate rows",
+    },
+    { label: "isnull", type: "method", info: "Detect missing values" },
+    { label: "isna", type: "method", info: "Detect missing values" },
+    {
+      label: "notnull",
+      type: "method",
+      info: "Detect existing (non-missing) values",
+    },
+    {
+      label: "notna",
+      type: "method",
+      info: "Detect existing (non-missing) values",
+    },
+
+    // Transformations
+    {
+      label: "sort_values",
+      type: "method",
+      info: "Sort by the values along either axis",
+    },
+    {
+      label: "sort_index",
+      type: "method",
+      info: "Sort object by labels (along an axis)",
+    },
+    {
+      label: "reset_index",
+      type: "method",
+      info: "Reset the index, or a level of it",
+    },
+    {
+      label: "set_index",
+      type: "method",
+      info: "Set the DataFrame index using existing columns",
+    },
+    {
+      label: "reindex",
+      type: "method",
+      info: "Conform Series/DataFrame to new index",
+    },
+    { label: "rename", type: "method", info: "Alter axes labels" },
+    {
+      label: "assign",
+      type: "method",
+      info: "Assign new columns to a DataFrame",
+    },
+    {
+      label: "pipe",
+      type: "method",
+      info: "Apply chainable functions that expect Series or DataFrames",
+    },
+
+    // Aggregation/GroupBy
+    {
+      label: "groupby",
+      type: "method",
+      info: "Group DataFrame using a mapper or by a Series of columns",
+    },
+    {
+      label: "agg",
+      type: "method",
+      info: "Aggregate using one or more operations",
+    },
+    {
+      label: "aggregate",
+      type: "method",
+      info: "Aggregate using one or more operations",
+    },
+    {
+      label: "apply",
+      type: "method",
+      info: "Apply a function along an axis of the DataFrame",
+    },
+    {
+      label: "applymap",
+      type: "method",
+      info: "Apply a function to a Dataframe elementwise",
+    },
+    {
+      label: "transform",
+      type: "method",
+      info: "Call func on self producing a DataFrame with transformed values",
+    },
+    {
+      label: "rolling",
+      type: "method",
+      info: "Provide rolling window calculations",
+    },
+    {
+      label: "expanding",
+      type: "method",
+      info: "Provide expanding transformations",
+    },
+    { label: "resample", type: "method", info: "Resample time-series data" },
+
+    // Statistical Methods
+    {
+      label: "sum",
+      type: "method",
+      info: "Return the sum of the values for the requested axis",
+    },
+    {
+      label: "mean",
+      type: "method",
+      info: "Return the mean of the values for the requested axis",
+    },
+    {
+      label: "median",
+      type: "method",
+      info: "Return the median of the values for the requested axis",
+    },
+    {
+      label: "mode",
+      type: "method",
+      info: "Get the mode(s) of each element along the selected axis",
+    },
+    {
+      label: "std",
+      type: "method",
+      info: "Return sample standard deviation over requested axis",
+    },
+    {
+      label: "var",
+      type: "method",
+      info: "Return unbiased variance over requested axis",
+    },
+    {
+      label: "min",
+      type: "method",
+      info: "Return the minimum of the values for the requested axis",
+    },
+    {
+      label: "max",
+      type: "method",
+      info: "Return the maximum of the values for the requested axis",
+    },
+    {
+      label: "count",
+      type: "method",
+      info: "Count non-NA cells for each column or row",
+    },
+    {
+      label: "nunique",
+      type: "method",
+      info: "Count number of distinct elements in specified axis",
+    },
+    {
+      label: "value_counts",
+      type: "method",
+      info: "Return a Series containing counts of unique values",
+    },
+    {
+      label: "corr",
+      type: "method",
+      info: "Compute pairwise correlation of columns",
+    },
+    {
+      label: "cov",
+      type: "method",
+      info: "Compute pairwise covariance of columns",
+    },
+    {
+      label: "rank",
+      type: "method",
+      info: "Compute numerical data ranks (1 through n) along axis",
+    },
+    {
+      label: "quantile",
+      type: "method",
+      info: "Return values at the given quantile over requested axis",
+    },
+    {
+      label: "clip",
+      type: "method",
+      info: "Trim values at input threshold(s)",
+    },
+
+    // Merging/Joining
+    {
+      label: "merge",
+      type: "method",
+      info: "Merge DataFrame or named Series objects",
+    },
+    {
+      label: "join",
+      type: "method",
+      info: "Join columns of another DataFrame",
+    },
+    {
+      label: "concat",
+      type: "method",
+      info: "Concatenate pandas objects along a particular axis",
+    },
+    {
+      label: "append",
+      type: "method",
+      info: "Append rows of other to the end of caller",
+    },
+    {
+      label: "update",
+      type: "method",
+      info: "Modify in place using non-NA values from another DataFrame",
+    },
+
+    // Reshaping
+    {
+      label: "pivot",
+      type: "method",
+      info: "Return reshaped DataFrame organized by given index / column values",
+    },
+    {
+      label: "pivot_table",
+      type: "method",
+      info: "Create a spreadsheet-style pivot table",
+    },
+    {
+      label: "melt",
+      type: "method",
+      info: "Unpivot a DataFrame from wide to long format",
+    },
+    {
+      label: "stack",
+      type: "method",
+      info: "Stack the prescribed level(s) from columns to index",
+    },
+    {
+      label: "unstack",
+      type: "method",
+      info: "Pivot a level of the (necessarily hierarchical) index labels",
+    },
+    { label: "transpose", type: "method", info: "Transpose index and columns" },
+    { label: "T", type: "property", info: "Transpose index and columns" },
+
+    // I/O
+    {
+      label: "to_csv",
+      type: "method",
+      info: "Write object to a comma-separated values (csv) file",
+    },
+    {
+      label: "to_excel",
+      type: "method",
+      info: "Write object to an Excel sheet",
+    },
+    {
+      label: "to_json",
+      type: "method",
+      info: "Convert the object to a JSON string",
+    },
+    {
+      label: "to_sql",
+      type: "method",
+      info: "Write records stored in a DataFrame to a SQL database",
+    },
+    {
+      label: "to_parquet",
+      type: "method",
+      info: "Write a DataFrame to the binary parquet format",
+    },
+    {
+      label: "to_dict",
+      type: "method",
+      info: "Convert the DataFrame to a dictionary",
+    },
+    {
+      label: "to_html",
+      type: "method",
+      info: "Render a DataFrame as an HTML table",
+    },
+    {
+      label: "to_string",
+      type: "method",
+      info: "Render a DataFrame to a console-friendly tabular output",
+    },
+    {
+      label: "to_numpy",
+      type: "method",
+      info: "Convert the DataFrame to a NumPy array",
+    },
+
+    // Time Series
+    {
+      label: "shift",
+      type: "method",
+      info: "Shift index by desired number of periods",
+    },
+    { label: "tshift", type: "method", info: "Shift the time index" },
+    {
+      label: "diff",
+      type: "method",
+      info: "First discrete difference of element",
+    },
+    {
+      label: "pct_change",
+      type: "method",
+      info: "Percentage change between the current and a prior element",
+    },
+
+    // Plotting (if matplotlib available)
+    {
+      label: "plot",
+      type: "method",
+      info: "Make plots of Series or DataFrame",
+    },
+    {
+      label: "hist",
+      type: "method",
+      info: "Make a histogram of the DataFrame's columns",
+    },
+    {
+      label: "boxplot",
+      type: "method",
+      info: "Make a box plot of the DataFrame columns",
+    },
+  ],
+
+  // Series methods (subset of DataFrame methods that make sense for Series)
+  Series: [
+    { label: "head", type: "method", info: "Return the first n values" },
+    { label: "tail", type: "method", info: "Return the last n values" },
+    {
+      label: "describe",
+      type: "method",
+      info: "Generate descriptive statistics",
+    },
+    {
+      label: "value_counts",
+      type: "method",
+      info: "Return a Series containing counts of unique values",
+    },
+    {
+      label: "unique",
+      type: "method",
+      info: "Return unique values of Series object",
+    },
+    {
+      label: "nunique",
+      type: "method",
+      info: "Return number of unique elements in the object",
+    },
+    { label: "sort_values", type: "method", info: "Sort by the values" },
+    {
+      label: "sort_index",
+      type: "method",
+      info: "Sort Series by index labels",
+    },
+    {
+      label: "reset_index",
+      type: "method",
+      info: "Generate a new DataFrame or Series with the index reset",
+    },
+    {
+      label: "dropna",
+      type: "method",
+      info: "Return a new Series with missing values removed",
+    },
+    {
+      label: "fillna",
+      type: "method",
+      info: "Fill NA/NaN values using the specified method",
+    },
+    {
+      label: "apply",
+      type: "method",
+      info: "Invoke function on values of Series",
+    },
+    {
+      label: "map",
+      type: "method",
+      info: "Map values of Series according to input correspondence",
+    },
+    {
+      label: "replace",
+      type: "method",
+      info: "Replace values given in to_replace with value",
+    },
+    {
+      label: "str",
+      type: "property",
+      info: "Vectorized string functions for Series and Index",
+    },
+    {
+      label: "dt",
+      type: "property",
+      info: "Accessor object for datetimelike properties",
+    },
+    {
+      label: "cat",
+      type: "property",
+      info: "Accessor object for categorical properties",
+    },
+  ],
+};
+
+function genericPandasCompletions(context: CompletionContext) {
+  const word = context.matchBefore(/[\w.]*/);
+  if (!word || (word.from === word.to && !context.explicit)) {
+    return null;
+  }
+
+  const text = word.text;
+
+  // Match patterns like "pd.", "df.", etc.
+  if (text.includes(".")) {
+    const parts = text.split(".");
+    const obj = parts.at(-2);
+    const partial = parts.at(-1) ?? "";
+
+    if (!obj || !pandasDefinitions[obj]) {
+      return null;
+    }
+
+    const options = pandasDefinitions[obj]
+      .filter(
+        (item) =>
+          !partial ||
+          item.label.toLowerCase().startsWith(partial.toLowerCase()),
+      )
+      .map((comp) => ({
+        ...comp,
+        info: undefined,
+        boost: comp.type === "method" ? 2 : comp.type === "function" ? 1 : 0,
+      }));
+
+    return {
+      filter: false,
+      from: word.from + text.length - partial.length,
+      options,
+    };
+  }
+}
+
+export const completion = [
+  pythonLanguage.data.of({
+    autocomplete: genericPandasCompletions,
+  }),
+  autocompletion(),
+];

enterprise/frontend/src/metabase-enterprise/transforms/components/RunButton/RunButton.tsx

@@ -2,7 +2,7 @@ import { type ReactNode, type Ref, forwardRef, useState } from "react";
 import { useUpdateEffect } from "react-use";
 import { t } from "ttag";
 
-import { Button, Icon, Loader } from "metabase/ui";
+import { Button, Group, Icon, Loader } from "metabase/ui";
 import type { TransformRun } from "metabase-types/api";
 
 const RECENT_TIMEOUT = 5000;
@@ -12,6 +12,8 @@ type RunButtonProps = {
   isLoading: boolean;
   isDisabled?: boolean;
   onRun: () => void;
+  onCancel?: () => void;
+  isCanceling?: boolean;
 };
 
 export const RunButton = forwardRef(function RunButton(
@@ -20,6 +22,8 @@ export const RunButton = forwardRef(function RunButton(
     isLoading,
     isDisabled: isExternallyDisabled = false,
     onRun,
+    onCancel,
+    isCanceling = false,
   }: RunButtonProps,
   ref: Ref<HTMLButtonElement>,
 ) {
@@ -37,6 +41,35 @@ export const RunButton = forwardRef(function RunButton(
     return () => clearTimeout(timeoutId);
   }, [run]);
 
+  const showCancelButton = (run?.status === "started" || isLoading) && onCancel;
+
+  if (showCancelButton) {
+    return (
+      <Group gap="sm">
+        <Button
+          ref={ref}
+          variant="filled"
+          color={color}
+          leftSection={leftSection}
+          disabled={isDisabled}
+          data-testid="run-button"
+          onClick={onRun}
+        >
+          {label}
+        </Button>
+        <Button
+          variant="outline"
+          color="error"
+          disabled={isCanceling}
+          data-testid="cancel-button"
+          onClick={onCancel}
+        >
+          {isCanceling ? t`Canceling…` : t`Cancel`}
+        </Button>
+      </Group>
+    );
+  }
+
   return (
     <Button
       ref={ref}

enterprise/frontend/src/metabase-enterprise/transforms/index.tsx

@@ -9,6 +9,7 @@ import { hasPremiumFeature } from "metabase-enterprise/settings";
 import { JobListPage } from "./pages/JobListPage";
 import { JobPage } from "./pages/JobPage";
 import { NewJobPage } from "./pages/NewJobPage";
+import { NewPythonTransformPage } from "./pages/NewPythonTransformPage";
 import { NewTransformQueryPage } from "./pages/NewTransformQueryPage";
 import { RunListPage } from "./pages/RunListPage";
 import { TransformListPage } from "./pages/TransformListPage";
@@ -32,7 +33,16 @@ if (hasPremiumFeature("transforms")) {
           <Route path="runs" component={RunListPage} />
           <Route path=":transformId" component={TransformPage} />
         </Route>
-        <Route path="new/:type" component={NewTransformQueryPage} />
+        <Route
+          path="new/:type"
+          component={(props: any) =>
+            props.params.type === "python" ? (
+              <NewPythonTransformPage />
+            ) : (
+              <NewTransformQueryPage params={props.params} />
+            )
+          }
+        />
         <Route path="new/card/:cardId" component={NewTransformQueryPage} />
         <Route path=":transformId/query" component={TransformQueryPage} />
       </Route>

enterprise/frontend/src/metabase-enterprise/transforms/pages/NewPythonTransformPage/CreatePythonTransformModal/CreatePythonTransformModal.tsx

@@ -0,0 +1,154 @@
+import { Field, Form, Formik } from "formik";
+import { t } from "ttag";
+import * as Yup from "yup";
+
+import { Button, Group, Modal, Stack, TextInput } from "metabase/ui";
+import { useCreateTransformMutation } from "metabase-enterprise/api";
+import type { PythonTransformSource, Transform } from "metabase-types/api";
+
+type PythonTransformApiSource = {
+  type: "python";
+  body: string;
+  "source-database": number;
+  "source-tables": Record<string, number>;
+};
+
+type CreatePythonTransformModalProps = {
+  source: PythonTransformSource;
+  onCreate: (transform: Transform) => void;
+  onClose: () => void;
+};
+
+type FormValues = {
+  name: string;
+  description: string | null;
+  targetSchema: string | null;
+  targetName: string;
+};
+
+const getValidationSchema = () =>
+  Yup.object({
+    name: Yup.string().required(t`Name is required`),
+    description: Yup.string().nullable(),
+    targetSchema: Yup.string().nullable(),
+    targetName: Yup.string().required(t`Table name is required`),
+  });
+
+export function CreatePythonTransformModal({
+  source,
+  onCreate,
+  onClose,
+}: CreatePythonTransformModalProps) {
+  const [createTransform] = useCreateTransformMutation();
+
+  const initialValues: FormValues = {
+    name: "",
+    description: null,
+    targetSchema: "public",
+    targetName: "",
+  };
+
+  const handleSubmit = async (values: FormValues) => {
+    try {
+      const transformedSource: PythonTransformApiSource = {
+        type: "python",
+        body: source.body,
+        "source-database": source["source-database"],
+        "source-tables": source["source-tables"] || {},
+      };
+
+      const transform = await createTransform({
+        name: values.name,
+        description: values.description,
+        source: transformedSource as any,
+        target: {
+          type: "table",
+          schema: values.targetSchema?.trim() || null,
+          name: values.targetName,
+          database: source["source-database"], // for now the same as source
+        },
+      }).unwrap();
+
+      onCreate(transform);
+    } catch (error) {
+      console.error("Error creating Python transform:", error);
+      // TODO: Show error toast
+    }
+  };
+
+  return (
+    <Modal
+      opened
+      onClose={onClose}
+      title={t`Create Python Transform`}
+      size="md"
+    >
+      <Formik
+        initialValues={initialValues}
+        validationSchema={getValidationSchema()}
+        onSubmit={handleSubmit}
+      >
+        {({ isSubmitting, isValid }) => (
+          <Form>
+            <Stack gap="md">
+              <Field name="name">
+                {({ field, meta }: any) => (
+                  <TextInput
+                    {...field}
+                    label={t`Transform name`}
+                    placeholder={t`My Python Transform`}
+                    error={meta.touched && meta.error}
+                  />
+                )}
+              </Field>
+
+              <Field name="description">
+                {({ field }: any) => (
+                  <TextInput
+                    {...field}
+                    label={t`Description (optional)`}
+                    placeholder={t`What does this transform do?`}
+                  />
+                )}
+              </Field>
+
+              <Field name="targetName">
+                {({ field, meta }: any) => (
+                  <TextInput
+                    {...field}
+                    label={t`Output table name`}
+                    placeholder={t`my_python_transform`}
+                    error={meta.touched && meta.error}
+                  />
+                )}
+              </Field>
+
+              <Field name="targetSchema">
+                {({ field }: any) => (
+                  <TextInput
+                    {...field}
+                    label={t`Schema (optional)`}
+                    placeholder={t`public`}
+                  />
+                )}
+              </Field>
+
+              <Group justify="flex-end" mt="md">
+                <Button variant="subtle" onClick={onClose}>
+                  {t`Cancel`}
+                </Button>
+                <Button
+                  type="submit"
+                  loading={isSubmitting}
+                  disabled={!isValid}
+                >
+                  {t`Create transform`}
+                </Button>
+              </Group>
+            </Stack>
+          </Form>
+        )}
+      </Formik>
+    </Modal>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/pages/NewPythonTransformPage/CreatePythonTransformModal/index.ts

@@ -0,0 +1 @@
+export { CreatePythonTransformModal } from "./CreatePythonTransformModal";

enterprise/frontend/src/metabase-enterprise/transforms/pages/NewPythonTransformPage/NewPythonTransformPage.tsx

@@ -0,0 +1,71 @@
+import { useDisclosure } from "@mantine/hooks";
+import { useState } from "react";
+import { push } from "react-router-redux";
+
+import { useDispatch } from "metabase/lib/redux";
+import type { PythonTransformSource, Transform } from "metabase-types/api";
+
+import { PythonTransformEditor } from "../../components/PythonTransformEditor";
+import { getTransformListUrl, getTransformUrl } from "../../urls";
+
+import { CreatePythonTransformModal } from "./CreatePythonTransformModal";
+
+const DEFAULT_PYTHON_SOURCE: PythonTransformSource = {
+  type: "python",
+  "source-database": 1, // Default to first database, will be updated by user
+  "source-tables": {}, // Will be populated when user selects tables
+  body: `# Write your Python transformation script here
+import pandas as pd
+
+def transform():
+    """
+    Your transformation function.
+
+    Select tables above to add them as function parameters.
+
+    Returns:
+        DataFrame to write to the destination table
+    """
+    # Your transformation logic here
+    return pd.DataFrame([{"message": "Hello from Python transform!"}])`,
+};
+
+export function NewPythonTransformPage() {
+  const [source, setSource] = useState<PythonTransformSource>(
+    DEFAULT_PYTHON_SOURCE,
+  );
+  const [isModalOpened, { open: openModal, close: closeModal }] =
+    useDisclosure();
+  const dispatch = useDispatch();
+
+  const handleCreate = (transform: Transform) => {
+    dispatch(push(getTransformUrl(transform.id)));
+  };
+
+  const handleSaveClick = (newSource: PythonTransformSource) => {
+    setSource(newSource);
+    openModal();
+  };
+
+  const handleCancelClick = () => {
+    dispatch(push(getTransformListUrl()));
+  };
+
+  return (
+    <>
+      <PythonTransformEditor
+        initialSource={DEFAULT_PYTHON_SOURCE}
+        isNew
+        onSave={handleSaveClick}
+        onCancel={handleCancelClick}
+      />
+      {isModalOpened && (
+        <CreatePythonTransformModal
+          source={source}
+          onCreate={handleCreate}
+          onClose={closeModal}
+        />
+      )}
+    </>
+  );
+}

enterprise/frontend/src/metabase-enterprise/transforms/pages/NewPythonTransformPage/index.ts

@@ -0,0 +1 @@
+export { NewPythonTransformPage } from "./NewPythonTransformPage";

enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/CreateTransformModal/CreateTransformModal.tsx

@@ -90,7 +90,10 @@ function CreateTransformForm({
   }
 
   const handleSubmit = async (values: NewTransformValues) => {
-    const request = getCreateRequest(query, values);
+    if (!databaseId) {
+      throw new Error("Database ID is required");
+    }
+    const request = getCreateRequest(query, values, databaseId);
     const transform = await createTransform(request).unwrap();
 
     trackTransformCreated({ transformId: transform.id });
@@ -149,6 +152,7 @@ function getInitialValues(schemas: string[]): NewTransformValues {
 function getCreateRequest(
   query: DatasetQuery,
   { name, description, targetName, targetSchema }: NewTransformValues,
+  databaseId: number,
 ): CreateTransformRequest {
   return {
     name: name,
@@ -161,6 +165,7 @@ function getCreateRequest(
       type: "table",
       name: targetName,
       schema: targetSchema,
+      database: databaseId,
     },
   };
 }

enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformQueryPage/NewTransformQueryPage.tsx

@@ -90,7 +90,7 @@ function getParsedParams({
   cardId,
 }: NewTransformQueryPageParams): NewTransformQueryPageParsedParams {
   return {
-    type: type === "native" ? "native" : "query",
+    type: type === "native" ? "native" : type === "python" ? "query" : "query",
     cardId: cardId != null ? Urls.extractEntityId(cardId) : undefined,
   };
 }

enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformListPage/CreateTransformMenu/CreateTransformMenu.tsx

@@ -85,6 +85,19 @@ export function CreateTransformMenu() {
               >
                 {t`SQL query`}
               </Menu.Item>
+              <Menu.Item
+                component={ForwardRefLink}
+                to={getNewTransformFromTypeUrl("python")}
+                leftSection={<Icon name="code_block" />}
+                onClick={() => {
+                  trackTransformCreate({
+                    triggeredFrom: "transform-page-create-menu",
+                    creationType: "python",
+                  });
+                }}
+              >
+                {t`Python script`}
+              </Menu.Item>
               <Menu.Item
                 leftSection={<Icon name="copy" />}
                 onClick={handleSavedQuestionClick}

enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/ManageSection/ManageSection.tsx

@@ -19,9 +19,12 @@ type ManageSectionProps = {
 };
 
 export function ManageSection({ transform }: ManageSectionProps) {
+  const sectionLabel =
+    transform.source.type === "python" ? t`Python Script` : t`Query`;
+
   return (
     <TitleSection
-      label={t`Query`}
+      label={sectionLabel}
       rightSection={
         <Group>
           <EditQueryButton transform={transform} />
@@ -30,7 +33,14 @@ export function ManageSection({ transform }: ManageSectionProps) {
       }
     >
       <Card p={0} shadow="none" withBorder>
-        <QueryView query={transform.source.query} />
+        {transform.source.type === "query" && (
+          <QueryView query={transform.source.query} />
+        )}
+        {transform.source.type === "python" && (
+          <div style={{ padding: "1rem" }}>
+            <pre>{transform.source.body}</pre>
+          </div>
+        )}
       </Card>
     </TitleSection>
   );
@@ -41,13 +51,16 @@ type EditQueryButtonProps = {
 };
 
 function EditQueryButton({ transform }: EditQueryButtonProps) {
+  const buttonText =
+    transform.source.type === "python" ? t`Edit script` : t`Edit query`;
+
   return (
     <Button
       component={Link}
       to={getTransformQueryUrl(transform.id)}
       leftSection={<Icon name="pencil_lines" aria-hidden />}
     >
-      {t`Edit query`}
+      {buttonText}
     </Button>
   );
 }

enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/RunSection/RunSection.tsx

@@ -5,6 +5,7 @@ import { parseTimestamp } from "metabase/lib/time-dayjs";
 import { useMetadataToasts } from "metabase/metadata/hooks";
 import { Anchor, Box, Divider, Group, Icon, Stack } from "metabase/ui";
 import {
+  useCancelTransformMutation,
   useLazyGetTransformQuery,
   useRunTransformMutation,
   useUpdateTransformMutation,
@@ -12,6 +13,7 @@ import {
 import { trackTranformTriggerManualRun } from "metabase-enterprise/transforms/analytics";
 import type { Transform, TransformTagId } from "metabase-types/api";
 
+import { LogOutput } from "../../../components/LogOutput";
 import { RunButton } from "../../../components/RunButton";
 import { RunErrorInfo } from "../../../components/RunErrorInfo";
 import { SplitSection } from "../../../components/SplitSection";
@@ -28,10 +30,16 @@ export function RunSection({ transform }: RunSectionProps) {
       label={t`Run this transform`}
       description={t`This transform will be run whenever the jobs it belongs to are scheduled.`}
     >
-      <Group p="lg" justify="space-between">
-        <RunStatusSection transform={transform} />
-        <RunButtonSection transform={transform} />
-      </Group>
+      <Stack>
+        <Group p="lg" justify="space-between">
+          <RunStatusSection transform={transform} />
+          <RunButtonSection transform={transform} />
+        </Group>
+        {transform?.last_run?.message &&
+          ["started", "succeeded"].some(
+            (s) => transform?.last_run?.status === s,
+          ) && <LogOutput content={transform.last_run.message} />}
+      </Stack>
       <Divider />
       <Group p="lg" gap="lg">
         <Stack gap="sm">
@@ -138,7 +146,9 @@ type RunButtonSectionProps = {
 function RunButtonSection({ transform }: RunButtonSectionProps) {
   const [fetchTransform, { isFetching }] = useLazyGetTransformQuery();
   const [runTransform, { isLoading: isRunning }] = useRunTransformMutation();
-  const { sendErrorToast } = useMetadataToasts();
+  const [cancelTransform, { isLoading: isCanceling }] =
+    useCancelTransformMutation();
+  const { sendErrorToast, sendSuccessToast } = useMetadataToasts();
 
   const handleRun = async () => {
     trackTranformTriggerManualRun({
@@ -156,11 +166,24 @@ function RunButtonSection({ transform }: RunButtonSectionProps) {
     return { error };
   };
 
+  const handleCancel = async () => {
+    const { error } = await cancelTransform(transform.id);
+    if (error) {
+      sendErrorToast(t`Failed to cancel transform`);
+    } else {
+      sendSuccessToast(t`Transform cancellation requested`);
+      // fetch the transform to get the updated status
+      fetchTransform(transform.id);
+    }
+  };
+
   return (
     <RunButton
       run={transform.last_run}
       isLoading={isFetching || isRunning}
       onRun={handleRun}
+      onCancel={handleCancel}
+      isCanceling={isCanceling}
     />
   );
 }

enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/SourceSection/SourceSection.tsx

@@ -0,0 +1,190 @@
+import { t } from "ttag";
+
+import { skipToken, useGetDatabaseQuery, useGetTableQuery } from "metabase/api";
+import Link from "metabase/common/components/Link";
+import CS from "metabase/css/core/index.css";
+import {
+  Divider,
+  Group,
+  Icon,
+  type IconName,
+  Loader,
+  Stack,
+  Text,
+} from "metabase/ui";
+import type { Transform } from "metabase-types/api";
+
+import { SplitSection } from "../../../components/SplitSection";
+import {
+  getBrowseDatabaseUrl,
+  getBrowseSchemaUrl,
+  getQueryBuilderUrl,
+} from "../../../urls";
+
+type SourceSectionProps = {
+  transform: Transform;
+};
+
+export function SourceSection({ transform }: SourceSectionProps) {
+  if (transform.source.type !== "python") {
+    return null;
+  }
+
+  return (
+    <SplitSection
+      label={t`Transform source`}
+      description={t`The data sources for this Python transform.`}
+    >
+      <Group p="lg">
+        <SourceInfo transform={transform} />
+      </Group>
+    </SplitSection>
+  );
+}
+
+type SourceInfoProps = {
+  transform: Transform;
+};
+
+function SourceInfo({ transform }: SourceInfoProps) {
+  const source = transform.source;
+  const sourceDatabaseId =
+    source.type === "python" ? source["source-database"] : undefined;
+  const sourceTables =
+    source.type === "python" ? source["source-tables"] : undefined;
+
+  const { data: database, isLoading: isDatabaseLoading } = useGetDatabaseQuery(
+    sourceDatabaseId ? { id: sourceDatabaseId } : skipToken,
+  );
+
+  if (!database) {
+    return null;
+  }
+
+  const hasMultipleTables =
+    sourceTables && Object.keys(sourceTables).length > 0;
+
+  if (isDatabaseLoading) {
+    return <Loader size="sm" />;
+  }
+
+  // Display for multiple tables
+  if (hasMultipleTables) {
+    return (
+      <Stack gap="sm">
+        {database && (
+          <Group gap="sm">
+            <SourceItemLink
+              label={database.name}
+              icon="database"
+              to={getBrowseDatabaseUrl(database.id)}
+              data-testid="source-database-link"
+            />
+          </Group>
+        )}
+        <Divider />
+        <Stack gap="xs">
+          {Object.entries(sourceTables).map(([alias, tableId]) => (
+            <TableWithAlias
+              key={alias}
+              alias={alias}
+              tableId={tableId as number}
+              databaseId={sourceDatabaseId as number}
+            />
+          ))}
+        </Stack>
+      </Stack>
+    );
+  }
+
+  // Just show database if no tables selected
+  return (
+    <Group gap="sm">
+      {database != null && (
+        <SourceItemLink
+          label={database.name}
+          icon="database"
+          to={getBrowseDatabaseUrl(database.id)}
+          data-testid="source-database-link"
+        />
+      )}
+    </Group>
+  );
+}
+
+type TableWithAliasProps = {
+  alias: string;
+  tableId: number;
+  databaseId: number;
+};
+
+function TableWithAlias({ alias, tableId, databaseId }: TableWithAliasProps) {
+  const { data: table, isLoading } = useGetTableQuery(
+    tableId ? { id: tableId } : skipToken,
+  );
+
+  if (isLoading) {
+    return <Loader size="xs" />;
+  }
+
+  return (
+    <Group gap="sm" align="flex-start">
+      <Group gap="xs">
+        {table?.schema && (
+          <>
+            <SourceItemLink
+              label={table.schema}
+              icon="folder"
+              to={getBrowseSchemaUrl(databaseId, table.schema)}
+              small
+            />
+            <SourceItemDivider />
+          </>
+        )}
+        <SourceItemLink
+          label={table?.display_name || table?.name || `Table ${tableId}`}
+          icon="table2"
+          to={table ? getQueryBuilderUrl(table.id, table.db_id) : undefined}
+          data-testid={`source-table-${alias}-link`}
+          small
+        />
+      </Group>
+    </Group>
+  );
+}
+
+type SourceItemLinkProps = {
+  label: string;
+  icon: IconName;
+  to?: string;
+  "data-testid"?: string;
+  small?: boolean;
+};
+
+function SourceItemLink({
+  label,
+  icon,
+  to,
+  "data-testid": dataTestId,
+  small = false,
+}: SourceItemLinkProps) {
+  return (
+    <Link
+      className={CS.link}
+      to={to ?? ""}
+      disabled={to == null}
+      data-testid={dataTestId}
+    >
+      <Group gap="xs">
+        <Icon name={icon} size={small ? 14 : 16} />
+        <Text c="inherit" size={small ? "sm" : "md"}>
+          {label}
+        </Text>
+      </Group>
+    </Link>
+  );
+}
+
+function SourceItemDivider() {
+  return <Icon name="chevronright" size={8} />;
+}

enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/SourceSection/index.ts

@@ -0,0 +1 @@
+export * from "./SourceSection";

enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TargetSection/TargetSection.tsx

@@ -54,7 +54,12 @@ type TargetInfoProps = {
 
 function TargetInfo({ transform }: TargetInfoProps) {
   const { source, target, table } = transform;
-  const { database: databaseId } = source.query;
+  const databaseId =
+    source.type === "query"
+      ? source.query.database
+      : source.type === "python"
+        ? source["source-database"]
+        : undefined;
   const { data, isLoading } = useGetDatabaseQuery(
     table == null && databaseId != null ? { id: databaseId } : skipToken,
   );

enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TargetSection/UpdateTargetModal/UpdateTargetModal.tsx

@@ -78,7 +78,12 @@ function UpdateTargetForm({
   onClose,
 }: UpdateTargetFormProps) {
   const { source, target, table } = transform;
-  const { database: databaseId } = source.query;
+  const databaseId =
+    source.type === "query"
+      ? source.query.database
+      : source.type === "python"
+        ? source["source-database"]
+        : undefined;
   const [updateTransform] = useUpdateTransformMutation();
   const [deleteTransformTarget] = useDeleteTransformTargetMutation();
   const initialValues = useMemo(() => getInitialValues(transform), [transform]);
@@ -97,10 +102,15 @@ function UpdateTargetForm({
   }
 
   const handleSubmit = async (values: EditTransformValues) => {
+    if (!databaseId) {
+      throw new Error("Database ID is required");
+    }
     if (shouldDeleteTarget) {
       await deleteTransformTarget(transform.id).unwrap();
     }
-    await updateTransform(getUpdateRequest(transform, values)).unwrap();
+    await updateTransform(
+      getUpdateRequest(transform, values, databaseId),
+    ).unwrap();
     onUpdate();
   };
 
@@ -179,6 +189,7 @@ function getSubmitButtonColor(shouldDeleteTarget: boolean) {
 function getUpdateRequest(
   { id }: Transform,
   { name, schema }: EditTransformValues,
+  databaseId: number,
 ): UpdateTransformRequest {
   return {
     id,
@@ -186,6 +197,7 @@ function getUpdateRequest(
       type: "table",
       name,
       schema,
+      database: databaseId,
     },
   };
 }

enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformPage/TransformPage.tsx

@@ -15,6 +15,7 @@ import { HeaderSection } from "./HeaderSection";
 import { ManageSection } from "./ManageSection";
 import { NameSection } from "./NameSection";
 import { RunSection } from "./RunSection";
+import { SourceSection } from "./SourceSection";
 import { TargetSection } from "./TargetSection";
 
 type TransformPageParams = {
@@ -59,6 +60,7 @@ export function TransformPage({ params }: TransformPageProps) {
         <NameSection transform={transform} />
       </Stack>
       <RunSection transform={transform} />
+      <SourceSection transform={transform} />
       <TargetSection transform={transform} />
       <ManageSection transform={transform} />
       <DependenciesSection transform={transform} />

enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformQueryPage/TransformQueryPage.tsx

@@ -10,7 +10,12 @@ import {
   useGetTransformQuery,
   useUpdateTransformMutation,
 } from "metabase-enterprise/api";
-import type { DatasetQuery, Transform } from "metabase-types/api";
+import { PythonTransformEditor } from "metabase-enterprise/transforms/components/PythonTransformEditor";
+import type {
+  DatasetQuery,
+  PythonTransformSource,
+  Transform,
+} from "metabase-types/api";
 
 import { QueryEditor } from "../../components/QueryEditor";
 import { getTransformUrl } from "../../urls";
@@ -53,7 +58,7 @@ export function TransformQueryPageBody({
   const dispatch = useDispatch();
   const { sendErrorToast, sendSuccessToast } = useMetadataToasts();
 
-  const handleSave = async (query: DatasetQuery) => {
+  const handleDatasetSave = async (query: DatasetQuery) => {
     const { error } = await updateTransform({
       id: transform.id,
       source: {
@@ -70,16 +75,41 @@ export function TransformQueryPageBody({
     }
   };
 
+  const handlePythonSave = async (source: PythonTransformSource) => {
+    const { error } = await updateTransform({
+      id: transform.id,
+      source: source,
+    });
+
+    if (error) {
+      sendErrorToast(t`Failed to update transform query`);
+    } else {
+      sendSuccessToast(t`Transform query updated`);
+      dispatch(push(getTransformUrl(transform.id)));
+    }
+  };
+
   const handleCancel = () => {
     dispatch(push(getTransformUrl(transform.id)));
   };
 
+  if (transform.source.type === "python") {
+    return (
+      <PythonTransformEditor
+        initialSource={transform.source}
+        isNew={false}
+        isSaving={isLoading}
+        onSave={handlePythonSave}
+        onCancel={handleCancel}
+      />
+    );
+  }
   return (
     <QueryEditor
       initialQuery={transform.source.query}
       isNew={false}
       isSaving={isLoading}
-      onSave={handleSave}
+      onSave={handleDatasetSave}
       onCancel={handleCancel}
     />
   );

enterprise/frontend/src/metabase-enterprise/transforms/urls.ts

@@ -15,7 +15,9 @@ export function getTransformListUrl() {
   return ROOT_URL;
 }
 
-export function getNewTransformFromTypeUrl(type: DatasetQuery["type"]) {
+export function getNewTransformFromTypeUrl(
+  type: DatasetQuery["type"] | "python",
+) {
   return `${ROOT_URL}/new/${type}`;
 }
 

example_db_transform.py

@@ -0,0 +1,33 @@
+import pandas as pd
+
+def transform(db):
+    """
+    Example transform function demonstrating database connectivity.
+    
+    The db parameter provides:
+    - db.read_table(table_name) - reads entire table
+    - db.read_table(table_name, schema) - reads table from specific schema
+    
+    This function would typically:
+    1. Read one or more tables using db.read_table()
+    2. Perform transformations, joins, aggregations, etc.
+    3. Return the final result as a pandas DataFrame
+    """
+    
+    # Example: Read data from multiple tables
+    # customers = db.read_table("customers")
+    # orders = db.read_table("orders") 
+    # products = db.read_table("products", schema="inventory")
+    
+    # Example: Perform transformations and joins
+    # result = customers.merge(orders, on="customer_id")
+    # result = result.merge(products, on="product_id")
+    
+    # For demonstration purposes, return a sample DataFrame
+    # In real usage, this would be the result of database operations
+    data = {
+        'customer_name': ['John Doe', 'Jane Smith', 'Bob Johnson'],
+        'total_orders': [5, 12, 3],
+        'total_spent': [299.99, 1450.50, 89.95]
+    }
+    return pd.DataFrame(data)

frontend/src/metabase-types/analytics/event.ts

@@ -294,7 +294,7 @@ export type TransformJobTriggerManualRunEvent = ValidateEvent<{
 export type TransformCreateEvent = ValidateEvent<{
   event: "transform_create";
   triggered_from: "transform-page-create-menu";
-  event_detail: "query" | "native" | "saved-question";
+  event_detail: "query" | "native" | "python" | "saved-question";
 }>;
 
 export type TransformCreatedEvent = ValidateEvent<{

frontend/src/metabase-types/api/mocks/transform.ts

@@ -9,13 +9,19 @@ import type {
 
 import { createMockStructuredDatasetQuery } from "./query";
 
-export function createMockTransformSource(
-  opts?: Partial<TransformSource>,
-): TransformSource {
+export function createMockTransformSource(): TransformSource {
   return {
     type: "query",
     query: createMockStructuredDatasetQuery(),
-    ...opts,
+  };
+}
+
+export function createMockPythonTransformSource(): TransformSource {
+  return {
+    type: "python",
+    body: "# Python script\nprint('Hello, world!')",
+    "source-database": 1,
+    "source-tables": {},
   };
 }
 
@@ -26,6 +32,7 @@ export function createMockTransformTarget(
     type: "table",
     name: "Table",
     schema: null,
+    database: 1,
     ...opts,
   };
 }

frontend/src/metabase-types/api/settings.ts

@@ -668,6 +668,14 @@ export interface EnterpriseSettings extends Settings {
   "saml-group-mappings": Record<string, GroupId[]> | null;
   "database-replication-enabled": boolean | null;
   "database-replication-connections"?: DatabaseReplicationConnections | null;
+  "python-execution-server-url"?: string | null;
+  "python-storage-s-3-endpoint"?: string | null;
+  "python-storage-s-3-region"?: string | null;
+  "python-storage-s-3-bucket"?: string | null;
+  "python-storage-s-3-access-key"?: string | null;
+  "python-storage-s-3-secret-key"?: string | null;
+  "python-storage-s-3-container-endpoint"?: string | null;
+  "python-storage-s-3-path-style-access"?: boolean | null;
   /**
    * @deprecated
    */

frontend/src/metabase-types/api/transform.ts

@@ -22,17 +22,26 @@ export type Transform = {
   last_run?: TransformRun | null;
 };
 
-export type TransformSource = {
+export type PythonTransformSource = {
+  type: "python";
+  body: string;
+  "source-database": number;
+  "source-tables": Record<string, number>; // alias -> table-id mapping
+};
+export type QueryTransformSource = {
   type: "query";
   query: DatasetQuery;
 };
 
+export type TransformSource = QueryTransformSource | PythonTransformSource;
+
 export type TransformTargetType = "table";
 
 export type TransformTarget = {
   type: TransformTargetType;
   name: string;
   schema: string | null;
+  database: number;
 };
 
 export type TransformRun = {

frontend/src/metabase/admin/settings/components/SettingsNav/SettingsNav.tsx

@@ -24,6 +24,7 @@ export function SettingsNav() {
   const hasSaml = useHasTokenFeature("sso_saml");
   const hasJwt = useHasTokenFeature("sso_jwt");
   const hasScim = useHasTokenFeature("scim");
+  const hasTransforms = useHasTokenFeature("transforms");
 
   return (
     <AdminNavWrapper>
@@ -88,6 +89,13 @@ export function SettingsNav() {
       </SettingsNavItem>
       <NavDivider />
       <SettingsNavItem path="uploads" label={t`Uploads`} icon="upload" />
+      {hasTransforms && (
+        <SettingsNavItem
+          path="python-runner"
+          label={t`Python Runner`}
+          icon="snippet"
+        />
+      )}
       <SettingsNavItem
         path="public-sharing"
         label={t`Public sharing`}

frontend/src/metabase/admin/settings/components/SettingsPages/PythonRunnerSettingsPage.tsx

@@ -0,0 +1,91 @@
+import { t } from "ttag";
+
+import {
+  SettingsPageWrapper,
+  SettingsSection,
+} from "metabase/admin/components/SettingsSection";
+import { useGetSettingsQuery } from "metabase/api";
+import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
+import { useHasTokenFeature } from "metabase/common/hooks";
+
+import { AdminSettingInput } from "../widgets/AdminSettingInput";
+
+export function PythonRunnerSettingsPage() {
+  // it seems we don't need to wire in _settingValues to the inputs - what is this magic??
+  const { data: _settingValues, isLoading } = useGetSettingsQuery();
+  const hasTransformsFeature = useHasTokenFeature("transforms");
+
+  if (isLoading) {
+    return <LoadingAndErrorWrapper loading />;
+  }
+
+  if (!hasTransformsFeature) {
+    return (
+      <SettingsPageWrapper title={t`Python Runner`}>
+        <SettingsSection>
+          <div className="text-medium">
+            {t`Python Runner configuration requires the transforms feature.`}
+          </div>
+        </SettingsSection>
+      </SettingsPageWrapper>
+    );
+  }
+
+  return (
+    <SettingsPageWrapper title={t`Python Runner`}>
+      <SettingsSection title={t`Service Configuration`}>
+        <AdminSettingInput
+          name="python-execution-server-url"
+          title={t`Python Execution Server URL`}
+          description={t`URL for the Python execution server that runs transform functions.`}
+          placeholder="http://localhost:5001"
+          inputType="text"
+        />
+      </SettingsSection>
+      <SettingsSection title={t`S3 Storage Configuration`}>
+        <AdminSettingInput
+          name="python-storage-s-3-endpoint"
+          title={t`S3 Endpoint`}
+          description={t`S3 endpoint URL for storing Python execution artifacts.`}
+          inputType="text"
+        />
+        <AdminSettingInput
+          name="python-storage-s-3-region"
+          title={t`S3 Region`}
+          description={t`AWS region for S3 storage.`}
+          inputType="text"
+        />
+        <AdminSettingInput
+          name="python-storage-s-3-bucket"
+          title={t`S3 Bucket`}
+          description={t`S3 bucket name for storing Python execution artifacts.`}
+          inputType="text"
+        />
+        <AdminSettingInput
+          name="python-storage-s-3-access-key"
+          title={t`S3 Access Key ID`}
+          description={t`AWS access key ID for S3 authentication.`}
+          inputType="password"
+        />
+        <AdminSettingInput
+          name="python-storage-s-3-secret-key"
+          title={t`S3 Secret Access Key`}
+          description={t`AWS secret access key for S3 authentication.`}
+          inputType="password"
+        />
+        <AdminSettingInput
+          name="python-storage-s-3-container-endpoint"
+          title={t`S3 Container Endpoint (Optional)`}
+          description={t`Alternative S3 endpoint accessible from containers. Leave empty if same as main endpoint.`}
+          inputType="text"
+        />
+        <AdminSettingInput
+          name="python-storage-s-3-path-style-access"
+          title={t`Use Path-Style Access`}
+          description={t`Use path-style access for S3 requests (required for LocalStack and some S3-compatible services).`}
+          inputType="boolean"
+        />
+      </SettingsSection>
+    </SettingsPageWrapper>
+  );
+}

frontend/src/metabase/admin/settingsRoutes.tsx

@@ -22,6 +22,7 @@ import { LocalizationSettingsPage } from "./settings/components/SettingsPages/Lo
 import { MapsSettingsPage } from "./settings/components/SettingsPages/MapsSettingsPage";
 import { NotificationSettingsPage } from "./settings/components/SettingsPages/NotificationSettingsPage";
 import { PublicSharingSettingsPage } from "./settings/components/SettingsPages/PublicSharingSettingsPage";
+import { PythonRunnerSettingsPage } from "./settings/components/SettingsPages/PythonRunnerSettingsPage";
 import { UpdatesSettingsPage } from "./settings/components/SettingsPages/UpdatesSettingsPage";
 import { UploadSettingsPage } from "./settings/components/SettingsPages/UploadSettingsPage";
 
@@ -63,6 +64,7 @@ export const getSettingsRoutes = () => (
     <Route path="maps" component={MapsSettingsPage} />
     <Route path="localization" component={LocalizationSettingsPage} />
     <Route path="uploads" component={UploadSettingsPage} />
+    <Route path="python-runner" component={PythonRunnerSettingsPage} />
     <Route path="public-sharing" component={PublicSharingSettingsPage} />
     <Route
       path="embedding-in-other-applications"

frontend/src/metabase/common/components/CodeEditor/CodeEditor.module.css

@@ -1,8 +0,0 @@
-.codeEditor {
-  :global {
-    .cm-gutter {
-      background-color: var(--mb-color-brand-light);
-      color: var(--mb-color-brand);
-    }
-  }
-}

frontend/src/metabase/common/components/CodeEditor/CodeEditor.tsx

@@ -1,8 +1,7 @@
-import cx from "classnames";
+import type { Extension } from "@uiw/react-codemirror";
 
 import { CodeMirror } from "metabase/common/components/CodeMirror";
 
-import S from "./CodeEditor.module.css";
 import type { CodeLanguage } from "./types";
 import { useExtensions } from "./utils";
 
@@ -15,6 +14,7 @@ type Props = {
   readOnly?: boolean;
   value: string;
   onChange?: (value: string) => void;
+  extensions?: Extension[];
 };
 
 export function CodeEditor({
@@ -26,8 +26,12 @@ export function CodeEditor({
   readOnly,
   value,
   onChange,
+  extensions: externalExtensions,
 }: Props) {
-  const extensions = useExtensions({ language });
+  const extensions = useExtensions({
+    language,
+    extensions: externalExtensions,
+  });
 
   return (
     <CodeMirror
@@ -37,7 +41,7 @@ export function CodeEditor({
         highlightActiveLine: false,
         highlightActiveLineGutter: false,
       }}
-      className={cx(S.codeEditor, className)}
+      className={className}
       extensions={extensions}
       id={id}
       readOnly={readOnly}

frontend/src/metabase/common/components/CodeEditor/utils.ts

@@ -2,7 +2,7 @@ import { html } from "@codemirror/lang-html";
 import { javascript } from "@codemirror/lang-javascript";
 import { json } from "@codemirror/lang-json";
 import { python } from "@codemirror/lang-python";
-import { StreamLanguage } from "@codemirror/language";
+import { StreamLanguage, indentUnit } from "@codemirror/language";
 import { clojure } from "@codemirror/legacy-modes/mode/clojure";
 import { pug } from "@codemirror/legacy-modes/mode/pug";
 import { ruby } from "@codemirror/legacy-modes/mode/ruby";
@@ -12,10 +12,19 @@ import { useMemo } from "react";
 
 import type { CodeLanguage } from "./types";
 
-export function useExtensions({ language }: { language?: CodeLanguage }) {
+export function useExtensions({
+  language,
+  extensions,
+}: {
+  language?: CodeLanguage;
+  extensions?: Extension[];
+}) {
   return useMemo(
-    () => [...(language ? [getLanguageExtension(language)] : [])],
-    [language],
+    () => [
+      ...(extensions ?? []),
+      ...(language ? [getLanguageExtension(language)] : []),
+    ],
+    [language, extensions],
   );
 }
 
@@ -28,7 +37,7 @@ export function getLanguageExtension(language: CodeLanguage): Extension {
     case "json":
       return json();
     case "python":
-      return python();
+      return [python(), indentUnit.of("    ")];
     case "mustache":
       return handlebars;
     case "pug":

modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj

@@ -12,6 +12,7 @@
    [metabase.driver.bigquery-cloud-sdk.query-processor :as bigquery.qp]
    [metabase.driver.common.table-rows-sample :as table-rows-sample]
    [metabase.driver.sql :as driver.sql]
+   [metabase.driver.sql-jdbc :as driver.sql-jdbc]
    [metabase.driver.sql-jdbc.sync.describe-database :as sql-jdbc.describe-database]
    [metabase.driver.sql.query-processor :as sql.qp]
    [metabase.driver.sql.util :as sql.u]
@@ -230,6 +231,30 @@
     "BIGNUMERIC" :type/Decimal
     :type/*))
 
+(defmulti ^:private type->database-type
+  "Internal type->database-type multimethod for BigQuery that dispatches on type."
+  {:arglists '([type])}
+  identity)
+
+(defmethod type->database-type :type/Array [_] [[:raw "ARRAY"]])
+(defmethod type->database-type :type/Boolean [_] [[:raw "BOOL"]])
+(defmethod type->database-type :type/Float [_] [[:raw "FLOAT64"]])
+(defmethod type->database-type :type/Integer [_] [[:raw "INT"]])
+(defmethod type->database-type :type/Number [_] [[:raw "INT"]])
+(defmethod type->database-type :type/Text [_] [[:raw "STRING"]])
+(defmethod type->database-type :type/TextLike [_] [[:raw "STRING"]])
+(defmethod type->database-type :type/Date [_] [[:raw "DATE"]])
+(defmethod type->database-type :type/DateTime [_] [[:raw "DATETIME"]])
+(defmethod type->database-type :type/DateTimeWithTZ [_] [[:raw "TIMESTAMP"]])
+(defmethod type->database-type :type/Time [_] [[:raw "TIME"]])
+(defmethod type->database-type :type/JSON [_] [[:raw "JSON"]])
+(defmethod type->database-type :type/SerializedJSON [_] [[:raw "JSON"]])
+(defmethod type->database-type :type/Decimal [_] [[:raw "BIGDECIMAL"]])
+
+(defmethod driver/type->database-type :bigquery-cloud-sdk
+  [_driver base-type]
+  (type->database-type base-type))
+
 (defn- field->database+base-type
   "Returns a normalized `database-type` and its `base-type` for a type from BigQuery Field type.
 
@@ -836,6 +861,21 @@
   (let [table-str (get-table-str table)]
     [(str "DROP TABLE IF EXISTS " table-str)]))
 
+(defmethod driver/create-table! :bigquery-cloud-sdk
+  [driver database-id table-name column-definitions & {:keys [primary-key]}]
+  (let [sql (#'driver.sql-jdbc/create-table!-sql driver table-name column-definitions :primary-key primary-key)]
+    (driver/execute-raw-queries! driver (t2/select-one :model/Database database-id) [sql])))
+
+(defmethod driver/drop-table! :bigquery-cloud-sdk
+  [driver database-id table-name]
+  (let [sql (driver/compile-drop-table driver table-name)]
+    (driver/execute-raw-queries! driver (t2/select-one :model/Database database-id) [sql])))
+
+(defmethod driver/insert-into! :bigquery-cloud-sdk
+  [driver db-id table-name column-names values]
+  (let [sqls (#'driver.sql-jdbc/insert-into!-sqls driver table-name column-names values true)]
+    (driver/execute-raw-queries! driver (t2/select-one :model/Database db-id) sqls)))
+
 (defmethod driver/execute-raw-queries! :bigquery-cloud-sdk
   [_driver connection-details queries]
   ;; connection-details is either database details directly (from transforms)

modules/drivers/bigquery-cloud-sdk/test/metabase/test/data/bigquery_cloud_sdk.clj

@@ -482,3 +482,11 @@
 
   (execute! "select name from `%s`.metabase_test_tracking.datasets order by accessed_at" (project-id))
   (database-exists?! (tx/get-dataset-definition (data.impl/resolve-dataset-definition *ns* 'test-data))))
+
+(defn ^:private get-test-data-name
+  []
+  (test-dataset-id
+   (tx/get-dataset-definition (or data.impl/*dbdef-used-to-create-db*
+                                  (tx/default-dataset :bigquery-cloud-sdk)))))
+
+(defmethod sql.tx/session-schema :bigquery-cloud-sdk [_driver] (get-test-data-name))

modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj

@@ -212,6 +212,27 @@
     :metabase.upload/datetime                 "Nullable(DateTime64(3))"
     :metabase.upload/offset-datetime          nil))
 
+(defmulti ^:private type->database-type
+  "Internal type->database-type multimethod for ClickHouse that dispatches on type."
+  {:arglists '([type])}
+  identity)
+
+(defmethod type->database-type :type/Boolean [_] ["Nullable(Boolean)"])
+(defmethod type->database-type :type/Float [_] ["Nullable(Float64)"])
+(defmethod type->database-type :type/Integer [_] ["Nullable(Int64)"])
+(defmethod type->database-type :type/Number [_] ["Nullable(Int64)"])
+(defmethod type->database-type :type/Text [_] ["Nullable(String)"])
+(defmethod type->database-type :type/TextLike [_] ["Nullable(String)"])
+(defmethod type->database-type :type/Date [_] ["Nullable(Date32)"])
+(defmethod type->database-type :type/DateTime [_] ["Nullable(DateTime64(3))"])
+(defmethod type->database-type :type/DateTimeWithTZ [_]
+  ;; ?
+  ["Nullable(DateTime64(3))"])
+
+(defmethod driver/type->database-type :clickhouse
+  [_driver base-type]
+  (type->database-type base-type))
+
 (defmethod driver/table-name-length-limit :clickhouse
   [_driver]
   ;; FIXME: This is a lie because you're really limited by a filesystems' limits, because Clickhouse uses

modules/drivers/redshift/src/metabase/driver/redshift.clj

@@ -539,6 +539,25 @@
     :metabase.upload/datetime                 [:timestamp]
     :metabase.upload/offset-datetime          [:timestamp-with-time-zone]))
 
+(defmulti ^:private type->database-type
+  "Internal type->database-type multimethod for Redshift that dispatches on type."
+  {:arglists '([type])}
+  identity)
+
+(defmethod type->database-type :type/TextLike [_] [[:varchar 65535]])
+(defmethod type->database-type :type/Text [_] [[:varchar 65535]])
+(defmethod type->database-type :type/Number [_] [:bigint])
+(defmethod type->database-type :type/Float [_] [(keyword "double precision")])
+(defmethod type->database-type :type/Boolean [_] [:boolean])
+(defmethod type->database-type :type/Date [_] [:date])
+(defmethod type->database-type :type/DateTime [_] [:timestamp])
+(defmethod type->database-type :type/DateTimeWithTZ [_] [:timestamp-with-time-zone])
+(defmethod type->database-type :type/Time [_] [:time])
+
+(defmethod driver/type->database-type :redshift
+  [_driver base-type]
+  (type->database-type base-type))
+
 (defmethod driver/allowed-promotions :redshift [_] {})
 
 (defmethod driver/table-name-length-limit :redshift

modules/drivers/snowflake/src/metabase/driver/snowflake.clj

@@ -262,7 +262,7 @@
     :FLOAT4                     :type/Float
     :FLOAT8                     :type/Float
     :DOUBLE                     :type/Float
-    (keyword "DOUBLE PRECISON") :type/Float
+    (keyword "DOUBLE PRECISION") :type/Float
     :REAL                       :type/Float
     :VARCHAR                    :type/Text
     :CHAR                       :type/Text
@@ -290,6 +290,27 @@
     :OBJECT                     :type/Dictionary
     :ARRAY                      :type/*} base-type))
 
+(defmulti ^:private type->database-type
+  "Internal type->database-type multimethod for Snowflake that dispatches on type."
+  {:arglists '([type])}
+  identity)
+
+(defmethod type->database-type :type/Array [_] [:ARRAY])
+(defmethod type->database-type :type/Boolean [_] [:BOOLEAN])
+(defmethod type->database-type :type/Date [_] [:DATE])
+(defmethod type->database-type :type/DateTime [_] [:DATETIME])
+(defmethod type->database-type :type/DateTimeWithLocalTZ [_] [:TIMESTAMPTZ])
+(defmethod type->database-type :type/DateTimeWithTZ [_] [:TIMESTAMPLTZ])
+(defmethod type->database-type :type/Decimal [_] [:DECIMAL])
+(defmethod type->database-type :type/Float [_] [:DOUBLE])
+(defmethod type->database-type :type/Number [_] [:BIGINT])
+(defmethod type->database-type :type/Text [_] [:TEXT])
+(defmethod type->database-type :type/Time [_] [:TIME])
+
+(defmethod driver/type->database-type :snowflake
+  [_driver base-type]
+  (type->database-type base-type))
+
 (defmethod sql.qp/unix-timestamp->honeysql [:snowflake :seconds]      [_ _ expr] [:to_timestamp_tz expr])
 (defmethod sql.qp/unix-timestamp->honeysql [:snowflake :milliseconds] [_ _ expr] [:to_timestamp_tz expr 3])
 (defmethod sql.qp/unix-timestamp->honeysql [:snowflake :microseconds] [_ _ expr] [:to_timestamp_tz expr 6])

modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj

@@ -119,6 +119,27 @@
     :xml              :type/*
     (keyword "int identity") :type/Integer} column-type)) ; auto-incrementing integer (ie pk) field
 
+(defmulti ^:private type->database-type
+  "Internal type->database-type multimethod for SQL Server that dispatches on type."
+  {:arglists '([type])}
+  identity)
+
+(defmethod type->database-type :type/Boolean [_] [:bit])
+(defmethod type->database-type :type/Date [_] [:date])
+(defmethod type->database-type :type/DateTime [_] [:datetime2])
+(defmethod type->database-type :type/DateTimeWithTZ [_] [:datetimeoffset])
+(defmethod type->database-type :type/Decimal [_] [:decimal])
+(defmethod type->database-type :type/Float [_] [:float])
+(defmethod type->database-type :type/Integer [_] [:int])
+(defmethod type->database-type :type/Number [_] [:bigint])
+(defmethod type->database-type :type/Text [_] [:text])
+(defmethod type->database-type :type/Time [_] [:time])
+(defmethod type->database-type :type/UUID [_] [:uniqueidentifier])
+
+(defmethod driver/type->database-type :sqlserver
+  [_driver base-type]
+  (type->database-type base-type))
+
 (defmethod sql-jdbc.conn/connection-details->spec :sqlserver
   [_ {:keys [user password db host port instance domain ssl]
       :or   {user "dbuser", password "dbpassword", db "", host "localhost"}

src/metabase/analytics/prometheus.clj

@@ -398,7 +398,56 @@
    (prometheus/counter :metabase-gsheets/connection-deleted
                        {:description "How many times the instance has deleted their Google Sheets connection."})
    (prometheus/counter :metabase-gsheets/connection-manually-synced
-                       {:description "How many times the instance has manually sync'ed their Google Sheets connection."})])
+                       {:description "How many times the instance has manually sync'ed their Google Sheets connection."})
+
+   ;; transform metrics
+   (prometheus/counter :metabase-transforms/job-runs-total
+                       {:description "Total number of transform job runs started."
+                        :labels [:job-id :run-method]})
+   (prometheus/counter :metabase-transforms/job-runs-completed
+                       {:description "Number of transform job runs that completed successfully."
+                        :labels [:job-id :run-method]})
+   (prometheus/counter :metabase-transforms/job-runs-failed
+                       {:description "Number of transform job runs that failed."
+                        :labels [:job-id :run-method]})
+   (prometheus/histogram :metabase-transforms/job-run-duration-ms
+                         {:description "Duration in milliseconds of transform job runs."
+                          :labels [:job-id :run-method]
+                          ;; 100ms -> 30 minutes
+                          :buckets [100 500 1000 5000 10000 30000 60000 300000 1800000]})
+   (prometheus/counter :metabase-transforms/stage-started
+                       {:description "Number of transform stages started."
+                        :labels [:job-run-id :stage-type :stage-label]})
+   (prometheus/counter :metabase-transforms/stage-completed
+                       {:description "Number of transform stages completed successfully."
+                        :labels [:job-run-id :stage-type :stage-label]})
+   (prometheus/counter :metabase-transforms/stage-failed
+                       {:description "Number of transform stages that failed."
+                        :labels [:job-run-id :stage-type :stage-label]})
+   (prometheus/histogram :metabase-transforms/stage-duration-ms
+                         {:description "Duration in milliseconds of individual transform stages."
+                          :labels [:job-run-id :stage-type :stage-label]
+                          ;; 10ms -> 10 minutes
+                          :buckets [10 100 500 1000 5000 10000 30000 60000 300000 600000]})
+   (prometheus/histogram :metabase-transforms/data-transfer-bytes
+                         {:description "Size in bytes of data transferred during transform stages."
+                          :labels [:job-run-id :stage-label]
+                          ;; 1KB -> 10GB
+                          :buckets [1000 10000 100000 1000000 10000000 100000000 1000000000 10000000000]})
+   (prometheus/histogram :metabase-transforms/data-transfer-rows
+                         {:description "Number of rows transferred during transform stages."
+                          :labels [:job-run-id :stage-label]
+                          ;; 10 -> 10M rows
+                          :buckets [10 100 1000 10000 100000 1000000 10000000]})
+   ;; Python-transform specific metrics
+   (prometheus/histogram :metabase-transforms/python-api-call-duration-ms
+                         {:description "Duration of Python runner API calls."
+                          :labels [:job-run-id]
+                          ;; 100ms -> 30 minutes
+                          :buckets [100 500 1000 5000 10000 30000 60000 300000 1800000]})
+   (prometheus/counter :metabase-transforms/python-api-calls-total
+                       {:description "Total number of Python runner API calls."
+                        :labels [:job-run-id :status]})])
 
 (defn- quartz-collectors
   []
@@ -575,7 +624,7 @@
   (spit "metrics" (iapetos.export/text-format (:registry system)))
 
   ;; See all metrics that match a given prefix:
-  ; (filter #(.startsWith % "metabase_search_") (clojure.string/split-lines (iapetos.export/text-format (:registry system))))
+                                        ; (filter #(.startsWith % "metabase_search_") (clojure.string/split-lines (iapetos.export/text-format (:registry system))))
 
   ;; need to restart the server to see the metrics? use:
   (shutdown!)

src/metabase/driver.clj

@@ -9,6 +9,8 @@
    [[metabase.driver.sql-jdbc]] for more details."
   #_{:clj-kondo/ignore [:metabase/modules]}
   (:require
+   [clojure.data.csv :as csv]
+   [clojure.java.io :as io]
    [clojure.set :as set]
    [clojure.string :as str]
    [metabase.auth-provider.core :as auth-provider]
@@ -1277,6 +1279,31 @@
   dispatch-on-initialized-driver
   :hierarchy #'hierarchy)
 
+(defmulti insert-from-source!
+  "Inserts data from a data source into an existing table.
+   This abstracts away the conversion and insertion process, allowing drivers
+   to optimize based on the data source type. Returns number of rows inserted. "
+  {:added "0.57.0", :arglists '([driver database-id table-name column-names data-source])}
+  (fn [driver _ _ _ data-source]
+    [(dispatch-on-initialized-driver driver) (:type data-source)])
+  :hierarchy #'hierarchy)
+
+(defmethod insert-from-source! [::driver :rows]
+  [driver db-id table-name column-names {:keys [data]}]
+  (when (seq data)
+    (insert-into! driver db-id table-name column-names data))
+  (count data))
+
+(defmethod insert-from-source! [::driver :csv-file]
+  [driver db-id table-name column-names {:keys [file]}]
+  (let [csv-rows (csv/read-csv (io/reader file))
+        header-rows (first csv-rows)
+        data-rows (map (fn [rows]
+                         (let [m (zipmap header-rows rows)]
+                           (mapv #(get m %) column-names)))
+                       (rest csv-rows))]
+    (insert-from-source! driver db-id table-name column-names {:type :rows :data data-rows})))
+
 (defmulti add-columns!
   "Add columns given by `column-definitions` to a table named `table-name`. If the table doesn't exist it will throw an error.
   `args` is an optional map with an optional key `primary-key`. The `primary-key` value is a vector of column names
@@ -1344,6 +1371,12 @@
   dispatch-on-initialized-driver
   :hierarchy #'hierarchy)
 
+(defmulti type->database-type
+  "Returns the database type for a given Metabase type (from the type hierarchy) as a HoneySQL spec."
+  {:added "0.57.0", :arglists '([driver base-type])}
+  (fn [driver _base-type] driver)
+  :hierarchy #'hierarchy)
+
 (defmulti allowed-promotions
   "Returns a mapping of which types a column can be implicitly relaxed to, based on the content of appended values.
   In the context of uploads, this permits certain appends or replacements of an existing csv table

src/metabase/driver/h2.clj

@@ -606,6 +606,28 @@
     :metabase.upload/datetime                 [:timestamp]
     :metabase.upload/offset-datetime          [:timestamp-with-time-zone]))
 
+(defmulti ^:private type->database-type
+  "Internal type->database-type multimethod for H2 that dispatches on type."
+  {:arglists '([type])}
+  identity)
+
+(defmethod type->database-type :type/TextLike [_] [:varchar])
+(defmethod type->database-type :type/Text [_] [:varchar])
+(defmethod type->database-type :type/Number [_] [:bigint])
+(defmethod type->database-type :type/Float [_] [(keyword "DOUBLE PRECISION")])
+(defmethod type->database-type :type/Decimal [_] [:decimal])
+(defmethod type->database-type :type/Boolean [_] [:boolean])
+(defmethod type->database-type :type/Date [_] [:date])
+(defmethod type->database-type :type/DateTime [_] [:timestamp])
+(defmethod type->database-type :type/DateTimeWithTZ [_] [:timestamp-with-time-zone])
+(defmethod type->database-type :type/Time [_] [:time])
+(defmethod type->database-type :type/TimeWithTZ [_] [:time-with-time-zone])
+(defmethod type->database-type :type/UUID [_] [:uuid])
+
+(defmethod driver/type->database-type :h2
+  [_driver base-type]
+  (type->database-type base-type))
+
 (defmethod driver/create-auto-pk-with-append-csv? :h2 [_driver] true)
 
 (defmethod driver/table-name-length-limit :h2

src/metabase/driver/mysql.clj

@@ -779,6 +779,28 @@
     :metabase.upload/datetime                 [:datetime]
     :metabase.upload/offset-datetime          [:timestamp]))
 
+(defmulti ^:private type->database-type
+  "Internal type->database-type multimethod for MySQL that dispatches on type."
+  {:arglists '([type])}
+  identity)
+
+(defmethod type->database-type :type/TextLike [_] [:text])
+(defmethod type->database-type :type/Text [_] [:text])
+(defmethod type->database-type :type/Number [_] [:bigint])
+(defmethod type->database-type :type/Float [_] [:double])
+(defmethod type->database-type :type/Decimal [_] [:decimal])
+(defmethod type->database-type :type/Boolean [_] [:boolean])
+(defmethod type->database-type :type/Date [_] [:date])
+(defmethod type->database-type :type/DateTime [_] [:datetime])
+(defmethod type->database-type :type/DateTimeWithTZ [_] [:timestamp])
+(defmethod type->database-type :type/Time [_] [:time])
+(defmethod type->database-type :type/JSON [_] [:json])
+(defmethod type->database-type :type/SerializedJSON [_] [:json])
+
+(defmethod driver/type->database-type :mysql
+  [_driver base-type]
+  (type->database-type base-type))
+
 (defmethod driver/allowed-promotions :mysql
   [_driver]
   {:metabase.upload/int     #{:metabase.upload/float}

src/metabase/driver/postgres.clj

@@ -1034,6 +1034,31 @@
                            :type   driver-api/qp.error-type.invalid-query}))
           (throw e))))))
 
+(defmulti ^:private type->database-type
+  "Internal type->database-type multimethod for Postgres that dispatches on type."
+  {:arglists '([type])}
+  identity)
+
+(defmethod type->database-type :type/TextLike [_] [:text])
+(defmethod type->database-type :type/Text [_] [:text])
+(defmethod type->database-type :type/Number [_] [:bigint])
+(defmethod type->database-type :type/Float [_] [:float])
+(defmethod type->database-type :type/Decimal [_] [:float])
+(defmethod type->database-type :type/Boolean [_] [:boolean])
+(defmethod type->database-type :type/Date [_] [:date])
+(defmethod type->database-type :type/DateTime [_] [:timestamp])
+(defmethod type->database-type :type/DateTimeWithTZ [_] [:timestamp-with-time-zone])
+(defmethod type->database-type :type/Time [_] [:time])
+(defmethod type->database-type :type/TimeWithTZ [_] [:time-with-time-zone])
+(defmethod type->database-type :type/UUID [_] [:uuid])
+(defmethod type->database-type :type/JSON [_] [:jsonb])
+(defmethod type->database-type :type/SerializedJSON [_] [:jsonb])
+(defmethod type->database-type :type/IPAddress [_] [:inet])
+
+(defmethod driver/type->database-type :postgres
+  [_driver base-type]
+  (type->database-type base-type))
+
 (defmethod driver/upload-type->database-type :postgres
   [_driver upload-type]
   (case upload-type

src/metabase/driver/sql_jdbc.clj

@@ -151,14 +151,16 @@
 (defmethod driver/create-table! :sql-jdbc
   [driver database-id table-name column-definitions & {:keys [primary-key]}]
   (let [sql (create-table!-sql driver table-name column-definitions :primary-key primary-key)]
-    (driver-api/execute-write-sql! database-id sql)))
+    (jdbc/with-db-transaction [conn (sql-jdbc.conn/db->pooled-connection-spec database-id)]
+      (jdbc/execute! conn sql))))
 
 (defmethod driver/drop-table! :sql-jdbc
   [driver db-id table-name]
   (let [sql (first (sql/format {:drop-table [:if-exists (keyword table-name)]}
                                :quoted true
                                :dialect (sql.qp/quote-style driver)))]
-    (driver-api/execute-write-sql! db-id sql)))
+    (jdbc/with-db-transaction [conn (sql-jdbc.conn/db->pooled-connection-spec db-id)]
+      (jdbc/execute! conn sql))))
 
 (defmethod driver/truncate! :sql-jdbc
   [driver db-id table-name]
@@ -169,8 +171,7 @@
     (jdbc/with-db-transaction [conn (sql-jdbc.conn/db->pooled-connection-spec db-id)]
       (jdbc/execute! conn sql))))
 
-(defmethod driver/insert-into! :sql-jdbc
-  [driver db-id table-name column-names values]
+(defn- insert-into!-sqls [driver table-name column-names values inline?]
   (let [;; We need to partition the insert into multiple statements for both performance and correctness.
         ;;
         ;; On Postgres with a large file, 100 (3.76m) was significantly faster than 50 (4.03m) and 25 (4.27m). 1,000 was a
@@ -184,12 +185,17 @@
         sqls       (map #(sql/format {:insert-into (keyword table-name)
                                       :columns     (quote-columns driver column-names)
                                       :values      %}
+                                     :inline inline?
                                      :quoted true
                                      :dialect dialect)
                         chunks)]
-    (jdbc/with-db-transaction [conn (sql-jdbc.conn/db->pooled-connection-spec db-id)]
-      (doseq [sql sqls]
-        (jdbc/execute! conn sql)))))
+    sqls))
+
+(defmethod driver/insert-into! :sql-jdbc
+  [driver db-id table-name column-names values]
+  (jdbc/with-db-transaction [conn (sql-jdbc.conn/db->pooled-connection-spec db-id)]
+    (doseq [sql (insert-into!-sqls driver table-name column-names values false)]
+      (jdbc/execute! conn sql))))
 
 (defmethod driver/add-columns! :sql-jdbc
   [driver db-id table-name column-definitions & {:keys [primary-key]}]

src/metabase/server/middleware/session.clj

@@ -198,7 +198,7 @@
     (u.password/verify-password passed-api-key "" api-key)
     (do-useless-hash)))
 
-(mu/defn- current-user-info-for-api-key :- [:maybe ::request.schema/current-user-info]
+(mu/defn current-user-info-for-api-key :- [:maybe ::request.schema/current-user-info]
   "Return User ID and superuser status for an API Key with `api-key-id"
   [api-key :- [:maybe :string]]
   (when (and api-key

src/metabase/upload/core.clj

@@ -7,9 +7,11 @@
 
 (p/import-vars
  [metabase.upload.impl
+  create-from-csv-and-sync!
   can-create-upload?
   current-database
   delete-upload!
   model-hydrate-based-on-upload
+  table-identifier
   update-action-schema
   update-csv!])

src/metabase/upload/impl.clj

@@ -527,7 +527,7 @@
          :num-rows          (count (rest rows))
          :generated-columns 0}))))
 
-(defn- create-from-csv-and-sync!
+(defn create-from-csv-and-sync!
   "This is separated from `create-csv-upload!` for testing"
   [{:keys [db filename file schema table-name display-name]}]
   (let [driver            (driver.u/database->driver db)

test/metabase/driver/common_test.clj

@@ -7,7 +7,8 @@
    [metabase.driver.util :as driver.u]
    [metabase.premium-features.core :as premium-features]
    [metabase.settings.core :as setting]
-   [metabase.test :as mt]))
+   [metabase.test :as mt]
+   [metabase.test.data.sql :as sql.tx]))
 
 (deftest ^:parallel base-type-inference-test
   (is (= :type/Text
@@ -88,3 +89,27 @@
                   [2 456.0 0.789 789.0]]
                  (mt/formatted-rows [int double double double]
                                     (mt/run-mbql-query json-decimals-table)))))))))
+
+(deftest insert-from-source!-test
+  (mt/test-drivers (mt/normal-drivers-with-feature :uploads)
+    (let [driver       driver/*driver*
+          db-id        (mt/id)
+          table-name   (mt/random-name)
+          schema-name  (sql.tx/session-schema driver)
+          qualified-table-name (if schema-name
+                                 (keyword schema-name table-name)
+                                 (keyword table-name))
+          column-definitions {"id" [:int], "name" [:text]}]
+      (mt/as-admin
+        (driver/create-table! driver db-id qualified-table-name column-definitions {})
+
+        (testing "insert-from-source! should insert new rows correctly"
+          (let [new-rows     [[2 "New Luke"] [3 "New Leia"]]
+                data-source  {:type :rows :data new-rows}
+                rows-inserted (driver/insert-from-source! driver db-id
+                                                          qualified-table-name
+                                                          (keys column-definitions)
+                                                          data-source)]
+            (is (= (count new-rows) rows-inserted))))
+
+        (driver/drop-table! driver db-id qualified-table-name)))))