support bigints

# Conflicts:
#	enterprise/backend/src/metabase_enterprise/transforms/api.clj
#	enterprise/backend/src/metabase_enterprise/transforms/canceling.clj
#	enterprise/backend/src/metabase_enterprise/transforms/jobs.clj
#	enterprise/backend/src/metabase_enterprise/transforms/ordering.clj
#	enterprise/backend/src/metabase_enterprise/transforms/util.clj
#	enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
#	enterprise/backend/test/metabase_enterprise/transforms/test_util.clj

.clj-kondo/config.edn

@@ -173,6 +173,7 @@
      metabase.driver.sql-jdbc.execute/execute-prepared-statement!
      metabase.notification.models/create-notification!
      metabase.pulse.send/send-pulse!
+     metabase-enterprise.transforms-python.python-runner-test/execute!
      ;; Safe in tests, where we typically use a private, temporary index per thread.
      metabase.search.core/reindex!
      metabase.util/run-count!
@@ -228,7 +229,7 @@
     (clojure.core.logic/fresh)
     (clojure.core.logic/matcha)
     (clojure.core.logic/run)
-    (clojure.test/is [query= =? malli= qv=])
+    (clojure.test/is [=? malli= qv=])
     (clojure.tools.macro/macrolet)
     instaparse.core/transform
     (metabase.server.streaming-response/streaming-response)
@@ -520,6 +521,7 @@
    metabase-enterprise.cache.config-test/with-temp-persist-models                                                            hooks.common/with-seven-bindings
    metabase-enterprise.semantic-search.db.connection/with-migrate-tx                                                         hooks.common/let-one-with-optional-value
    metabase-enterprise.serialization.test-util/with-temp-dpc                                                                 hooks.toucan2.tools.with-temp/with-temp
+   metabase-enterprise.transforms-python.drivers-test/with-test-table                                                        hooks.common/with-two-bindings
    metabase.api.common.internal-test/with-jetty-server                                                                       hooks.common/with-one-binding
    metabase.api.macros/defendpoint                                                                                           hooks.metabase.api.macros/defendpoint
    metabase.app-db.schema-migrations-test.impl/test-migrations                                                               hooks.metabase.app-db.schema-migrations-test.impl/test-migrations
@@ -625,7 +627,7 @@
    metabase.test/with-temp                                                                                                   hooks.toucan2.tools.with-temp/with-temp
    metabase.test/with-temp-file                                                                                              hooks.metabase.test.util/with-temp-file
    metabase.test/with-temporary-setting-values                                                                               hooks.metabase.test.util/with-temporary-setting-values
-   metabase.users.api-test/with-temp-user-email!                                                                              hooks.common/with-one-binding
+   metabase.users.api-test/with-temp-user-email!                                                                             hooks.common/with-one-binding
    metabase.util.i18n/deferred-trs                                                                                           hooks.metabase.util.i18n/deferred-trs
    metabase.util.i18n/deferred-tru                                                                                           hooks.metabase.util.i18n/deferred-tru
    metabase.util.i18n/trs                                                                                                    hooks.metabase.util.i18n/trs
@@ -714,13 +716,18 @@
   {:pattern "^metabase\\.lib\\..*"
    :name    metabase-lib-excluding-legacy-mbql}
   ;;
-  ;; QP source namespaces: `metabase.query-processor.*`, `metabase-enterprise.sandbox.query-processor.*`, and
-  ;; `metabase.driver.*`, excluding `-test` namespaces.
+  ;; QP and driver source namespaces: `metabase.query-processor.*`, `metabase-enterprise.sandbox.query-processor.*`,
+  ;; and `metabase.driver.*`, excluding `-test` namespaces.
   ;;
   {:pattern "metabase(?:(?:(?:-enterprise\\.sandbox)?\\.query-processor\\.)|(?:\\.driver\\.)).*(?<!-test)$"
    :name    qp-and-driver-source-namespaces}
   ;;
-  ;; driver namespaces
+  ;; QP source namespaces
+  ;;
+  {:pattern "metabase(?:(?:-enterprise\\.sandbox)?\\.query-processor\\.).*(?<!-test)$"
+   :name    qp-source-namespaces}
+  ;;
+  ;; driver namespaces, including tests
   ;;
   {:pattern "metabase\\.driver.*$"
    :name    driver-namespaces}
@@ -762,8 +769,8 @@
   source-namespaces
   {:linters
    {:metabase/validate-escaped-single-quotes-in-i18n {:level :warning}
-    :metabase/validate-string-or-str-args-to-i18n {:level :warning}
-    :metabase/discourage-millis-duration {:level :warning}
+    :metabase/validate-string-or-str-args-to-i18n    {:level :warning}
+    :metabase/discourage-millis-duration             {:level :warning}
 
     :discouraged-var
     {clojure.core/with-redefs {:message "Don't use with-redefs outside of tests"}
@@ -782,7 +789,7 @@
                              clojure.string/lower-case {:level :off}
                              clojure.string/upper-case {:level :off}}}}
 
-  ;; enable all the extra linters we haven't enabled in the core app yet in MLv2. It's all new code so we can adhere
+  ;; enable all the extra linters we haven't enabled in the core app yet in Lib. It's all new code so we can adhere
   ;; to the new stricter rules from day 1
   metabase-lib
   {:linters
@@ -802,19 +809,25 @@
      metabase.test/with-temp            {:message "Don't use application database inside Lib code"}}
 
     :discouraged-namespace
-    {metabase.driver            {:message "MLv2 code should be independent of driver code/methods"}
-     metabase.test              {:message "MLv2 tests should only use MLv2 test helpers"}
-     metabase.test.data         {:message "MLv2 tests should only use MLv2 test helpers"}
-     metabase.util.malli.schema {:message "Prefer metabase.lib.schema schemas in MLv2 code"}
-     toucan.db                  {:message "Don't use application database inside MLv2 code"}
-     toucan.models              {:message "Don't use application database inside MLv2 code"}
-     toucan.util.test           {:message "Don't use application database inside MLv2 code"}
-     toucan2.core               {:message "Don't use application database inside MLv2 code"}}}}
+    {metabase.driver            {:message "Lib code should be independent of driver code/methods"}
+     metabase.test              {:message "Lib tests should only use Lib test helpers"}
+     metabase.test.data         {:message "Lib tests should only use Lib test helpers"}
+     metabase.util.malli.schema {:message "Prefer metabase.lib.schema schemas in Lib code"}
+     toucan.db                  {:message "Don't use application database inside Lib code"}
+     toucan.models              {:message "Don't use application database inside Lib code"}
+     toucan.util.test           {:message "Don't use application database inside Lib code"}
+     toucan2.core               {:message "Don't use application database inside Lib code"}}}}
 
   metabase-lib-excluding-legacy-mbql
   {:linters
    {:discouraged-var
-    {metabase.legacy-mbql.util/unique-name-generator {:message "Use metabase.lib.util/unique-name-generator or non-truncating-unique-name-generator"}}}}
+    {metabase.legacy-mbql.util/unique-name-generator {:message "Use metabase.lib.util/unique-name-generator or non-truncating-unique-name-generator"}}}
+
+   :discouraged-namespace
+   {metabase.legacy-mbql.js         {:message "Avoid legacy MBQL utils in Lib; move things into Lib if needded."}
+    metabase.legacy-mbql.jvm-util   {:message "Avoid legacy MBQL utils in Lib; move things into Lib if needded."}
+    metabase.legacy-mbql.predicates {:message "Avoid legacy MBQL utils in Lib; move things into Lib if needded."}
+    metabase.legacy-mbql.util       {:message "Avoid legacy MBQL utils in Lib; move things into Lib if needded."}}}
 
   driver-namespaces
   {:linters
@@ -831,19 +844,31 @@
    {:discouraged-namespace
     {metabase.query-processor.store {:level :off}
      metabase.util.malli.schema     {:message "Prefer metabase.lib.schema or metabase.legacy-mbql.schema schemas in QP/driver code"}
-     toucan.db                      {:message "Don't use application database directly in QP code; use QP Store/metadata provider"}
-     toucan.models                  {:message "Don't use application database directly in QP code; use QP Store/metadata provider"}
-     toucan.util.test               {:message "Don't use application database directly in QP code; use QP Store/metadata provider"}
-     toucan2.core                   {:message "Don't use application database directly in QP code; use QP Store/metadata provider"}}
+     toucan.db                      {:message "Don't use application database directly in QP code; use the MetadataProvider"}
+     toucan.models                  {:message "Don't use application database directly in QP code; use the MetadataProvider"}
+     toucan.util.test               {:message "Don't use application database directly in QP code; use the MetadataProvider"}
+     toucan2.core                   {:message "Don't use application database directly in QP code; use the MetadataProvider"}}
 
     :discouraged-var
     {medley.core/map-keys                 {:message "Use metabase.util.performance/update-keys"}
      medley.core/map-vals                 {:message "Use clojure.core/update-vals"}
-     metabase.app-db.core/query           {:message "Don't use application database directly in QP code; use QP Store/metadata provider"}
-     metabase.app-db.core/reducible-query {:message "Don't use application database directly in QP code; use QP Store/metadata provider"}
+     metabase.app-db.core/query           {:message "Don't use application database directly in QP code; use the MetadataProvider"}
+     metabase.app-db.core/reducible-query {:message "Don't use application database directly in QP code; use the MetadataProvider"}
      metabase.lib.convert/->legacy-MBQL   {:level :off}
      metabase.lib.core/->legacy-MBQL      {:level :off}}}}
 
+  qp-source-namespaces
+  {:linters
+   {:discouraged-namespace
+    {metabase.legacy-mbql.js             {:message "Avoid legacy MBQL namespaces in the QP, use Lib instead"}
+     metabase.legacy-mbql.jvm-util       {:message "Avoid legacy MBQL namespaces in the QP, use Lib instead"}
+     metabase.legacy-mbql.normalize      {:message "Avoid legacy MBQL namespaces in the QP, use Lib instead"}
+     metabase.legacy-mbql.predicates     {:message "Avoid legacy MBQL namespaces in the QP, use Lib instead"}
+     metabase.legacy-mbql.schema         {:message "Avoid legacy MBQL namespaces in the QP, use Lib instead"}
+     metabase.legacy-mbql.schema.helpers {:message "Avoid legacy MBQL namespaces in the QP, use Lib instead"}
+     metabase.legacy-mbql.schema.macros  {:message "Avoid legacy MBQL namespaces in the QP, use Lib instead"}
+     metabase.legacy-mbql.util           {:message "Avoid legacy MBQL namespaces in the QP, use Lib instead"}}}}
+
   general-qp-and-driver-test-namespaces
   {:linters
    {:discouraged-var

.clj-kondo/config/modules/config.edn

@@ -1625,6 +1625,17 @@
            util
            enterprise/harbormaster}}
 
+  enterprise/comments
+  {:team "Workflows"
+   :api  #{metabase-enterprise.comments.api
+           metabase-enterprise.comments.init}
+   :uses #{api
+           events
+           models
+           request
+           users
+           util}}
+
   enterprise/content-translation
   {:team "UX West"
    :api  #{metabase-enterprise.content-translation.dictionary
@@ -1723,6 +1734,17 @@
            settings
            util}}
 
+  enterprise/embedding-hub
+  {:team "Embedding"
+   :api  #{metabase-enterprise.embedding-hub.api}
+   :uses #{api
+           appearance
+           audit-app
+           embedding
+           premium-features
+           util
+           enterprise/sso}}
+
   enterprise/gsheets
   {:team "Cloud"
    :api  #{metabase-enterprise.gsheets.api
@@ -1960,8 +1982,8 @@
    :api  #{metabase-enterprise.transforms.api
            metabase-enterprise.transforms.core
            metabase-enterprise.transforms.init
-           ;; TODO: Remove this from API once we remove the python-runner callback to metabase
-           metabase-enterprise.transforms.settings}
+           metabase-enterprise.transforms.instrumentation
+           metabase-enterprise.transforms.util}
    :uses #{analytics
            api
            app-db
@@ -1979,18 +2001,25 @@
            sync
            task
            task-history
-           ;; TODO: Remove this once we're using a better file format than CSV
            upload
-           util}}
+           util
+           enterprise/transforms-python}}
 
-  enterprise/python-transform
-  {:team "Transform"
-   :api  #{metabase-enterprise.python-transform.api
-           metabase-enterprise.python-transform.core
-           metabase-enterprise.python-transform.init}
+  enterprise/transforms-python
+  {:team "Querying"
+   :api  #{metabase-enterprise.transforms-python.api
+           metabase-enterprise.transforms-python.execute
+           metabase-enterprise.transforms-python.init
+           metabase-enterprise.transforms-python.settings}
    :uses #{api
+           app-db
+           config
+           driver
            models
-           util}}
+           query-processor
+           settings
+           util
+           enterprise/transforms}}
 
   enterprise/upload-management
   {:team "UX West"

.github/scripts/setup-python-runner.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+set -e
+
+# Parse command line arguments
+SKIP_START=false
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --skip-start)
+      SKIP_START=true
+      shift
+      ;;
+    *)
+      echo "Unknown option: $1"
+      echo "Usage: $0 [--skip-start]"
+      exit 1
+      ;;
+  esac
+done
+
+echo "Cloning python-runner repository..."
+git clone --depth 1 https://${METABASE_AUTOMATION_USER_TOKEN}@github.com/metabase/python-runner-container.git python-runner
+
+if [ "$SKIP_START" = false ]; then
+  echo "Starting python-runner..."
+  cd python-runner
+  make run-ci
+  cd ..
+fi

.github/workflows/app-db.yml

@@ -35,21 +35,21 @@ jobs:
             build-static-viz: false
             test-args: >-
               :only '"enterprise/backend/test"'
-            exclude-tag: ':mb/driver-tests'
+            exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
           - name: EE App DB Tests (Part 1)
             build-static-viz: true
             test-args: >-
               :only '"test"'
               :partition/total 2
               :partition/index 0
-            exclude-tag: ':mb/driver-tests'
+            exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
           - name: EE App DB Tests (Part 2)
             build-static-viz: true
             test-args: >-
               :only '"test"'
               :partition/total 2
               :partition/index 1
-            exclude-tag: ':mb/driver-tests'
+            exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
 
     services:
       mysql:
@@ -147,21 +147,21 @@ jobs:
             build-static-viz: false
             test-args: >-
               :only '"enterprise/backend/test"'
-            exclude-tag: ':mb/driver-tests'
+            exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
           - name: EE App DB Tests (Part 1)
             build-static-viz: true
             test-args: >-
               :only '"test"'
               :partition/total 2
               :partition/index 0
-            exclude-tag: ':mb/driver-tests'
+            exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
           - name: EE App DB Tests (Part 2)
             build-static-viz: true
             test-args: >-
               :only '"test"'
               :partition/total 2
               :partition/index 1
-            exclude-tag: ':mb/driver-tests'
+            exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
 
     services:
       mysql:
@@ -259,11 +259,13 @@ jobs:
             build-static-viz: false
             test-args: >-
               :only-tags [:mb/driver-tests]
+              :exclude-tags [:mb/transforms-python-test]
+            exclude-tag: ':mb/transforms-python-test'
           - name: Enterprise Tests
             build-static-viz: false
             test-args: >-
               :only '"enterprise/backend/test"'
-            exclude-tag: ':mb/driver-tests'
+            exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
           - name: EE App DB Tests (Part 1)
             build-static-viz: true
             test-args: >-
@@ -271,14 +273,14 @@ jobs:
               :exclude-tags [:mb/driver-tests]
               :partition/total 2
               :partition/index 0
-            exclude-tag: ':mb/driver-tests'
+            exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
           - name: EE App DB Tests (Part 2)
             build-static-viz: true
             test-args: >-
               :only '"test"'
               :partition/total 2
               :partition/index 1
-            exclude-tag: ':mb/driver-tests'
+            exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
     name: "${{ matrix.version.name }} ${{ matrix.job.name }}"
     env:
       # Only run old migrations tests on pushes to master or release branches. All other branches should skip tests
@@ -328,6 +330,10 @@ jobs:
           test-args: >-
             ${{ matrix.job.test-args }}
             :exclude-tags '[ ${{ matrix.job.exclude-tag }} ${{ env.__ADDITIONAL_EXCLUDED_TAG__ }}]'
+      - name: Setup python-runner
+        env:
+          METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+        run: ./.github/scripts/setup-python-runner.sh
       - name: Upload Test Results
         uses: ./.github/actions/upload-test-results
         if: always()

.github/workflows/drivers.yml

@@ -90,6 +90,7 @@ jobs:
         junit-name: 'be-tests-athena-ee'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -106,6 +107,18 @@ jobs:
     if: ${{ !inputs.skip }}
     runs-on: ubuntu-22.04
     timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        job:
+          - name: Driver Tests
+            test-args: >-
+              :only-tags [:mb/driver-tests]
+              :exclude-tags [:mb/transforms-python-test]
+          - name: Transforms Python Tests
+            test-args: >-
+              :only-tags [:mb/transforms-python-test]
+            needs-python-runner: true
     env:
       CI: 'true'
       DRIVERS: bigquery-cloud-sdk
@@ -116,15 +129,19 @@ jobs:
       MB_BIGQUERY_TEST_REFRESH_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_REFRESH_TOKEN }}
       MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON }}
       MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING }}
-    name: BigQuery
+    name: BigQuery ${{ matrix.job.name }}
     steps:
     - uses: actions/checkout@v4
+    - name: Setup python-runner
+      if: ${{ matrix.job.needs-python-runner == true }}
+      env:
+        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      run: ./.github/scripts/setup-python-runner.sh
     - name: Test BigQuery Cloud SDK driver
       uses: ./.github/actions/test-driver
       with:
         junit-name: 'be-tests-bigquery-cloud-sdk-ee'
-        test-args: >-
-          :only-tags [:mb/driver-tests]
+        test-args: ${{ matrix.job.test-args }}
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -141,6 +158,18 @@ jobs:
     if: ${{ !inputs.skip }}
     runs-on: ubuntu-22.04
     timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        job:
+          - name: Driver Tests
+            test-args: >-
+              :only-tags [:mb/driver-tests]
+              :exclude-tags [:mb/transforms-python-test]
+          - name: Transforms Python Tests
+            test-args: >-
+              :only-tags [:mb/transforms-python-test]
+            needs-python-runner: true
     env:
       CI: 'true'
       DRIVERS: clickhouse
@@ -148,9 +177,14 @@ jobs:
       MB_CLICKHOUSE_TEST_PORT: 8123
       MB_CLICKHOUSE_TEST_OLD_PORT: 8124
       MB_CLICKHOUSE_TEST_NGINX_PORT: 8127
-    name: Clickhouse
+    name: Clickhouse ${{ matrix.job.name }}
     steps:
     - uses: actions/checkout@v4
+    - name: Setup python-runner
+      if: ${{ matrix.job.needs-python-runner == true }}
+      env:
+        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      run: ./.github/scripts/setup-python-runner.sh
     - name: Add ClickHouse TLS instance to /etc/hosts
       run: |
         sudo echo "127.0.0.1 server.clickhouseconnect.test" | sudo tee -a /etc/hosts
@@ -170,8 +204,7 @@ jobs:
       uses: ./.github/actions/test-driver
       with:
         junit-name: 'be-tests-clickhouse-ee'
-        test-args: >-
-          :only-tags [:mb/driver-tests]
+        test-args: ${{ matrix.job.test-args }}
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -208,6 +241,7 @@ jobs:
         junit-name: 'be-tests-druid-ee'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -244,6 +278,7 @@ jobs:
         junit-name: 'be-tests-druid-jdbc-ee'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -280,6 +315,7 @@ jobs:
         junit-name: 'be-tests-databricks-ee'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -317,6 +353,7 @@ jobs:
         junit-name: 'be-tests-databricks-multi-catalog-ee'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -338,18 +375,17 @@ jobs:
     name: H2
     steps:
     - uses: actions/checkout@v4
-      with:
-        submodules: true
-        token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
-    - name: Start python-runner
-      working-directory: python-runner
-      run: make run
+    - name: Setup python-runner
+      env:
+        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      run: ./.github/scripts/setup-python-runner.sh
     - name: Test H2 driver
       uses: ./.github/actions/test-driver
       with:
         junit-name: 'be-tests-h2'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -394,6 +430,11 @@ jobs:
           - name: Driver Tests
             test-args: >-
               :only-tags [:mb/driver-tests]
+              :exclude-tags [:mb/transforms-python-test]
+          - name: Transforms Python Tests
+            test-args: >-
+              :only-tags [:mb/transforms-python-test]
+            needs-python-runner: true
 
     services:
       mysql:
@@ -448,12 +489,11 @@ jobs:
     name: "${{ matrix.version.name }} ${{ matrix.job.name }}"
     steps:
       - uses: actions/checkout@v4
-        with:
-          submodules: true
-          token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
-      - name: Start python-runner
-        working-directory: python-runner
-        run: make run
+      - name: Setup python-runner
+      #if: ${{ matrix.job.needs-python-runner == true }}
+        env:
+          METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+        run: ./.github/scripts/setup-python-runner.sh
       - name: Test ${{ matrix.version.name }}
         uses: ./.github/actions/test-driver
         with:
@@ -488,6 +528,15 @@ jobs:
           - name: MongoDB Latest
             junit-name: be-tests-mongo-latest-ee
             image: mongo:latest
+        job:
+          - name: Driver Tests
+            test-args: >-
+              :exclude-tags '[:mongo-sharded-cluster-tests :mb/transforms-python-test]'
+              :only-tags [:mb/driver-tests]
+          - name: Transforms Python Tests
+            test-args: >-
+              :only-tags [:mb/transforms-python-test]
+            needs-python-runner: true
     env:
       CI: 'true'
       DRIVERS: mongo
@@ -501,16 +550,19 @@ jobs:
         env:
           MONGO_INITDB_ROOT_USERNAME: metabase
           MONGO_INITDB_ROOT_PASSWORD: metasample123
-    name: ${{ matrix.version.name }}
+    name: ${{ matrix.version.name }} ${{ matrix.job.name }}
     steps:
     - uses: actions/checkout@v4
+    - name: Setup python-runner
+      if: ${{ matrix.job.needs-python-runner == true }}
+      env:
+        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      run: ./.github/scripts/setup-python-runner.sh
     - name: Test ${{ matrix.version.name }}
       uses: ./.github/actions/test-driver
       with:
         junit-name: ${{ matrix.version.junit-name }}
-        test-args: >-
-          :exclude-tags [:mongo-sharded-cluster-tests]
-          :only-tags [:mb/driver-tests]
+        test-args: ${{ matrix.job.test-args }}
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -574,7 +626,7 @@ jobs:
         with:
           junit-name: ${{ matrix.version.junit-name }}
           test-args: >-
-            :exclude-tags [:mongo-sharded-cluster-tests]
+            :exclude-tags '[:mongo-sharded-cluster-tests :mb/transforms-python-test]'
             :only-tags [:mb/driver-tests]
       - name: Upload Test Results
         uses: ./.github/actions/upload-test-results
@@ -682,6 +734,7 @@ jobs:
         junit-name: ${{ matrix.version.junit-name }}
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -716,6 +769,11 @@ jobs:
           - name: Driver Tests
             test-args: >-
               :only-tags [:mb/driver-tests]
+              :exclude-tags [:mb/transforms-python-test]
+          - name: Transforms Python Tests
+            test-args: >-
+              :only-tags [:mb/transforms-python-test]
+            needs-python-runner: true
     name: "${{ matrix.version.name }} ${{ matrix.job.name }}"
     env:
       # Only run old migrations tests on pushes to master or release branches. All other branches should skip tests
@@ -757,12 +815,11 @@ jobs:
           POSTGRES_HOST_AUTH_METHOD: trust
     steps:
       - uses: actions/checkout@v4
-        with:
-          submodules: true
-          token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
-      - name: Start python-runner
-        working-directory: python-runner
-        run: make run
+      - name: Setup python-runner
+      #if: ${{ matrix.job.needs-python-runner == true }}
+        env:
+          METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+        run: ./.github/scripts/setup-python-runner.sh
       - name: Test ${{ matrix.version.name }} (${{ matrix.job.name }})
         uses: ./.github/actions/test-driver
         with:
@@ -839,6 +896,7 @@ jobs:
         junit-name: 'be-tests-presto-jdbc-ee'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -868,8 +926,14 @@ jobs:
             skip: false
             test-args: >-
               :only-tags [:mb/driver-tests]
-              :exclude-tags [:mb/upload-tests]
+              :exclude-tags '[:mb/upload-tests :mb/transforms-python-test]'
               :fail-fast? true
+          - name: Redshift Transforms Python Tests
+            skip: false
+            test-args: >-
+              :only-tags [:mb/transforms-python-test]
+              :fail-fast? true
+            needs-python-runner: true
 
     env:
       CI: 'true'
@@ -882,6 +946,11 @@ jobs:
     name: ${{ matrix.job.name }}
     steps:
     - uses: actions/checkout@v4
+    - name: Setup python-runner
+      if: ${{ matrix.job.needs-python-runner == true }}
+      env:
+        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      run: ./.github/scripts/setup-python-runner.sh
     - name: Test ${{ matrix.job.name }}
       if: ${{ !matrix.job.skip }}
       uses: ./.github/actions/test-driver
@@ -905,6 +974,18 @@ jobs:
     if: ${{ !inputs.skip }}
     runs-on: ubuntu-22.04
     timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        job:
+          - name: Driver Tests
+            test-args: >-
+              :only-tags [:mb/driver-tests]
+              :exclude-tags [:mb/transforms-python-test]
+          - name: Transforms Python Tests
+            test-args: >-
+              :only-tags [:mb/transforms-python-test]
+            needs-python-runner: true
     env:
       CI: 'true'
       DRIVERS: snowflake
@@ -923,15 +1004,19 @@ jobs:
       MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_CUSTOM_USER: RSA_ROLE_TEST_CUSTOM_USER
       MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_ROLE: RSA_ROLE_TEST_ROLE
       MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_DB: RSA_ROLE_TEST_DB
-    name: Snowflake
+    name: Snowflake ${{ matrix.job.name }}
     steps:
     - uses: actions/checkout@v4
+    - name: Setup python-runner
+      if: ${{ matrix.job.needs-python-runner == true }}
+      env:
+        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      run: ./.github/scripts/setup-python-runner.sh
     - name: Test Snowflake driver
       uses: ./.github/actions/test-driver
       with:
         junit-name: 'be-tests-snowflake-ee'
-        test-args: >-
-          :only-tags [:mb/driver-tests]
+        test-args: ${{ matrix.job.test-args }}
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -966,6 +1051,7 @@ jobs:
         junit-name: 'be-tests-sparksql-ee'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -995,6 +1081,7 @@ jobs:
         junit-name: 'be-tests-sqlite-ee'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -1022,6 +1109,15 @@ jobs:
           - name: SQL Server 2022
             junit-name: be-tests-sqlserver-2022-ee
             image: mcr.microsoft.com/mssql/server:2022-latest
+        job:
+          - name: Driver Tests
+            test-args: >-
+              :only-tags [:mb/driver-tests]
+              :exclude-tags [:mb/transforms-python-test]
+          - name: Transforms Python Tests
+            test-args: >-
+              :only-tags [:mb/transforms-python-test]
+            needs-python-runner: true
     env:
       CI: 'true'
       DRIVERS: sqlserver
@@ -1037,15 +1133,19 @@ jobs:
           ACCEPT_EULA: Y
           SA_PASSWORD: 'P@ssw0rd'
           MSSQL_MEMORY_LIMIT_MB: 1024
-    name: ${{ matrix.version.name }}
+    name: ${{ matrix.version.name }} ${{ matrix.job.name }}
     steps:
     - uses: actions/checkout@v4
+    - name: Setup python-runner
+      if: ${{ matrix.job.needs-python-runner == true }}
+      env:
+        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      run: ./.github/scripts/setup-python-runner.sh
     - name: Test ${{ matrix.version.name }}
       uses: ./.github/actions/test-driver
       with:
         junit-name: ${{ matrix.version.junit-name }}
-        test-args: >-
-          :only-tags [:mb/driver-tests]
+        test-args: ${{ matrix.job.test-args }}
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()

.github/workflows/e2e-test.yml

@@ -60,6 +60,14 @@ jobs:
       CYPRESS_IS_EMBEDDING_SDK: "${{ inputs.name == 'embedding-sdk' }}"
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
+          token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+
+      - name: Setup python-runner
+        env:
+          METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+        run: ./.github/scripts/setup-python-runner.sh
 
       - name: Prepare Docker containers
         uses: ./.github/actions/e2e-prepare-containers
@@ -143,7 +151,7 @@ jobs:
         run: |
           echo "Running tests with specs: $SPEC_PATH"
           node e2e/runner/run_cypress_ci.js e2e \
-          --env grepTags="${{inputs.tags}}+-@skip",grepOmitFiltered=true \
+          --env grepTags="${{inputs.tags && format('{0}+-@skip', inputs.tags) || '-@skip'}}",grepOmitFiltered=true \
           --spec "$SPEC_PATH" \
           --browser ${{ steps.cypress-prep.outputs.chrome-path }}