Optimized 1 images, saved 55.5 KB (35.2% smaller)

I think this was just a mistake - there's no need to URL-encode this and
it actually breaks the redirect.

.clj-kondo/config.edn

@@ -173,6 +173,7 @@
      metabase.driver.sql-jdbc.execute/execute-prepared-statement!
      metabase.notification.models/create-notification!
      metabase.pulse.send/send-pulse!
+     metabase-enterprise.transforms-python.python-runner-test/execute!
      ;; Safe in tests, where we typically use a private, temporary index per thread.
      metabase.search.core/reindex!
      metabase.util/run-count!
@@ -418,6 +419,7 @@
   metabase-enterprise.transforms.api-test/with-transform-tags                           clojure.core/let
   metabase-enterprise.transforms.api-test/with-worker-runs                              clojure.core/let
   metabase-enterprise.transforms.test-util/with-transform-cleanup!                      clojure.core/let
+  metabase-enterprise.transforms.execute/with-transform-lifecycle                       clojure.core/let
   metabase.actions.test-util/with-actions                                               clojure.core/let
   metabase.api.common/let-404                                                           clojure.core/let
   metabase.app-db.custom-migrations.util/with-temp-scheduler!                           clojure.core/fn
@@ -519,6 +521,7 @@
    metabase-enterprise.cache.config-test/with-temp-persist-models                                                            hooks.common/with-seven-bindings
    metabase-enterprise.semantic-search.db.connection/with-migrate-tx                                                         hooks.common/let-one-with-optional-value
    metabase-enterprise.serialization.test-util/with-temp-dpc                                                                 hooks.toucan2.tools.with-temp/with-temp
+   metabase-enterprise.transforms-python.drivers-test/with-test-table                                                        hooks.common/with-two-bindings
    metabase.api.common.internal-test/with-jetty-server                                                                       hooks.common/with-one-binding
    metabase.api.macros/defendpoint                                                                                           hooks.metabase.api.macros/defendpoint
    metabase.app-db.schema-migrations-test.impl/test-migrations                                                               hooks.metabase.app-db.schema-migrations-test.impl/test-migrations

.clj-kondo/config/modules/config.edn

@@ -894,6 +894,19 @@
            enterprise/advanced-permissions
            enterprise/sandbox}}
 
+  python-runner
+  {:team "Workflows"
+   :api  #{metabase.python-runner.api
+           metabase.python-runner.core
+           metabase.python-runner.init}
+   :uses #{api
+           driver
+           models
+           query-processor
+           request
+           util
+           enterprise/transforms}}
+
   queries
   {:team "Querying"
    :api  #{metabase.queries.api
@@ -939,6 +952,7 @@
    :uses #{api
            legacy-mbql
            lib
+           lib-be
            models
            permissions
            query-processor
@@ -1227,6 +1241,7 @@
            events
            lib
            models
+           premium-features
            query-processor
            settings
            task
@@ -1472,6 +1487,7 @@
            driver
            legacy-mbql
            lib
+           lib-be
            models
            queries
            query-permissions
@@ -1982,6 +1998,7 @@
    :api  #{metabase-enterprise.transforms.api
            metabase-enterprise.transforms.core
            metabase-enterprise.transforms.init
+           metabase-enterprise.transforms.instrumentation
            metabase-enterprise.transforms.util}
    :uses #{analytics
            api
@@ -2007,6 +2024,7 @@
   enterprise/transforms-python
   {:team "Querying"
    :api  #{metabase-enterprise.transforms-python.api
+           metabase-enterprise.transforms-python.execute
            metabase-enterprise.transforms-python.init
            metabase-enterprise.transforms-python.settings}
    :uses #{api

.github/PULL_REQUEST_TEMPLATE.md

@@ -3,7 +3,7 @@
 
 > **Warning**
 >
-> If that is your first contribution to Metabase, please sign the [Contributor License Agreement](https://docs.google.com/a/metabase.com/forms/d/1oV38o7b9ONFSwuzwmERRMi9SYrhYeOrkbmNaq9pOJ_E/viewform) (unless it's a tiny documentation change). Also, if you're attempting to fix a translation issue, please submit your changes to our [Crowdin project](https://crowdin.com/project/metabase-i18n) instead of opening a PR.
+> If that is your first contribution to Metabase, please sign the [Contributor License Agreement](https://docs.google.com/a/metabase.com/forms/d/1oV38o7b9ONFSwuzwmERRMi9SYrhYeOrkbmNaq9pOJ_E/viewform). Also, if you're attempting to fix a translation issue, please submit your changes to our [Crowdin project](https://crowdin.com/project/metabase-i18n) instead of opening a PR.
 
 Closes https://github.com/metabase/metabase/issues/[issue_number]
 

.github/scripts/setup-python-runner.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+set -e
+
+# Parse command line arguments
+SKIP_START=false
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --skip-start)
+      SKIP_START=true
+      shift
+      ;;
+    *)
+      echo "Unknown option: $1"
+      echo "Usage: $0 [--skip-start]"
+      exit 1
+      ;;
+  esac
+done
+
+echo "Cloning python-runner repository..."
+git clone --depth 1 https://${METABASE_AUTOMATION_USER_TOKEN}@github.com/metabase/python-runner-container.git python-runner
+
+if [ "$SKIP_START" = false ]; then
+  echo "Starting python-runner..."
+  cd python-runner
+  make run-ci
+  cd ..
+fi

.github/workflows/app-db.yml

@@ -35,21 +35,21 @@ jobs:
             build-static-viz: false
             test-args: >-
               :only '"enterprise/backend/test"'
-            exclude-tag: ':mb/driver-tests'
+            exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
           - name: EE App DB Tests (Part 1)
             build-static-viz: true
             test-args: >-
               :only '"test"'
               :partition/total 2
               :partition/index 0
-            exclude-tag: ':mb/driver-tests'
+            exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
           - name: EE App DB Tests (Part 2)
             build-static-viz: true
             test-args: >-
               :only '"test"'
               :partition/total 2
               :partition/index 1
-            exclude-tag: ':mb/driver-tests'
+            exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
 
     services:
       mysql:
@@ -147,21 +147,21 @@ jobs:
             build-static-viz: false
             test-args: >-
               :only '"enterprise/backend/test"'
-            exclude-tag: ':mb/driver-tests'
+            exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
           - name: EE App DB Tests (Part 1)
             build-static-viz: true
             test-args: >-
               :only '"test"'
               :partition/total 2
               :partition/index 0
-            exclude-tag: ':mb/driver-tests'
+            exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
           - name: EE App DB Tests (Part 2)
             build-static-viz: true
             test-args: >-
               :only '"test"'
               :partition/total 2
               :partition/index 1
-            exclude-tag: ':mb/driver-tests'
+            exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
 
     services:
       mysql:
@@ -259,11 +259,13 @@ jobs:
             build-static-viz: false
             test-args: >-
               :only-tags [:mb/driver-tests]
+              :exclude-tags [:mb/transforms-python-test]
+            exclude-tag: ':mb/transforms-python-test'
           - name: Enterprise Tests
             build-static-viz: false
             test-args: >-
               :only '"enterprise/backend/test"'
-            exclude-tag: ':mb/driver-tests'
+            exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
           - name: EE App DB Tests (Part 1)
             build-static-viz: true
             test-args: >-
@@ -271,14 +273,14 @@ jobs:
               :exclude-tags [:mb/driver-tests]
               :partition/total 2
               :partition/index 0
-            exclude-tag: ':mb/driver-tests'
+            exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
           - name: EE App DB Tests (Part 2)
             build-static-viz: true
             test-args: >-
               :only '"test"'
               :partition/total 2
               :partition/index 1
-            exclude-tag: ':mb/driver-tests'
+            exclude-tag: ':mb/driver-tests :mb/transforms-python-test'
     name: "${{ matrix.version.name }} ${{ matrix.job.name }}"
     env:
       # Only run old migrations tests on pushes to master or release branches. All other branches should skip tests

.github/workflows/drivers.yml

@@ -90,6 +90,7 @@ jobs:
         junit-name: 'be-tests-athena-ee'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -106,6 +107,19 @@ jobs:
     if: ${{ !inputs.skip }}
     runs-on: ubuntu-22.04
     timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        job:
+          - name: Driver Tests
+            test-args: >-
+              :only-tags [:mb/driver-tests]
+              :exclude-tags [:mb/transforms-python-test]
+# DS 2025-09-26 temporarily disabled while we diagnose flakes
+#          - name: Transforms Python Tests
+#            test-args: >-
+#              :only-tags [:mb/transforms-python-test]
+#            needs-python-runner: true
     env:
       CI: 'true'
       DRIVERS: bigquery-cloud-sdk
@@ -116,15 +130,19 @@ jobs:
       MB_BIGQUERY_TEST_REFRESH_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_REFRESH_TOKEN }}
       MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON }}
       MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING }}
-    name: BigQuery
+    name: BigQuery ${{ matrix.job.name }}
     steps:
     - uses: actions/checkout@v4
+    - name: Setup python-runner
+      if: ${{ matrix.job.needs-python-runner == true }}
+      env:
+        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      run: ./.github/scripts/setup-python-runner.sh
     - name: Test BigQuery Cloud SDK driver
       uses: ./.github/actions/test-driver
       with:
         junit-name: 'be-tests-bigquery-cloud-sdk-ee'
-        test-args: >-
-          :only-tags [:mb/driver-tests]
+        test-args: ${{ matrix.job.test-args }}
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -141,6 +159,18 @@ jobs:
     if: ${{ !inputs.skip }}
     runs-on: ubuntu-22.04
     timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        job:
+          - name: Driver Tests
+            test-args: >-
+              :only-tags [:mb/driver-tests]
+              :exclude-tags [:mb/transforms-python-test]
+          - name: Transforms Python Tests
+            test-args: >-
+              :only-tags [:mb/transforms-python-test]
+            needs-python-runner: true
     env:
       CI: 'true'
       DRIVERS: clickhouse
@@ -148,9 +178,14 @@ jobs:
       MB_CLICKHOUSE_TEST_PORT: 8123
       MB_CLICKHOUSE_TEST_OLD_PORT: 8124
       MB_CLICKHOUSE_TEST_NGINX_PORT: 8127
-    name: Clickhouse
+    name: Clickhouse ${{ matrix.job.name }}
     steps:
     - uses: actions/checkout@v4
+    - name: Setup python-runner
+      if: ${{ matrix.job.needs-python-runner == true }}
+      env:
+        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      run: ./.github/scripts/setup-python-runner.sh
     - name: Add ClickHouse TLS instance to /etc/hosts
       run: |
         sudo echo "127.0.0.1 server.clickhouseconnect.test" | sudo tee -a /etc/hosts
@@ -170,8 +205,7 @@ jobs:
       uses: ./.github/actions/test-driver
       with:
         junit-name: 'be-tests-clickhouse-ee'
-        test-args: >-
-          :only-tags [:mb/driver-tests]
+        test-args: ${{ matrix.job.test-args }}
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -208,6 +242,7 @@ jobs:
         junit-name: 'be-tests-druid-ee'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -244,6 +279,7 @@ jobs:
         junit-name: 'be-tests-druid-jdbc-ee'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -280,6 +316,7 @@ jobs:
         junit-name: 'be-tests-databricks-ee'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -317,6 +354,7 @@ jobs:
         junit-name: 'be-tests-databricks-multi-catalog-ee'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -338,12 +376,17 @@ jobs:
     name: H2
     steps:
     - uses: actions/checkout@v4
+    - name: Setup python-runner
+      env:
+        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      run: ./.github/scripts/setup-python-runner.sh
     - name: Test H2 driver
       uses: ./.github/actions/test-driver
       with:
         junit-name: 'be-tests-h2'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -388,6 +431,11 @@ jobs:
           - name: Driver Tests
             test-args: >-
               :only-tags [:mb/driver-tests]
+              :exclude-tags [:mb/transforms-python-test]
+          - name: Transforms Python Tests
+            test-args: >-
+              :only-tags [:mb/transforms-python-test]
+            needs-python-runner: true
 
     services:
       mysql:
@@ -442,6 +490,11 @@ jobs:
     name: "${{ matrix.version.name }} ${{ matrix.job.name }}"
     steps:
       - uses: actions/checkout@v4
+      - name: Setup python-runner
+      #if: ${{ matrix.job.needs-python-runner == true }}
+        env:
+          METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+        run: ./.github/scripts/setup-python-runner.sh
       - name: Test ${{ matrix.version.name }}
         uses: ./.github/actions/test-driver
         with:
@@ -476,6 +529,15 @@ jobs:
           - name: MongoDB Latest
             junit-name: be-tests-mongo-latest-ee
             image: mongo:latest
+        job:
+          - name: Driver Tests
+            test-args: >-
+              :exclude-tags '[:mongo-sharded-cluster-tests :mb/transforms-python-test]'
+              :only-tags [:mb/driver-tests]
+          - name: Transforms Python Tests
+            test-args: >-
+              :only-tags [:mb/transforms-python-test]
+            needs-python-runner: true
     env:
       CI: 'true'
       DRIVERS: mongo
@@ -489,16 +551,19 @@ jobs:
         env:
           MONGO_INITDB_ROOT_USERNAME: metabase
           MONGO_INITDB_ROOT_PASSWORD: metasample123
-    name: ${{ matrix.version.name }}
+    name: ${{ matrix.version.name }} ${{ matrix.job.name }}
     steps:
     - uses: actions/checkout@v4
+    - name: Setup python-runner
+      if: ${{ matrix.job.needs-python-runner == true }}
+      env:
+        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      run: ./.github/scripts/setup-python-runner.sh
     - name: Test ${{ matrix.version.name }}
       uses: ./.github/actions/test-driver
       with:
         junit-name: ${{ matrix.version.junit-name }}
-        test-args: >-
-          :exclude-tags [:mongo-sharded-cluster-tests]
-          :only-tags [:mb/driver-tests]
+        test-args: ${{ matrix.job.test-args }}
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -562,7 +627,7 @@ jobs:
         with:
           junit-name: ${{ matrix.version.junit-name }}
           test-args: >-
-            :exclude-tags [:mongo-sharded-cluster-tests]
+            :exclude-tags '[:mongo-sharded-cluster-tests :mb/transforms-python-test]'
             :only-tags [:mb/driver-tests]
       - name: Upload Test Results
         uses: ./.github/actions/upload-test-results
@@ -670,6 +735,7 @@ jobs:
         junit-name: ${{ matrix.version.junit-name }}
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -704,6 +770,11 @@ jobs:
           - name: Driver Tests
             test-args: >-
               :only-tags [:mb/driver-tests]
+              :exclude-tags [:mb/transforms-python-test]
+          - name: Transforms Python Tests
+            test-args: >-
+              :only-tags [:mb/transforms-python-test]
+            needs-python-runner: true
     name: "${{ matrix.version.name }} ${{ matrix.job.name }}"
     env:
       # Only run old migrations tests on pushes to master or release branches. All other branches should skip tests
@@ -745,6 +816,11 @@ jobs:
           POSTGRES_HOST_AUTH_METHOD: trust
     steps:
       - uses: actions/checkout@v4
+      - name: Setup python-runner
+      #if: ${{ matrix.job.needs-python-runner == true }}
+        env:
+          METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+        run: ./.github/scripts/setup-python-runner.sh
       - name: Test ${{ matrix.version.name }} (${{ matrix.job.name }})
         uses: ./.github/actions/test-driver
         with:
@@ -821,6 +897,7 @@ jobs:
         junit-name: 'be-tests-presto-jdbc-ee'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -850,8 +927,14 @@ jobs:
             skip: false
             test-args: >-
               :only-tags [:mb/driver-tests]
-              :exclude-tags [:mb/upload-tests]
+              :exclude-tags '[:mb/upload-tests :mb/transforms-python-test]'
               :fail-fast? true
+          - name: Redshift Transforms Python Tests
+            skip: false
+            test-args: >-
+              :only-tags [:mb/transforms-python-test]
+              :fail-fast? true
+            needs-python-runner: true
 
     env:
       CI: 'true'
@@ -864,6 +947,11 @@ jobs:
     name: ${{ matrix.job.name }}
     steps:
     - uses: actions/checkout@v4
+    - name: Setup python-runner
+      if: ${{ matrix.job.needs-python-runner == true }}
+      env:
+        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      run: ./.github/scripts/setup-python-runner.sh
     - name: Test ${{ matrix.job.name }}
       if: ${{ !matrix.job.skip }}
       uses: ./.github/actions/test-driver
@@ -887,6 +975,19 @@ jobs:
     if: ${{ !inputs.skip }}
     runs-on: ubuntu-22.04
     timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        job:
+          - name: Driver Tests
+            test-args: >-
+              :only-tags [:mb/driver-tests]
+              :exclude-tags [:mb/transforms-python-test]
+# DS 2025-09-26 temporarily disabled while we diagnose flakes
+#          - name: Transforms Python Tests
+#            test-args: >-
+#              :only-tags [:mb/transforms-python-test]
+#            needs-python-runner: true
     env:
       CI: 'true'
       DRIVERS: snowflake
@@ -905,15 +1006,19 @@ jobs:
       MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_CUSTOM_USER: RSA_ROLE_TEST_CUSTOM_USER
       MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_ROLE: RSA_ROLE_TEST_ROLE
       MB_SNOWFLAKE_TEST_RSA_ROLE_TEST_DB: RSA_ROLE_TEST_DB
-    name: Snowflake
+    name: Snowflake ${{ matrix.job.name }}
     steps:
     - uses: actions/checkout@v4
+    - name: Setup python-runner
+      if: ${{ matrix.job.needs-python-runner == true }}
+      env:
+        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      run: ./.github/scripts/setup-python-runner.sh
     - name: Test Snowflake driver
       uses: ./.github/actions/test-driver
       with:
         junit-name: 'be-tests-snowflake-ee'
-        test-args: >-
-          :only-tags [:mb/driver-tests]
+        test-args: ${{ matrix.job.test-args }}
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -948,6 +1053,7 @@ jobs:
         junit-name: 'be-tests-sparksql-ee'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -977,6 +1083,7 @@ jobs:
         junit-name: 'be-tests-sqlite-ee'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -1004,6 +1111,15 @@ jobs:
           - name: SQL Server 2022
             junit-name: be-tests-sqlserver-2022-ee
             image: mcr.microsoft.com/mssql/server:2022-latest
+        job:
+          - name: Driver Tests
+            test-args: >-
+              :only-tags [:mb/driver-tests]
+              :exclude-tags [:mb/transforms-python-test]
+          - name: Transforms Python Tests
+            test-args: >-
+              :only-tags [:mb/transforms-python-test]
+            needs-python-runner: true
     env:
       CI: 'true'
       DRIVERS: sqlserver
@@ -1019,15 +1135,19 @@ jobs:
           ACCEPT_EULA: Y
           SA_PASSWORD: 'P@ssw0rd'
           MSSQL_MEMORY_LIMIT_MB: 1024
-    name: ${{ matrix.version.name }}
+    name: ${{ matrix.version.name }} ${{ matrix.job.name }}
     steps:
     - uses: actions/checkout@v4
+    - name: Setup python-runner
+      if: ${{ matrix.job.needs-python-runner == true }}
+      env:
+        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      run: ./.github/scripts/setup-python-runner.sh
     - name: Test ${{ matrix.version.name }}
       uses: ./.github/actions/test-driver
       with:
         junit-name: ${{ matrix.version.junit-name }}
-        test-args: >-
-          :only-tags [:mb/driver-tests]
+        test-args: ${{ matrix.job.test-args }}
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()
@@ -1136,6 +1256,7 @@ jobs:
         junit-name: 'be-tests-vertica-ee'
         test-args: >-
           :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
     - name: Upload Test Results
       uses: ./.github/actions/upload-test-results
       if: always()

.github/workflows/e2e-test.yml

@@ -60,6 +60,14 @@ jobs:
       CYPRESS_IS_EMBEDDING_SDK: "${{ inputs.name == 'embedding-sdk' }}"
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
+          token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+
+      - name: Setup python-runner
+        env:
+          METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+        run: ./.github/scripts/setup-python-runner.sh
 
       - name: Prepare Docker containers
         uses: ./.github/actions/e2e-prepare-containers

deps.edn

@@ -198,6 +198,8 @@
                                                            org.eclipse.jetty.ee9/jetty-ee9-servlet
                                                            org.eclipse.jetty.ee9.websocket/jetty-ee9-websocket-jetty-server]}
   slingshot/slingshot                       {:mvn/version "0.12.2"}             ; enhanced throw/catch, used by other deps
+  software.amazon.awssdk/s3                 {:mvn/version "2.33.0"}             ; AWS SDK v2 for S3
+  software.amazon.awssdk/sts                {:mvn/version "2.33.0"}             ; Required for S3 client to use service account
   stencil/stencil                           {:mvn/version "0.5.0"}              ; Mustache templates for Clojure
   user-agent/user-agent                     {:mvn/version "0.1.1"}              ; User-Agent string parser, for Login History page & elsewhere
   weavejester/dependency                    {:mvn/version "1.0.0"}              ; Dependency graphs and topological sorting

docs/CONTRIBUTING.md

@@ -61,7 +61,7 @@ Once one or more people have started to work on a feature, it should be marked [
 
 ### E) Verification and merging
 
-All PRs that involve more than an insignificant change should be reviewed. See our [Code Review Process](./developers-guide/code-reviews.md).
+All PRs that involve more than an insignificant change should be reviewed. See our [Code Review Process](https://www.metabase.com/docs/master/developers-guide/code-reviews).
 
 If all goes well, the feature gets coded up, verified and then the pull request gets merged! High-fives all around.
 
@@ -95,7 +95,7 @@ By our definition, "Bugs" are situations where the program doesn't do what it wa
 
 There are a lot of docs, which means keeping them up to date is hard. If you notice inconsistencies, errors, or outdated information, please help us keep them current!
 
-Note that **we cannot accept translations for documentation at this time**. We support [in-app translations](./configuring-metabase/localization.md), and only support languages that have 100% coverage. But 1) the in-app text is orders of magnitude shorter than our docs, 2) it changes at a slower pace, and 3) we have a lot of people help out. We may consider supporting docs in multiple languages in the future, but for now we need to focus our resources on improving our existing documentation (and expanding it to include all of the new features we're adding).
+Note that **we cannot accept translations for documentation at this time**. We support [in-app translations](https://www.metabase.com/docs/master/configuring-metabase/localization), and only support languages that have 100% coverage. But 1) the in-app text is orders of magnitude shorter than our docs, 2) it changes at a slower pace, and 3) we have a lot of people help out. We may consider supporting docs in multiple languages in the future, but for now we need to focus our resources on improving our existing documentation (and expanding it to include all of the new features we're adding).
 
 ### Working on features
 

docs/ai/settings.md

@@ -1,9 +1,9 @@
 ---
-title: Metabot settings
+title: Metabot AI settings
 summary: Configure Metabot settings, including which collections Metabot can access, and learn tips for improving Metabot's performance through data modeling and metadata.
 ---
 
-# Metabot settings
+# Metabot AI settings
 
 {% include beta-blockquote.html
    message="For now, <a href='https://www.metabase.com/features/metabot-ai'>Metabot</a> is only available as an add-on for Pro and Enterprise plans on Metabase Cloud."
@@ -13,21 +13,26 @@ _Settings > Admin settings > AI_
 
 This page covers admin settings for Metabase's AI assistant, [Metabot](./metabot.md).
 
-## Configure Metabot
-
-To help Metabot focus on the data you care about most, select the collection containing the models and metrics you'd like it to work with.
-
-## Pick a collection for Metabot to have access to
-
 ![Admin settings for AI Metabot](./images/ai-settings.png)
 
-To select a collection for Metabot:
+## Verified content
+
+Admins on Pro and Enterprise plans can tell Metabot to only work with [models](../data-modeling/models.md) and [metrics](../data-modeling/metrics.md) that have been [verified](../exploration-and-organization/content-verification.md).
+
+Restricting Metabot to verified models and metrics (and only models and metrics) can help Metabot produce more reliable answers, since you know someone has at least vetted the data Metabot can use.
+
+## When embedding Metabot, you can pick a collection for Metabot to have access to
+
+When embedding Metabot in your app, you can select a collection for Metabot:
 
 1. Go to **Settings** > **Admin settings** > **AI**.
-2. Click the **Collection** dropdown.
+2. Click **Embedded Metabot**.
+3. In the **Collection Embedded Metabot can use** section, click **Pick a collection**.
 3. Select the collection that contains the models and metrics you want Metabot to use.
 
-Metabot will use the models and metrics in that collection to help answer questions and generate queries. You can change this collection at any time.
+Metabot will use the models and metrics in that collection to help answer questions and generate queries. You can change this collection at any time. To give Metabot access to all collections, you can set the collection to the root collection, called "Our Analytics" (the default).
+
+Alternatively (or additionally), you can restrict Metabot to [verified content](#verified-content).
 
 ## Tips for making the most of Metabot
 
@@ -66,7 +71,7 @@ Metabase will try to set semantic types automatically, but you should confirm th
 
 ### Curate prompt suggestions
 
-When you select a collection for Metabot to "learn", Metabot will suggest a series of prompts based on the content it finds in that collection. These give people a feel for the kinds of things people can ask Metabot to do.
+When you select a collection for Metabot to "learn", Metabot will suggest a series of prompts based on the content it finds in that collection. These prompts just give people a feel for the kinds of things people can ask Metabot to do.
 
 Admins can run these generated prompts to test the answers, or trash the individual prompts if they're not useful or misleading. You can also regenerate all the prompts with a click.
 

docs/api.html

@@ -83,30 +83,6 @@ redirect_from:
   },
   "components" : {
     "schemas" : {
-      "api.tiles.route-params" : {
-        "type" : "object",
-        "properties" : {
-          "lat-field" : {
-            "type" : "string"
-          },
-          "lon-field" : {
-            "type" : "string"
-          },
-          "x" : {
-            "description" : "value must be an integer.",
-            "type" : "integer"
-          },
-          "y" : {
-            "description" : "value must be an integer.",
-            "type" : "integer"
-          },
-          "zoom" : {
-            "description" : "value must be an integer.",
-            "type" : "integer"
-          }
-        },
-        "required" : [ "zoom", "x", "y", "lat-field", "lon-field" ]
-      },
       "gsheets.response" : {
         "anyOf" : [ {
           "type" : "object",
@@ -1984,26 +1960,52 @@ redirect_from:
         "enum" : [ "none", "global-schedule" ]
       },
       "metabase-enterprise.transforms.api.transform-source" : {
-        "type" : "object",
-        "properties" : {
-          "query" : {
-            "type" : "object",
-            "properties" : {
-              "database" : {
+        "oneOf" : [ {
+          "type" : "object",
+          "properties" : {
+            "query" : {
+              "type" : "object",
+              "properties" : {
+                "database" : {
+                  "type" : "integer"
+                }
+              },
+              "required" : [ "database" ]
+            },
+            "type" : {
+              "const" : "query"
+            }
+          },
+          "required" : [ "type", "query" ]
+        }, {
+          "type" : "object",
+          "properties" : {
+            "body" : {
+              "type" : "string"
+            },
+            "source-database" : {
+              "type" : "integer"
+            },
+            "source-tables" : {
+              "type" : "object",
+              "additionalProperties" : {
                 "type" : "integer"
               }
             },
-            "required" : [ "database" ]
+            "type" : {
+              "const" : "python"
+            }
           },
-          "type" : {
-            "const" : "query"
-          }
-        },
-        "required" : [ "type", "query" ]
+          "required" : [ "source-tables", "type", "body" ],
+          "additionalProperties" : false
+        } ]
       },
       "metabase-enterprise.transforms.api.transform-target" : {
         "type" : "object",
         "properties" : {
+          "database" : {
+            "type" : "integer"
+          },
           "name" : {
             "type" : "string"
           },
@@ -3135,7 +3137,7 @@ redirect_from:
             "$ref" : "#/components/schemas/metabase.lib.schema.common.non-blank-string"
           },
           "id" : {
-            "$ref" : "#/components/schemas/metabase.lib.schema.common.non-blank-string"
+            "$ref" : "#/components/schemas/metabase.lib.schema.template-tag.id"
           },
           "name" : {
             "$ref" : "#/components/schemas/metabase.lib.schema.common.non-blank-string"
@@ -3174,7 +3176,7 @@ redirect_from:
             "$ref" : "#/components/schemas/metabase.lib.schema.common.non-blank-string"
           },
           "id" : {
-            "$ref" : "#/components/schemas/metabase.lib.schema.common.non-blank-string"
+            "$ref" : "#/components/schemas/metabase.lib.schema.template-tag.id"
           },
           "name" : {
             "$ref" : "#/components/schemas/metabase.lib.schema.common.non-blank-string"
@@ -3200,7 +3202,7 @@ redirect_from:
             "$ref" : "#/components/schemas/metabase.lib.schema.common.non-blank-string"
           },
           "id" : {
-            "$ref" : "#/components/schemas/metabase.lib.schema.common.non-blank-string"
+            "$ref" : "#/components/schemas/metabase.lib.schema.template-tag.id"
           },
           "name" : {
             "$ref" : "#/components/schemas/metabase.lib.schema.common.non-blank-string"
@@ -3228,7 +3230,7 @@ redirect_from:
             "$ref" : "#/components/schemas/metabase.lib.schema.common.non-blank-string"
           },
           "id" : {
-            "$ref" : "#/components/schemas/metabase.lib.schema.common.non-blank-string"
+            "$ref" : "#/components/schemas/metabase.lib.schema.template-tag.id"
           },
           "name" : {
             "$ref" : "#/components/schemas/metabase.lib.schema.common.non-blank-string"
@@ -3254,7 +3256,7 @@ redirect_from:
             "$ref" : "#/components/schemas/metabase.lib.schema.common.non-blank-string"
           },
           "id" : {
-            "$ref" : "#/components/schemas/metabase.lib.schema.common.non-blank-string"
+            "$ref" : "#/components/schemas/metabase.lib.schema.template-tag.id"
           },
           "name" : {
             "$ref" : "#/components/schemas/metabase.lib.schema.common.non-blank-string"
@@ -4882,6 +4884,14 @@ redirect_from:
         "type" : "string",
         "enum" : [ "date", "number", "boolean", "text" ]
       },
+      "metabase.lib.schema.template-tag.id" : {
+        "oneOf" : [ {
+          "type" : "string",
+          "format" : "uuid"
+        }, {
+          "$ref" : "#/components/schemas/metabase.lib.schema.common.non-blank-string"
+        } ]
+      },
       "metabase.lib.schema.temporal-bucketing.unit" : {
         "type" : "string",
         "enum" : [ "day", "day-of-month", "day-of-week", "day-of-year", "default", "hour", "hour-of-day", "millisecond", "minute", "minute-of-hour", "month", "month-of-year", "quarter", "quarter-of-year", "second", "second-of-minute", "week", "week-of-year", "year", "year-of-era" ]
@@ -17599,11 +17609,18 @@ redirect_from:
         "tags" : [ "/api/embed" ]
       }
     },
-    "/api/embed/tiles/card/{token}/{zoom}/{x}/{y}/{lat-field}/{lon-field}" : {
+    "/api/embed/tiles/card/{token}/{zoom}/{x}/{y}" : {
       "get" : {
-        "summary" : "GET /api/embed/tiles/card/{token}/{zoom}/{x}/{y}/{lat-field}/{lon-field}",
+        "summary" : "GET /api/embed/tiles/card/{token}/{zoom}/{x}/{y}",
         "description" : "Generates a single tile image for an embedded Card using the map visualization.",
         "parameters" : [ {
+          "in" : "path",
+          "name" : "token",
+          "required" : true,
+          "schema" : {
+            "type" : "string"
+          }
+        }, {
           "in" : "path",
           "name" : "zoom",
           "required" : true,
@@ -17627,27 +17644,6 @@ redirect_from:
             "type" : "integer"
           },
           "description" : "value must be an integer."
-        }, {
-          "in" : "path",
-          "name" : "lat-field",
-          "required" : true,
-          "schema" : {
-            "type" : "string"
-          }
-        }, {
-          "in" : "path",
-          "name" : "lon-field",
-          "required" : true,
-          "schema" : {
-            "type" : "string"
-          }
-        }, {
-          "in" : "path",
-          "name" : "token",
-          "required" : true,
-          "schema" : {
-            "type" : "string"
-          }
         }, {
           "in" : "query",
           "name" : "parameters",
@@ -17656,6 +17652,20 @@ redirect_from:
             "type" : "string"
           },
           "description" : "value must be a valid JSON string."
+        }, {
+          "in" : "query",
+          "name" : "latField",
+          "required" : true,
+          "schema" : {
+            "type" : "string"
+          }
+        }, {
+          "in" : "query",
+          "name" : "lonField",
+          "required" : true,
+          "schema" : {
+            "type" : "string"
+          }
         } ],
         "responses" : {
           "2XX" : {
@@ -17671,49 +17681,11 @@ redirect_from:
         "tags" : [ "/api/embed" ]
       }
     },
-    "/api/embed/tiles/dashboard/{token}/dashcard/{dashcard-id}/card/{card-id}/{zoom}/{x}/{y}/{lat-field}/{lon-field}" : {
+    "/api/embed/tiles/dashboard/{token}/dashcard/{dashcard-id}/card/{card-id}/{zoom}/{x}/{y}" : {
       "get" : {
-        "summary" : "GET /api/embed/tiles/dashboard/{token}/dashcard/{dashcard-id}/card/{card-id}/{zoom}/{x}/{y}/{lat-field}/{lon-field}",
+        "summary" : "GET /api/embed/tiles/dashboard/{token}/dashcard/{dashcard-id}/card/{card-id}/{zoom}/{x}/{y}",
         "description" : "Generates a single tile image for a Card on an embedded Dashboard using the map visualization.",
         "parameters" : [ {
-          "in" : "path",
-          "name" : "zoom",
-          "required" : true,
-          "schema" : {
-            "type" : "integer"
-          },
-          "description" : "value must be an integer."
-        }, {
-          "in" : "path",
-          "name" : "x",
-          "required" : true,
-          "schema" : {
-            "type" : "integer"
-          },
-          "description" : "value must be an integer."
-        }, {
-          "in" : "path",
-          "name" : "y",
-          "required" : true,
-          "schema" : {
-            "type" : "integer"
-          },
-          "description" : "value must be an integer."
-        }, {
-          "in" : "path",
-          "name" : "lat-field",
-          "required" : true,
-          "schema" : {
-            "type" : "string"
-          }
-        }, {
-          "in" : "path",
-          "name" : "lon-field",
-          "required" : true,
-          "schema" : {
-            "type" : "string"
-          }
-        }, {
           "in" : "path",
           "name" : "token",
           "required" : true,
@@ -17738,6 +17710,30 @@ redirect_from:
             "minimum" : 1
           },
           "description" : "value must be an integer greater than zero."
+        }, {
+          "in" : "path",
+          "name" : "zoom",
+          "required" : true,
+          "schema" : {
+            "type" : "integer"
+          },
+          "description" : "value must be an integer."
+        }, {
+          "in" : "path",
+          "name" : "x",
+          "required" : true,
+          "schema" : {
+            "type" : "integer"
+          },
+          "description" : "value must be an integer."
+        }, {
+          "in" : "path",
+          "name" : "y",
+          "required" : true,
+          "schema" : {
+            "type" : "integer"
+          },
+          "description" : "value must be an integer."
         }, {
           "in" : "query",
           "name" : "parameters",
@@ -17746,6 +17742,20 @@ redirect_from:
             "type" : "string"
           },
           "description" : "value must be a valid JSON string."
+        }, {
+          "in" : "query",
+          "name" : "latField",
+          "required" : true,
+          "schema" : {
+            "type" : "string"
+          }
+        }, {
+          "in" : "query",
+          "name" : "lonField",
+          "required" : true,
+          "schema" : {
+            "type" : "string"
+          }
         } ],
         "responses" : {
           "2XX" : {
@@ -20755,6 +20765,168 @@ redirect_from:
         "tags" : [ "/api/preview_embed" ]
       }
     },
+    "/api/preview_embed/tiles/card/{token}/{zoom}/{x}/{y}" : {
+      "get" : {
+        "summary" : "GET /api/preview_embed/tiles/card/{token}/{zoom}/{x}/{y}",
+        "description" : "Generates a single tile image for an embedded Card using the map visualization.",
+        "parameters" : [ {
+          "in" : "path",
+          "name" : "token",
+          "required" : true,
+          "schema" : {
+            "type" : "string"
+          }
+        }, {
+          "in" : "path",
+          "name" : "zoom",
+          "required" : true,
+          "schema" : {
+            "type" : "integer"
+          },
+          "description" : "value must be an integer."
+        }, {
+          "in" : "path",
+          "name" : "x",
+          "required" : true,
+          "schema" : {
+            "type" : "integer"
+          },
+          "description" : "value must be an integer."
+        }, {
+          "in" : "path",
+          "name" : "y",
+          "required" : true,
+          "schema" : {
+            "type" : "integer"
+          },
+          "description" : "value must be an integer."
+        }, {
+          "in" : "query",
+          "name" : "parameters",
+          "required" : false,
+          "schema" : {
+            "type" : "string"
+          },
+          "description" : "value must be a valid JSON string."
+        }, {
+          "in" : "query",
+          "name" : "latField",
+          "required" : true,
+          "schema" : {
+            "type" : "string"
+          }
+        }, {
+          "in" : "query",
+          "name" : "lonField",
+          "required" : true,
+          "schema" : {
+            "type" : "string"
+          }
+        } ],
+        "responses" : {
+          "2XX" : {
+            "description" : "Successful response"
+          },
+          "4XX" : {
+            "description" : "Client error response"
+          },
+          "5XX" : {
+            "description" : "Server error response"
+          }
+        },
+        "tags" : [ "/api/preview_embed" ]
+      }
+    },
+    "/api/preview_embed/tiles/dashboard/{token}/dashcard/{dashcard-id}/card/{card-id}/{zoom}/{x}/{y}" : {
+      "get" : {
+        "summary" : "GET /api/preview_embed/tiles/dashboard/{token}/dashcard/{dashcard-id}/card/{card-id}/{zoom}/{x}/{y}",
+        "description" : "Generates a single tile image for a Card on an embedded Dashboard using the map visualization.",
+        "parameters" : [ {
+          "in" : "path",
+          "name" : "token",
+          "required" : true,
+          "schema" : {
+            "type" : "string"
+          }
+        }, {
+          "in" : "path",
+          "name" : "dashcard-id",
+          "required" : true,
+          "schema" : {
+            "type" : "integer",
+            "minimum" : 1
+          },
+          "description" : "value must be an integer greater than zero."
+        }, {
+          "in" : "path",
+          "name" : "card-id",
+          "required" : true,
+          "schema" : {
+            "type" : "integer",
+            "minimum" : 1
+          },
+          "description" : "value must be an integer greater than zero."
+        }, {
+          "in" : "path",
+          "name" : "zoom",
+          "required" : true,
+          "schema" : {
+            "type" : "integer"
+          },
+          "description" : "value must be an integer."
+        }, {
+          "in" : "path",
+          "name" : "x",
+          "required" : true,
+          "schema" : {
+            "type" : "integer"
+          },
+          "description" : "value must be an integer."
+        }, {
+          "in" : "path",
+          "name" : "y",
+          "required" : true,
+          "schema" : {
+            "type" : "integer"
+          },
+          "description" : "value must be an integer."
+        }, {
+          "in" : "query",
+          "name" : "parameters",
+          "required" : false,
+          "schema" : {
+            "type" : "string"
+          },
+          "description" : "value must be a valid JSON string."
+        }, {
+          "in" : "query",
+          "name" : "latField",
+          "required" : true,
+          "schema" : {
+            "type" : "string"
+          }
+        }, {
+          "in" : "query",
+          "name" : "lonField",
+          "required" : true,
+          "schema" : {
+            "type" : "string"
+          }
+        } ],
+        "responses" : {
+          "2XX" : {
+            "description" : "Successful response"
+          },
+          "4XX" : {
+            "description" : "Client error response"
+          },
+          "5XX" : {
+            "description" : "Server error response"
+          }
+        },
+        "tags" : [ "/api/preview_embed" ]
+      }
+    },
     "/api/product-feedback/" : {
       "post" : {
         "summary" : "POST /api/product-feedback/",
@@ -21635,11 +21807,20 @@ redirect_from:
         "tags" : [ "/api/public" ]
       }
     },
-    "/api/public/tiles/card/{uuid}/{zoom}/{x}/{y}/{lat-field}/{lon-field}" : {
+    "/api/public/tiles/card/{uuid}/{zoom}/{x}/{y}" : {
       "get" : {
-        "summary" : "GET /api/public/tiles/card/{uuid}/{zoom}/{x}/{y}/{lat-field}/{lon-field}",
+        "summary" : "GET /api/public/tiles/card/{uuid}/{zoom}/{x}/{y}",
         "description" : "Generates a single tile image for a publicly-accessible Card using the map visualization. Does not require auth\n  credentials. Public sharing must be enabled.",
         "parameters" : [ {
+          "in" : "path",
+          "name" : "uuid",
+          "required" : true,
+          "schema" : {
+            "type" : "string",
+            "pattern" : "[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}"
+          },
+          "description" : "value must be a valid UUID."
+        }, {
           "in" : "path",
           "name" : "zoom",
           "required" : true,
@@ -21663,29 +21844,6 @@ redirect_from:
             "type" : "integer"
           },
           "description" : "value must be an integer."
-        }, {
-          "in" : "path",
-          "name" : "lat-field",
-          "required" : true,
-          "schema" : {
-            "type" : "string"
-          }
-        }, {
-          "in" : "path",
-          "name" : "lon-field",
-          "required" : true,
-          "schema" : {
-            "type" : "string"
-          }
-        }, {
-          "in" : "path",
-          "name" : "uuid",
-          "required" : true,
-          "schema" : {
-            "type" : "string",
-            "pattern" : "[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}"
-          },
-          "description" : "value must be a valid UUID."
         }, {
           "in" : "query",
           "name" : "parameters",
@@ -21694,6 +21852,20 @@ redirect_from:
             "type" : "string"
           },
           "description" : "value must be a valid JSON string."
+        }, {
+          "in" : "query",
+          "name" : "latField",
+          "required" : true,
+          "schema" : {
+            "type" : "string"
+          }
+        }, {
+          "in" : "query",
+          "name" : "lonField",
+          "required" : true,
+          "schema" : {
+            "type" : "string"
+          }
         } ],
         "responses" : {
           "2XX" : {
@@ -21709,49 +21881,11 @@ redirect_from:
         "tags" : [ "/api/public" ]
       }
     },
-    "/api/public/tiles/dashboard/{uuid}/dashcard/{dashcard-id}/card/{card-id}/{zoom}/{x}/{y}/{lat-field}/{lon-field}" : {
+    "/api/public/tiles/dashboard/{uuid}/dashcard/{dashcard-id}/card/{card-id}/{zoom}/{x}/{y}" : {
       "get" : {
-        "summary" : "GET /api/public/tiles/dashboard/{uuid}/dashcard/{dashcard-id}/card/{card-id}/{zoom}/{x}/{y}/{lat-field}/{lon-field}",
+        "summary" : "GET /api/public/tiles/dashboard/{uuid}/dashcard/{dashcard-id}/card/{card-id}/{zoom}/{x}/{y}",
         "description" : "Generates a single tile image for a Card using the map visualization in a publicly-accessible Dashboard. Does not\n  require auth credentials. Public sharing must be enabled.",
         "parameters" : [ {
-          "in" : "path",
-          "name" : "zoom",
-          "required" : true,
-          "schema" : {
-            "type" : "integer"
-          },
-          "description" : "value must be an integer."
-        }, {
-          "in" : "path",
-          "name" : "x",
-          "required" : true,
-          "schema" : {
-            "type" : "integer"
-          },
-          "description" : "value must be an integer."
-        }, {
-          "in" : "path",
-          "name" : "y",
-          "required" : true,
-          "schema" : {
-            "type" : "integer"
-          },
-          "description" : "value must be an integer."
-        }, {
-          "in" : "path",
-          "name" : "lat-field",
-          "required" : true,
-          "schema" : {
-            "type" : "string"
-          }
-        }, {
-          "in" : "path",
-          "name" : "lon-field",
-          "required" : true,
-          "schema" : {
-            "type" : "string"
-          }
-        }, {
           "in" : "path",
           "name" : "uuid",
           "required" : true,
@@ -21778,6 +21912,30 @@ redirect_from:
             "minimum" : 1
           },
           "description" : "value must be an integer greater than zero."
+        }, {
+          "in" : "path",
+          "name" : "zoom",
+          "required" : true,
+          "schema" : {
+            "type" : "integer"
+          },
+          "description" : "value must be an integer."
+        }, {
+          "in" : "path",
+          "name" : "x",
+          "required" : true,
+          "schema" : {
+            "type" : "integer"
+          },
+          "description" : "value must be an integer."
+        }, {
+          "in" : "path",
+          "name" : "y",
+          "required" : true,
+          "schema" : {
+            "type" : "integer"
+          },
+          "description" : "value must be an integer."
         }, {
           "in" : "query",
           "name" : "parameters",
@@ -21786,6 +21944,20 @@ redirect_from:
             "type" : "string"
           },
           "description" : "value must be a valid JSON string."
+        }, {
+          "in" : "query",
+          "name" : "latField",
+          "required" : true,
+          "schema" : {
+            "type" : "string"
+          }
+        }, {
+          "in" : "query",
+          "name" : "lonField",
+          "required" : true,
+          "schema" : {
+            "type" : "string"
+          }
         } ],
         "responses" : {
           "2XX" : {
@@ -24180,11 +24352,20 @@ redirect_from:
         "tags" : [ "/api/task" ]
       }
     },
-    "/api/tiles/{card-id}/{zoom}/{x}/{y}/{lat-field}/{lon-field}" : {
+    "/api/tiles/{card-id}/{zoom}/{x}/{y}" : {
       "get" : {
-        "summary" : "GET /api/tiles/{card-id}/{zoom}/{x}/{y}/{lat-field}/{lon-field}",
+        "summary" : "GET /api/tiles/{card-id}/{zoom}/{x}/{y}",
         "description" : "Generates a single tile image for a saved Card.",
         "parameters" : [ {
+          "in" : "path",
+          "name" : "card-id",
+          "required" : true,
+          "schema" : {
+            "type" : "integer",
+            "minimum" : 1
+          },
+          "description" : "value must be an integer greater than zero."
+        }, {
           "in" : "path",
           "name" : "zoom",
           "required" : true,
@@ -24208,29 +24389,6 @@ redirect_from:
             "type" : "integer"
           },
           "description" : "value must be an integer."
-        }, {
-          "in" : "path",
-          "name" : "lat-field",
-          "required" : true,
-          "schema" : {
-            "type" : "string"
-          }
-        }, {
-          "in" : "path",
-          "name" : "lon-field",
-          "required" : true,
-          "schema" : {
-            "type" : "string"
-          }
-        }, {
-          "in" : "path",
-          "name" : "card-id",
-          "required" : true,
-          "schema" : {
-            "type" : "integer",
-            "minimum" : 1
-          },
-          "description" : "value must be an integer greater than zero."
         }, {
           "in" : "query",
           "name" : "parameters",
@@ -24239,6 +24397,20 @@ redirect_from:
             "type" : "string"
           },
           "description" : "value must be a valid JSON string."
+        }, {
+          "in" : "query",
+          "name" : "latField",
+          "required" : true,
+          "schema" : {
+            "type" : "string"
+          }
+        }, {
+          "in" : "query",
+          "name" : "lonField",
+          "required" : true,
+          "schema" : {
+            "type" : "string"
+          }
         } ],
         "responses" : {
           "2XX" : {
@@ -24254,49 +24426,11 @@ redirect_from:
         "tags" : [ "/api/tiles" ]
       }
     },
-    "/api/tiles/{dashboard-id}/dashcard/{dashcard-id}/card/{card-id}/{zoom}/{x}/{y}/{lat-field}/{lon-field}" : {
+    "/api/tiles/{dashboard-id}/dashcard/{dashcard-id}/card/{card-id}/{zoom}/{x}/{y}" : {
       "get" : {
-        "summary" : "GET /api/tiles/{dashboard-id}/dashcard/{dashcard-id}/card/{card-id}/{zoom}/{x}/{y}/{lat-field}/{lon-field}",
+        "summary" : "GET /api/tiles/{dashboard-id}/dashcard/{dashcard-id}/card/{card-id}/{zoom}/{x}/{y}",
         "description" : "Generates a single tile image for a dashcard.",
         "parameters" : [ {
-          "in" : "path",
-          "name" : "zoom",
-          "required" : true,
-          "schema" : {
-            "type" : "integer"
-          },
-          "description" : "value must be an integer."
-        }, {
-          "in" : "path",
-          "name" : "x",
-          "required" : true,
-          "schema" : {
-            "type" : "integer"
-          },
-          "description" : "value must be an integer."
-        }, {
-          "in" : "path",
-          "name" : "y",
-          "required" : true,
-          "schema" : {
-            "type" : "integer"
-          },
-          "description" : "value must be an integer."
-        }, {
-          "in" : "path",
-          "name" : "lat-field",
-          "required" : true,
-          "schema" : {
-            "type" : "string"
-          }
-        }, {
-          "in" : "path",
-          "name" : "lon-field",
-          "required" : true,
-          "schema" : {
-            "type" : "string"
-          }
-        }, {
           "in" : "path",
           "name" : "dashboard-id",
           "required" : true,
@@ -24323,6 +24457,30 @@ redirect_from:
             "minimum" : 1
           },
           "description" : "value must be an integer greater than zero."
+        }, {
+          "in" : "path",
+          "name" : "zoom",
+          "required" : true,
+          "schema" : {
+            "type" : "integer"
+          },
+          "description" : "value must be an integer."
+        }, {
+          "in" : "path",
+          "name" : "x",
+          "required" : true,
+          "schema" : {
+            "type" : "integer"
+          },
+          "description" : "value must be an integer."
+        }, {
+          "in" : "path",
+          "name" : "y",
+          "required" : true,
+          "schema" : {
+            "type" : "integer"
+          },
+          "description" : "value must be an integer."
         }, {
           "in" : "query",
           "name" : "parameters",
@@ -24331,6 +24489,20 @@ redirect_from:
             "type" : "string"
           },
           "description" : "value must be a valid JSON string."
+        }, {
+          "in" : "query",
+          "name" : "latField",
+          "required" : true,
+          "schema" : {
+            "type" : "string"
+          }
+        }, {
+          "in" : "query",
+          "name" : "lonField",
+          "required" : true,
+          "schema" : {
+            "type" : "string"
+          }
         } ],
         "responses" : {
           "2XX" : {
@@ -24346,11 +24518,35 @@ redirect_from:
         "tags" : [ "/api/tiles" ]
       }
     },
-    "/api/tiles/{zoom}/{x}/{y}/{lat-field}/{lon-field}" : {
+    "/api/tiles/{zoom}/{x}/{y}" : {
       "get" : {
-        "summary" : "GET /api/tiles/{zoom}/{x}/{y}/{lat-field}/{lon-field}",
+        "summary" : "GET /api/tiles/{zoom}/{x}/{y}",
         "description" : "Generates a single tile image for an ad-hoc query.",
         "parameters" : [ {
+          "in" : "path",
+          "name" : "zoom",
+          "required" : true,
+          "schema" : {
+            "type" : "integer"
+          },
+          "description" : "value must be an integer."
+        }, {
+          "in" : "path",
+          "name" : "x",
+          "required" : true,
+          "schema" : {
+            "type" : "integer"
+          },
+          "description" : "value must be an integer."
+        }, {
+          "in" : "path",
+          "name" : "y",
+          "required" : true,
+          "schema" : {
+            "type" : "integer"
+          },
+          "description" : "value must be an integer."
+        }, {
           "in" : "query",
           "name" : "query",
           "required" : true,
@@ -24358,6 +24554,20 @@ redirect_from:
             "type" : "string"
           },
           "description" : "value must be a valid JSON string."
+        }, {
+          "in" : "query",
+          "name" : "latField",
+          "required" : true,
+          "schema" : {
+            "type" : "string"
+          }
+        }, {
+          "in" : "query",
+          "name" : "lonField",
+          "required" : true,
+          "schema" : {
+            "type" : "string"
+          }
         } ],
         "responses" : {
           "2XX" : {

docs/cloud/google-sheets.md

@@ -80,6 +80,7 @@ Here's what you need to know when syncing Google Sheets:
 - **Special character replacement**. Some characters just don't play nice with databases (like "?"). When we find these in column names, we'll replace them with "x" to keep things running smoothly.
 - **Renamed files will create new tables**. If you rename files in your folder or tabs in your sheets, we'll treat them as brand new tables and import them fresh.
 - **New columns sync automatically**. Adding new columns to your sheets? No problem — they'll show up in Metabase as expected.
+- **Renaming sheet columns will create new table columns**. If you rename a column in a spreadsheet, Metabase will create a new column with the new name in the synced table. The original column will remain in the table but it'll become empty. You'll be able to hide the old column in Table Metadata settings.
 - **Empty sheets won't import**. We won't import completely empty sheets or sheets that only have column headers. There needs to be some actual data in there.
 - **Google Sheets must have unique names**. If files in the synced Google Drive folder (and its subfolders) have the same name (e.g., one sheet is in the root folder, another sheet in a subfolder), the sheets might not sync properly.
 

docs/developers-guide/community-drivers.md

@@ -43,6 +43,7 @@ Anyone can build a community driver. These are the currently known third-party d
 | [Neo4j](https://github.com/StronkMan/metabase-neo4j-driver)                           | ![GitHub stars](https://img.shields.io/github/stars/StronkMan/metabase-neo4j-driver)               | ![GitHub (Pre-)Release Date](https://img.shields.io/github/release-date-pre/StronkMan/metabase-neo4j-driver)               |
 | [Netsuite SuiteAnalytics Connect](https://github.com/ericcj/metabase-netsuite-driver) | ![GitHub stars](https://img.shields.io/github/stars/ericcj/metabase-netsuite-driver)               | ![GitHub (Pre-)Release Date](https://img.shields.io/github/release-date-pre/ericcj/metabase-netsuite-driver)               |
 | [Peaka](https://github.com/peakacom/metabase-driver)                                  | ![GitHub stars](https://img.shields.io/github/stars/peakacom/metabase-driver)                      | ![GitHub (Pre-)Release Date](https://img.shields.io/github/release-date-pre/peakacom/metabase-driver)                      |
+| [SPARQL](https://github.com/jhisse/metabase-sparql-driver)                            | ![GitHub stars](https://img.shields.io/github/stars/jhisse/metabase-sparql-driver)                 | ![GitHub (Pre-)Release Date](https://img.shields.io/github/release-date-pre/jhisse/metabase-sparql-driver)                 |
 | [Teradata](https://github.com/swisscom-bigdata/metabase-teradata-driver)              | ![GitHub stars](https://img.shields.io/github/stars/swisscom-bigdata/metabase-teradata-driver)     | ![GitHub (Pre-)Release Date](https://img.shields.io/github/release-date-pre/swisscom-bigdata/metabase-teradata-driver)     |
 
 If you don't see a driver for your database, try looking in the comments of the [issue related to the database](https://github.com/metabase/metabase/labels/Database%2F). You might also find more drivers by searching on GitHub for "Metabase driver".

docs/developers-guide/devenv.md

@@ -7,7 +7,7 @@ title: Development environment
 The Metabase application has two basic components:
 
 1. A backend written in Clojure which contains a REST API as well as all the relevant code for talking to databases and processing queries.
-2. A frontend written as a Javascript single-page application which provides the web UI.
+2. A frontend written as a JavaScript single-page application which provides the web UI.
 
 Both components are built and assembled together into a single JAR file. In the directory where you run the JAR, you can create a JAR file (if Metabase hasn't already created it) and add drivers in there (the drivers are also JARs).
 
@@ -147,6 +147,17 @@ You can also start a REPL another way (e.g., through your editor) and then call:
 To start the server (at `localhost:3000`). This will also set up or migrate your application database. To actually
 use Metabase, don't forget to start the frontend as well (e.g. with `yarn build-hot`).
 
+### Multiple Instances
+
+By default Rspack runs the development server on port `8088`. You can run multiple instances of Metabase on the same machine by specifying a different port for each instance.
+
+Frontend:
+- If you are running the frontend with `yarn build-hot`, set the `MB_FRONTEND_DEV_PORT` environment variable: `MB_FRONTEND_DEV_PORT=8089 MB_EDITION=ee yarn build-hot`
+- If you are building the frontend statically with `yarn build`, there is nothing different to do
+
+Backend:
+- Set the `MB_JETTY_PORT` environment variable and `MB_FRONTEND_DEV_PORT` to the same one as for the frontend.
+
 ### The application database
 
 By default, Metabase uses H2 for its application database, but we recommend using Postgres. This is configured with

docs/developers-guide/driver-changelog.md

@@ -20,7 +20,22 @@ title: Driver interface changelog
 - Added `metabase.driver/rename-tables!*` multimethod for atomic table renaming operations. Takes a map of {from-table to-table}
   pairs that has been topologically sorted.
 
-- Added the driver multi-methods `metabase.driver/schema-exists?` and `metabase.driver/create-schema-if-needed!` which should be implemented by drivers that support `:schemas` and `:transforms/table`.
+- Added `metabase.driver/rename-table!` multimethod for table renaming operations. Takes a single from-table to-table pair.
+  Fallbacks to singleton call to `rename-tables!*` if available.
+
+- Added the driver multi-methods `metabase.driver/schema-exists?` and `metabase.driver/create-schema-if-needed!`
+  which should be implemented by drivers that support `:schemas` and `:transforms/table`.
+
+- Added `metabase.driver/type->database-type` multimethod that returns the database type for a given Metabase
+  type (from the type hierarchy) as a HoneySQL spec. This method handles general Metabase base types.
+
+- Added `metabase.driver/insert-from-source!` multimethod that abstracts data insertion from various sources
+  into existing tables. This multimethod dispatches on both the driver and the data source type
+  (`:rows` or `:jsonl-file`). It allows drivers to optimize based on the data source type and returns the number
+  of rows inserted. Default implementations are provided for both `:rows` and `:jsonl-file` source types.
+
+- Added `metabase.driver/insert-col->val` multimethod to parse values for insertion based on driver, data source type
+  and column definition. Drivers should implement this when their insertion mechanism needs values converted to proper types.
 
 - `metabase.driver/humanize-connection-error-message` now takes a list of all messages in the exception chain,
   instead of just the top-level message as a string.

docs/documents/introduction.md

@@ -42,9 +42,9 @@ If however, you delete the document's copy, that copy is permanently deleted. It
 
 ### Charts generated by Metabot only exist in the document
 
-If Metabot generates a chart, Metabase saves that question to the document itself. If you delete it, it's gone forever. You won't be able to recover the question from the trash.
+If [Metabot](../ai/metabot.md) generates a chart, Metabase saves that question to the document itself. If you delete it, it's gone forever. You won't be able to recover the question from the trash.
 
-### Editing charts
+## Editing charts
 
 To edit a chart, click the **three-dot** menu in the chart's upper right. You can:
 
@@ -54,6 +54,22 @@ To edit a chart, click the **three-dot** menu in the chart's upper right. You ca
 
 Edits only affect the chart in the document. If you added the chart from a collection, Metabase will have created a copy of the chart. Any changes made to the chart in the document won't affect the original question.
 
+## Arranging charts
+
+![Document layout](./images/document-layout.png)
+
+To rearrange charts, simply click and hold on the chart and drag it to where you want it in the document.
+
+You can put up to three charts side by side (more than three charts side-by-side gets crowded). To resize charts, you can click on the edges to adjust their size. 
+
+When there are multiple charts in the same row, you can resize them horizontally by clicking between the charts and sliding left or right to resize.
+
+To resize a chart's height, click the bottom of the chart and resize the chart up or down.
+
+For now, you can only put charts side-by-side (not text elements).
+
+After arranging your charts, save your changes!
+
 ## Formatting text
 
 Documents support Markdown syntax for:
@@ -71,9 +87,13 @@ If you paste a link into a document, Metabase will turn it into a clickable link
 
 ## Commenting on documents
 
-To comment on a document, click the chat bubble next to the section you want to comment on.
+To comment on a text element, click the chat bubble next to the section. You can only comment on whole sections; you can't highlight text to comment on.
 
-You can comment on whole sections like charts, headings, or text blocks. You can't highlight portions of text to comment on.
+To comment on a chart:
+
+1. Hover over the chart.
+2. Click the three-dot menu.
+3. Select **Comment**.
 
 You can:
 
@@ -84,7 +104,9 @@ You can:
 
 ### Comment notifications
 
-If someone mentions you in a thread, or responds to a comment in a thread you're in, Metabase will send you an email to let you know. Currently, there's no central place to view all of your threads and mentions, and no way to disable email notifications for comments.
+If someone mentions you in a thread, or responds to a comment in a thread you're in, Metabase will send you an email to let you know.
+
+Currently, there's no central place to view all of your threads and mentions, and no way to disable email notifications for comments.
 
 ### Deleting comments
 

docs/embedding/embedded-analytics-js.md

@@ -23,20 +23,25 @@ Currently you can choose to embed:
 
 ## Quickstart
 
+You can also follow the setup guide directly in Metabase in **Admin > Embedding > Setup guide**. We're recording the steps here for convenience.
+
 ### 1. Enable Embedded Analytics JS
 
-1. In Metabase, go to **Admin Settings > Embedding > Modular embedding**.
+1. In Metabase, go to **Admin > Embedding > Modular embedding**.
 2. Toggle on **Embedded Analytics JS**.
 3. Under **Cross-Origin Resource Sharing (CORS)**, add the URLs of the websites where you want to embed Metabase (such as `https://*.example.com`). For testing embeds, you can use `localhost` which is always included in CORS policy.
 
 ### 2. Create a new embed
 
-1. In Metabase, go to **Admin Settings > Embedding > Quick start** and select **Create embed**. Note that this will only be visible to admins.
+1. In Metabase, go to **Admin > Embedding > Setup guide > Embed in your code**, and select **Try it out** next to **Embedded analytics JS**.
+
+   If you're planning to embed an existing question or dashboard, you can instead go straight to that question or dashboard, click on the **Share** button, and choose **Embedded Analytics JS**.
+
 2. Choose the _type_ of entity to embed:
-    - Dashboard
-    - Question
-    - Exploration (which will embed the Metabase query builder)
-    - Browser
+   - Dashboard
+   - Question
+   - Exploration (which will embed the Metabase query builder)
+   - Browser
 3. Next, select the entity you want to embed. For browser, pick the collection you want people to start from.
 
 Once you've selected what you want to embed, click Next to customize your embed.
@@ -109,13 +114,13 @@ Note the `defer` attribute and the reference to your Metabase URL in the script
 
 If you're embedding multiple entities on the same page, you only need to include the `<script>` tags once globally.
 
-You can also generate the code snippet for Embedded Analytics JS interactively in Metabase through **Admin Settings > Embedding > Setup guide > Create embed**. Check out the [Quickstart](#quickstart).
+You can also generate the code snippet for Embedded Analytics JS interactively in Metabase through **Admin > Embedding > Setup guide > Embed in your code**. Check out the [quickstart](#quickstart).
 
 ## Customizing embeds
 
 The exact customization options you see will depend on what type of entity you're embedding.
 
-When you're creating a new embed using **Admin Settings > Embedding > Setup guide > Create embed**, you'll see the following customization options in the interactive creation flow. These options correspond to parameters in [components](#components).
+When you're creating a new embed using **Admin > Embedding > Setup guide > Embed in your code**, you'll see the following customization options in the interactive creation flow. These options correspond to parameters in [components](#components).
 
 - **Allow people to drill through on data points**: determines whether people can interact with the chart (or charts on a dashboard). Interactivity includes [drilling down](https://www.metabase.com/learn/metabase-basics/querying-and-dashboards/questions/drill-through) to individual records from aggregated questions, filtering on click, zooming in, etc. Disabling drill-through for an embedded _question_ also disables people's ability to add filters and summaries.
 
@@ -141,6 +146,8 @@ To define the configuration that applies to every embed on the page, use the `de
 
 - `apiKey: mb_YourAPIKey` (optional, for development only) - another way to preview embeds locally using an API key.
 
+- `fetchRequestToken: () => Promise<{ jwt: string }>` (optional) - you can customize how the SDK fetches the refresh token for JWT authentication by specifying the `fetchRequestToken` function. See [customizing JWT authentication](./sdk/authentication.md#customizing-jwt-authentication).
+
 ### Theming
 
 You can specify colors, fonts, spacing, and other appearance options using the `theme` parameter in your embed configuration.
@@ -360,7 +367,7 @@ To render a question (chart).
 
 To render a collection browser so people can navigate a collection and open dashboards or questions.
 
- ```html
+```html
 <metabase-browser initial-collection="14" read-only="false"></metabase-browser>
 ```
 

docs/embedding/interactive-embedding-quick-start-guide.md

@@ -4,6 +4,8 @@ title: "Interactive embedding quickstart"
 
 # Interactive embedding quickstart
 
+> If you are just starting out with Metabase embedding, consider using [Embedded Analytics JS](./embedded-analytics-js.md) - an improved, more customizable option for embedding interactive Metabase elements. Interactive embedding remains fully supported.
+
 You'll embed the full Metabase application in your app. Once logged in, people can view a Metabase dashboard in your web app, and be able to use the full Metabase application to explore their data, and only their data.
 
 {% include shared/in-page-promo-embedding-workshop.html %}
@@ -27,9 +29,9 @@ You could also use the dashboard's [Entity ID](../installation-and-operation/ser
 
 ### Enable interactive embedding
 
-In Metabase, click on the **gear** icon in the upper right and go to **Admin settings** > **Settings** > **Embedding**. Look for **Interactive Embedding** and click on the **Enabled** toggle.
+In Metabase, click on the **gear** icon in the upper right and go to **Admin > Embedding > Interactive Embedding** and toggle on **Enable interactive embedding**.
 
-Click on the **Configure** button under **Interactive Embedding**. Under **Authorized origins**, add the URL of the website or web app where you want to embed Metabase. If you're running your app locally, you can add localhost and specify the port number, e.g. `http://localhost:8080`.
+Under **Authorized origins**, add the URL of the website or web app where you want to embed Metabase. If you're running your app locally, you can add localhost and specify the port number, e.g. `http://localhost:8080`.
 
 #### SameSite configuration
 
@@ -39,7 +41,7 @@ If you're embedding Metabase in a different domain, you may need to [set the ses
 
 #### Enable authentication with JWT
 
-While still in the Admin panel's **Settings** section, click on **Authentication**.
+While still in the **Interactive embedding** section, click on **Authentication** under **Related settings**.
 
 On the card that says **JWT**, click the **Setup** button (you may have to scroll down to view the JWT card).
 
@@ -167,7 +169,7 @@ In Metabase, click the **gear** icon and go to **Admin settings** > **People** >
 
 You'll map this string in the `groups` key to a Metabase group, so that when the person signs in via SSO, Metabase automatically assigns them to the appropriate Metabase group.
 
-In Metabase's admin section, go to **Settings** > **Authentication**. Scroll to the **JWT** card and click **Edit**.
+In Metabase's admin section, go to **Authentication > JWT** and click **Edit**.
 
 In the **Group schema** section, toggle on **Synchronize group memberships**. If the names of groups in the `groups` array match Metabase group names exactly (e.g. both are `"Customer Acme"`), then the groups will be mapped automatically.
 

docs/embedding/interactive-embedding.md

@@ -13,6 +13,8 @@ redirect_from:
 
 **Interactive embedding** is what you want if you want to offer [multi-tenant, self-service analytics](https://www.metabase.com/learn/metabase-basics/embedding/multi-tenant-self-service-analytics).
 
+> If you are just starting out with Metabase embedding, consider using [Embedded Analytics JS](./embedded-analytics-js.md) instead of interactive embedding - it's an improved, more customizable option for embedding interactive Metabase elements. Interactive embedding remains fully supported.
+
 Interactive embedding is the only type of embedding that integrates with your [permissions](../permissions/introduction.md) and [SSO](../people-and-groups/start.md#authentication) to give people the right level of access to [query](https://www.metabase.com/glossary/query-builder) and [drill-down](https://www.metabase.com/learn/metabase-basics/querying-and-dashboards/questions/drill-through) into your data.
 
 ## Interactive embedding demo
@@ -38,10 +40,9 @@ If you have your app running locally, and you're using the Pro Cloud version, or
 
 ## Enabling interactive embedding in Metabase
 
-1. Go to **Settings** > **Admin settings** > **Embedding**.
-2. Click **Enable**.
-3. Click **Interactive embedding**.
-4. Under **Authorized origins**, add the URL of the website or web app where you want to embed Metabase (such as `https://*.example.com`).
+1. Go to **Admin > Embedding > Interactive**.
+2. Click **Enable interactive embedding**.
+3. Under **Authorized origins**, add the URL of the website or web app where you want to embed Metabase (such as `https://*.example.com`).
 
 ## Setting up embedding on your website
 
@@ -129,7 +130,7 @@ Note that your interactive embed must be compatible with Safari to run on _any_
 
 If you want to embed Metabase in another domain (say, if Metabase is hosted at `metabase.yourcompany.com`, but you want to embed Metabase at `yourcompany.github.io`), you can tell Metabase to set the session cookie's SameSite value to "none".
 
-You can set session cookie's SameSite value in **Admin settings** > **Embedding** > **Interactive embedding** > **SameSite cookie setting**.
+You can set session cookie's SameSite value in **Admin settings** > **Embedding** > **Interactive** > **SameSite cookie setting**.
 
 SameSite values include:
 

docs/embedding/interactive-ui-components.md

@@ -7,6 +7,8 @@ description: Customize the UI components in your interactive Metabase embed by a
 
 To change the interface of your interactive embed, you can add parameters to the end of your embedding URL. If you want to change the colors or fonts in your embed, see [Customizing appearance](../configuring-metabase/appearance.md).
 
+> If you are just starting out with Metabase embedding, consider using [Embedded Analytics JS](./embedded-analytics-js.md) instead of interactive embedding - it's an improved, more customizable option for embedding interactive Metabase elements. Interactive embedding remains fully supported.
+
 For example, you can disable Metabase's [top nav bar](#top_nav) and [side nav menu](#side_nav) like this:
 
 ```

docs/embedding/introduction.md

@@ -12,23 +12,11 @@ Here are the different ways you can embed Metabase.
 
 {% include shared/in-page-promo-embedding-workshop.html %}
 
-## Embedded analytics SDK with React
-
-With the [Embedded analytics SDK](./sdk/introduction.md), you can embed individual Metabase components with React (like standalone charts, dashboards, the query builder, and more). You can manage access and interactivity per component, and you have advanced customization for seamless styling.
-
-**When to use the Embedded analytics SDK**: you want the most control over how you embed Metabase in your React app.
-
 ## Embedded analytics JS
 
-With [Embedded analytics JS](./embedded-analytics-js.md), you can embed individual Metabase components in your web app with JavaScript — no React required. Choose from dashboards, questions, or the query builder, and configure per‑component options like drill‑through, parameters, downloads, and theming.
+With [Embedded analytics JS](./embedded-analytics-js.md), you can embed individual Metabase components in your web app with JavaScript — no React required. Choose from dashboards, questions, or the query builder, and configure per‑component options like drill‑through, parameters, downloads, and theming. Embedded Analytics JS integrates with [SSO](securing-embeds.md) and [data permissions](../permissions/embedding.md).
 
-**When to use Embedded analytics JS**: you’re not using React (or want a drop‑in script) and want to embed Metabase components with per‑component controls and theming.
-
-## Interactive embedding
-
-Interactive embedding is the only kind of embedding that [integrates with SSO and data permissions](./interactive-embedding.md) to enable true self-service access to the underlying data.
-
-**When to use interactive embedding**: you want to [offer multi-tenant, self-service analytics](https://www.metabase.com/blog/why-full-app-embedding). With interactive embedding, people can create their own questions, dashboards, models, and more, with row and column security policies determining which data they can access.
+**When to use Embedded analytics JS**: You want to [offer multi-tenant, self-service analytics](https://www.metabase.com/blog/why-full-app-embedding), you’re not using React (or want a drop‑in script) and want to embed Metabase components with per‑component controls and theming.
 
 ## Static embedding
 
@@ -42,31 +30,41 @@ If you'd like to share your data with the good people of the internet, admins ca
 
 **When to use public links and embeds**: public links and embeds are good for one-off charts and dashboards. Admins can use them when you just need to show someone a chart or dashboard without giving people access to your Metabase. And you don't care who sees the data; you want to make those stats available to everyone.
 
+## Interactive embedding
+
+[Interactive embedding](./interactive-embedding.md) allows you to embed the entire Metabase app in an iframe, and integrate Metabase SSO with your app's authentication.
+
 ## Comparison of embedding types
 
-| Action                                                                                                                          | [SDK](./sdk/introduction.md) | [JS](./embedded-analytics-js.md) | [Interactive](./interactive-embedding.md) | [Static](./static-embedding.md) | [Public](../embedding/public-links.md) |
-| ------------------------------------------------------------------------------------------------------------------------------- | ---------------------------- | -------------------------------- | ----------------------------------------- | ------------------------------- | -------------------------------------- |
-| Display charts and dashboards                                                                                                   | ✅                           | ✅                               | ✅                                        | ✅                              | ✅                                     |
-| Display interactive [filter widgets](https://www.metabase.com/glossary/filter-widget)                                           | ✅                           | ✅                               | ✅                                        | ✅                              | ✅                                     |
-| Export results\*                                                                                                                | ✅                           | ✅                               | ✅                                        | ✅                              | ✅                                     |
-| Restrict data with [locked filters](./static-embedding-parameters.md#restricting-data-in-a-static-embed-with-locked-parameters) | ❌                           | ❌                               | ❌                                        | ✅                              | ❌                                     |
-| Restrict data with [row and column security](../permissions/row-and-column-security.md)                                         | ✅                           | ✅                               | ✅                                        | ❌                              | ❌                                     |
-| Use the [drill-through menu](https://www.metabase.com/learn/metabase-basics/querying-and-dashboards/questions/drill-through)    | ✅                           | ✅                               | ✅                                        | ❌                              | ❌                                     |
-| Self-serve via [query builder](https://www.metabase.com/glossary/query-builder)                                                 | ✅                           | ✅                               | ✅                                        | ❌                              | ❌                                     |
-| View usage of embeds with [usage analytics](../usage-and-performance-tools/usage-analytics.md)                                  | ✅                           | ✅                               | ✅                                        | ❌                              | ❌                                     |
-| [Actions on dashboards](../dashboards/actions.md)                                                                               | ✅                           | ✅                               | ✅                                        | ❌                              | ❌                                     |
-| Embed individual Metabase components                                                                                            | ✅                           | ✅                               | ❌                                        | ❌                              | ❌                                     |
-| Manage access and interactivity per component                                                                                   | ✅                           | ✅                               | ❌                                        | ❌                              | ❌                                     |
+| Action                                                                                                                          | [React SDK](./sdk/introduction.md) | [JS](./embedded-analytics-js.md) | [Interactive](./interactive-embedding.md) | [Static](./static-embedding.md) | [Public](../embedding/public-links.md) |
+| ------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------- | -------------------------------- | ----------------------------------------- | ------------------------------- | -------------------------------------- |
+| Display charts and dashboards                                                                                                   | ✅                                 | ✅                               | ✅                                        | ✅                              | ✅                                     |
+| Display interactive [filter widgets](https://www.metabase.com/glossary/filter-widget)                                           | ✅                                 | ✅                               | ✅                                        | ✅                              | ✅                                     |
+| Export results\*                                                                                                                | ✅                                 | ✅                               | ✅                                        | ✅                              | ✅                                     |
+| Restrict data with [locked filters](./static-embedding-parameters.md#restricting-data-in-a-static-embed-with-locked-parameters) | ❌                                 | ❌                               | ❌                                        | ✅                              | ❌                                     |
+| [Data segregation](../permissions/embedding.md)                                                                                 | ✅                                 | ✅                               | ✅                                        | ❌                              | ❌                                     |
+| Use the [drill-through menu](https://www.metabase.com/learn/metabase-basics/querying-and-dashboards/questions/drill-through)    | ✅                                 | ✅                               | ✅                                        | ❌                              | ❌                                     |
+| Self-serve via [query builder](../questions/query-builder/editor.md)                                                            | ✅                                 | ✅                               | ✅                                        | ❌                              | ❌                                     |
+| [Basic appearance customization](../configuring-metabase/appearance.md)\*\*                                                     | ✅                                 | ✅                               | ✅                                        | ✅                              | ✅                                     |
+| [Advanced theming](./sdk/appearance.md)                                                                                         | ✅                                 | ✅                               | ❌                                        | ❌                              | ❌                                     |
+| View usage of embeds with [usage analytics](../usage-and-performance-tools/usage-analytics.md)                                  | ✅                                 | ✅                               | ✅                                        | ❌                              | ❌                                     |
+| [Actions on dashboards](../dashboards/actions.md)                                                                               | ✅                                 | ✅                               | ✅                                        | ❌                              | ❌                                     |
+| Embed individual Metabase components                                                                                            | ✅                                 | ✅                               | ❌                                        | ❌                              | ❌                                     |
+| Manage access and interactivity per component                                                                                   | ✅                                 | ✅                               | ❌                                        | ❌                              | ❌                                     |
+| Custom layouts                                                                                                                  | ✅                                 | ❌                               | ❌                                        | ❌                              | ❌                                     |
+| Customize behavior with [plugins](./sdk/plugins.md)                                                                             | ✅                                 | ❌                               | ❌                                        | ❌                              | ❌                                     |
 
 \* Each embedding type allows data downloads by default, but only [Pro and Enterprise](https://www.metabase.com/pricing/) plans can disable data downloads.
 
+\*\* Requires [Pro and Enterprise](https://www.metabase.com/pricing/) for any embedding type.
+
 ### Embedded analytics SDK vs JS
 
 When deciding between the Embedded analytics SDK and Embedded analytics JS: if your app uses React, you should use the SDK. Otherwise, use the JS library. The JS library uses the SDK under the hood, but you can have more control with React and the SDK.
 
-## Switching from static to interactive embedding
+## Switching from static embedding to Embedded Analytics JS
 
-[Interactive embedding](./interactive-embedding.md) requires authentication via single sign-on (SSO), so you'll need to set that up both in your Metabase and in your application's server. Check out our [Interactive embedding quick start](../embedding/interactive-embedding-quick-start-guide.md).
+[Embedded Analytics JS](./embedded-analytics-js.md) requires authentication via single sign-on (SSO), so you'll need to set that up both in your Metabase and in your application's server. Check out our [Modular embedding authentication](../embedding/sdk/authentication.md).
 
 ## Further reading
 

docs/embedding/public-links.md

@@ -65,9 +65,9 @@ You can simulate drill-through on a public dashboard by setting up a [custom cli
 6. Optional: pass a filter value from the primary dashboard to the destination dashboard by adding a query parameter to the end of the destination URL:
 
 ```
-{%raw%}
+{% raw %}
 /public/dashboard/?child_filter_name={{parent_column_name}}
-{%endraw%}
+{% endraw %}
 ```
 
 For example, if you have a primary public dashboard that displays **Invoices** data, you can pass the **Plan** name (on click) to a destination public dashboard that displays **Accounts** data:

docs/embedding/sdk/authentication.md

@@ -89,7 +89,7 @@ You can add some middleware in your backend to handle cross-domain requests.
 
 ## Customizing JWT authentication
 
-You can customize how the SDK fetches the refresh token by specifying the `fetchRefreshToken` function with the `defineMetabaseAuthConfig` function:
+You can customize how the SDK fetches the request token by specifying the `fetchRequestToken` function with the `defineMetabaseAuthConfig` function:
 
 ```typescript
 {% include_file "{{ dirname }}/snippets/authentication/auth-config-jwt.tsx" snippet="example" %}

docs/embedding/sdk/introduction.md

@@ -36,7 +36,9 @@ To use the SDK, you'll need to enable the SDK in Metabase, and install the SDK i
 
 ### Enable the SDK in Metabase
 
-Enable the Embedded analytics SDK by going to **Admin settings > Settings > Embedding**. Toggle on the SDK, and hit **Configure**. Enter the origins for your website or app where you want to allow SDK embedding, separated by a space. Localhost is automatically included.
+1. Enable the Embedded analytics SDK by going to **Admin settings > Embedding > Modular**.
+2. Toggle on **SDK for React**.
+3. In **Cross-Origin Resource Sharing (CORS)**, enter the origins for your website or app where you want to allow SDK embedding, separated by a space. Localhost is automatically included.
 
 ### Install the SDK in your React application
 

docs/embedding/sdk/quickstart-with-sample-app.md

@@ -98,11 +98,11 @@ To enable SSO with JWT when self-hosting, you'll need to [activate your license]
 
 ## Enable embedding in Metabase
 
-From any Metabase page, click on the **gear** icon in the upper right and select **Admin Settings** > **Settings** > **Embedding**.
+From any Metabase page, click on the **gear** icon in the upper right and select **Admin Settings** > **Embedding**.
 
 Turn on:
 
-- Embedded analytics SDK
+- Embedded analytics SDK (it's in in the **Modular** section)
 - Static embedding
 
 Otherwise, this whole thing is hopeless.

docs/embedding/sdk/quickstart.md

@@ -36,11 +36,11 @@ To embed a dashboard in your app using the SDK, you'll need to:
 
 ## 1. Enable the SDK in Metabase
 
-In Metabase, click on the gear icon in the upper right and navigate to **Admin Settings > Settings > Embedding** and enable the Embedded analytics SDK.
+In Metabase, click on the gear icon in the upper right and navigate to **Admin Settings > Embedding > Modular** and enable the **SDK for React**.
 
 ## 2. Create an API key in Metabase
 
-Still in the Admin's Settings tab, navigate to the **Authentication** section in the sidebar and click on the **API keys** tab. [Create a new API key](../../people-and-groups/api-keys.md).
+Still in the Admin console, go to **Settings > Authentication** and click on the **API keys** tab. [Create a new API key](../../people-and-groups/api-keys.md).
 
 - Key name: "Embedded analytics SDK" (just to make the key easy to identify).
 - Group: select “Admin” (since this is only for local testing).

docs/embedding/sdk/version.md

@@ -30,8 +30,7 @@ Metabase Cloud upgrades your instance automatically as new versions roll out. Bu
 
 To manually pin your version of Metabase:
 
-1. Go to **Admin settings > Settings > Embedding**.
-2. Go to the Embedded analytics SDK card.
-3. Scroll to **Version pinning** and click **Request version pinning**.
+1. Go to **Admin > Embedding > Modular**.
+2. Scroll to **Version pinning** and click **Request version pinning**.
 
 This will open a mailto link to our support team.

docs/embedding/start.md

@@ -10,25 +10,13 @@ redirect_from:
 
 What is embedding, and how does it work?
 
-## [Interactive embedding](./interactive-embedding.md)
-
-The solution to self-service customer analytics: embed the full Metabase app in your app. Interactive embedding integrates with your data permissions to let people slice and dice data on their own using Metabase's query builder.
-
-## [Interactive embedding quickstart](./interactive-embedding-quick-start-guide.md)
-
-You'll embed the full Metabase application in your app. Once logged in, people can view a Metabase dashboard in your web app, and be able to use the full Metabase application to explore their data, and only their data.
-
-## [Interactive UI components](./interactive-ui-components.md)
-
-Customize the UI components in your interactive embed by adding parameters to the embedding URL.
-
 ## [Embedded analytics SDK](./sdk/introduction.md)
 
 With the Embedded analytics SDK, you can embed individual Metabase components with React (like standalone charts, dashboards, the query builder, and more). You can manage access and interactivity per component, and you have advanced customization for seamless styling.
 
 ## [Embedded analytics JS](./embedded-analytics-js.md)
 
-Embed dashboards, questions, or the query builder in your app with JavaScript (no React required). Built on the Embedded analytics SDK with per-component controls and theming.
+Embed dashboards, questions, or the query builder in your app with JavaScript (no React required). Built on the Embedded analytics SDK with per-component controls and theming. Integrate your app's auth with Metabase SSO.
 
 ## [Embedded analytics SDK quickstart](./sdk/quickstart.md)
 
@@ -42,6 +30,18 @@ Also known as Signed Embedding, Static embedding is a secure way to embed charts
 
 You can pass parameters between Metabase and your website via the embedding URL to specify how Metabase items should look and behave inside the iframe on your website.
 
+## [Interactive embedding](./interactive-embedding.md)
+
+Interactive embedding allows you to embed full Metabase app in an iframe. Interactive embedding integrates with your data permissions to let people slice and dice data on their own using Metabase's query builder.
+
+## [Interactive embedding quickstart](./interactive-embedding-quick-start-guide.md)
+
+You'll embed the full Metabase application in your app. Once logged in, people can view a Metabase dashboard in your web app, and be able to use the full Metabase application to explore their data, and only their data.
+
+## [Interactive UI components](./interactive-ui-components.md)
+
+Customize the UI components in your interactive embed by adding parameters to the embedding URL.
+
 ## [Public embeds](./public-links.md)
 
 Admins can also create unsecured public links or embeds of questions and dashboards.

docs/embedding/static-embedding.md

@@ -12,7 +12,7 @@ Also known as: standalone embedding, or signed embedding.
 
 In general, embedding works by displaying a Metabase URL inside an iframe in your website. A **static embed** (or signed embed) is an iframe that's loading a Metabase URL secured with a signed JSON Web Token (JWT). Metabase will only load the URL if the request supplies a JWT signed with the secret shared between your app and your Metabase. The JWT also includes a reference to the resource to load, e.g., the dashboard ID, and any values for locked parameters.
 
-You can't use static embeds with [row and column security](../permissions/row-and-column-security.md), [drill-through](https://www.metabase.com/learn/metabase-basics/querying-and-dashboards/questions/drill-through), and user-specific data isn't captured in [usage analytics](../usage-and-performance-tools/usage-analytics.md) because signed JWTs don't create user sessions (server-side sessions). For those features, check out [interactive embedding](./interactive-embedding.md).
+You can't use static embeds with [row and column security](../permissions/row-and-column-security.md), [drill-through](https://www.metabase.com/learn/metabase-basics/querying-and-dashboards/questions/drill-through), and user-specific data isn't captured in [usage analytics](../usage-and-performance-tools/usage-analytics.md) because signed JWTs don't create user sessions (server-side sessions). For those features, check out [Embedded analytics JS](./embedded-analytics-js.md).
 
 You can, however, restrict data in static embeds for specific people or groups by [locking parameters](./static-embedding-parameters.md#restricting-data-in-a-static-embed-with-locked-parameters).
 
@@ -34,12 +34,12 @@ your_metabase_embedding_url/your_signed_jwt?filter=true
 
 The signed JWT is generated using your [Metabase secret key](#regenerating-the-static-embedding-secret-key). The secret key tells Metabase that the request for filtered data can be trusted, so it's safe to display the results at the new embedding URL. Note that this secret key is shared for all static embeds, so whoever has access to that key will have access to all embedded artifacts.
 
-If you want to embed charts with additional interactive features, like [drill-down](https://www.metabase.com/learn/metabase-basics/querying-and-dashboards/questions/drill-through) and [self-service querying](../questions/query-builder/editor.md), see [Interactive embedding](./interactive-embedding.md).
+If you want to embed charts with additional interactive features, like [drill-down](https://www.metabase.com/learn/metabase-basics/querying-and-dashboards/questions/drill-through) and [self-service querying](../questions/query-builder/editor.md), see [Embedded analytics JS](./embedded-analytics-js.md).
 
 ## Turning on the embedding feature in Metabase
 
-1. Go to **Settings** > **Admin settings** > **Embedding**.
-2. Toggle the **Enable embedding**.
+1. Go to **Settings** > **Admin settings** > **Embedding > Static**.
+2. Toggle **Enable static embedding**.
 
 ## Making a question or dashboard embeddable
 
@@ -127,7 +127,7 @@ If you change the [parameters](./static-embedding-parameters.md) of your embedde
 
 ## Disabling embedding for a question or dashboard
 
-You can find a list of all static embeds of questions and dashboards from **Admin settings** > **Embedding** > **Static embedding** > **Manage**.
+You can find a list of all static embeds of questions and dashboards from **Admin settings** > **Embedding** > **Static**.
 
 1. Visit the embeddable question or dashboard.
 2. Click on the **sharing icon** (square with an arrow pointing to the top right).
@@ -164,9 +164,8 @@ The banner appears on static embeds created with Metabase's open-source version.
 
 Your embedding secret key is used to sign JWTs for all of your [embedding URLs](#adding-the-embedding-url-to-your-website).
 
-1. Go to **Settings** > **Admin settings** > **Embedding**.
-2. On the **Static embedding** card, click on **Manage**.
-3. Under **Regenerate secret key**, click **Regenerate key**.
+1. Go to **Admin** > **Embedding** > **Static embedding**.
+2. Under **Regenerate secret key**, click **Regenerate key**.
 
 This key is shared across all static embeds. Whoever has access to this key could get access to all embedded artifacts, so keep this key secure. If you regenerate this key, you'll need to update your server code with the new key.
 

docs/embedding/translations.md

@@ -7,7 +7,7 @@ summary: Upload a translation dictionary to translate questions and dashboards i
 
 {% include plans-blockquote.html feature="Translation of embedded content" convert_pro_link_to_embbedding=true %}
 
-For now, translations are only available for [static embeds](./static-embedding.md), not Interactive embedding or the Embedded analytics SDK.
+For now, translations are only available for [static embeds](./static-embedding.md), not Interactive embedding or the Embedded analytics JS/SDK.
 
 You can upload a translation dictionary to translate strings both in Metabase content (like dashboard titles) and in the data itself (like column names and values).
 
@@ -21,6 +21,11 @@ The dictionary must be a CSV with these columns:
 
 > Don't put any sensitive data in the dictionary, since anyone can see the dictionary—including viewers of public links.
 
+To add a translation dictionary:
+
+1. Go to **Admin > Embedding > Static**.
+2. Under **Translate embedded dashboards and question**, click **Upload translation dictionary**.
+
 Uploading a new dictionary will replace the existing dictionary.
 
 To remove a translation dictionary, upload a blank dictionary.
@@ -64,9 +69,9 @@ Exact translations also apply to strings that use punctuation and special charac
 
 Translations are case-insensitive. For example, if your translation dictionary has a translation:
 
-| Language | String      | Translation  |
-| -------- | ----------- | ------------ |
-| pt-BR    | Examples    | Exemplos     |
+| Language | String   | Translation |
+| -------- | -------- | ----------- |
+| pt-BR    | Examples | Exemplos    |
 
 `Examples`, `examples`, `EXAMPLES` would all be translated as `Exemplos`. That is, Metabase won't preserve the original case; it will output the exact translation string in the dictionary.
 

docs/exploration-and-organization/exploration.md

@@ -25,7 +25,7 @@ You can use the command palette to:
 
 So anytime you want to do or find anything in Metabase, just hit `cmd/ctrl + k` and start typing what you want to do.
 
-> The command palette is currently unavailable in [interactive embedding](../embedding/interactive-embedding.md) contexts.
+> The command palette is currently unavailable in [Embedded analytics JS](../embedding/embedded-analytics-js.md) and [interactive embedding](../embedding/interactive-embedding.md) contexts.
 
 ## Advanced search
 

docs/installation-and-operation/serialization.md

@@ -296,7 +296,7 @@ This ID refers to the collection where the question was saved. In a real export,
 
 ### Entity IDs work with embedding
 
-Metabase supports working with [Entity IDs](#metabase-uses-entity-ids-to-identify-and-reference-metabase-items) for questions, dashboards, and collections in [Static Embedding](../embedding/static-embedding.md), [Interactive embedding](../embedding/interactive-embedding.md), and the [Embedded Analytics SDK](../embedding/sdk/introduction.md).
+Metabase supports working with [Entity IDs](#metabase-uses-entity-ids-to-identify-and-reference-metabase-items) for questions, dashboards, and collections in [Static Embedding](../embedding/static-embedding.md), [Embedded analytics JS](../embedding/embedded-analytics-js.md), [Interactive embedding](../embedding/interactive-embedding.md), and the [Embedded analytics SDK](../embedding/sdk/introduction.md).
 
 A high-level workflow for using Entity IDs when embedding Metabase in your app would look something like:
 

docs/permissions/row-and-column-security.md

@@ -1,5 +1,6 @@
 ---
 title: Row and column security
+summary: Control what data different groups can see by filtering rows and columns based on user attributes. Useful for multi-tenant analytics.
 redirect_from:
   - /docs/latest/enterprise-guide/data-sandboxes
   - /docs/latest/permissions/data-sandboxes
@@ -46,10 +47,10 @@ Row and column security show specific data to each person based on their [user a
 
 You can **restrict rows** by filtering a column using a [user attribute value](#choosing-user-attributes-for-row-and-column-security).
 
-For example, you can filter the Accounts table for a group so that:
+For example, you can filter the Accounts table for a group so that the user attribute filters the table:
 
-- A person with the user attribute value "Basic" will see rows where `Plan = "Basic"` (rows where the Plan column matches the value "Basic").
-- A person with the user attribute value "Premium" will see the rows where `Plan = "Premium"` (rows where the Plan column matches the value "Premium").
+- "Basic" will see rows where `Plan = "Basic"` (rows where the Plan column matches the value "Basic").
+- "Premium" will see the rows where `Plan = "Premium"` (rows where the Plan column matches the value "Premium").
 
 ### Custom row and column security: use a SQL question to create a custom "view" of a table
 
@@ -57,7 +58,7 @@ To **restrict rows _and_ columns**, you can use a SQL question to filter the tab
 
 For example, say your original Accounts table includes the columns: `ID`, `Email`, `Plan`, and `Created At`. If you want to hide the Email column, you can create a "Restricted Accounts" SQL question with the columns: `ID`, `Plan`, and `Created At`.
 
-You can use a question to filter tables to:
+You can use a question to:
 
 - [Display an edited column instead of hiding the column](#displaying-edited-columns).
 - [Pass a user attribute to a SQL parameter](#restricting-rows-with-user-attributes-using-a-sql-variable).
@@ -69,10 +70,10 @@ You can use a question to filter tables to:
 
 Row security displays a filtered table, in place of an original table, to a specific group. How Metabase filters that table depends on the value in each person's user attribute.
 
-For example, you can set up a row-level security so that:
+For example, you can set up a row-level security so that the user attribute `plan` shows different rows for different values:
 
-- Someone with the user attribute with key of "plan" and a value of "Basic" will see a version of the Accounts table with a filter for `Plan = "Basic"` (that is, only the rows where the Plan column matches the value "Basic").
-- Someone with a "plan" user attribute set to "Premium" will see a different version of the Accounts table with the filter `Plan = "Premium"` applied.
+- "Basic" will see a version of the Accounts table with a filter for `Plan = "Basic"` (that is, only the rows where the Plan column matches the value "Basic").
+- "Premium" will see a different version of the Accounts table with the filter `Plan = "Premium"` applied.
 
 ## Choosing user attributes for row and column security
 
@@ -179,7 +180,7 @@ In steps 9-10 of the [row restriction setup](#restricting-rows-with-user-attribu
 WHERE plan = "Basic"
 ```
 
-Note that the parameters must be required for SQL questions used to create a custom views. E.g., you cannot use an optional parameter; the following won't work:
+Note that the parameters must be required for SQL questions used to create custom views. For example, you can't use an optional parameter; the following won't work:
 
 ```sql
 {%raw%}
@@ -189,6 +190,42 @@ Note that the parameters must be required for SQL questions used to create a cus
 
 Learn more about [SQL parameters](../questions/native-editor/sql-parameters.md)
 
+### Advanced row-level security: filtering tables for people that have multiple IDs
+
+For example, say have a table like this:
+
+| User_ID | Value |
+|---------|-------|
+| 1       | 10    |
+| 1       | 50    |
+| 2       | 5     |
+| 2       | 50    |
+| 3       | 5     |
+| 3       | 5     |
+
+If you want to give someone access to multiple user IDs (e.g., the person should see rows for both `User_ID` 1 and 2.), you can set up a user attribute, like `user_id` that can handle comma-separated values like "1,2". 
+
+1. Create a SQL question that parses the comma-separated string and filters the table:
+
+```sql
+{%raw%}
+SELECT *
+FROM users_with_values
+WHERE user_id = ANY(STRING_TO_ARRAY(REGEXP_REPLACE(TRIM({{user_id}}), '\\s*,\\s*', ','), ','))
+{% endraw %}
+```
+
+This query:
+
+- Trims whitespace from the user attribute value
+- Replaces any spaces around commas with just commas
+- Converts the comma-separated string to an array
+- Filters rows where the user_id matches any value in the array
+
+The `STRING_TO_ARRAY()` and `REGEXP_REPLACE()` functions are PostgreSQL-specific. To see which functions your database supports, see your database's documentation.
+
+3. Set up the row and column security using this SQL question. See [setting up column security](#setting-up-column-security).
+
 ## Preventing row and column security permissions conflicts
 
 Some Metabase permissions can conflict with row and column security to give more permissive or more restrictive data access than you intended.
@@ -199,7 +236,7 @@ The Email column may get exposed to someone if:
 
 - The person belongs to [multiple row and column security policies](#multiple-row-and-column-security-permissions).
 - Someone else in a non-secured group shares the Email column from:
-  - A [SQL question](../questions/native-editor/writing-sql.md).
+  - A [SQL question](../questions/native-editor/writing-sql.md)
   - A [public link](#public-sharing)
   - An [alert, or dashboard subscription](../permissions/notifications.md)
 
@@ -243,12 +280,13 @@ To prevent this from happening, you'll have to [disable public sharing](../embed
 
 Metabase can only create row and column security using the group membership or user attributes of people who are logged in. Since public links don't require logins, Metabase won't have enough info to apply permissions.
 
-## Limitations
+## Limitations of row and column security
 
-Row and column security is limited to questions built with the [query builder](../questions/query-builder/editor.md).
+Some things to keep in mind when using row and column security.
 
 ### Groups with native query permissions (access to the SQL editor) can bypass row and column security
 
+Row and column security is limited to the [query builder](../questions/query-builder/editor.md).
 You can't set up [native query persmissons](./data.md#create-queries-permissions) for groups with row and column security.
 
 To enforce row-level permissions with the native query editor, check out [impersonation](./impersonation.md).
@@ -263,6 +301,22 @@ Since Metabase can't parse SQL queries, the results of SQL questions will always
 
 MongoDB only supports [row-level security](#row-level-security-filter-by-a-column-in-the-table). Row and column security permissions are unavailable for Apache Druid.
 
+### Advanced data types require a workaround
+
+If you're trying to set up row security on a column with an advanced data type (like enums or arrays), you'll need to convert that data to a basic type first. Options include:
+
+#### Option 1: Use SQL to cast the advanced data type to a basic SQL type
+
+Create a SQL question that casts the advanced data type column to a basic data type, then use that question for your row and column security setup.
+
+#### Option 2: Create a database view
+
+If you can't use SQL casting in Metabase, create a view in your database that converts the advanced data type to a basic type, then set up row and column security on that view instead of the original table. You'll also need to block the original table.
+
+#### Option 3: Use transforms 
+
+ Use a [transform](../data-modeling/transforms.md) to create a table that casts the advanced data type to a basic type. Then set up row and column security on the transformed table instead. You'll also need to block the original table.
+
 ## Further reading
 
 - [Row and column security examples](./row-and-column-security-examples.md)

docs/troubleshooting-guide/timezones.md

@@ -25,7 +25,7 @@ To fix this problem you'll need answers to these questions:
 Once you have these answers, look for cases like these:
 
 1. Your question or chart is comparing or sorting values with inconsistent or missing time zones. For example, if a flight's departure and arrival times are reported in local time, it can appear to arrive before it has left.
-2. Your question is aggregating timetsamps with different time zones: for example, the "daily" totals for your website's traffic include more than 24 hours worth of data because you are using the local dates from East Asia, Europe, and the Americas.
+2. Your question is aggregating timestamps with different time zones: for example, the "daily" totals for your website's traffic include more than 24 hours worth of data because you are using the local dates from East Asia, Europe, and the Americas.
 
 Once you think you have identified a problem, drill down to understand exactly what time zone conversion is causing the underlying problem. For example, suppose you're looking at a time series with daily values; if your error is happening with weekly totals, you can:
 

e2e/support/document-initial-data.ts

@@ -0,0 +1,165 @@
+import {
+  ORDERS_BY_YEAR_QUESTION_ID,
+  ORDERS_COUNT_QUESTION_ID,
+  ORDERS_QUESTION_ID,
+} from "e2e/support/cypress_sample_instance_data";
+
+export const DOCUMENT_WITH_TWO_CARDS = {
+  content: [
+    {
+      type: "paragraph",
+      content: [
+        {
+          type: "text",
+          text: "Testing drag and drop functionality",
+        },
+      ],
+      attrs: {
+        _id: "1",
+      },
+    },
+    {
+      type: "resizeNode",
+      attrs: {
+        height: 350,
+        minHeight: 280,
+        _id: "2",
+      },
+      content: [
+        {
+          type: "cardEmbed",
+          attrs: {
+            id: ORDERS_QUESTION_ID,
+            name: null,
+            _id: "2a",
+          },
+        },
+      ],
+    },
+    {
+      type: "paragraph",
+      content: [
+        {
+          type: "text",
+          text: "Some text between cards",
+        },
+      ],
+      attrs: {
+        _id: "3",
+      },
+    },
+    {
+      type: "resizeNode",
+      attrs: {
+        height: 350,
+        minHeight: 280,
+        _id: "4",
+      },
+      content: [
+        {
+          type: "cardEmbed",
+          attrs: {
+            id: ORDERS_COUNT_QUESTION_ID,
+            name: null,
+            _id: "4a",
+          },
+        },
+      ],
+    },
+    {
+      type: "paragraph",
+      attrs: {
+        _id: "5",
+      },
+    },
+  ],
+  type: "doc",
+};
+
+export const DOCUMENT_WITH_THREE_CARDS_AND_COLUMNS = {
+  content: [
+    {
+      type: "paragraph",
+      content: [
+        {
+          type: "text",
+          text: "Advanced drag and drop scenarios",
+        },
+      ],
+      attrs: {
+        _id: "1",
+      },
+    },
+    {
+      type: "resizeNode",
+      attrs: {
+        height: 350,
+        minHeight: 280,
+        _id: "2",
+      },
+      content: [
+        {
+          type: "flexContainer",
+          attrs: {
+            _id: "2a",
+          },
+          content: [
+            {
+              type: "cardEmbed",
+              attrs: {
+                id: ORDERS_QUESTION_ID,
+                name: null,
+                _id: "2a1",
+              },
+            },
+            {
+              type: "cardEmbed",
+              attrs: {
+                id: ORDERS_COUNT_QUESTION_ID,
+                name: null,
+                _id: "2a2",
+              },
+            },
+          ],
+        },
+      ],
+    },
+    {
+      type: "paragraph",
+      content: [
+        {
+          type: "text",
+          text: "Standalone card below",
+        },
+      ],
+      attrs: {
+        _id: "3",
+      },
+    },
+    {
+      type: "resizeNode",
+      attrs: {
+        height: 350,
+        minHeight: 280,
+        _id: "4",
+      },
+      content: [
+        {
+          type: "cardEmbed",
+          attrs: {
+            id: ORDERS_BY_YEAR_QUESTION_ID,
+            name: null,
+            _id: "4a",
+          },
+        },
+      ],
+    },
+    {
+      type: "paragraph",
+      attrs: {
+        _id: "5",
+      },
+    },
+  ],
+  type: "doc",
+};

e2e/support/helpers/api/index.ts

@@ -38,4 +38,4 @@ export { getCurrentUser } from "./getCurrentUser";
 export { remapDisplayValueToFK } from "./remapDisplayValueToFK";
 export { updateComment } from "./updateComment";
 export { updateDashboardCards } from "./updateDashboardCards";
-export { updateSetting } from "./updateSetting";
+export { updateSetting, updateEnterpriseSetting } from "./updateSetting";

e2e/support/helpers/api/updateSetting.ts

@@ -1,4 +1,4 @@
-import type { Settings } from "metabase-types/api";
+import type { EnterpriseSettings, Settings } from "metabase-types/api";
 
 export const updateSetting = <
   TKey extends keyof Settings,
@@ -9,3 +9,13 @@ export const updateSetting = <
 ): Cypress.Chainable<Cypress.Response<never>> => {
   return cy.request<never>("PUT", `/api/setting/${setting}`, { value });
 };
+
+export const updateEnterpriseSetting = <
+  TKey extends keyof EnterpriseSettings,
+  TValue extends EnterpriseSettings[TKey],
+>(
+  setting: TKey,
+  value: TValue,
+): Cypress.Chainable<Cypress.Response<never>> => {
+  return cy.request<never>("PUT", `/api/setting/${setting}`, { value });
+};

e2e/support/helpers/e2e-codemirror-helpers.ts

@@ -0,0 +1,223 @@
+type TypeOptions = {
+  delay?: number;
+  focus?: boolean;
+  allowFastSet?: boolean;
+};
+
+export function codeMirrorHelpers<T extends object>(testId: string, extra: T) {
+  const helpers = {
+    get() {
+      cy.findAllByTestId("loading-indicator").should("not.exist");
+      return cy.get(`[data-testid=${testId}] .cm-content`);
+    },
+    focus() {
+      helpers.get().should("be.visible").click("right");
+      helpers.get().get(".cm-editor").should("have.class", "cm-focused");
+      return helpers;
+    },
+    blur() {
+      helpers.get().get(".cm-content").blur();
+      return helpers;
+    },
+    selectAll() {
+      const isMac = Cypress.platform === "darwin";
+      const metaKey = isMac ? "Meta" : "Control";
+      helpers.focus();
+      cy.realPress([metaKey, "A"]);
+      return helpers;
+    },
+    clear() {
+      helpers.selectAll();
+      cy.realPress(["Backspace"]);
+      return helpers;
+    },
+    type(
+      text: string,
+      { focus = true, delay = 10, allowFastSet = false }: TypeOptions = {},
+    ) {
+      if (focus) {
+        helpers.focus();
+      }
+
+      if (allowFastSet) {
+        // Enter the formula in one go
+        // HACK: we do invoke("text") instead of type() because type() does not work on
+        // CodeMirror elements in Cypress. realType() would work but some of the formulas
+        // contain special characters that are not supported by realType().
+        helpers.get().invoke("text", text);
+
+        // invoke("text") does not trigger the validator, so we need to trigger it manually
+        // by typing something
+        helpers.type(" {backspace}");
+
+        return helpers;
+      }
+
+      const isMac = Cypress.platform === "darwin";
+      const metaKey = isMac ? "Meta" : "Control";
+
+      const parts = text.replaceAll("{{", "{{}{{}").split(/(\{[^}]+\})/);
+
+      function insert(text: string) {
+        cy.realType(text, { pressDelay: delay });
+      }
+
+      // HACK: realType does not accept a lot of common escape sequences,
+      // so we implement them manually here for the native editor.
+      parts.forEach((part) => {
+        switch (part.toLowerCase()) {
+          case "":
+            return;
+
+          case "{clear}":
+            return helpers.clear();
+
+          case "{selectall}":
+            return helpers.selectAll();
+
+          case "{leftarrow}":
+            return cy.realPress(["ArrowLeft"]);
+
+          case "{rightarrow}":
+            return cy.realPress(["ArrowRight"]);
+
+          case "{downarrow}":
+            return cy.realPress(["ArrowDown"]);
+
+          case "{uparrow}":
+            return cy.realPress(["ArrowUp"]);
+
+          case "{enter}":
+            return cy.realPress(["Enter"]);
+
+          case "{home}":
+          case "{movetostart}":
+            return cy.realPress(["Home"]);
+
+          case "{end}":
+          case "{movetoend}":
+            return cy.realPress(["End"]);
+
+          case "{backspace}":
+            return cy.realPress(["Backspace"]);
+
+          case "{tab}":
+            return cy.realPress(["Tab"]);
+
+          case "{nextcompletion}":
+            cy.wait(50);
+            return cy.realPress([metaKey, "j"]);
+
+          case "{prevcompletion}":
+            cy.wait(50);
+            return cy.realPress([metaKey, "k"]);
+
+          case "{{}":
+            return cy.realType("{");
+        }
+
+        if (part.startsWith("{") && part.endsWith("}")) {
+          // unknown escape sequence, let's try typing it
+          insert("{");
+          insert(part.slice(1));
+          return;
+        }
+
+        // realType does not support → arrow, so we replace it with ->, which the editor
+        // expands into →
+        const unexpanded = part.replaceAll(/→/g, "->");
+        const alphabet =
+          "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789^[]()-,.;_!@#$%&*+=/<>\" ':;\\\n{}";
+        if (unexpanded.split("").some((char) => !alphabet.includes(char))) {
+          throw new Error(
+            `unknown character in codeMirrorHelpers.type in ${part}`,
+          );
+        }
+
+        insert(unexpanded);
+      });
+
+      return helpers;
+    },
+    textbox() {
+      return helpers.get().get("[role='textbox']");
+    },
+    value() {
+      // Get the multiline text content of the editor
+      return helpers
+        .textbox()
+        .get(".cm-line")
+        .then((lines) => {
+          const text: string[] = [];
+          lines.each((_, line) => {
+            const placeholder = line.querySelector(".cm-placeholder");
+            if (placeholder) {
+              return;
+            }
+            text.push(line.textContent ?? "");
+          });
+          return text.join("\n");
+        });
+    },
+    completions() {
+      return cy.get(".cm-tooltip-autocomplete").should("be.visible");
+    },
+    completion(label: string) {
+      return cy.get(".cm-completionLabel").contains(label).parent();
+    },
+    acceptCompletion(key: "enter" | "tab" = "enter") {
+      helpers.completions().should("be.visible");
+
+      // Avoid flakiness with CodeMirror not accepting the suggestion immediately
+      cy.wait(300);
+      helpers.type(`{${key}}`, { focus: false });
+    },
+    selectCompletion(name: string) {
+      helpers.completions().should("be.visible");
+
+      // Avoid flakiness with CodeMirror not accepting the suggestion immediately
+      cy.wait(300);
+      helpers.completion(name).click();
+    },
+    paste(content: string) {
+      helpers.textbox().then((el) => {
+        const clipboardData = new DataTransfer();
+        clipboardData.setData("text/plain", content);
+
+        const pasteEvent = new ClipboardEvent("paste", {
+          bubbles: true,
+          cancelable: true,
+          clipboardData,
+        });
+
+        el[0].dispatchEvent(pasteEvent);
+      });
+    },
+    ...extra,
+  };
+
+  return helpers;
+}
+
+export function codeMirrorEditor() {
+  return cy.get(".cm-content");
+}
+
+export function codeMirrorValue() {
+  // Get the multiline text content of the editor
+  return codeMirrorEditor()
+    .get("[role='textbox']")
+    .get(".cm-line")
+    .then((lines) => {
+      const text: string[] = [];
+      lines.each((_, line) => {
+        text.push(line.textContent ?? "");
+      });
+      const value = text.join("\n");
+      const placeholder = "SELECT * FROM TABLE_NAME";
+      if (value === placeholder) {
+        return "";
+      }
+      return value;
+    });
+}

e2e/support/helpers/e2e-comment-helpers.ts

@@ -21,16 +21,22 @@ function getDocumentNodeButton({
   targetId,
   childTargetId,
   hasComments,
+  isCardEmbedNode,
 }: {
   targetId: DocumentId;
   childTargetId: string;
   hasComments?: boolean;
+  isCardEmbedNode?: boolean;
 }) {
   const threadUrl = `/document/${targetId}/comments/${childTargetId}${hasComments ? "" : "?new=true"}`;
 
-  return getDocumentNodeButtons().filter(
-    (_, element) => element.getAttribute("href") === threadUrl,
-  );
+  if (isCardEmbedNode) {
+    return cy.findByRole("button", { name: "Comments" });
+  } else {
+    return getDocumentNodeButtons().filter(
+      (_, element) => element.getAttribute("href") === threadUrl,
+    );
+  }
 }
 
 function getDocumentNodeButtons() {

e2e/support/helpers/e2e-custom-column-helpers.ts

@@ -1,5 +1,7 @@
 import { popover } from "e2e/support/helpers/e2e-ui-elements-helpers";
 
+import { codeMirrorHelpers } from "./e2e-codemirror-helpers";
+
 export function expressionEditorWidget() {
   return cy.findByTestId("expression-editor");
 }
@@ -8,9 +10,6 @@ export function expressionEditorTextfield() {
   return CustomExpressionEditor.get();
 }
 
-const isMac = Cypress.platform === "darwin";
-const metaKey = isMac ? "Meta" : "Control";
-
 export function enterCustomColumnDetails({
   formula,
   name,
@@ -71,224 +70,43 @@ export function enterCustomColumnDetails({
   }
 }
 
-export const CustomExpressionEditor = {
-  get() {
-    return cy.findByTestId("custom-expression-query-editor");
-  },
-  /**
-   * @param {string} text the formula to type
-   * @param {Object} options
-   * @param {boolean} [options.focus]
-   *   true by default. Set to false to stop the helper from explicitly focussing the formula input.
-   * @param {boolean} [options.allowFastSet]
-   *   true by default. However, if you need to examine the popover in the test, it should be set to false so the popover is not dismissed
-   *   Because CodeMirror uses a contenteditable div, and it is not possible to use cy.type() on it, we emulate .type with realPress.
-   *   This does not always work, since realPress() does not support all characters. Setting this to true will enable an escape hatch
-   *   that uses cy.invoke('text') under the hood, to allow for formulas that contain unsupported characters.
-   *   This has some other sideeffects however, so use it sparingly.
-   */
-  type(
-    text: string,
-    {
-      allowFastSet = false,
-      focus = true,
-      delay = 0,
-    }: {
-      focus?: boolean;
-      allowFastSet?: boolean;
-      delay?: number;
-    } = {},
-  ) {
-    if (focus) {
-      CustomExpressionEditor.focus();
-    }
-
-    if (allowFastSet) {
-      // Enter the formula in one go
-      // HACK: we do invoke("text") instead of type() because type() does not work on
-      // CodeMirror elements in Cypress. realType() would work but some of the formulas
-      // contain special characters that are not supported by realType().
-      CustomExpressionEditor.get().findByRole("textbox").invoke("text", text);
-
-      // invoke("text") does not trigger the validator, so we need to trigger it manually
-      // by typing something
-      CustomExpressionEditor.type(" {backspace}");
-
+export const CustomExpressionEditor = codeMirrorHelpers(
+  "custom-expression-query-editor",
+  {
+    formatButton() {
+      return cy.findByLabelText("Auto-format");
+    },
+    format() {
+      CustomExpressionEditor.formatButton().should("be.visible");
+      CustomExpressionEditor.formatButton().click();
       return CustomExpressionEditor;
-    }
-
-    const parts = text.replaceAll("{{", "{{}{{}").split(/(\{[^}]+\})/);
-
-    parts.forEach((part) => {
-      cy.wait(delay);
-      switch (part.toLowerCase()) {
-        case "":
-          return;
-
-        case "{clear}":
-          return CustomExpressionEditor.clear();
-
-        case "{selectall}":
-          return CustomExpressionEditor.selectAll();
-
-        case "{leftarrow}":
-          return cy.realPress(["ArrowLeft"]);
-
-        case "{rightarrow}":
-          return cy.realPress(["ArrowRight"]);
-
-        case "{downarrow}":
-          return cy.realPress(["ArrowDown"]);
-
-        case "{uparrow}":
-          return cy.realPress(["ArrowUp"]);
-
-        case "{enter}":
-          return cy.realPress(["Enter"]);
-
-        case "{home}":
-        case "{movetostart}":
-          return cy.realPress(["Home"]);
-
-        case "{end}":
-        case "{movetoend}":
-          return cy.realPress(["End"]);
-
-        case "{backspace}":
-          return cy.realPress(["Backspace"]);
-
-        case "{tab}":
-          return cy.realPress(["Tab"]);
-
-        case "{nextcompletion}":
-          cy.wait(50);
-          return cy.realPress([metaKey, "j"]);
-
-        case "{prevcompletion}":
-          cy.wait(50);
-          return cy.realPress([metaKey, "k"]);
-
-        case "{{}":
-          return cy.realType("{");
-      }
-
-      if (part.startsWith("{") && part.endsWith("}")) {
-        // unknown escape sequence, let's try typing it
-        throw new Error(
-          `unknown escape sequence in CustomExpressionEditor.type: ${part}`,
-        );
-      }
-
-      // realType does not support → arrow, so we replace it with ->, which the editor
-      // expands into →
-      const unexpanded = part.replaceAll(/→/g, "->");
-
-      const alphabet =
-        "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789^[]()-,.;_!@#$%&*+=/<>\" ':;\\\n";
-      if (unexpanded.split("").some((char) => !alphabet.includes(char))) {
-        throw new Error(
-          `unknown character in CustomExpressionEditor.type in ${part}`,
-        );
-      }
-
-      cy.realType(unexpanded, { delay });
-    });
-    return CustomExpressionEditor;
-  },
-  focus() {
-    CustomExpressionEditor.get().click("right");
-    return CustomExpressionEditor;
-  },
-  blur() {
-    // click outside the expression editor
-    cy.findByTestId("expression-editor").click("bottomRight", { force: true });
-    return CustomExpressionEditor;
-  },
-  formatButton() {
-    return cy.findByLabelText("Auto-format");
-  },
-  format() {
-    CustomExpressionEditor.formatButton().should("be.visible");
-    CustomExpressionEditor.formatButton().click();
-    return CustomExpressionEditor;
-  },
-  selectAll() {
-    CustomExpressionEditor.focus();
-    cy.realPress([metaKey, "A"]);
-    return CustomExpressionEditor;
-  },
-  clear() {
-    CustomExpressionEditor.selectAll();
-    cy.realPress(["Backspace"]);
-    return CustomExpressionEditor;
-  },
-  textbox() {
-    return CustomExpressionEditor.get().get("[role='textbox']");
-  },
-  value() {
-    // Get the multiline text content of the editor
-    return CustomExpressionEditor.textbox()
-      .get(".cm-line")
-      .then((lines) => {
-        const text: string[] = [];
-        lines.each((_, line) => {
-          text.push(line.textContent ?? "");
-        });
-        const value = text.join("\n");
-        const placeholder = "Type your expression, press '[' for columns…";
-        if (value === placeholder) {
-          return "";
-        }
-        return value;
+    },
+    helpTextHeader() {
+      return cy.findByTestId("expression-helper-popover-structure");
+    },
+    helpText() {
+      return cy.findByTestId("expression-helper");
+    },
+    nameInput() {
+      return cy.findByTestId("expression-name");
+    },
+    functionBrowser() {
+      return cy.findByTestId("expression-editor-function-browser");
+    },
+    completions() {
+      return cy.findByTestId("custom-expression-editor-suggestions");
+    },
+    completion(name: string) {
+      return CustomExpressionEditor.completions()
+        .findAllByRole("option")
+        .contains(name)
+        .first();
+    },
+    blur() {
+      cy.findByTestId("expression-editor").click("bottomRight", {
+        force: true,
       });
+      return CustomExpressionEditor;
+    },
   },
-  completions() {
-    return cy.findByTestId("custom-expression-editor-suggestions");
-  },
-  completion(name: string) {
-    return CustomExpressionEditor.completions()
-      .findAllByRole("option")
-      .contains(name)
-      .first();
-  },
-  acceptCompletion(key: "enter" | "tab" = "enter") {
-    CustomExpressionEditor.completions().should("be.visible");
-
-    // Avoid flakiness with CodeMirror not accepting the suggestion immediately
-    cy.wait(300);
-    CustomExpressionEditor.type(`{${key}}`, { focus: false });
-  },
-  selectCompletion(name: string) {
-    CustomExpressionEditor.completions().should("be.visible");
-
-    // Avoid flakiness with CodeMirror not accepting the suggestion immediately
-    cy.wait(300);
-    CustomExpressionEditor.completion(name).click();
-  },
-  helpTextHeader() {
-    return cy.findByTestId("expression-helper-popover-structure");
-  },
-  helpText() {
-    return cy.findByTestId("expression-helper");
-  },
-  paste(content: string) {
-    CustomExpressionEditor.textbox().then((el) => {
-      const clipboardData = new DataTransfer();
-      clipboardData.setData("text/plain", content);
-
-      const pasteEvent = new ClipboardEvent("paste", {
-        bubbles: true,
-        cancelable: true,
-        clipboardData,
-      });
-
-      el[0].dispatchEvent(pasteEvent);
-    });
-  },
-  nameInput() {
-    return cy.findByTestId("expression-name");
-  },
-  functionBrowser() {
-    return cy.findByTestId("expression-editor-function-browser");
-  },
-};
+);

e2e/support/helpers/e2e-document-helpers.ts

@@ -37,10 +37,22 @@ export const documentMetabotSuggestionItem = (name: RegExp | string) =>
 export const commandSuggestionItem = (name: RegExp | string) =>
   commandSuggestionDialog().findByRole("option", { name });
 
-export const getDocumentCard = (name: string) =>
-  documentContent()
-    .findAllByTestId("document-card-embed")
-    .contains('[data-testid="document-card-embed"]', name);
+export const getDocumentCard = (name: string, expectCardExists = true) => {
+  const cardsResult = documentContent()
+    .findAllByTestId("card-embed-title")
+    .filter((index, el) => {
+      // Filter elements based on custom logic, e.g., text content, class, etc.
+      return el.innerText === name;
+    })
+    .should("have.length", expectCardExists ? 1 : 0);
+
+  return expectCardExists
+    ? cardsResult.closest('[data-testid="document-card-embed"]')
+    : cardsResult;
+};
+
+export const getDocumentCardResizeContainer = (name: string) =>
+  getDocumentCard(name).closest('[data-type="resizeNode"]');
 
 export const assertDocumentCardVizType = (name: string, type: string) =>
   getDocumentCard(name).find(`[data-viz-ui-name=${type}]`);
@@ -102,3 +114,124 @@ const visitDocumentById = (id: DocumentId) => {
 
   cy.wait(`@${alias}`);
 };
+
+export function dragAndDropCardOnAnotherCard(
+  sourceCardTitle: string,
+  targetCardTitle: string,
+  {
+    side = "left",
+  }: {
+    side?: "left" | "right";
+  } = {},
+) {
+  // Perform drag and drop: drag sourceCard card onto targetCard
+  getDocumentCard(sourceCardTitle)
+    .should("exist")
+    .then(($ordersCard) => {
+      getDocumentCard(targetCardTitle)
+        .should("exist")
+        .then(($ordersCountCard) => {
+          const sourceRect = $ordersCard[0].getBoundingClientRect();
+
+          // Start drag from center of Orders card
+          getDocumentCard(sourceCardTitle).trigger("mousedown", {
+            x: 10,
+            y: 10,
+            scrollBehavior: false,
+            force: true,
+          });
+          const dataTransfer = new DataTransfer();
+          getDocumentCard(sourceCardTitle).trigger("dragstart", {
+            dataTransfer,
+            clientX: sourceRect.left + 10,
+            clientY: sourceRect.top + 10,
+            scrollBehavior: false,
+            force: true,
+          });
+
+          const targetRect = $ordersCountCard[0].getBoundingClientRect();
+          // Calculate position for target card side drop (20% from left or 20% from right based on 'side' param)
+          const sideX =
+            targetRect.left + targetRect.width * (side === "left" ? 0.2 : 0.8);
+          const centerY = targetRect.top + 10;
+          documentContent().trigger("mousemove", {
+            clientX: sideX,
+            clientY: centerY,
+            scrollBehavior: false,
+          });
+          getDocumentCard(targetCardTitle).trigger("dragover", {
+            clientX: sideX,
+            clientY: centerY,
+            scrollBehavior: false,
+            force: true,
+          });
+
+          getDocumentCard(sourceCardTitle).realMouseUp({
+            scrollBehavior: false,
+          });
+          getDocumentCard(targetCardTitle).trigger("drop", {
+            dataTransfer,
+            clientX: sideX,
+            clientY: centerY,
+            scrollBehavior: false,
+            force: true,
+          });
+          getDocumentCard(sourceCardTitle).trigger("dragend", {
+            scrollBehavior: false,
+            force: true,
+          });
+        });
+    });
+}
+
+export function documentUndo() {
+  const macOSX = Cypress.platform === "darwin";
+
+  documentContent()
+    .get('[contenteditable="true"]')
+    .click()
+    .type(macOSX ? "{cmd+z}" : "{ctrl+z}", {
+      scrollBehavior: false,
+      force: true,
+    });
+}
+
+export function getDragHandleForDocumentResizeNode(element: Cypress.Chainable) {
+  return element.findByTestId("resize-node-drag-handle");
+}
+
+export function documentDoDrag(
+  _handle: Cypress.Chainable<JQuery<HTMLElement>>,
+  diff: { x?: number; y?: number },
+) {
+  _handle.then((handle) => {
+    const { x: deltaX = 0, y: deltaY = 0 } = diff;
+
+    const rect = handle[0].getBoundingClientRect();
+
+    cy.log(`x: ${rect.x}, y: ${rect.y}, diff: ${diff}`);
+
+    cy.wrap(handle).trigger("mousedown", {
+      button: 0,
+      clientX: rect.x,
+      clientY: rect.y,
+      force: true,
+    });
+    cy.get("body")
+      .trigger("mousemove", {
+        button: 0,
+        clientX: rect.x + deltaX,
+        clientY: rect.y + deltaY,
+        force: true,
+      })
+      .trigger("mouseup");
+  });
+}
+
+export function getFlexContainerForCard(name: string) {
+  return getDocumentCard(name).closest('[data-type="flexContainer"]');
+}
+
+export function getResizeHandlesForFlexContianer(element: Cypress.Chainable) {
+  return element.findAllByTestId("flex-container-drag-handle");
+}

e2e/support/helpers/e2e-embedding-iframe-sdk-helpers.ts

@@ -24,6 +24,7 @@ export interface BaseEmbedTestPageOptions {
     instanceUrl?: string;
     apiKey?: string;
     useExistingUserSession?: boolean;
+    fetchRequestToken?: () => Promise<{ jwt: string }>;
     theme?: MetabaseTheme;
     preferredAuthMethod?: "jwt" | "saml";
     locale?: string;
@@ -39,7 +40,7 @@ export interface BaseEmbedTestPageOptions {
     afterEmbed?: string;
   };
 
-  onVisitPage?(): void;
+  onVisitPage?(win: Cypress.AUTWindow): void;
 }
 
 export interface MetabaseElement {
@@ -267,14 +268,7 @@ export const getNewEmbedConfigurationScript = ({
   useExistingUserSession,
   preferredAuthMethod,
   locale,
-}: {
-  instanceUrl?: string;
-  theme?: MetabaseTheme;
-  apiKey?: string;
-  useExistingUserSession?: boolean;
-  preferredAuthMethod?: "jwt" | "saml";
-  locale?: string;
-} = {}) => {
+}: BaseEmbedTestPageOptions["metabaseConfig"] = {}) => {
   const config = {
     instanceUrl,
     apiKey,

e2e/support/helpers/e2e-native-editor-helpers.ts

@@ -1,42 +1,6 @@
+import { codeMirrorHelpers } from "./e2e-codemirror-helpers";
 import { popover } from "./e2e-ui-elements-helpers";
 
-function nativeEditor() {
-  cy.findAllByTestId("loading-indicator").should("not.exist");
-  return cy.get("[data-testid=native-query-editor] .cm-content");
-}
-
-function focusNativeEditor() {
-  nativeEditor().should("be.visible").click();
-
-  nativeEditor().get(".cm-editor").should("have.class", "cm-focused");
-
-  return nativeEditor();
-}
-
-function blurNativeEditor() {
-  nativeEditor().get(".cm-editor").blur();
-}
-
-function nativeEditorCompletions() {
-  return cy.get(".cm-tooltip-autocomplete").should("be.visible");
-}
-
-function nativeEditorCompletion(label: string) {
-  return cy.get(".cm-completionLabel").contains(label).parent();
-}
-
-function nativeEditorSelectAll() {
-  const isMac = Cypress.platform === "darwin";
-  const metaKey = isMac ? "Meta" : "Control";
-  focusNativeEditor().realPress([metaKey, "A"]);
-  cy.get(".cm-selectionBackground").should("exist");
-}
-
-function clearNativeEditor() {
-  nativeEditorSelectAll();
-  cy.realPress(["Backspace"]);
-}
-
 export function nativeEditorDataSource() {
   return cy.findAllByTestId("gui-builder-data").first();
 }
@@ -46,139 +10,7 @@ export function selectNativeEditorDataSource(name: string) {
   popover().findByText(name).click();
 }
 
-type TypeOptions = {
-  delay?: number;
-  focus?: boolean;
-};
-
-function nativeEditorType(
-  text: string,
-  { delay = 10, focus = true }: TypeOptions = {},
-) {
-  if (focus) {
-    focusNativeEditor();
-  }
-
-  const isMac = Cypress.platform === "darwin";
-  const metaKey = isMac ? "Meta" : "Control";
-
-  const parts = text.replaceAll("{{", "{{}{{}").split(/(\{[^}]+\})/);
-
-  function type(text: string) {
-    cy.realType(text, {
-      pressDelay: delay,
-    });
-  }
-
-  // HACK: realType does not accept a lot of common escape sequences,
-  // so we implement them manually here for the native editor.
-  parts.forEach((part) => {
-    switch (part.toLowerCase()) {
-      case "":
-        return;
-
-      case "{clear}":
-        return clearNativeEditor();
-
-      case "{selectall}":
-        return nativeEditorSelectAll();
-
-      case "{leftarrow}":
-        return cy.realPress(["ArrowLeft"]);
-
-      case "{rightarrow}":
-        return cy.realPress(["ArrowRight"]);
-
-      case "{downarrow}":
-        return cy.realPress(["ArrowDown"]);
-
-      case "{uparrow}":
-        return cy.realPress(["ArrowUp"]);
-
-      case "{enter}":
-        return cy.realPress(["Enter"]);
-
-      case "{home}":
-      case "{movetostart}":
-        return cy.realPress(["Home"]);
-
-      case "{end}":
-      case "{movetoend}":
-        return cy.realPress(["End"]);
-
-      case "{backspace}":
-        return cy.realPress(["Backspace"]);
-
-      case "{tab}":
-        return cy.realPress(["Tab"]);
-
-      case "{nextcompletion}":
-        cy.wait(50);
-        return cy.realPress([metaKey, "j"]);
-
-      case "{prevcompletion}":
-        cy.wait(50);
-        return cy.realPress([metaKey, "k"]);
-
-      case "{{}":
-        return cy.realType("{");
-    }
-
-    if (part.startsWith("{") && part.endsWith("}")) {
-      // unknown escape sequence, let's try typing it
-      type("{");
-      type(part.slice(1));
-      return;
-    }
-
-    type(part);
-  });
-
-  return nativeEditor();
-}
-
-export const NativeEditor = {
-  get: nativeEditor,
-  type(text: string, options: TypeOptions = {}) {
-    nativeEditorType(text, options);
-    return NativeEditor;
-  },
-  focus() {
-    focusNativeEditor();
-    return NativeEditor;
-  },
-  blur() {
-    blurNativeEditor();
-    return NativeEditor;
-  },
-  selectAll() {
-    nativeEditorSelectAll();
-    return NativeEditor;
-  },
-  clear() {
-    clearNativeEditor();
-    return NativeEditor;
-  },
-  textbox() {
-    return NativeEditor.get().get("[role='textbox']");
-  },
-  value() {
-    // Get the multiline text content of the editor
-    return NativeEditor.textbox()
-      .get(".cm-line")
-      .then((lines) => {
-        const text: string[] = [];
-        lines.each((_, line) => {
-          text.push(line.textContent ?? "");
-        });
-        const value = text.join("\n");
-        const placeholder = "SELECT * FROM TABLE_NAME";
-        if (value === placeholder) {
-          return "";
-        }
-        return value;
-      });
-  },
-  completions: nativeEditorCompletions,
-  completion: nativeEditorCompletion,
-};
+export const NativeEditor = codeMirrorHelpers("native-query-editor", {
+  dataSource: nativeEditorDataSource,
+  selectDataSource: selectNativeEditorDataSource,
+});

e2e/support/helpers/e2e-python-helpers.ts

@@ -0,0 +1,3 @@
+import { codeMirrorHelpers } from "./e2e-codemirror-helpers";
+
+export const PythonEditor = codeMirrorHelpers("python-editor", {});

e2e/support/helpers/embedding-sdk-testing/embedding-sdk-helpers.ts

@@ -11,6 +11,19 @@ import {
 } from "e2e/support/helpers";
 import { enableJwtAuth } from "e2e/support/helpers/e2e-jwt-helpers";
 
+export const getSignedJwtForUser = async (user = USERS.admin) => {
+  const secret = new TextEncoder().encode(JWT_SHARED_SECRET);
+
+  return new jose.SignJWT({
+    email: user.email,
+    first_name: user.first_name,
+    last_name: user.last_name,
+    exp: Math.round(Date.now() / 1000) + 60 * 10, // 10 minutes expiration
+  })
+    .setProtectedHeader({ alg: "HS256" })
+    .sign(secret);
+};
+
 export const mockAuthProviderAndJwtSignIn = (user = USERS.admin) => {
   cy.intercept("GET", `${AUTH_PROVIDER_URL}**`, async (req) => {
     try {
@@ -26,15 +39,7 @@ export const mockAuthProviderAndJwtSignIn = (user = USERS.admin) => {
         return;
       }
 
-      const secret = new TextEncoder().encode(JWT_SHARED_SECRET);
-      const jwt = await new jose.SignJWT({
-        email: user.email,
-        first_name: user.first_name,
-        last_name: user.last_name,
-        exp: Math.round(Date.now() / 1000) + 60 * 10, // 10 minutes expiration
-      })
-        .setProtectedHeader({ alg: "HS256" })
-        .sign(secret);
+      const jwt = await getSignedJwtForUser(user);
 
       req.reply({
         statusCode: 200,

e2e/support/helpers/index.ts

@@ -6,6 +6,7 @@ export * from "./e2e-bi-basics-helpers";
 export * from "./e2e-boolean-helpers";
 export * from "./e2e-browser-helpers";
 export * from "./e2e-cloud-helpers";
+export * from "./e2e-codemirror-helpers";
 export * from "./e2e-collection-helpers";
 export * from "./e2e-command-palette-helpers";
 export * from "./e2e-comment-helpers";
@@ -35,6 +36,7 @@ export * from "./e2e-native-editor-helpers";
 export * from "./e2e-notebook-helpers";
 export * from "./e2e-notification-helpers";
 export * from "./e2e-permissions-helpers";
+export * from "./e2e-python-helpers";
 export * from "./e2e-qa-databases-helpers";
 export * from "./e2e-relative-date-picker-helpers";
 export * from "./e2e-request-helpers";

e2e/test-component/scenarios/embedding-sdk/styles-tests.cy.spec.tsx

@@ -4,9 +4,11 @@ import {
   InteractiveDashboard,
   InteractiveQuestion,
   MetabaseProvider,
+  type MetabaseTheme,
   StaticQuestion,
   defineMetabaseTheme,
 } from "@metabase/embedding-sdk-react";
+import { useState } from "react";
 
 import { SAMPLE_DATABASE } from "e2e/support/cypress_sample_database";
 import { ORDERS_QUESTION_ID } from "e2e/support/cypress_sample_instance_data";
@@ -132,6 +134,42 @@ describe("scenarios > embedding-sdk > styles", () => {
         .invoke("css", "color")
         .should("equal", "rgb(255, 0, 0)");
     });
+
+    it('should be able to reset theme colors by setting it to "undefined" (EMB-696)', () => {
+      const THEME = defineMetabaseTheme({
+        colors: {
+          "text-primary": "#0000ff",
+        },
+      });
+      function TestComponent() {
+        const [theme, setTheme] = useState<MetabaseTheme | undefined>(
+          undefined,
+        );
+        return (
+          <MetabaseProvider
+            authConfig={DEFAULT_SDK_AUTH_PROVIDER_CONFIG}
+            theme={theme}
+          >
+            <button onClick={() => setTheme(THEME)}>Set theme</button>
+            <button onClick={() => setTheme(undefined)}>Remove theme</button>
+            <InteractiveQuestion questionId={ORDERS_QUESTION_ID} />
+          </MetabaseProvider>
+        );
+      }
+      cy.mount(<TestComponent />);
+
+      getSdkRoot().within(() => {
+        cy.findByRole("button", { name: "Set theme" }).click();
+        cy.findByText("Orders")
+          .invoke("css", "color")
+          .should("equal", "rgb(0, 0, 255)");
+        cy.findByRole("button", { name: "Remove theme" }).click();
+        cy.findByText("Orders")
+          .invoke("css", "color")
+          // --mb-color-text-primary
+          .should("equal", "rgb(76, 87, 115)");
+      });
+    });
   });
 
   describe("style leaking", () => {

e2e/test/scenarios/admin/databases.cy.spec.js

@@ -122,9 +122,9 @@ describe("admin > database > add", () => {
             .click({ force: true })
             .should("have.attr", "data-checked", "true");
 
-          cy.findByLabelText(
+          cy.findByDisplayValue(
             "Never, I'll do this manually if I need to",
-          ).should("have.attr", "aria-selected", "true");
+          ).should("exist");
 
           // make sure tooltips behave as expected
           cy.findByLabelText("Host")
@@ -243,11 +243,9 @@ describe("admin > database > add", () => {
           "true",
         );
 
-        cy.findByLabelText("Never, I'll do this manually if I need to").should(
-          "have.attr",
-          "aria-selected",
-          "true",
-        );
+        cy.findByDisplayValue(
+          "Never, I'll do this manually if I need to",
+        ).should("exist");
       });
     });
 
@@ -711,17 +709,15 @@ describe("scenarios > admin > databases > sample database", () => {
     });
 
     // "lets you change the cache_field_values period"
-    cy.findByLabelText("Never, I'll do this manually if I need to").should(
-      "have.attr",
-      "aria-selected",
-      "true",
-    );
+    cy.findByDisplayValue("Never, I'll do this manually if I need to")
+      .should("be.visible")
+      .click();
 
-    cy.findByLabelText("Regularly, on a schedule")
-      .click()
-      .within(() => {
-        cy.findByText("Daily").click();
-      });
+    H.popover().findByText("Regularly, on a schedule").click();
+    cy.findAllByRole("button", { name: /Daily/ })
+      .should("have.length", 2)
+      .eq(1)
+      .click();
     H.popover().findByText("Weekly").click();
 
     cy.button("Save changes").click();
@@ -735,7 +731,8 @@ describe("scenarios > admin > databases > sample database", () => {
     });
 
     // "lets you change the cache_field_values to 'Only when adding a new filter widget'"
-    cy.findByLabelText("Only when adding a new filter widget").click();
+    cy.findByDisplayValue("Regularly, on a schedule").click();
+    H.popover().findByText("Only when adding a new filter widget").click();
     cy.button("Save changes", { timeout: 10000 }).click();
     cy.wait("@databaseUpdate").then(({ response: { body } }) => {
       editDatabase();
@@ -744,7 +741,8 @@ describe("scenarios > admin > databases > sample database", () => {
     });
 
     // and back to never
-    cy.findByLabelText("Never, I'll do this manually if I need to").click();
+    cy.findByDisplayValue("Only when adding a new filter widget").click();
+    H.popover().findByText("Never, I'll do this manually if I need to").click();
     cy.button("Save changes", { timeout: 10000 }).click();
     cy.wait("@databaseUpdate").then(({ response: { body } }) => {
       editDatabase();
@@ -769,7 +767,8 @@ describe("scenarios > admin > databases > sample database", () => {
     });
 
     editDatabase();
-    cy.findByLabelText("Regularly, on a schedule").click();
+    cy.findByDisplayValue("Never, I'll do this manually if I need to").click();
+    H.popover().findByText("Regularly, on a schedule").click();
     cy.button("Save changes").click();
     cy.wait("@databaseUpdate").then(({ request: { body } }) => {
       expect(body.is_full_sync).to.equal(true);
@@ -783,7 +782,8 @@ describe("scenarios > admin > databases > sample database", () => {
     });
 
     editDatabase();
-    cy.findByLabelText("Only when adding a new filter widget").click();
+    cy.findByDisplayValue("Regularly, on a schedule").click();
+    H.popover().findByText("Only when adding a new filter widget").click();
     cy.button("Save changes").click();
     cy.wait("@databaseUpdate").then(({ request: { body } }) => {
       expect(body.is_full_sync).to.equal(false);

e2e/test/scenarios/admin/transforms.cy.spec.ts

@@ -1,10 +1,13 @@
 const { H } = cy;
 
+import dedent from "ts-dedent";
+
 import { SAMPLE_DB_ID, WRITABLE_DB_ID } from "e2e/support/cypress_data";
 import { SAMPLE_DATABASE } from "e2e/support/cypress_sample_database";
 import type {
   CardType,
   ListTransformRunsResponse,
+  PythonTransformTableAliases,
   TransformTagId,
 } from "metabase-types/api";
 
@@ -127,7 +130,7 @@ H.describeWithSnowplowEE("scenarios > admin > transforms", () => {
     it("should be able to use the data reference and snippets when writing a SQL transform", () => {
       H.createSnippet({
         name: "snippet1",
-        content: "1",
+        content: "'foo'",
       });
 
       visitTransformListPage();
@@ -175,12 +178,135 @@ H.describeWithSnowplowEE("scenarios > admin > transforms", () => {
           .click();
 
         editorSidebar().should("not.exist");
+
+        cy.findByTestId("native-query-editor-action-buttons")
+          .findByLabelText("Preview the query")
+          .click();
+
+        H.modal().findByText("'foo'").should("be.visible");
       }
 
       testDataReference();
       testSnippets();
     });
 
+    it(
+      "should be possible to create and run a Python transform",
+      { tags: ["@transforms-python"] },
+      () => {
+        setPythonRunnerSettings();
+        cy.log("create a new transform");
+        visitTransformListPage();
+        getTransformListPage().button("Create a transform").click();
+        H.popover().findByText("Python script").click();
+
+        H.expectUnstructuredSnowplowEvent({
+          event: "transform_create",
+          event_detail: "python",
+          triggered_from: "transform-page-create-menu",
+        });
+
+        cy.findByTestId("python-data-picker")
+          .findByText("Select a database")
+          .click();
+
+        cy.log("Unsupported databases should be disabled");
+        H.popover()
+          .findByRole("option", { name: "Sample Database" })
+          .should("have.attr", "aria-disabled", "true");
+
+        cy.log("Select database");
+        H.popover().findByText(DB_NAME).click();
+
+        getPythonDataPicker().button("Select a table…").click();
+        H.entityPickerModal().findByText("Animals").click();
+
+        getPythonDataPicker().within(() => {
+          cy.findByText("Writable Postgres12 / Schema A").should("be.visible");
+          cy.findByText("Animals").should("be.visible");
+          cy.findByPlaceholderText("Enter alias").should(
+            "have.value",
+            "animals",
+          );
+          cy.findByPlaceholderText("Enter alias")
+            .clear()
+            .type("foo bar")
+            .blur();
+          cy.findByPlaceholderText("Enter alias").should(
+            "have.value",
+            "foo_bar",
+          );
+        });
+
+        H.PythonEditor.value()
+          .should("contain", "def transform(foo_bar):")
+          .should(
+            "contain",
+            'foo_bar: DataFrame containing the data from the "Writable Postgres12.Schema A.Animals" table',
+          );
+
+        getPythonDataPicker().within(() => {
+          cy.findByText("Add a table").click();
+          cy.button("Select a table…").click();
+        });
+
+        H.entityPickerModal().within(() => {
+          cy.log("Selecting the same table should not be possible");
+          cy.findByText("Animals")
+            .parent()
+            .parent()
+            .parent()
+            .should("have.attr", "data-disabled", "true");
+
+          cy.findByText("Schema B").click();
+          cy.findByText("Animals").click();
+        });
+
+        getPythonDataPicker().within(() => {
+          cy.icon("refresh").click();
+          cy.findAllByPlaceholderText("Enter alias")
+            .first()
+            .should("have.value", "animals_1")
+            .clear()
+            .type("foo bar")
+            .blur()
+            .should("have.value", "foo_bar");
+          cy.findAllByPlaceholderText("Enter alias")
+            .eq(1)
+            .should("have.value", "animals")
+            .clear()
+            .type("foo bar")
+            .blur()
+            .should("have.value", "foo_bar_1");
+        });
+
+        getQueryEditor().button("Save").click();
+
+        H.modal().within(() => {
+          cy.findByLabelText("Name").clear().type("Python transform");
+          cy.findByLabelText("Table name").clear().type("python_transform");
+          cy.button("Save").click();
+        });
+
+        H.expectUnstructuredSnowplowEvent({
+          event: "transform_created",
+        });
+
+        cy.log("run the transform and make sure its table can be queried");
+        runTransformAndWaitForSuccess();
+        H.expectUnstructuredSnowplowEvent({
+          event: "transform_trigger_manual_run",
+          triggered_from: "transform-page",
+        });
+
+        getRunSection().should("contain", "Executing Python transform");
+
+        getTableLink().click();
+        H.queryBuilderHeader().findByText(DB_NAME).should("be.visible");
+        H.assertQueryBuilderRowCount(1);
+      },
+    );
+
     it("should be able to create and run a transform from a question or a model", () => {
       function testCardSource({
         type,
@@ -511,6 +637,114 @@ LIMIT
     });
   });
 
+  describe("filtering", () => {
+    it("should be able to filter transforms", () => {
+      createMbqlTransform({ visitTransform: true });
+      runJobAndWaitForSuccess();
+
+      createSqlTransform({
+        sourceQuery: `SELECT * FROM "${TARGET_SCHEMA}"."${SOURCE_TABLE}"`,
+        targetTable: TARGET_TABLE_2,
+        visitTransform: true,
+      });
+      getTagsInput().click();
+      H.popover().findByText("hourly").click();
+      H.undoToast().findByText("Transform tags updated").should("be.visible");
+
+      visitTransformListPage();
+
+      function testLastRunDateFilter() {
+        cy.log("no filters");
+        getContentTable().within(() => {
+          cy.findByText("MBQL transform").should("be.visible");
+          cy.findByText("SQL transform").should("be.visible");
+        });
+
+        cy.log("last run at - add a filter");
+        getLastRunDateFilterWidget().click();
+        H.popover().within(() => {
+          cy.findByText("Relative date range…").click();
+          cy.findByText("Current").click();
+          cy.findByText("Week").click();
+        });
+
+        getLastRunDateFilterWidget().should("contain", "This week");
+        getContentTable().within(() => {
+          cy.findByText("MBQL transform").should("be.visible");
+          cy.findByText("SQL transform").should("not.exist");
+        });
+
+        cy.log("last run at filter - remove filter");
+        getLastRunDateFilterWidget().button("Remove filter").click();
+        getContentTable().within(() => {
+          cy.findByText("MBQL transform").should("be.visible");
+          cy.findByText("SQL transform").should("be.visible");
+        });
+      }
+
+      function testLastRunStatusFilter() {
+        cy.log("no filters");
+        getContentTable().within(() => {
+          cy.findByText("MBQL transform").should("be.visible");
+          cy.findByText("SQL transform").should("be.visible");
+        });
+
+        cy.log("last run status - add a filter");
+        getLastRunStatusFilterWidget().click();
+        H.popover().within(() => {
+          cy.findByText("Success").click();
+          cy.findByText("Add filter").click();
+        });
+
+        getLastRunStatusFilterWidget().should("contain", "Success");
+        getContentTable().within(() => {
+          cy.findByText("MBQL transform").should("be.visible");
+          cy.findByText("SQL transform").should("not.exist");
+        });
+
+        cy.log("last run status filter - remove filter");
+        getLastRunStatusFilterWidget().button("Remove filter").click();
+        getContentTable().within(() => {
+          cy.findByText("MBQL transform").should("be.visible");
+          cy.findByText("SQL transform").should("be.visible");
+        });
+      }
+
+      function testTagFilter() {
+        cy.log("no filters");
+        getContentTable().within(() => {
+          cy.findByText("MBQL transform").should("be.visible");
+          cy.findByText("SQL transform").should("be.visible");
+        });
+
+        cy.log("tags - add a filter");
+        getTagFilterWidget().click();
+        H.popover().within(() => {
+          cy.findByText("hourly").click();
+          cy.findByText("daily").click();
+          cy.button("Add filter").click();
+        });
+
+        getTagFilterWidget().should("contain", "2 tags");
+        getContentTable().within(() => {
+          cy.findByText("SQL transform").should("be.visible");
+          cy.findByText("MBQL transform").should("not.exist");
+        });
+
+        cy.log("last run status filter - remove filter");
+        getTagFilterWidget().button("Remove filter").click();
+        getContentTable().within(() => {
+          cy.findByText("MBQL transform").should("be.visible");
+          cy.findByText("SQL transform").should("be.visible");
+        });
+      }
+
+      testLastRunDateFilter();
+      testLastRunStatusFilter();
+      testTagFilter();
+    });
+  });
+
   describe("name and description", () => {
     it("should be able to edit the name and description after creation", () => {
       createMbqlTransform({ visitTransform: true });
@@ -990,6 +1224,7 @@ LIMIT
                 },
                 target: {
                   type: "table",
+                  database: WRITABLE_DB_ID,
                   name: TARGET_TABLE,
                   schema: TARGET_SCHEMA,
                 },
@@ -1103,6 +1338,7 @@ LIMIT
             },
             target: {
               type: "table",
+              database: WRITABLE_DB_ID,
               name: TARGET_TABLE,
               schema: TARGET_SCHEMA,
             },
@@ -1163,6 +1399,38 @@ LIMIT
       H.queryBuilderHeader().findByText(DB_NAME).should("be.visible");
       H.assertQueryBuilderRowCount(1);
     });
+
+    it("should be able to update a Python query", () => {
+      setPythonRunnerSettings();
+      cy.log("create a new transform");
+      H.getTableId({ name: "Animals", databaseId: WRITABLE_DB_ID }).then(
+        (id) => {
+          createPythonTransform({
+            body: dedent`
+          import pandas as pd
+
+          def transform(foo):
+            return pd.DataFrame([{"foo": 42 }])
+        `,
+            sourceTables: { foo: id },
+            visitTransform: true,
+          });
+        },
+      );
+
+      cy.log("update the query");
+      getTransformPage().findByRole("link", { name: "Edit script" }).click();
+      H.PythonEditor.type("{backspace}{backspace}{backspace} + 10 }])");
+
+      getQueryEditor().button("Save changes").click();
+      cy.wait("@updateTransform");
+
+      cy.log("run the transform and make sure the query has changed");
+      runTransformAndWaitForSuccess();
+      getTableLink().click();
+      H.queryBuilderHeader().findByText(DB_NAME).should("be.visible");
+      H.assertQueryBuilderRowCount(1);
+    });
   });
 
   describe("runs", () => {
@@ -1279,6 +1547,7 @@ LIMIT
           },
           target: {
             type: "table",
+            database: WRITABLE_DB_ID,
             name: TARGET_TABLE,
             schema: TARGET_SCHEMA,
           },
@@ -1389,6 +1658,136 @@ LIMIT
       H.main().findByText("Dependencies").should("not.exist");
     });
   });
+
+  describe("python > common library", () => {
+    it(
+      "should be possible to edit and save the common library",
+      { tags: ["@transforms-python"] },
+      () => {
+        visitCommonLibrary();
+
+        cy.log("updating the library should be possible");
+        H.PythonEditor.clear().type(
+          dedent`
+          def useful_calculation(a, b):
+          return a + b
+        `,
+        );
+        getLibraryEditorHeader().findByText("Save").click();
+
+        cy.log("the contents should be saved properly");
+        visitCommonLibrary();
+        H.PythonEditor.value().should(
+          "eq",
+          dedent`
+          def useful_calculation(a, b):
+              return a + b
+          `,
+        );
+
+        cy.log("reverting the changes should be possible");
+        H.PythonEditor.clear().type("# oops");
+        getLibraryEditorHeader().findByText("Revert").click();
+        H.PythonEditor.value().should(
+          "eq",
+          dedent`
+          def useful_calculation(a, b):
+              return a + b
+          `,
+        );
+      },
+    );
+
+    it(
+      "should be possible to use the common library",
+      { tags: ["@transforms-python"] },
+      () => {
+        setPythonRunnerSettings();
+        createPythonLibrary(
+          "common.py",
+          dedent`
+            def useful_calculation(a, b):
+              return a + b
+          `,
+        );
+
+        visitTransformListPage();
+        getTransformListPage().button("Create a transform").click();
+        H.popover().findByText("Python script").click();
+
+        H.PythonEditor.clear().type(
+          dedent`
+          import pandas as pd
+
+          def transform():
+          return pd.DataFrame([{"foo": common.useful_calculation(1, 2)}])
+        `,
+        );
+
+        getQueryEditor().findByText("Import common library").click();
+        H.PythonEditor.value().should("contain", "import common");
+
+        cy.findByTestId("python-data-picker")
+          .findByText("Select a database")
+          .click();
+
+        H.popover().findByText(DB_NAME).click();
+
+        cy.findByTestId("python-data-picker")
+          .findByText("Select a table…")
+          .click();
+
+        H.entityPickerModal().within(() => {
+          cy.findByText("Schema a").click();
+          cy.findByText("Animals").click();
+        });
+
+        getQueryEditor().button("Save").click();
+
+        H.modal().within(() => {
+          cy.findByLabelText("Name").clear().type("Python transform");
+          cy.findByLabelText("Table name").clear().type("python_transform");
+          cy.button("Save").click();
+        });
+
+        runTransformAndWaitForSuccess();
+        getTableLink().click();
+        H.queryBuilderHeader()
+          .findByText("Python Transform")
+          .should("be.visible");
+        H.assertQueryBuilderRowCount(1);
+        cy.findByTestId("scalar-value").should("have.text", "3");
+        H.expectUnstructuredSnowplowEvent({
+          event: "transform_created",
+        });
+
+        cy.log("update the common library and run the transform again");
+        cy.go("back");
+        createPythonLibrary(
+          "common.py",
+          dedent`
+            def useful_calculation(a, b):
+              return a + b + 40
+          `,
+        );
+        runTransformAndWaitForSuccess();
+        getTableLink().click();
+        H.queryBuilderHeader()
+          .findByText("Python Transform")
+          .should("be.visible");
+        H.assertQueryBuilderRowCount(1);
+        cy.findByTestId("scalar-value").should("have.text", "43");
+      },
+    );
+
+    function visitCommonLibrary(path = "common.py") {
+      cy.visit(`/admin/transforms/library/${path}`);
+    }
+
+    function getLibraryEditorHeader() {
+      return cy.findByTestId("library-editor-header");
+    }
+  });
 });
 
 describe("scenarios > admin > transforms > databases without :schemas", () => {
@@ -1709,13 +2108,11 @@ describe("scenarios > admin > transforms > jobs", () => {
     it("should be able to filter jobs ", () => {
       cy.log("run hourly job so know that was recently run");
       visitJobListPage();
-
       getContentTable().findByText("Hourly job").click();
       runJobAndWaitForSuccess();
-
       visitJobListPage();
 
-      function testLastRunFilter() {
+      function testLastRunDateFilter() {
         cy.log("no filters");
         getContentTable().within(() => {
           cy.findByText("Hourly job").should("be.visible");
@@ -1725,14 +2122,14 @@ describe("scenarios > admin > transforms > jobs", () => {
         });
 
         cy.log("last run at - add a filter");
-        getLastRunAtFilterWidget().click();
+        getLastRunDateFilterWidget().click();
         H.popover().findByText("Previous month").click();
 
-        getLastRunAtFilterWidget().should("contain", "Previous month");
+        getLastRunDateFilterWidget().should("contain", "Previous month");
         getContentTable().should("not.exist");
 
         cy.log("last run at filter - remove filter");
-        getLastRunAtFilterWidget().button("Remove filter").click();
+        getLastRunDateFilterWidget().button("Remove filter").click();
         getContentTable().within(() => {
           cy.findByText("Hourly job").should("be.visible");
           cy.findByText("Daily job").should("be.visible");
@@ -1741,7 +2138,42 @@ describe("scenarios > admin > transforms > jobs", () => {
         });
       }
 
-      function testNextRunFilter() {
+      function testLastRunStatusFilter() {
+        cy.log("no filters");
+        getContentTable().within(() => {
+          cy.findByText("Hourly job").should("be.visible");
+          cy.findByText("Daily job").should("be.visible");
+          cy.findByText("Weekly job").should("be.visible");
+          cy.findByText("Monthly job").should("be.visible");
+        });
+
+        cy.log("last run status - add a filter");
+        getLastRunStatusFilterWidget().click();
+        H.popover().within(() => {
+          cy.findByText("In progress").click();
+          cy.findByText("Success").click();
+          cy.button("Add filter").click();
+        });
+
+        getLastRunStatusFilterWidget().should("contain", "2 statuses");
+        getContentTable().within(() => {
+          cy.findByText("Hourly job").should("be.visible");
+          cy.findByText("Daily job").should("not.exist");
+          cy.findByText("Weekly job").should("not.exist");
+          cy.findByText("Monthly job").should("not.exist");
+        });
+
+        cy.log("last run at filter - remove filter");
+        getLastRunStatusFilterWidget().button("Remove filter").click();
+        getContentTable().within(() => {
+          cy.findByText("Hourly job").should("be.visible");
+          cy.findByText("Daily job").should("be.visible");
+          cy.findByText("Weekly job").should("be.visible");
+          cy.findByText("Monthly job").should("be.visible");
+        });
+      }
+
+      function testNextRunDateFilter() {
         cy.log("no filters");
         getContentTable().within(() => {
           cy.findByText("Hourly job").should("be.visible");
@@ -1751,7 +2183,7 @@ describe("scenarios > admin > transforms > jobs", () => {
         });
 
         cy.log("next run - add a filter");
-        getNextRunFilterWidget().click();
+        getNextRunDateFilterWidget().click();
         H.popover().within(() => {
           cy.findByText("Fixed date range…").click();
           cy.findByLabelText("Start date").clear().type("12/10/2024");
@@ -1759,14 +2191,14 @@ describe("scenarios > admin > transforms > jobs", () => {
           cy.button("Apply").click();
         });
 
-        getNextRunFilterWidget().should(
+        getNextRunDateFilterWidget().should(
           "contain",
           "December 10, 2024 - January 5, 2025",
         );
         getContentTable().should("not.exist");
 
         cy.log("next run filter - remove filter");
-        getNextRunFilterWidget().button("Remove filter").click();
+        getNextRunDateFilterWidget().button("Remove filter").click();
         getContentTable().within(() => {
           cy.findByText("Hourly job").should("be.visible");
           cy.findByText("Daily job").should("be.visible");
@@ -1839,8 +2271,9 @@ describe("scenarios > admin > transforms > jobs", () => {
         });
       }
 
-      testLastRunFilter();
-      testNextRunFilter();
+      testLastRunDateFilter();
+      testLastRunStatusFilter();
+      testNextRunDateFilter();
       testTagFilter();
     });
   });
@@ -2329,11 +2762,15 @@ function getTransformFilterWidget() {
   return cy.findByRole("group", { name: "Transform" });
 }
 
-function getLastRunAtFilterWidget() {
+function getLastRunDateFilterWidget() {
   return cy.findByRole("group", { name: "Last run at" });
 }
 
-function getNextRunFilterWidget() {
+function getLastRunStatusFilterWidget() {
+  return cy.findByRole("group", { name: "Last run status" });
+}
+
+function getNextRunDateFilterWidget() {
   return cy.findByRole("group", { name: "Next run at" });
 }
 
@@ -2350,11 +2787,11 @@ function getRunMethodFilterWidget() {
 }
 
 function getStartAtFilterWidget() {
-  return cy.findByRole("group", { name: "Start at" });
+  return cy.findByRole("group", { name: "Started at" });
 }
 
 function getEndAtFilterWidget() {
-  return cy.findByRole("group", { name: "End at" });
+  return cy.findByRole("group", { name: "Ended at" });
 }
 
 function visitTransformListPage() {
@@ -2424,6 +2861,7 @@ function createMbqlTransform({
         },
         target: {
           type: "table",
+          database: WRITABLE_DB_ID,
           name: targetTable,
           schema: targetSchema,
         },
@@ -2462,6 +2900,42 @@ function createSqlTransform({
       },
       target: {
         type: "table",
+        database: WRITABLE_DB_ID,
+        name: targetTable,
+        schema: targetSchema,
+      },
+      tag_ids: tagIds,
+    },
+    { wrapId: true, visitTransform },
+  );
+}
+function createPythonTransform({
+  body,
+  sourceTables,
+  targetTable = TARGET_TABLE,
+  targetSchema = TARGET_SCHEMA,
+  tagIds,
+  visitTransform,
+}: {
+  body: string;
+  sourceTables: PythonTransformTableAliases;
+  targetTable?: string;
+  targetSchema?: string;
+  tagIds?: TransformTagId[];
+  visitTransform?: boolean;
+}) {
+  H.createTransform(
+    {
+      name: "Python transform",
+      source: {
+        type: "python",
+        "source-database": WRITABLE_DB_ID,
+        "source-tables": sourceTables,
+        body,
+      },
+      target: {
+        type: "table",
+        database: WRITABLE_DB_ID,
         name: targetTable,
         schema: targetSchema,
       },
@@ -2525,3 +2999,35 @@ function assertOptionNotSelected(name: string) {
 function editorSidebar() {
   return cy.findByTestId("editor-sidebar");
 }
+
+function getPythonDataPicker() {
+  return cy.findByTestId("python-data-picker");
+}
+
+function createPythonLibrary(path: string, source: string) {
+  cy.request("PUT", `/api/ee/transforms-python/library/${path}`, {
+    source,
+  });
+}
+
+function setPythonRunnerSettings() {
+  H.updateEnterpriseSetting("python-runner-url", "http://localhost:5001");
+  H.updateEnterpriseSetting("python-runner-api-token", "dev-token-12345");
+  H.updateEnterpriseSetting(
+    "python-storage-s-3-endpoint",
+    "http://localhost:4566",
+  );
+  H.updateEnterpriseSetting("python-storage-s-3-region", "us-east-1");
+  H.updateEnterpriseSetting(
+    "python-storage-s-3-bucket",
+    "metabase-python-runner",
+  );
+  H.updateEnterpriseSetting("python-storage-s-3-prefix", "test-prefix");
+  H.updateEnterpriseSetting("python-storage-s-3-access-key", "test");
+  H.updateEnterpriseSetting("python-storage-s-3-secret-key", "test");
+  H.updateEnterpriseSetting(
+    "python-storage-s-3-container-endpoint",
+    "http://localstack:4566",
+  );
+  H.updateEnterpriseSetting("python-storage-s-3-path-style-access", true);
+}

e2e/test/scenarios/dashboard-cards/dashboard-drill.cy.spec.js

@@ -553,7 +553,10 @@ describe("scenarios > dashboard > dashboard drill", () => {
     cy.wait("@dataset").then((xhr) => {
       expect(xhr.response.body.error).not.to.exist;
     });
-    cy.findByTestId("object-detail").findByText("Fantastic Wool Shirt");
+    cy.findByTestId("object-detail")
+      .findAllByText("Fantastic Wool Shirt")
+      .should("have.length", 2)
+      .and("be.visible");
   });
 
   it("should apply correct date range on a graph drill-through (metabase#13785)", () => {

e2e/test/scenarios/dashboard/dashboard.cy.spec.js

@@ -645,6 +645,33 @@ describe("scenarios > dashboard", () => {
         H.getDashboardCard(1).contains("bottom");
       },
     );
+
+    it("should not save the dashboard when the user clicks 'Discard changes'", () => {
+      // Navigate to the dashboard via client-side navigation (to trigger the client-side "Discard changes" prompt)
+      cy.visit("/");
+      cy.findByTestId("main-navbar-root").findByText("Our analytics").click();
+      cy.findByTestId("collection-table")
+        .findByText(originalDashboardName)
+        .click();
+
+      cy.log("Make a change to the dashboard");
+      H.editDashboard();
+      cy.findByTestId("dashboard-empty-state")
+        .findByText("Add a chart")
+        .click();
+      H.sidebar().findByText("Orders, Count").click();
+      H.getDashboardCards().should("have.length", 1);
+
+      cy.log("Navigate back and discard changes");
+      cy.go("back");
+      H.modal().button("Discard changes").click();
+      cy.findByTestId("collection-table")
+        .findByText(originalDashboardName)
+        .click();
+
+      cy.log("Verify changes were not saved");
+      cy.findByTestId("dashboard-empty-state").should("exist");
+    });
   });
 
   describe("iframe cards", () => {

e2e/test/scenarios/detail-view/detail-view.cy.spec.ts

@@ -44,7 +44,9 @@ describe("detail view", () => {
       });
 
       DetailView.verifyDetails([
+        ["ID", "1"],
         ["Ean", "1018947080336"],
+        ["Title", "Rustic Paper Wallet"],
         ["Category", "Gizmo"],
         ["Vendor", "Swaniawski, Casper and Hilll"],
         ["Price", "29.46"],
@@ -122,6 +124,7 @@ describe("detail view", () => {
       });
 
       DetailView.verifyDetails([
+        ["ID", "1"],
         ["User ID", "1"],
         ["Product ID", "14"],
         ["Subtotal", "37.65"],
@@ -137,6 +140,7 @@ describe("detail view", () => {
         ["Products → Ean", "8833419218504"],
         ["Products → Price", "25.1"],
         ["Products → Rating", "4"],
+        ["Products → Title", "Awesome Concrete Shoes"],
         ["Products → Vendor", "McClure-Lockman"],
       ]);
 
@@ -273,6 +277,7 @@ describe("detail view", () => {
       });
 
       DetailView.verifyDetails([
+        ["ID", "1"],
         ["User", "Hudson Borer"],
         ["Product", "Product: Awesome Concrete Shoes"],
         ["Subtotal ($)", "37.65"],
@@ -281,17 +286,19 @@ describe("detail view", () => {
         ["Discount ($)", "empty"],
         ["Created At", "February 11, 2025, 9:40 PM"],
         ["Quantity", "two"],
+        ["Order image", "https://example.com/order/1.jpg"],
         ["Product image", "https://example.com/product/14.jpg"],
         ["Products → Category", "Widget"],
         ["Products → Created At", "December 31, 2023, 2:41 PM"],
         ["Products → Ean", "8833419218504"],
         ["Products → Price", "25.1"],
         ["Products → Rating", "4"],
+        ["Products → Title", "Awesome Concrete Shoes"],
         ["Products → Vendor", "McClure-Lockman"],
       ]);
 
       cy.log("user id remapped to user name");
-      DetailView.getDetailsRowValue({ index: 0, rowsCount: 15 }).within(() => {
+      DetailView.getDetailsRowValue({ index: 1, rowsCount: 18 }).within(() => {
         cy.findByRole("link", { name: "Hudson Borer" })
           .should("be.visible")
           .and("have.attr", "href", `/table/${PEOPLE_ID}/detail/1`);
@@ -300,7 +307,7 @@ describe("detail view", () => {
       cy.log(
         "product id remapped to product title, and custom view_as setting",
       );
-      DetailView.getDetailsRowValue({ index: 1, rowsCount: 15 }).within(() => {
+      DetailView.getDetailsRowValue({ index: 2, rowsCount: 18 }).within(() => {
         cy.findByRole("link", { name: "Product: Awesome Concrete Shoes" })
           .should("be.visible")
           .and("have.attr", "href", "https://example.com/14")
@@ -314,13 +321,13 @@ describe("detail view", () => {
       });
 
       cy.log("image should be rendered in a frame with a link");
-      DetailView.getDetailsRowValue({ index: 8, rowsCount: 15 }).within(() => {
+      DetailView.getDetailsRowValue({ index: 10, rowsCount: 18 }).within(() => {
         cy.findByRole("img")
           .should("exist") // do not wait for image to load
           .and("have.attr", "src", "https://example.com/product/14.jpg");
 
         cy.findByRole("link")
-          .should("be.visible")
+          .should("exist")
           .and("have.text", "https://example.com/product/14.jpg")
           .and("have.attr", "href", "https://example.com/product/14.jpg")
           .and("have.attr", "target", "_blank")
@@ -332,7 +339,7 @@ describe("detail view", () => {
   it("displays emails as links", () => {
     DetailView.visitTable(PEOPLE_ID, 1);
 
-    DetailView.getDetailsRowValue({ index: 1, rowsCount: 11 }).within(() => {
+    DetailView.getDetailsRowValue({ index: 2, rowsCount: 13 }).within(() => {
       cy.findByRole("link", { name: "borer-hudson@yahoo.com" }).should(
         "have.attr",
         "href",

e2e/test/scenarios/documents/card-embed-node.cy.spec.ts

@@ -0,0 +1,593 @@
+import {
+  DOCUMENT_WITH_THREE_CARDS_AND_COLUMNS,
+  DOCUMENT_WITH_TWO_CARDS,
+} from "e2e/support/document-initial-data";
+
+const { H } = cy;
+
+describe("documents card embed node custom logic", () => {
+  beforeEach(() => {
+    H.restore();
+    cy.signInAsAdmin();
+    H.activateToken("bleeding-edge");
+  });
+
+  describe("cardEmbed drag and drop", () => {
+    beforeEach(() => {
+      H.createDocument({
+        name: "DnD Test Document",
+        document: DOCUMENT_WITH_TWO_CARDS,
+        collection_id: null,
+        alias: "document",
+        idAlias: "documentId",
+      });
+
+      cy.get("@documentId").then((id) => cy.visit(`/document/${id}`));
+    });
+
+    it("should create a flexContainer when dropping one cardEmbed onto another standalone cardEmbed", () => {
+      // Wait for cards to load
+      H.getDocumentCard("Orders")
+        .should("be.visible")
+        .findByTestId("table-root")
+        .should("exist");
+      H.getDocumentCard("Orders, Count")
+        .should("be.visible")
+        .findByTestId("table-root")
+        .should("exist");
+
+      // Verify no flexContainer exists initially
+      H.documentContent()
+        .find('[data-type="flexContainer"]')
+        .should("not.exist");
+
+      H.dragAndDropCardOnAnotherCard("Orders", "Orders, Count");
+
+      // Verify flexContainer was created
+      H.documentContent().find('[data-type="flexContainer"]').should("exist");
+
+      // Verify both cards are now inside the flexContainer
+      H.documentContent()
+        .find('[data-type="flexContainer"]')
+        .within(() => {
+          assertFlexContainerCardsOrder(["Orders", "Orders, Count"]);
+        });
+
+      // Verify the flexContainer is wrapped in a resizeNode
+      H.documentContent()
+        .find('[data-type="resizeNode"]')
+        .should("exist")
+        .within(() => {
+          cy.get('[data-type="flexContainer"]').should("exist");
+        });
+
+      // Verify that originally separate cards are now side by side
+      H.documentContent()
+        .find('[data-type="flexContainer"]')
+        .within(() => {
+          cy.findAllByTestId("document-card-embed").should("have.length", 2);
+        });
+    });
+
+    it("should handle drag and drop with proper drop side positioning", () => {
+      // Wait for cards to load
+      H.getDocumentCard("Orders")
+        .should("be.visible")
+        .findByTestId("table-root")
+        .should("exist");
+      H.getDocumentCard("Orders, Count")
+        .should("be.visible")
+        .findByTestId("table-root")
+        .should("exist");
+
+      H.dragAndDropCardOnAnotherCard("Orders", "Orders, Count", {
+        side: "right",
+      });
+
+      // Verify flexContainer was created with correct order
+      H.documentContent()
+        .find('[data-type="flexContainer"]')
+        .should("exist")
+        .within(() => {
+          // Orders, Count should be first (left), Orders should be second (right)
+          assertFlexContainerCardsOrder(["Orders, Count", "Orders"]);
+        });
+    });
+
+    it("should prevent dropping a card onto itself", () => {
+      // Wait for cards to load
+      H.getDocumentCard("Orders")
+        .should("be.visible")
+        .findByTestId("table-root")
+        .should("exist");
+
+      // Verify initial state - no flexContainer
+      H.documentContent()
+        .find('[data-type="flexContainer"]')
+        .should("not.exist");
+
+      // Attempt to drag and drop the card onto itself
+      H.dragAndDropCardOnAnotherCard("Orders", "Orders");
+
+      // Verify no flexContainer was created
+      H.documentContent()
+        .find('[data-type="flexContainer"]')
+        .should("not.exist");
+
+      // Verify the card is still standalone
+      H.getDocumentCard("Orders").should("exist");
+    });
+  });
+
+  describe("advanced flexContainer scenarios", () => {
+    beforeEach(() => {
+      H.createDocument({
+        name: "Advanced DnD Test Document",
+        document: DOCUMENT_WITH_THREE_CARDS_AND_COLUMNS,
+        collection_id: null,
+        alias: "document",
+        idAlias: "documentId",
+      });
+
+      cy.get("@documentId").then((id) => cy.visit(`/document/${id}`));
+    });
+
+    it("should allow you to resize the cards inside a flex container", () => {
+      getCardWidths(["Orders", "Orders, Count"], (first, second) => {
+        cy.wrap(first).as("ogWidth1");
+        cy.wrap(second).as("ogWidth2");
+        expect(first).to.be.closeTo(second, 3);
+      });
+
+      H.getDocumentCard("Orders").as("ORDERS_CARD");
+      H.getDocumentCard("Orders, Count").as("ORDERS_COUNT_CARD");
+
+      const flexContainer = H.getFlexContainerForCard("Orders");
+
+      const handles = H.getResizeHandlesForFlexContianer(flexContainer);
+
+      H.documentDoDrag(handles.eq(0), { x: 100 });
+
+      getCardWidths(["Orders", "Orders, Count"], (first, second) => {
+        cy.get("@ogWidth1").then((_first) => {
+          cy.get("@ogWidth2").then((_second) => {
+            cy.log("compare that changes are close to the drag distance");
+            expect((_first as unknown as number) + 100).to.be.closeTo(first, 3);
+            expect((_second as unknown as number) - 100).to.be.closeTo(
+              second,
+              3,
+            );
+
+            expect(first).to.be.closeTo(second + 200, 3);
+          });
+        });
+      });
+    });
+
+    it("should add a third card to an existing flexContainer with 2 cards", () => {
+      // Wait for all cards to load
+      H.getDocumentCard("Orders")
+        .should("be.visible")
+        .findByTestId("table-root")
+        .should("exist");
+      H.getDocumentCard("Orders, Count")
+        .should("be.visible")
+        .findByTestId("table-root")
+        .should("exist");
+      H.getDocumentCard("Orders, Count, Grouped by Created At (year)")
+        .should("be.visible")
+        .findByTestId("chart-container")
+        .should("exist");
+
+      // Verify initial state - flexContainer exists with 2 cards
+      H.documentContent()
+        .find('[data-type="flexContainer"]')
+        .should("exist")
+        .within(() => {
+          cy.findAllByTestId("document-card-embed").should("have.length", 2);
+        });
+
+      // Drag the standalone card (Orders by Year) onto one of the cards in the flexContainer
+      H.dragAndDropCardOnAnotherCard(
+        "Orders, Count, Grouped by Created At (year)",
+        "Orders",
+        { side: "left" },
+      );
+
+      // Verify the flexContainer now has 3 cards
+
+      H.documentContent()
+        .find('[data-type="flexContainer"]')
+        .should("exist")
+        .within(() => {
+          assertFlexContainerCardsOrder([
+            "Orders, Count, Grouped by Created At (year)",
+            "Orders",
+            "Orders, Count",
+          ]);
+        });
+
+      H.documentUndo();
+
+      // Drag the standalone card (Orders by Year) onto one of the cards in the flexContainer
+      H.dragAndDropCardOnAnotherCard(
+        "Orders, Count, Grouped by Created At (year)",
+        "Orders",
+        { side: "right" },
+      );
+
+      H.documentContent()
+        .find('[data-type="flexContainer"]')
+        .should("exist")
+        .within(() => {
+          assertFlexContainerCardsOrder([
+            "Orders",
+            "Orders, Count, Grouped by Created At (year)",
+            "Orders, Count",
+          ]);
+        });
+
+      H.documentUndo();
+
+      // Drag the standalone card (Orders by Year) onto one of the cards in the flexContainer
+      H.dragAndDropCardOnAnotherCard(
+        "Orders, Count, Grouped by Created At (year)",
+        "Orders, Count",
+        { side: "right" },
+      );
+
+      H.documentContent()
+        .find('[data-type="flexContainer"]')
+        .should("exist")
+        .within(() => {
+          assertFlexContainerCardsOrder([
+            "Orders",
+            "Orders, Count",
+            "Orders, Count, Grouped by Created At (year)",
+          ]);
+        });
+
+      cy.log("changing the widths of 2 cards should leave the 3rd alone");
+
+      const cardNames = [
+        "Orders",
+        "Orders, Count",
+        "Orders, Count, Grouped by Created At (year)",
+      ];
+
+      getCardWidths(cardNames, (first, second, third) => {
+        cy.wrap(first).as("_first");
+        cy.wrap(second).as("_second");
+        cy.wrap(third).as("_third");
+        expect(first).to.be.closeTo(second, 10);
+        expect(first).to.be.closeTo(third as unknown as number, 10);
+      });
+
+      const flexContainer = H.getFlexContainerForCard("Orders");
+
+      const handles = H.getResizeHandlesForFlexContianer(flexContainer);
+
+      H.documentDoDrag(handles.eq(1), { x: 50 });
+
+      getCardWidths(cardNames, (first, second, third) => {
+        cy.get("@_first").then((_first) => {
+          cy.get("@_second").then((_second) => {
+            cy.get("@_third").then((_third) => {
+              expect(first).to.be.closeTo(_first as unknown as number, 3);
+              expect(second).to.be.greaterThan(_second as unknown as number);
+              expect(third).to.be.lessThan(_third as unknown as number);
+            });
+          });
+        });
+      });
+    });
+
+    it("should prevent adding a fourth card to a flexContainer with 3 cards", () => {
+      // Wait for all cards to load
+      H.getDocumentCard("Orders")
+        .should("be.visible")
+        .findByTestId("table-root")
+        .should("exist");
+      H.getDocumentCard("Orders, Count")
+        .should("be.visible")
+        .findByTestId("table-root")
+        .should("exist");
+
+      // First, add the third card to reach the limit
+      H.dragAndDropCardOnAnotherCard(
+        "Orders, Count, Grouped by Created At (year)",
+        "Orders",
+        { side: "right" },
+      );
+
+      // Verify we have 3 cards in the flexContainer
+      H.documentContent()
+        .find('[data-type="flexContainer"]')
+        .within(() => {
+          cy.findAllByTestId("document-card-embed").should("have.length", 3);
+        });
+
+      // Add another card to try to exceed the limit
+      addNewStandaloneCard("Orders Model", "model");
+
+      // Wait for the new card to be added
+      H.documentContent()
+        .findAllByTestId("document-card-embed")
+        .should("have.length", 4); // 3 in flexContainer + 1 new standalone
+
+      // Try to drag the new standalone card onto the flexContainer
+      H.dragAndDropCardOnAnotherCard("Orders", "Orders, Count", {
+        side: "left",
+      });
+
+      // Verify the flexContainer still has only 3 cards (drop should be rejected)
+      H.documentContent()
+        .find('[data-type="flexContainer"]')
+        .within(() => {
+          cy.findAllByTestId("document-card-embed").should("have.length", 3);
+        });
+
+      // Verify the standalone card is still separate
+      H.documentContent()
+        .findAllByTestId("document-card-embed")
+        .should("have.length", 4); // Still 4 total, with 1 standalone
+    });
+
+    it("should reorder cards within the same flexContainer", () => {
+      // Wait for all cards to load
+      H.getDocumentCard("Orders")
+        .should("be.visible")
+        .findByTestId("table-root")
+        .should("exist");
+      H.getDocumentCard("Orders, Count")
+        .should("be.visible")
+        .findByTestId("table-root")
+        .should("exist");
+
+      // Verify initial order: Orders | Orders, Count
+      H.documentContent()
+        .find('[data-type="flexContainer"]')
+        .should("exist")
+        .within(() => {
+          assertFlexContainerCardsOrder(["Orders", "Orders, Count"]);
+        });
+
+      // Drag Orders to the right side of Orders, Count to reorder them
+      H.dragAndDropCardOnAnotherCard("Orders", "Orders, Count", {
+        side: "right",
+      });
+
+      // Verify new order: Orders, Count | Orders
+      H.documentContent()
+        .find('[data-type="flexContainer"]')
+        .should("exist")
+        .within(() => {
+          assertFlexContainerCardsOrder(["Orders, Count", "Orders"]);
+        });
+
+      H.dragAndDropCardOnAnotherCard("Orders", "Orders, Count", {
+        side: "left",
+      });
+
+      H.documentContent()
+        .find('[data-type="flexContainer"]')
+        .should("exist")
+        .within(() => {
+          assertFlexContainerCardsOrder(["Orders", "Orders, Count"]);
+        });
+    });
+
+    it("should handle moving cards between different flexContainers", () => {
+      // First create another flexContainer by dragging the standalone card onto a new location
+      // Add another standalone card first
+      addNewStandaloneCard("Orders Model", "model");
+
+      // Wait for the new card
+      H.documentContent()
+        .findAllByTestId("document-card-embed")
+        .should("have.length", 4);
+
+      // Create a second flexContainer by dropping Orders by Year onto the new Orders card
+      H.dragAndDropCardOnAnotherCard(
+        "Orders, Count, Grouped by Created At (year)",
+        "Orders Model",
+      );
+
+      // Verify we now have 2 flexContainers
+      H.documentContent()
+        .find('[data-type="flexContainer"]')
+        .should("have.length", 2);
+
+      // Move a card from the first flexContainer to the second one
+      H.dragAndDropCardOnAnotherCard(
+        "Orders, Count",
+        "Orders, Count, Grouped by Created At (year)",
+        { side: "right" },
+      );
+
+      // Verify the first flexContainer now has only 1 card (should be unwrapped)
+      H.documentContent()
+        .find('[data-type="flexContainer"]')
+        .should("have.length", 1);
+
+      // Verify the remaining flexContainer has 3 cards
+      H.documentContent()
+        .find('[data-type="flexContainer"]')
+        .within(() => {
+          cy.findAllByTestId("document-card-embed").should("have.length", 3);
+        });
+
+      H.dragAndDropCardOnAnotherCard(
+        "Orders, Count, Grouped by Created At (year)",
+        "Orders",
+        { side: "right" },
+      );
+
+      H.documentContent()
+        .find('[data-type="flexContainer"]')
+        .should("have.length", 2);
+    });
+  });
+
+  describe("deleting a cardEmbed", () => {
+    it("should delete a cardEmbed when selected and Backspace is pressed", () => {
+      H.createDocument({
+        name: "DnD Test Document",
+        document: DOCUMENT_WITH_TWO_CARDS,
+        collection_id: null,
+        alias: "document",
+        idAlias: "documentId",
+      });
+
+      cy.get("@documentId").then((id) => cy.visit(`/document/${id}`));
+
+      // Wait for cards to load
+      H.getDocumentCard("Orders")
+        .should("be.visible")
+        .findByTestId("table-root")
+        .should("exist");
+      H.getDocumentCard("Orders, Count")
+        .should("be.visible")
+        .findByTestId("table-root")
+        .should("exist");
+
+      // Verify initial state - we have 2 standalone cards
+      H.documentContent()
+        .findAllByTestId("document-card-embed")
+        .should("have.length", 2);
+
+      // Click on the Orders card to select it
+      H.getDocumentCard("Orders").realClick({ position: "top" });
+
+      // Press Backspace to delete the selected card
+      cy.realPress("Backspace");
+
+      // Verify the Orders card has been deleted
+      H.getDocumentCard("Orders", false).should("not.exist");
+
+      // Verify only one card remains
+      H.documentContent()
+        .findAllByTestId("document-card-embed")
+        .should("have.length", 1);
+
+      // Verify the remaining card is Orders, Count
+      H.getDocumentCard("Orders, Count").should("exist").and("be.visible");
+    });
+
+    it("should delete a cardEmbed from a flexContainer when selected and Backspace is pressed", () => {
+      H.createDocument({
+        name: "DnD Test Document",
+        document: DOCUMENT_WITH_THREE_CARDS_AND_COLUMNS,
+        collection_id: null,
+        alias: "document",
+        idAlias: "documentId",
+      });
+
+      cy.get("@documentId").then((id) => cy.visit(`/document/${id}`));
+
+      // First create a flexContainer by dropping one card onto another
+      H.getDocumentCard("Orders")
+        .should("be.visible")
+        .findByTestId("table-root")
+        .should("exist");
+      H.getDocumentCard("Orders, Count")
+        .should("be.visible")
+        .findByTestId("table-root")
+        .should("exist");
+
+      // Create flexContainer
+      H.dragAndDropCardOnAnotherCard(
+        "Orders, Count, Grouped by Created At (year)",
+        "Orders",
+        { side: "right" },
+      );
+
+      // Verify flexContainer was created with 2 cards
+      H.documentContent()
+        .find('[data-type="flexContainer"]')
+        .should("exist")
+        .within(() => {
+          cy.findAllByTestId("document-card-embed").should("have.length", 3);
+        });
+
+      // Click on one of the cards in the flexContainer to select it
+      H.getDocumentCard("Orders").realClick({ position: "top" });
+
+      // Press Backspace to delete the selected card
+      cy.realPress("Backspace");
+
+      // Verify the Orders card has been deleted from the flexContainer
+      H.getDocumentCard("Orders", false).should("not.exist");
+
+      H.documentContent()
+        .findAllByTestId("document-card-embed")
+        .should("have.length", 2);
+
+      H.documentContent().get('[data-type="flexContainer"]').should("exist");
+
+      // Click on one of the cards in the flexContainer to select it
+      H.getDocumentCard("Orders, Count").realClick({ position: "top" });
+
+      // Press Backspace to delete the selected card
+      cy.realPress("Backspace");
+
+      // Verify the flexContainer now has only 1 card and should be unwrapped back to standalone
+      H.documentContent()
+        .get('[data-type="flexContainer"]')
+        .should("not.exist"); // FlexContainer should be unwrapped when only 1 card remains
+
+      // Verify only the Orders, Count card remains as a standalone card
+      H.documentContent()
+        .findAllByTestId("document-card-embed")
+        .should("have.length", 1);
+      H.getDocumentCard("Orders, Count, Grouped by Created At (year)").should(
+        "exist",
+      );
+    });
+  });
+});
+
+function assertFlexContainerCardsOrder(expectedCardTitles: string[]) {
+  for (let i = 0; i < expectedCardTitles.length; i++) {
+    cy.findAllByTestId("document-card-embed")
+      .should("have.length", expectedCardTitles.length)
+      .eq(i)
+      .findByTestId("card-embed-title")
+      .should("contain.text", expectedCardTitles[i]);
+  }
+}
+
+function addNewStandaloneCard(
+  cardName: string,
+  cardType: "question" | "model",
+) {
+  cy.get(".node-paragraph.is-empty").click();
+  H.addToDocument("/", false);
+  H.commandSuggestionItem("Chart").click();
+  H.entityPickerModalTab(
+    cardType === "question" ? "Questions" : "Models",
+  ).click();
+  H.entityPickerModalItem(1, cardName).click();
+}
+
+function getCardWidths(
+  cardNames: string[],
+  cb: (val1: number, val2: number, val3?: number) => void,
+) {
+  const [firstCardName, secondCardName, thirdCardName] = cardNames;
+  H.getDocumentCard(firstCardName).then((firstCard) => {
+    H.getDocumentCard(secondCardName).then((secondCard) => {
+      if (thirdCardName) {
+        H.getDocumentCard(thirdCardName).then((thirdCard) => {
+          cb(
+            firstCard.width() as number,
+            secondCard.width() as number,
+            thirdCard.width() as number,
+          );
+        });
+      } else {
+        cb(firstCard.width() as number, secondCard.width() as number);
+      }
+    });
+  });
+}

e2e/test/scenarios/documents/comments.cy.spec.ts

@@ -50,27 +50,38 @@ H.describeWithSnowplowEE("document comments", () => {
       testCommentingOnNode(documentId, BLOCKQUOTE_ID, getBlockquote);
       testCommentingOnNode(documentId, ORDERED_LIST_ID, getOrderedList);
       testCommentingOnNode(documentId, CODE_BLOCK_ID, getCodeBlock);
-      testCommentingOnNode(documentId, CARD_EMBED_ID, getEmbed);
+      testCommentingOnNode(documentId, CARD_EMBED_ID, getEmbed, {
+        isCardEmbedNode: true,
+      });
     });
 
     function testCommentingOnNode<E extends HTMLElement>(
       targetId: DocumentId,
       childTargetId: string,
       getNodeElement: () => Cypress.Chainable<JQuery<E>>,
+      { isCardEmbedNode = false } = {},
     ) {
       cy.get("body").click(0, 0);
-      Comments.getDocumentNodeButton({ targetId, childTargetId }).should(
-        "not.be.visible",
-      );
 
-      getNodeElement()
-        .closest("[data-node-view-wrapper]")
-        .should("have.attr", "aria-expanded", "false");
-      getNodeElement().scrollIntoView();
-      getNodeElement().realHover();
-      Comments.getDocumentNodeButton({ targetId, childTargetId })
-        .should("be.visible")
-        .click();
+      if (isCardEmbedNode) {
+        getNodeElement().scrollIntoView();
+        getNodeElement().icon("ellipsis").click();
+
+        H.menu().findByText("Comment").click();
+      } else {
+        Comments.getDocumentNodeButton({ targetId, childTargetId }).should(
+          "not.be.visible",
+        );
+
+        getNodeElement()
+          .closest("[data-node-view-wrapper]")
+          .should("have.attr", "aria-expanded", "false");
+        getNodeElement().scrollIntoView();
+        getNodeElement().realHover();
+        Comments.getDocumentNodeButton({ targetId, childTargetId })
+          .should("be.visible")
+          .click();
+      }
 
       Comments.getSidebar().within(() => {
         cy.findByRole("heading", { name: "Comments about this" }).should(
@@ -95,6 +106,7 @@ H.describeWithSnowplowEE("document comments", () => {
         targetId,
         childTargetId,
         hasComments: true,
+        isCardEmbedNode,
       })
         .should("be.visible")
         .and("contain.text", "1");
@@ -112,6 +124,7 @@ H.describeWithSnowplowEE("document comments", () => {
         targetId,
         childTargetId,
         hasComments: true,
+        isCardEmbedNode,
       }).should("be.visible");
     }
   });
@@ -1801,12 +1814,21 @@ function createLoremIpsumDocument() {
           ],
         },
         {
-          type: "cardEmbed",
+          type: "resizeNode",
           attrs: {
-            id: ORDERS_QUESTION_ID,
-            name: null,
-            _id: CARD_EMBED_ID,
+            height: 350,
+            minHeight: 280,
           },
+          content: [
+            {
+              type: "cardEmbed",
+              attrs: {
+                id: ORDERS_QUESTION_ID,
+                name: null,
+                _id: CARD_EMBED_ID,
+              },
+            },
+          ],
         },
         {
           type: "paragraph",

e2e/test/scenarios/documents/documents.cy.spec.ts

@@ -193,12 +193,21 @@ H.describeWithSnowplowEE("documents", () => {
                 },
               },
               {
-                type: "cardEmbed",
+                type: "resizeNode",
                 attrs: {
-                  id: ORDERS_QUESTION_ID,
-                  name: null,
-                  _id: "2",
+                  height: 442,
+                  minHeight: 280,
                 },
+                content: [
+                  {
+                    type: "cardEmbed",
+                    attrs: {
+                      id: ORDERS_QUESTION_ID,
+                      name: null,
+                      _id: "2",
+                    },
+                  },
+                ],
               },
               {
                 type: "paragraph",
@@ -217,12 +226,16 @@ H.describeWithSnowplowEE("documents", () => {
 
       it("renders a 'not found' message if the copied card has been permanently deleted", () => {
         cy.get<Document>("@document").then(({ id, document: { content } }) => {
-          const cardEmbed = content?.find((n) => n.type === "cardEmbed");
+          const resizeNode = content?.find((n) => n.type === "resizeNode");
+          const cardEmbed = resizeNode?.content?.[0];
           const clonedCardId = cardEmbed?.attrs?.id;
           cy.request("DELETE", `/api/card/${clonedCardId}`);
           cy.visit(`/document/${id}`);
         });
-        H.getDocumentCard("Couldn't find this chart.").should("exist");
+        cy.findByTestId("document-card-embed").should(
+          "have.text",
+          "Couldn't find this chart.",
+        );
       });
 
       it("read only access", () => {
@@ -684,12 +697,48 @@ H.describeWithSnowplowEE("documents", () => {
           });
         });
 
-        H.getDocumentCard(ORDERS_COUNT_BY_PRODUCT_CATEGORY.name).should(
-          "not.exist",
-        );
+        H.documentContent()
+          .findAllByTestId("card-embed-title")
+          .contains(ORDERS_COUNT_BY_PRODUCT_CATEGORY.name)
+          .should("not.exist");
+
         H.getDocumentCard("Orders").should("exist");
       });
 
+      it("should support resizing cards", () => {
+        H.documentContent().click();
+        H.addToDocument("/", false);
+
+        cy.log("search via type");
+        H.addToDocument("Accounts", false);
+        H.commandSuggestionDialog().should(
+          "contain.text",
+          ACCOUNTS_COUNT_BY_CREATED_AT.name,
+        );
+
+        cy.realPress("{downarrow}");
+        H.addToDocument("\n", false);
+
+        H.getDocumentCard(ACCOUNTS_COUNT_BY_CREATED_AT.name).then((el) => {
+          const ogHeight = el.height();
+          const resizeNode = H.getDocumentCardResizeContainer(
+            ACCOUNTS_COUNT_BY_CREATED_AT.name,
+          );
+
+          H.documentDoDrag(H.getDragHandleForDocumentResizeNode(resizeNode), {
+            y: 200,
+          });
+
+          H.getDocumentCard(ACCOUNTS_COUNT_BY_CREATED_AT.name).then((el) => {
+            const newHeight = el.height();
+
+            cy.log(`${ogHeight}, ${newHeight}`);
+
+            expect(newHeight).to.be.lessThan(ogHeight as number);
+          });
+        });
+      });
+
       it("should copy an added card on save", () => {
         cy.intercept({
           method: "PUT",

e2e/test/scenarios/embedding/sdk-iframe-embedding/authentication.cy.spec.ts

@@ -1,5 +1,7 @@
+import { USERS } from "e2e/support/cypress_data";
 import { ORDERS_DASHBOARD_ID } from "e2e/support/cypress_sample_instance_data";
 import {
+  getSignedJwtForUser,
   mockAuthSsoEndpointForSamlAuthProvider,
   stubWindowOpenForSamlPopup,
 } from "e2e/support/helpers/embedding-sdk-testing";
@@ -124,6 +126,62 @@ describe("scenarios > embedding > sdk iframe embedding > authentication", () =>
     assertDashboardLoaded(frame);
   });
 
+  it("can login via JWT and a custom fetch request token function", () => {
+    H.prepareSdkIframeEmbedTest({ enabledAuthMethods: ["jwt"], signOut: true });
+
+    const frame = H.loadSdkIframeEmbedTestPage({
+      onVisitPage: (win) => {
+        (win as any).metabaseConfig = {
+          ...(win as any).metabaseConfig,
+          fetchRequestToken: async () => {
+            const jwt = await getSignedJwtForUser(USERS.admin);
+
+            return { jwt };
+          },
+        };
+      },
+      elements: [
+        {
+          component: "metabase-dashboard",
+          attributes: {
+            dashboardId: ORDERS_DASHBOARD_ID,
+          },
+        },
+      ],
+    });
+
+    assertDashboardLoaded(frame);
+  });
+
+  it("shows error message if login via JWT and a custom fetch request token is failing", () => {
+    H.prepareSdkIframeEmbedTest({ enabledAuthMethods: ["jwt"], signOut: true });
+
+    const frame = H.loadSdkIframeEmbedTestPage({
+      onVisitPage: (win) => {
+        (win as any).metabaseConfig = {
+          ...(win as any).metabaseConfig,
+          fetchRequestToken: async () => {
+            return { jwt: "" };
+          },
+        };
+      },
+      elements: [
+        {
+          component: "metabase-dashboard",
+          attributes: {
+            dashboardId: ORDERS_DASHBOARD_ID,
+          },
+        },
+      ],
+    });
+
+    frame.within(() => {
+      cy.findByTestId("sdk-error-container")
+        .findByText(/Failed to fetch JWT token/)
+        .should("exist");
+    });
+  });
+
   it("can login via SAML", () => {
     mockAuthSsoEndpointForSamlAuthProvider();
     H.prepareSdkIframeEmbedTest({ enabledAuthMethods: [], signOut: true });

e2e/test/scenarios/embedding/sdk-iframe-embedding/sdk-pdf-downloads.cy.spec.ts

@@ -0,0 +1,97 @@
+const { H } = cy;
+import { ORDERS_DASHBOARD_ID } from "e2e/support/cypress_sample_instance_data";
+import type { BaseEmbedTestPageOptions } from "e2e/support/helpers";
+
+H.describeWithSnowplowEE(
+  "scenarios > embedding > sdk iframe embedding > pdf downloads",
+  () => {
+    const setup = (options: Partial<BaseEmbedTestPageOptions>) => {
+      const frame = H.loadSdkIframeEmbedTestPage({
+        elements: [
+          {
+            component: "metabase-dashboard",
+            attributes: {
+              dashboardId: ORDERS_DASHBOARD_ID,
+              withDownloads: true,
+            },
+          },
+        ],
+        ...options,
+      });
+
+      cy.wait("@getDashCardQuery");
+
+      // Check that the dashboard loaded fine
+      frame.within(() => {
+        cy.findByText("Orders in a dashboard").should("be.visible");
+        cy.findByText("Orders").should("be.visible");
+        H.assertTableRowsCount(2000);
+      });
+
+      return frame;
+    };
+
+    beforeEach(() => {
+      H.resetSnowplow();
+      cy.deleteDownloadsFolder();
+    });
+
+    afterEach(() => {
+      H.expectNoBadSnowplowEvents();
+    });
+
+    describe("Dashboard PDF downloads", () => {
+      beforeEach(() => {
+        H.prepareSdkIframeEmbedTest({ signOut: true });
+      });
+
+      it("should download dashboard as PDF with analytics tracking", () => {
+        const frame = setup({
+          metabaseConfig: {
+            theme: {
+              components: {
+                dashboard: {
+                  backgroundColor: "transparent",
+                },
+              },
+            },
+          },
+        });
+
+        frame
+          .should("contain", "Orders in a dashboard")
+          .findByLabelText("Download as PDF")
+          .should("be.visible")
+          .click();
+
+        // Verify PDF file download
+        cy.verifyDownload("Orders in a dashboard.pdf");
+
+        // Verify analytics tracking
+        H.expectUnstructuredSnowplowEvent({
+          dashboard_accessed_via: "sdk-embed",
+          event: "dashboard_pdf_exported",
+        });
+      });
+
+      it("should hide PDF download button when with-downloads is false", () => {
+        const frame = setup({
+          elements: [
+            {
+              component: "metabase-dashboard",
+              attributes: {
+                dashboardId: ORDERS_DASHBOARD_ID,
+                withDownloads: false,
+              },
+            },
+          ],
+        });
+
+        frame
+          .should("contain", "Orders in a dashboard")
+          .findByLabelText("Download as PDF")
+          .should("not.exist");
+      });
+    });
+  },
+);

e2e/test/scenarios/models/model-indexes.cy.spec.js

@@ -139,7 +139,7 @@ describe("scenarios > model indexes", () => {
       cy.findByRole("heading", { name: "Small Marble Shoes" }).should(
         "be.visible",
       );
-      cy.findAllByText("Small Marble Shoes").should("have.length", 1);
+      cy.findAllByText("Small Marble Shoes").should("have.length", 2);
       cy.findByText("Doohickey").should("be.visible");
     });
   });
@@ -176,7 +176,7 @@ describe("scenarios > model indexes", () => {
     cy.findByTestId("object-detail").within(() => {
       cy.findByText(/We're a little lost/i).should("not.exist");
       cy.findByRole("heading", { name: "Anais Zieme" }).should("be.visible");
-      cy.findAllByText("Anais Zieme").should("have.length", 1);
+      cy.findAllByText("Anais Zieme").should("have.length", 2);
     });
   });
 
@@ -196,7 +196,7 @@ describe("scenarios > model indexes", () => {
       cy.findByRole("heading", { name: "Small Marble Shoes" }).should(
         "be.visible",
       );
-      cy.findAllByText("Small Marble Shoes").should("have.length", 1);
+      cy.findAllByText("Small Marble Shoes").should("have.length", 2);
       cy.findByText("Doohickey").should("be.visible");
     });
 

e2e/test/scenarios/native-filters/native-filters-reproductions.cy.spec.js

@@ -1,5 +1,9 @@
 const { H } = cy;
-import { SAMPLE_DB_ID, USER_GROUPS } from "e2e/support/cypress_data";
+import {
+  SAMPLE_DB_ID,
+  USER_GROUPS,
+  WRITABLE_DB_ID,
+} from "e2e/support/cypress_data";
 import { SAMPLE_DATABASE } from "e2e/support/cypress_sample_database";
 
 import * as FieldFilter from "./helpers/e2e-field-filter-helpers";
@@ -1272,3 +1276,30 @@ describe("issue 58061", () => {
     });
   });
 });
+
+describe("issue 63537", () => {
+  beforeEach(() => {
+    H.restore("postgres-writable");
+    H.resetTestTable({ type: "postgres", table: "many_data_types" });
+    cy.signInAsAdmin();
+    H.resyncDatabase({ dbId: WRITABLE_DB_ID, tableName: "many_data_types" });
+  });
+
+  it("should allow to use postgres enums in field filters (metabase#63537)", () => {
+    H.startNewNativeQuestion({ database: WRITABLE_DB_ID });
+    H.NativeEditor.type("SELECT * FROM many_data_types WHERE {{f}}");
+    SQLFilter.openTypePickerFromDefaultFilterType();
+    SQLFilter.chooseType("Field Filter");
+    FieldFilter.mapTo({
+      table: "Many Data Types",
+      field: "Enum",
+    });
+    H.filterWidget().click();
+    H.popover().within(() => {
+      cy.findByText("beta").click();
+      cy.button("Add filter").click();
+    });
+    H.runNativeQuery();
+    H.assertQueryBuilderRowCount(2);
+  });
+});

e2e/test/scenarios/native/native.cy.spec.js

@@ -578,8 +578,9 @@ describe("scenarios > question > native", () => {
     cy.button("Preview the query").click();
     cy.wait("@datasetNative");
 
-    // eslint-disable-next-line no-unscoped-text-selectors -- deprecated usage
-    cy.findByText(/where CATEGORY='Gadget'/).should("be.visible");
+    H.modal().within(() => {
+      H.codeMirrorValue().should("contain", "CATEGORY = 'Gadget'");
+    });
   });
 
   it("should show errors when previewing a query", () => {

e2e/test/scenarios/native/snippet-tags.cy.spec.ts

@@ -338,6 +338,31 @@ describe("scenarios > native > snippet tags", () => {
       H.assertTableRowsCount(53);
     });
   });
+
+  it("should handle snippet tags with trailing spaces correctly", () => {
+    cy.log("create a snippet");
+    H.createSnippet({
+      name: "category filter",
+      content: "category = {{category}}",
+    });
+
+    H.startNewNativeQuestion();
+
+    cy.log("type a query with snippet tags containing trailing spaces");
+    H.NativeEditor.type(
+      "select id from products where {{snippet: category filter }}",
+    );
+
+    cy.log("verify snippet reference is recognized");
+    getEditorTopBar().within(() => {
+      cy.findByPlaceholderText("Category").should("be.visible");
+    });
+
+    cy.log("verify the query can be run with the parameter");
+    getEditorTopBar().findByPlaceholderText("Category").type("Widget");
+    H.runNativeQuery();
+    H.assertQueryBuilderRowCount(54);
+  });
 });
 
 function getEditorSidebar() {

e2e/test/scenarios/native/snippets.cy.spec.js

@@ -183,7 +183,9 @@ describe("scenarios > question > snippets", () => {
     cy.findByTestId("native-query-top-bar")
       .findByLabelText("Preview the query")
       .click();
-    H.modal().findByText("select 'foo'").should("be.visible");
+    H.modal().within(() => {
+      H.codeMirrorValue().should("eq", "select\n  'foo'");
+    });
   });
 });
 

e2e/test/scenarios/search/search-snowplow.cy.spec.js

@@ -54,6 +54,7 @@ H.describeWithSnowplow("scenarios > search > snowplow", () => {
               search_engine: P.string,
               request_id: P.string,
               entity_model: P.string,
+              entity_id: P.number,
               search_term_hash: P.string,
               search_term: null,
             },

e2e/test/scenarios/visualizations-tabular/object_detail.cy.spec.js

@@ -429,28 +429,30 @@ describe("scenarios > question > object details", { tags: "@slow" }, () => {
     H.openObjectDetail(0);
 
     cy.findByTestId("object-detail").within(() => {
-      cy.findByText("Link to review 1")
-        .should("be.visible")
-        .should("have.attr", "href")
+      cy.findAllByText("Link to review 1")
+        .should("have.length", 2)
+        .and("be.visible")
+        .and("have.attr", "href")
         .and("eq", "https://metabase.test?review=1");
 
       cy.findByText("Rating: 5")
         .should("be.visible")
-        .should("have.attr", "href")
+        .and("have.attr", "href")
         .and("eq", "https://metabase.test?rating=5");
     });
 
     cy.findByLabelText("Next row").click();
 
     cy.findByTestId("object-detail").within(() => {
-      cy.findByText("Link to review 2")
-        .should("be.visible")
-        .should("have.attr", "href")
+      cy.findAllByText("Link to review 2")
+        .should("have.length", 2)
+        .and("be.visible")
+        .and("have.attr", "href")
         .and("eq", "https://metabase.test?review=2");
 
       cy.findByText("Rating: 4")
         .should("be.visible")
-        .should("have.attr", "href")
+        .and("have.attr", "href")
         .and("eq", "https://metabase.test?rating=4");
     });
   });
@@ -574,7 +576,7 @@ describe("scenarios > question > object details", { tags: "@slow" }, () => {
   });
 
   describe("detail page links - questions", () => {
-    it("no primary keys", () => {
+    it("no primary keys (WRK-900)", () => {
       H.visitQuestionAdhoc({
         display: "table",
         dataset_query: {
@@ -596,6 +598,10 @@ describe("scenarios > question > object details", { tags: "@slow" }, () => {
       cy.findByTestId("object-detail").within(() => {
         cy.findByLabelText("Copy link to this record").should("not.exist");
         cy.findByLabelText("Open in full page").should("not.exist");
+
+        cy.log("should not show relationships when there is no PK (WRK-900)");
+        cy.findByText(/is connected to/).should("not.exist");
+        cy.findByRole("link", { name: /Orders/ }).should("not.exist");
       });
     });
 
@@ -673,7 +679,7 @@ describe("scenarios > question > object details", { tags: "@slow" }, () => {
   });
 
   describe("detail page links - models", () => {
-    it("no primary keys", () => {
+    it("no primary keys (WRK-900)", () => {
       H.createQuestion(
         {
           type: "model",
@@ -694,6 +700,10 @@ describe("scenarios > question > object details", { tags: "@slow" }, () => {
       cy.findByTestId("object-detail").within(() => {
         cy.findByLabelText("Copy link to this record").should("not.exist");
         cy.findByLabelText("Open in full page").should("not.exist");
+
+        cy.log("should not show relationships when there is no PK (WRK-900)");
+        cy.findByText(/is connected to/).should("not.exist");
+        cy.findByRole("link", { name: /Orders/ }).should("not.exist");
       });
     });
 

e2e/test/scenarios/visualizations-tabular/table.cy.spec.js

@@ -154,7 +154,7 @@ describe("scenarios > visualizations > table", () => {
     H.openObjectDetail(5);
 
     // Ensure click on row index opens the object detail
-    H.modal().findByText("6").should("be.visible");
+    H.modal().findAllByText("6").should("have.length", 2).and("be.visible");
 
     // Close object detail modal
     cy.realType("{esc}");

enterprise/backend/src/metabase_enterprise/advanced_config/file/users.clj

@@ -56,4 +56,5 @@
 (defmethod advanced-config.file.i/initialize-section! :users
   [_section-name users]
   (doseq [user users]
-    (init-from-config-file! user)))
+    ;; we're lower-casing emails in :model/User, so we should do the same here
+    (init-from-config-file! (update user :email u/lower-case-en))))

enterprise/backend/src/metabase_enterprise/metabot_v3/task/suggested_prompts_generator.clj

@@ -20,16 +20,17 @@
 (def ^:private generator-trigger-key (triggers/key "metabase.task.metabot-v3.suggested-prompts-generator.trigger"))
 
 (defn- maybe-generate-suggested-prompts! []
-  (let [metabot-eid (get-in metabot-v3.config/metabot-config
-                            [metabot-v3.config/internal-metabot-id :entity-id])
-        metabot-id (t2/select-one-pk :model/Metabot :entity_id metabot-eid)
-        suggested-prompts-cnt (t2/count :model/MetabotPrompt :metabot_id metabot-id)]
-    (if (zero? suggested-prompts-cnt)
-      (do
-        (log/info "No suggested prompts found. Generating suggested prompts.")
-        (metabot-v3.suggested-prompts/generate-sample-prompts metabot-id)
-        (log/info "Suggested prompts generated successfully."))
-      (log/info "Suggested prompts are present. Not generating."))))
+  (when (premium-features/has-feature? :metabot-v3)
+    (let [metabot-eid (get-in metabot-v3.config/metabot-config
+                              [metabot-v3.config/internal-metabot-id :entity-id])
+          metabot-id (t2/select-one-pk :model/Metabot :entity_id metabot-eid)
+          suggested-prompts-cnt (t2/count :model/MetabotPrompt :metabot_id metabot-id)]
+      (if (zero? suggested-prompts-cnt)
+        (do
+          (log/info "No suggested prompts found. Generating suggested prompts.")
+          (metabot-v3.suggested-prompts/generate-sample-prompts metabot-id)
+          (log/info "Suggested prompts generated successfully."))
+        (log/info "Suggested prompts are present. Not generating.")))))
 
 (task/defjob ^{DisallowConcurrentExecution true
                :doc "Initial _suggested prompts_ generation for internal Metabot."}

enterprise/backend/src/metabase_enterprise/scim/v2/api.clj

@@ -68,7 +68,9 @@
    [:Operations
     [:sequential [:map
                   [:op ms/NonBlankString]
-                  [:value [:or ms/NonBlankString ms/BooleanValue]]]]]])
+                  [:value [:or [:map-of [:or :keyword :string]
+                                [:or ms/NonBlankString ms/BooleanValue]]
+                           ms/NonBlankString ms/BooleanValue]]]]]])
 
 (def SCIMGroup
   "Malli schema for a SCIM group."
@@ -280,6 +282,18 @@
                                :status      400
                                :status-code 400})))))))))
 
+(defn- patch->user-updates
+  [acc path value]
+  (if (and (nil? path) (map? value))
+    (reduce-kv patch->user-updates acc value)
+    (let [path-str (some-> path name)]
+      (case path-str
+        "active"          (assoc acc :is_active (Boolean/valueOf (u/lower-case-en value)))
+        "userName"        (assoc acc :email value)
+        "name.givenName"  (assoc acc :first_name value)
+        "name.familyName" (assoc acc :last_name value)
+        (throw-scim-error 400 (format "Unsupported path: %s" path-str))))))
+
 (api.macros/defendpoint :patch ["/Users/:id" :id #"[^/]+"]
   "Activate or deactivate a user. Supports specific replace operations, but not arbitrary patches."
   [{:keys [id]} :- [:map
@@ -292,14 +306,8 @@
             updates (reduce
                      (fn [acc operation]
                        (let [{:keys [op path value]} operation]
-                         (if (= (u/lower-case-en op) "replace")
-                           (case path
-                             "active"          (assoc acc :is_active (Boolean/valueOf (u/lower-case-en value)))
-                             "userName"        (assoc acc :email value)
-                             "name.givenName"  (assoc acc :first_name value)
-                             "name.familyName" (assoc acc :last_name value)
-                             (throw-scim-error 400 (format "Unsupported path: %s" path)))
-                           acc)))
+                         (cond-> acc
+                           (= (u/lower-case-en op) "replace") (patch->user-updates path value))))
                      {}
                      (:Operations patch-ops))]
         (t2/update! :model/User (u/the-id user) updates)

enterprise/backend/src/metabase_enterprise/semantic_search/task/index_cleanup.clj

@@ -8,10 +8,9 @@
    [honey.sql :as sql]
    [honey.sql.helpers :as sql.helpers]
    [java-time.api :as t]
-   [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
    [metabase-enterprise.semantic-search.env :as semantic.env]
    [metabase-enterprise.semantic-search.settings :as semantic.settings]
-   [metabase.premium-features.core :as premium-features]
+   [metabase-enterprise.semantic-search.util :as semantic.u]
    [metabase.task.core :as task]
    [metabase.util.log :as log]
    [next.jdbc :as jdbc]
@@ -165,11 +164,12 @@
 
 (defn- cleanup-stale-indexes-and-gate-tombstones!
   []
-  (let [pgvector             (semantic.env/get-pgvector-datasource!)
-        index-metadata       (semantic.env/get-index-metadata)]
-    (cleanup-stale-indexes! pgvector index-metadata)
-    (cleanup-old-gate-tombstones! pgvector index-metadata)
-    (cleanup-orphan-repair-tables! pgvector)))
+  (when (semantic.u/semantic-search-available?)
+    (let [pgvector             (semantic.env/get-pgvector-datasource!)
+          index-metadata       (semantic.env/get-index-metadata)]
+      (cleanup-stale-indexes! pgvector index-metadata)
+      (cleanup-old-gate-tombstones! pgvector index-metadata)
+      (cleanup-orphan-repair-tables! pgvector))))
 
 (def ^:private cleanup-job-key (jobs/key "metabase.task.semantic-index-cleanup.job"))
 (def ^:private cleanup-trigger-key (triggers/key "metabase.task.semantic-index-cleanup.trigger"))
@@ -180,8 +180,7 @@
   (cleanup-stale-indexes-and-gate-tombstones!))
 
 (defmethod task/init! ::SemanticIndexCleanup [_]
-  (when (and (string? (not-empty semantic.db.datasource/db-url))
-             (premium-features/has-feature? :semantic-search))
+  (when (semantic.u/semantic-search-available?)
     (let [job (jobs/build
                (jobs/of-type SemanticIndexCleanup)
                (jobs/with-identity cleanup-job-key))

enterprise/backend/src/metabase_enterprise/semantic_search/task/index_repair.clj

@@ -5,8 +5,7 @@
    [clojurewerkz.quartzite.schedule.cron :as cron]
    [clojurewerkz.quartzite.triggers :as triggers]
    [metabase-enterprise.semantic-search.core :as semantic-search.core]
-   [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
-   [metabase.premium-features.core :as premium-features]
+   [metabase-enterprise.semantic-search.util :as semantic.u]
    [metabase.search.ingestion :as search.ingestion]
    [metabase.task.core :as task]
    [metabase.util.log :as log])
@@ -21,17 +20,17 @@
 (task/defjob ^{DisallowConcurrentExecution true
                :doc "Runs repair-index! to maintain semantic search consistency"}
   SemanticIndexRepair [_ctx]
-  (log/with-context {:quartz-job-type 'SemanticIndexRepair}
-    (try
-      (log/info "Starting semantic search index repair")
-      (semantic-search.core/repair-index! (search.ingestion/searchable-documents))
-      (log/info "Completed semantic search index repair")
-      (catch Exception e
-        (log/error e "Failed to complete semantic search index repair")))))
+  (when (semantic.u/semantic-search-available?)
+    (log/with-context {:quartz-job-type 'SemanticIndexRepair}
+      (try
+        (log/info "Starting semantic search index repair")
+        (semantic-search.core/repair-index! (search.ingestion/searchable-documents))
+        (log/info "Completed semantic search index repair")
+        (catch Exception e
+          (log/error e "Failed to complete semantic search index repair"))))))
 
 (defmethod task/init! ::SemanticIndexRepair [_]
-  (when (and (string? (not-empty semantic.db.datasource/db-url))
-             (premium-features/has-feature? :semantic-search))
+  (when (semantic.u/semantic-search-available?)
     (let [job (jobs/build
                (jobs/of-type SemanticIndexRepair)
                (jobs/with-identity repair-job-key))

enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj

@@ -3,10 +3,9 @@
    [clojurewerkz.quartzite.jobs :as jobs]
    [clojurewerkz.quartzite.schedule.simple :as simple]
    [clojurewerkz.quartzite.triggers :as triggers]
-   [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
    [metabase-enterprise.semantic-search.env :as semantic.env]
    [metabase-enterprise.semantic-search.indexer :as semantic-search.indexer]
-   [metabase.premium-features.core :as premium-features]
+   [metabase-enterprise.semantic-search.util :as semantic.u]
    [metabase.search.engine :as search.engine]
    [metabase.task.core :as task]
    [metabase.util.log :as log])
@@ -32,14 +31,15 @@
  SemanticSearchIndexer []
   org.quartz.Job
   (execute [_ _]
-    (log/with-context {:quartz-job-type 'SemanticSearchIndexer}
-      (when (search.engine/supported-engine? :search.engine/semantic)
-        (try
-          (vreset! execution-thread-ref (Thread/currentThread))
-          (semantic-search.indexer/quartz-job-run! (semantic.env/get-pgvector-datasource!) (semantic.env/get-index-metadata))
-          (finally
-            (locking execution-thread-ref
-              (vreset! execution-thread-ref nil)))))))
+    (when (semantic.u/semantic-search-available?)
+      (log/with-context {:quartz-job-type 'SemanticSearchIndexer}
+        (when (search.engine/supported-engine? :search.engine/semantic)
+          (try
+            (vreset! execution-thread-ref (Thread/currentThread))
+            (semantic-search.indexer/quartz-job-run! (semantic.env/get-pgvector-datasource!) (semantic.env/get-index-metadata))
+            (finally
+              (locking execution-thread-ref
+                (vreset! execution-thread-ref nil))))))))
   org.quartz.InterruptableJob
   (interrupt [_]
    ;; locking required here to avoid racing with the unset in the finally
@@ -52,8 +52,7 @@
 (def ^:private ^Duration run-frequency (Duration/parse "PT20S"))
 
 (defmethod task/init! ::SemanticSearchIndexer [_]
-  (when (and (string? (not-empty semantic.db.datasource/db-url))
-             (premium-features/has-feature? :semantic-search))
+  (when (semantic.u/semantic-search-available?)
     (let [job         (jobs/build
                        (jobs/of-type SemanticSearchIndexer)
                        (jobs/store-durably)

enterprise/backend/src/metabase_enterprise/semantic_search/task/metric_collector.clj

@@ -4,12 +4,10 @@
    [clojurewerkz.quartzite.schedule.simple :as simple]
    [clojurewerkz.quartzite.triggers :as triggers]
    [honey.sql :as sql]
-   [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
    [metabase-enterprise.semantic-search.dlq :as semantic.dlq]
    [metabase-enterprise.semantic-search.env :as semantic.env]
-   [metabase-enterprise.semantic-search.util :as semantic.util]
+   [metabase-enterprise.semantic-search.util :as semantic.u]
    [metabase.analytics.core :as analytics]
-   [metabase.premium-features.core :as premium-features]
    [metabase.task.core :as task]
    [metabase.util.log :as log]
    [next.jdbc :as jdbc]
@@ -37,7 +35,7 @@
   (let [{:keys [gate-table-name]} (semantic.env/get-index-metadata)]
     (assert (string? gate-table-name))
     (log/debugf "Checking size of gate table %s" gate-table-name)
-    (if (semantic.util/table-exists? pgvector gate-table-name)
+    (if (semantic.u/table-exists? pgvector gate-table-name)
       (let [table-size (row-count pgvector gate-table-name)]
         (log/debugf "Setting `semantic-gate-size` metric to %d" table-size)
         (analytics/set! :metabase-search/semantic-gate-size table-size)
@@ -46,7 +44,7 @@
 
 (defn- active-index-id
   [pgvector index-metadata]
-  (if (semantic.util/table-exists? pgvector (:control-table-name index-metadata))
+  (if (semantic.u/table-exists? pgvector (:control-table-name index-metadata))
     (:id (jdbc/execute-one!
           pgvector
           (sql/format {:select [[:active_id :id]]
@@ -59,7 +57,7 @@
   (if-some [active-index-id (active-index-id pgvector index-metadata)]
     (let [dlq-table-name (name (semantic.dlq/dlq-table-name-kw index-metadata active-index-id))]
       (log/debugf "Checking size of DLQ table %s" dlq-table-name)
-      (when (semantic.util/table-exists? pgvector dlq-table-name)
+      (when (semantic.u/table-exists? pgvector dlq-table-name)
         (let [table-size (row-count pgvector dlq-table-name)]
           (log/debugf "Setting `semantic-dlq-size` metric to %d" table-size)
           (analytics/set! :metabase-search/semantic-dlq-size table-size)
@@ -67,10 +65,11 @@
     (log/warn "DLQ table does not exist. Index may not have been initialized.")))
 
 (defn- collect-metrics! []
-  (let [pgvector (semantic.env/get-pgvector-datasource!)
-        index-metadata (semantic.env/get-index-metadata)]
-    (collect-gate-size! pgvector)
-    (collect-dlq-size! pgvector index-metadata)))
+  (when (semantic.u/semantic-search-available?)
+    (let [pgvector (semantic.env/get-pgvector-datasource!)
+          index-metadata (semantic.env/get-index-metadata)]
+      (collect-gate-size! pgvector)
+      (collect-dlq-size! pgvector index-metadata))))
 
 (task/defjob ^{DisallowConcurrentExecution true
                :doc "Collect expensive semantic search metrics"}
@@ -81,8 +80,7 @@
 
 (defmethod task/init! ::SemanticMetricCollector
   [_]
-  (when (and (string? (not-empty semantic.db.datasource/db-url))
-             (premium-features/has-feature? :semantic-search))
+  (when (semantic.u/semantic-search-available?)
     (let [job (jobs/build
                (jobs/of-type SemanticMetricCollector)
                (jobs/with-identity collector-job-key))
@@ -95,6 +93,3 @@
                      (simple/repeat-forever)))
                    (triggers/start-now))]
       (task/schedule-task! job trigger))))
-
-(comment
-  (task/trigger-now! collector-job-key))

enterprise/backend/src/metabase_enterprise/semantic_search/task/usage_trimmer.clj

@@ -3,7 +3,7 @@
    [clojurewerkz.quartzite.jobs :as jobs]
    [clojurewerkz.quartzite.schedule.cron :as cron]
    [clojurewerkz.quartzite.triggers :as triggers]
-   [metabase.premium-features.core :as premium-features]
+   [metabase-enterprise.semantic-search.util :as semantic.u]
    [metabase.task.core :as task]
    [metabase.util.log :as log]
    [toucan2.core :as t2])
@@ -20,10 +20,11 @@
 
 (defn- trim-old-token-data!
   []
-  (log/info "Attempting to delete old semantic search usage data.")
-  (let [t (Timestamp/valueOf (.minusMonths (java.time.LocalDateTime/now) storage-months))]
-    (t2/delete! :model/SemanticSearchTokenTracking {:where [:< :created_at t]}))
-  (log/info "Semantic search old data cleanup successful."))
+  (when (semantic.u/semantic-search-available?)
+    (log/info "Attempting to delete old semantic search usage data.")
+    (let [t (Timestamp/valueOf (.minusMonths (java.time.LocalDateTime/now) storage-months))]
+      (t2/delete! :model/SemanticSearchTokenTracking {:where [:< :created_at t]}))
+    (log/info "Semantic search old data cleanup successful.")))
 
 (task/defjob ^{DisallowConcurrentExecution true
                :doc "Clean up inactive semantic search index tables"}
@@ -31,8 +32,8 @@
   (trim-old-token-data!))
 
 (defmethod task/init! ::SemanticSearchUsageTrimmer
-  []
-  (when (premium-features/has-feature? :semantic-search)
+  [_]
+  (when (semantic.u/semantic-search-available?)
     (let [job (jobs/build
                (jobs/of-type SemanticSearchUsageTrimmer)
                (jobs/with-identity trimmer-job-key))

enterprise/backend/src/metabase_enterprise/semantic_search/util.clj

@@ -1,5 +1,7 @@
 (ns metabase-enterprise.semantic-search.util
   (:require
+   [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
+   [metabase.premium-features.core :as premium-features]
    [next.jdbc :as jdbc]
    [next.jdbc.result-set :as jdbc.rs]))
 
@@ -11,3 +13,9 @@
                           table-name]
                          {:builder-fn jdbc.rs/as-unqualified-lower-maps})
       (:table_exists false)))
+
+(defn semantic-search-available?
+  "Predicate to check whether semantic search is available on the instance."
+  []
+  (and (string? (not-empty semantic.db.datasource/db-url))
+       (premium-features/has-feature? :semantic-search)))

enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj

@@ -72,27 +72,28 @@
       (format "%d: %s" coll-id names))
     "[no collection]"))
 
-(defn- card-label [card-id]
-  (let [card (t2/select-one [:model/Card :collection_id :name] :id card-id)]
-    (format "Card %d (%s from collection %s)" card-id (:name card) (collection-label (:collection_id card)))))
+(defn- entity-label [{:keys [model id]}]
+  (let [entity (t2/select-one [model :collection_id :name] :id id)]
+    (format "%s %d (%s from collection %s)" (name model) id (:name entity) (collection-label (:collection_id entity)))))
 
 (defn- parse-target [[model-name id :as target]]
   (if (string? id)
     [model-name (serdes/eid->id model-name id)]
     target))
 
-(defn- escape-analysis [{colls "Collection" cards "Card"} nodes]
+(defn- escape-analysis [{colls "Collection" cards "Card" :as _by-model} nodes]
   (log/tracef "Running escape analysis for %d colls and %d cards" (count colls) (count cards))
   (when-let [colls (-> colls set not-empty)]
-    (let [known-cards (t2/select-pks-set :model/Card {:where [:or
-                                                              [:in :collection_id colls]
-                                                              (when (contains? colls nil)
-                                                                [:= :collection_id nil])]})
-          escaped     (->> (set/difference (set cards) known-cards)
-                           (mapv (fn [id]
-                                   (-> (get nodes ["Card" id])
-                                       (assoc :escapee id)))))]
-      escaped)))
+    (let [clause       {:where [:or
+                                [:in :collection_id colls]
+                                (when (contains? colls nil)
+                                  [:= :collection_id nil])]}
+          possible-pks (t2/select-pks-set :model/Card clause)]
+      (->> (set/difference (set cards) possible-pks)
+           (mapv (fn [id]
+                   (-> (get nodes ["Card" id])
+                       (assoc :escapee {:model :model/Card
+                                        :id    id}))))))))
 
 (defn- log-escape-report! [escaped]
   (let [dashboards (group-by #(get % "Dashboard") escaped)]
@@ -100,15 +101,28 @@
       (log/warnf "Failed to export Dashboard %d (%s) containing Card saved outside requested collections: %s"
                  dash-id
                  (t2/select-one-fn :name :model/Dashboard :id dash-id)
-                 (str/join ", " (map #(card-label (:escapee %)) escapes))))
+                 (str/join ", " (map #(entity-label (:escapee %)) escapes))))
     (when-let [other (not-empty (get dashboards nil))]
       (log/warnf "Failed to export Cards based on questions outside requested collections: %s"
                  (str/join ", " (for [item other]
                                   (format "%s -> %s"
                                           (if (get item "Card")
-                                            (card-label (get item "Card"))
+                                            (entity-label {:model :model/Card :id (get item "Card")})
                                             (dissoc item :escapee))
-                                          (card-label (:escapee item)))))))))
+                                          (entity-label (:escapee item)))))))))
+
+(defn- resolve-targets
+  "Returns all targets (for either supplied initial `targets` or for supplied `user-id`)."
+  [targets user-id]
+  (let [initial-targets (if (seq targets)
+                          (mapv parse-target targets)
+                          (mapv vector (repeat "Collection") (collection-set-for-user user-id)))
+        ;; a map of `{[model-name id] {source-model source-id ...}}`
+        targets         (u/traverse initial-targets #(serdes/descendants (first %) (second %)))]
+    ;; due to traverse argument we'd lose original source entities, lets track them
+    (merge-with into
+                targets
+                (u/traverse (map key targets) #(serdes/required (first %) (second %))))))
 
 (defn- extract-subtrees
   "Extracts the targeted entities and all their descendants into a reducible stream of extracted maps.
@@ -125,16 +139,10 @@
   `opts` are passed down to [[serdes/extract-all]] for each model."
   [{:keys [targets user-id] :as opts}]
   (log/tracef "Extracting subtrees with options: %s" (pr-str opts))
-  (let [inner-targets (if (seq targets)
-                        (mapv parse-target targets)
-                        (mapv vector (repeat "Collection") (collection-set-for-user user-id)))
-        ;; nodes are a map of `{[model-name id] {dep-model dep-id ...}}`
-        nodes         (set/union
-                       (u/traverse inner-targets #(serdes/ascendants (first %) (second %)))
-                       (u/traverse inner-targets #(serdes/descendants (first %) (second %))))
+  (let [nodes    (resolve-targets targets user-id)
         ;; by model is a map of `{model-name [ids ...]}`
-        by-model      (u/group-by first second (keys nodes))
-        escaped       (escape-analysis by-model nodes)]
+        by-model (u/group-by first second (keys nodes))
+        escaped  (escape-analysis by-model nodes)]
     (if (seq escaped)
       (log-escape-report! escaped)
       (let [models         (model-set opts)

enterprise/backend/src/metabase_enterprise/serialization/v2/storage.clj

@@ -4,7 +4,10 @@
    [clojure.string :as str]
    [metabase-enterprise.serialization.dump :refer [spit-yaml!]]
    [metabase.models.serialization :as serdes]
-   [metabase.util.log :as log]))
+   [metabase.util.log :as log])
+  (:import
+   [java.io File]
+   [java.nio.file Path]))
 
 (set! *warn-on-reflection* true)
 
@@ -16,8 +19,7 @@
       (str/replace "/"  "__SLASH__")
       (str/replace "\\" "__BACKSLASH__")))
 
-(defn- file
-  [ctx entity]
+(defn- file ^File [ctx entity]
   (let [;; Get the desired [[serdes/storage-path]].
         base-path   (serdes/storage-path entity ctx)
         dirnames    (drop-last base-path)
@@ -26,9 +28,12 @@
     (apply io/file (:root-dir ctx) (map escape-segment (concat dirnames [basename])))))
 
 (defn- store-entity! [opts entity]
-  (log/info "Storing" {:path (serdes/log-path-str (:serdes/meta entity))})
-  (spit-yaml! (file opts entity) entity)
-  (:serdes/meta entity))
+  (let [f (file opts entity)]
+    (log/info "Storing" {:path (serdes/log-path-str (:serdes/meta entity))
+                         :file (str (.relativize (Path/of (str (:root-dir opts)) (make-array String 0))
+                                                 (.toPath f)))})
+    (spit-yaml! f entity)
+    (:serdes/meta entity)))
 
 (defn- store-settings! [{:keys [root-dir]} settings]
   (when (seq settings)

enterprise/backend/src/metabase_enterprise/sso/integrations/jwt.clj

@@ -16,9 +16,7 @@
    [metabase.sso.core :as sso]
    [metabase.util.i18n :refer [tru]]
    [ring.util.response :as response]
-   [toucan2.core :as t2])
-  (:import
-   (java.net URLEncoder)))
+   [toucan2.core :as t2]))
 
 (set! *warn-on-reflection* true)
 
@@ -99,7 +97,7 @@
 
 (defn- session-data
   [jwt {{redirect :return_to} :params, :as request}]
-  (let [redirect-url (or redirect (URLEncoder/encode "/"))]
+  (let [redirect-url (or redirect "/")]
     (sso-utils/check-sso-redirect redirect-url)
     (let [jwt-data     (try
                          (jwt/unsign jwt (sso-settings/jwt-shared-secret)

enterprise/backend/src/metabase_enterprise/transforms/api.clj

@@ -1,7 +1,9 @@
 (ns metabase-enterprise.transforms.api
   (:require
+   [metabase-enterprise.transforms-python.execute :as transforms-python.execute]
    [metabase-enterprise.transforms.api.transform-job]
    [metabase-enterprise.transforms.api.transform-tag]
+   [metabase-enterprise.transforms.canceling :as transforms.canceling]
    [metabase-enterprise.transforms.execute :as transforms.execute]
    [metabase-enterprise.transforms.models.transform :as transform.model]
    [metabase-enterprise.transforms.models.transform-run :as transform-run]
@@ -29,12 +31,21 @@
 (set! *warn-on-reflection* true)
 
 (mr/def ::transform-source
-  [:map
-   [:type [:= "query"]]
-   [:query [:map [:database :int]]]])
+  [:multi {:dispatch (comp keyword :type)}
+   [:query
+    [:map
+     [:type [:= "query"]]
+     [:query [:map [:database :int]]]]]
+   [:python
+    [:map {:closed true}
+     [:source-database {:optional true} :int]
+     [:source-tables   [:map-of :string :int]]
+     [:type [:= "python"]]
+     [:body :string]]]])
 
 (mr/def ::transform-target
   [:map
+   [:database {:optional true} :int]
    [:type [:enum "table"]]
    [:schema {:optional true} [:or ms/NonBlankString :nil]]
    [:name :string]])
@@ -55,14 +66,10 @@
              :schema "transforms"
              :name "gadget_products"}}])
 
-(defn- source-database-id
-  [transform]
-  (-> transform :source :query :database))
-
 (defn- check-database-feature
   [transform]
-  (let [database (api/check-400 (t2/select-one :model/Database (source-database-id transform))
-                                (deferred-tru "The source database cannot be found."))
+  (let [database (api/check-400 (t2/select-one :model/Database (transforms.util/target-database-id transform))
+                                (deferred-tru "The target database cannot be found."))
         feature (transforms.util/required-database-feature transform)]
     (api/check-400 (not (:is_sample database))
                    (deferred-tru "Cannot run transforms on the sample database."))
@@ -73,14 +80,27 @@
     (api/check-400 (not (transforms.util/db-routing-enabled? database))
                    (deferred-tru "Transforms are not supported on databases with DB routing enabled."))))
 
+(defn- check-feature-enabled!
+  [transform]
+  (api/check (transforms.util/check-feature-enabled transform)
+             [402 (deferred-tru "Premium features required for this transform type are not enabled.")]))
+
 (api.macros/defendpoint :get "/"
   "Get a list of transforms."
   [_route-params
-   _query-params]
+   {:keys [last_run_start_time last_run_statuses tag_ids]} :-
+   [:map
+    [:last_run_start_time {:optional true} [:maybe ms/NonBlankString]]
+    [:last_run_statuses {:optional true} [:maybe (ms/QueryVectorOf [:enum "started" "succeeded" "failed" "timeout"])]]
+    [:tag_ids {:optional true} [:maybe (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]]]]
   (api/check-superuser)
-  (-> (t2/select :model/Transform)
-      (t2/hydrate :last_run :transform_tag_ids)
-      (->> (map #(update % :last_run transforms.util/localize-run-timestamps)))))
+  (let [transforms (t2/select :model/Transform)]
+    (into []
+          (comp (transforms.util/->date-field-filter-xf [:last_run :start_time] last_run_start_time)
+                (transforms.util/->status-filter-xf [:last_run :status] last_run_statuses)
+                (transforms.util/->tag-filter-xf [:tag_ids] tag_ids)
+                (map #(update % :last_run transforms.util/localize-run-timestamps)))
+          (t2/hydrate transforms :last_run :transform_tag_ids))))
 
 (api.macros/defendpoint :post "/"
   "Create a new transform."
@@ -95,6 +115,7 @@
             [:tag_ids {:optional true} [:sequential ms/PositiveInt]]]]
   (api/check-superuser)
   (check-database-feature body)
+  (check-feature-enabled! body)
   (api/check (not (transforms.util/target-table-exists? body))
              403
              (deferred-tru "A table with that name already exists."))
@@ -115,8 +136,7 @@
   (log/info "get transform" id)
   (api/check-superuser)
   (let [{:keys [target] :as transform} (api/check-404 (t2/select-one :model/Transform id))
-        database-id (source-database-id transform)
-        target-table (transforms.util/target-table database-id target :active true)]
+        target-table (transforms.util/target-table (transforms.util/target-database-id transform) target :active true)]
     (-> transform
         (t2/hydrate :last_run :transform_tag_ids)
         (u/update-some :last_run transforms.util/localize-run-timestamps)
@@ -173,10 +193,13 @@
     (let [old (t2/select-one :model/Transform id)
           new (merge old body)
           target-fields #(-> % :target (select-keys [:schema :name]))]
+      ;; we must validate on a full transform object
+      (check-feature-enabled! new)
       (check-database-feature new)
-      (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]
-        (throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)
-                        {:status-code 400})))
+      (when (transforms.util/query-transform? old)
+        (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]
+          (throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)
+                          {:status-code 400}))))
       (api/check (not (and (not= (target-fields old) (target-fields new))
                            (transforms.util/target-table-exists? new)))
                  403
@@ -212,8 +235,11 @@
                     [:id ms/PositiveInt]]]
   (log/info "canceling transform " id)
   (api/check-superuser)
-  (let [run (api/check-404 (transform-run/running-run-for-transform-id id))]
-    (transform-run-cancelation/mark-cancel-started-run! (:id run)))
+  (let [transform (api/check-404 (t2/select-one :model/Transform id))
+        run (api/check-404 (transform-run/running-run-for-transform-id id))]
+    (transform-run-cancelation/mark-cancel-started-run! (:id run))
+    (when (transforms.util/python-transform? transform)
+      (transforms.canceling/cancel-run! (:id run))))
   nil)
 
 (api.macros/defendpoint :post "/:id/run"
@@ -223,10 +249,15 @@
   (log/info "run transform" id)
   (api/check-superuser)
   (let [transform (api/check-404 (t2/select-one :model/Transform id))
+        _         (check-feature-enabled! transform)
         start-promise (promise)]
-    (u.jvm/in-virtual-thread*
-     (transforms.execute/run-mbql-transform! transform {:start-promise start-promise
-                                                        :run-method :manual}))
+    (if (transforms.util/python-transform? transform)
+      (u.jvm/in-virtual-thread*
+       (transforms-python.execute/execute-python-transform! transform {:start-promise start-promise
+                                                                       :run-method :manual}))
+      (u.jvm/in-virtual-thread*
+       (transforms.execute/run-mbql-transform! transform {:start-promise start-promise
+                                                          :run-method :manual})))
     (when (instance? Throwable @start-promise)
       (throw @start-promise))
     (let [result @start-promise

enterprise/backend/src/metabase_enterprise/transforms/api/transform_job.clj

@@ -8,8 +8,6 @@
    [metabase.api.common :as api]
    [metabase.api.macros :as api.macros]
    [metabase.api.routes.common :refer [+auth]]
-   [metabase.driver.common.parameters.dates :as params.dates]
-   [metabase.util.date-2 :as u.date]
    [metabase.util.i18n :refer [deferred-tru]]
    [metabase.util.jvm :as u.jvm]
    [metabase.util.log :as log]
@@ -143,23 +141,6 @@
     (assoc job :next_run {:start_time (str (transforms.util/->instant start-time))})
     job))
 
-(defn- matching-timestamp?
-  [job field-path {:keys [start end]}]
-  (when-let [field-instant (transforms.util/->instant (get-in job field-path))]
-    (let [start-instant (some-> start u.date/parse transforms.util/->instant)
-          end-instant (some-> end u.date/parse transforms.util/->instant)]
-      (and (or (nil? start)
-               (not (.isBefore field-instant start-instant)))
-           (or (nil? end)
-               (.isAfter end-instant field-instant))))))
-
-(defn- ->date-field-filter-xf
-  [field-path filter-value]
-  (let [range (some-> filter-value (params.dates/date-string->range {:inclusive-end? false}))]
-    (if range
-      (filter #(matching-timestamp? % field-path range))
-      identity)))
-
 (api.macros/defendpoint :get "/:job-id/transforms"
   "Get the transforms of job specified by the job's ID."
   [{:keys [job-id]} :- [:map
@@ -172,22 +153,21 @@
 (api.macros/defendpoint :get "/"
   "Get all transform jobs."
   [_route-params
-   {:keys [last_run_start_time next_run_start_time transform_tag_ids]} :-
+   {:keys [last_run_start_time next_run_start_time last_run_statuses tag_ids]} :-
    [:map
     [:last_run_start_time {:optional true} [:maybe ms/NonBlankString]]
     [:next_run_start_time {:optional true} [:maybe ms/NonBlankString]]
-    [:transform_tag_ids   {:optional true} [:maybe (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]]]]
+    [:last_run_statuses {:optional true} [:maybe (ms/QueryVectorOf [:enum "started" "succeeded" "failed" "timeout"])]]
+    [:tag_ids {:optional true} [:maybe (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]]]]
   (log/info "Getting all transform jobs")
   (api/check-superuser)
-  (let [jobs (t2/select :model/TransformJob {:order-by [[:created_at :desc]]})
-        transform-tag-ids (-> transform_tag_ids set not-empty)]
+  (let [jobs (t2/select :model/TransformJob {:order-by [[:created_at :desc]]})]
     (into []
           (comp (map add-next-run)
-                (->date-field-filter-xf [:last_run :start_time] last_run_start_time)
-                (->date-field-filter-xf [:next_run :start_time] next_run_start_time)
-                (if transform-tag-ids
-                  (filter #(some transform-tag-ids (:tag_ids %)))
-                  identity)
+                (transforms.util/->date-field-filter-xf [:last_run :start_time] last_run_start_time)
+                (transforms.util/->date-field-filter-xf [:next_run :start_time] next_run_start_time)
+                (transforms.util/->status-filter-xf [:last_run :status] last_run_statuses)
+                (transforms.util/->tag-filter-xf [:tag_ids] tag_ids)
                 (map #(update % :last_run transforms.util/localize-run-timestamps))
                 (map #(update % :next_run transforms.util/localize-run-timestamps)))
           (t2/hydrate jobs :tag_ids :last_run))))

enterprise/backend/src/metabase_enterprise/transforms/canceling.clj

@@ -49,7 +49,9 @@
    (chan-signal-cancel! run-id)
    (transform-run/timeout-run! run-id)))
 
-(defn- cancel-run! [run-id]
+(defn cancel-run!
+  "Cancel a run with id."
+  [run-id]
   (when (chan-signal-cancel! run-id)
     (transform-run/cancel-run! run-id)))
 

enterprise/backend/src/metabase_enterprise/transforms/execute.clj

@@ -1,5 +1,6 @@
 (ns metabase-enterprise.transforms.execute
   (:require
+   [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
    [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.driver :as driver]
    [metabase.driver.util :as driver.u]
@@ -54,7 +55,8 @@
          (transforms.util/run-cancelable-transform!
           run-id driver transform-details
           (fn [_cancel-chan] (driver/run-transform! driver transform-details opts)))
-         (transforms.util/sync-target! target database run-id)))
+         (transforms.instrumentation/with-stage-timing [run-id :table-sync]
+           (transforms.util/sync-target! target database run-id))))
      (catch Throwable t
        (log/error t "Error executing transform")
        (when start-promise

enterprise/backend/src/metabase_enterprise/transforms/instrumentation.clj

@@ -0,0 +1,193 @@
+(ns metabase-enterprise.transforms.instrumentation
+  (:require
+   [metabase.analytics.prometheus :as prometheus]
+   [metabase.util :as u]
+   [metabase.util.log :as log]
+   [metabase.util.malli :as mu]
+   [metabase.util.malli.registry :as mr]))
+
+(set! *warn-on-reflection* true)
+
+(mr/def ::stage-type
+  [:enum :export :computation :import])
+
+(def ^:private label-to-stage
+  {:dwh-to-file      :export
+   :file-to-s3       :export
+   :file-to-dwh      :import
+   :python-execution :computation
+   :mbql-query       :computation
+   :table-sync       :import})
+
+(mr/def ::stage-label
+  [:enum
+   :dwh-to-file
+   :file-to-s3
+   :file-to-dwh
+
+   :python-execution
+   :mbql-query
+   :table-sync])
+
+(mu/defn record-stage-start!
+  "Record the start of a transform stage."
+  [job-run-id :- pos-int?
+   stage-label :- ::stage-label]
+  (let [stage-type (label-to-stage stage-label)]
+    (log/infof "Transform stage started: run-id=%d type=%s label=%s" job-run-id (name stage-type) (name stage-label))
+    (prometheus/inc! :metabase-transforms/stage-started
+                     {:job-run-id (str job-run-id)
+                      :stage-type (name stage-type)
+                      :stage-label (name stage-label)})))
+
+(mu/defn record-stage-completion!
+  "Record the successful completion of a transform stage."
+  [job-run-id :- pos-int?
+   stage-label :- ::stage-label
+   duration-ms :- int?]
+  (let [stage-type (label-to-stage stage-label)]
+    (log/infof "Transform stage completed: run-id=%d type=%s label=%s duration=%dms" job-run-id (name stage-type) (name stage-label) duration-ms)
+    (let [labels {:job-run-id (str job-run-id)
+                  :stage-type (name stage-type)
+                  :stage-label (name stage-label)}]
+      (prometheus/inc! :metabase-transforms/stage-completed labels)
+      (prometheus/observe! :metabase-transforms/stage-duration-ms labels duration-ms))))
+
+(mu/defn record-stage-failure!
+  "Record the failure of a transform stage."
+  [job-run-id :- pos-int?
+   stage-label :- ::stage-label
+   duration-ms :- int?]
+  (let [stage-type (label-to-stage stage-label)]
+    (log/warnf "Transform stage failed: run-id=%d type=%s label=%s duration=%s" job-run-id (name stage-type) (name stage-label) (str duration-ms "ms"))
+    (let [labels {:job-run-id (str job-run-id)
+                  :stage-type (name stage-type)
+                  :stage-label (name stage-label)}]
+      (prometheus/inc! :metabase-transforms/stage-failed labels)
+      (when duration-ms
+        (prometheus/observe! :metabase-transforms/stage-duration-ms labels duration-ms)))))
+
+(defn with-timing
+  "Generic timing function that executes body while recording metrics.
+
+   Takes a map of optional functions:
+   - :start-fn - function to call when starting
+   - :success-fn - function to call on success (receives duration-ms as last arg)
+   - :error-fn - function to call on error (receives duration-ms as last arg)
+
+   Usage:
+   (with-timing {:start-fn #(record-stage-start! %1 %3)
+                 :success-fn #(record-stage-completion! %1 %2 %3)
+                 :error-fn #(record-stage-failure! %1 %2 %3)}
+                [job-run-id stage-label]
+                #(do-work)) "
+  [callbacks args body-fn]
+  (let [start-time (u/start-timer)]
+    (when-let [start-fn (:start-fn callbacks)]
+      (apply start-fn args))
+    (try
+      (let [result (body-fn)]
+        (when-let [success-fn (:success-fn callbacks)]
+          (apply success-fn (conj (vec args) (long (u/since-ms start-time)))))
+        result)
+      (catch Throwable t
+        (when-let [error-fn (:error-fn callbacks)]
+          (apply error-fn (conj (vec args) (long (u/since-ms start-time)))))
+        (throw t)))))
+
+(defmacro with-stage-timing
+  "Execute body while timing a transform stage. Automatically records start, completion or failure.
+
+   Usage:
+   (with-stage-timing [job-run-id :dwh-to-file]
+     ;; stage implementation
+     result)"
+  [args & body]
+  `(with-timing {:start-fn record-stage-start!
+                 :success-fn record-stage-completion!
+                 :error-fn record-stage-failure!}
+     ~args
+     (^:once fn* [] ~@body)))
+
+(mu/defn record-data-transfer!
+  "Record metrics about data transfer (size and rows)."
+  [job-run-id :- pos-int?
+   stage-label :- ::stage-label
+   bytes :- [:maybe int?]
+   rows :- [:maybe int?]]
+  (if (and (nil? bytes) (nil? rows))
+    (log/warnf "Data transfer recorded: run-id=%d stage=%s but no bytes or rows provided"
+               job-run-id
+               (name stage-label))
+    (log/infof "Data transfer recorded: run-id=%d stage=%s%s%s"
+               job-run-id
+               (name stage-label)
+               (if bytes (str " bytes=" bytes) "")
+               (if rows (str " rows=" rows) "")))
+  (let [labels {:job-run-id (str job-run-id)
+                :stage-label (name stage-label)}]
+    (when bytes
+      (prometheus/observe! :metabase-transforms/data-transfer-bytes labels bytes))
+    (when rows
+      (prometheus/observe! :metabase-transforms/data-transfer-rows labels rows))))
+
+(defn record-job-start!
+  "Record the start of a transform job run."
+  [job-id run-method]
+  (log/infof "Transform job started: job-id=%d run-method=%s" job-id (name run-method))
+  (prometheus/inc! :metabase-transforms/job-runs-total
+                   {:job-id (str job-id)
+                    :run-method (name run-method)}))
+
+(defn record-job-completion!
+  "Record the successful completion of a transform job run."
+  [job-id run-method duration-ms]
+  (log/infof "Transform job completed: job-id=%d run-method=%s duration=%dms" job-id (name run-method) duration-ms)
+  (let [labels {:job-id (str job-id)
+                :run-method (name run-method)}]
+    (prometheus/inc! :metabase-transforms/job-runs-completed labels)
+    (prometheus/observe! :metabase-transforms/job-run-duration-ms labels duration-ms)))
+
+(defn record-job-failure!
+  "Record the failure of a transform job run."
+  [job-id run-method duration-ms]
+  (log/warnf "Transform job failed: job-id=%d run-method=%s duration=%dms" job-id (name run-method) duration-ms)
+  (let [labels {:job-id (str job-id)
+                :run-method (name run-method)}]
+    (prometheus/inc! :metabase-transforms/job-runs-failed labels)
+    (prometheus/observe! :metabase-transforms/job-run-duration-ms labels duration-ms)))
+
+(defmacro with-job-timing
+  "Execute body while timing a transform job run. Automatically records start, completion or failure.
+
+   Usage:
+   (with-job-timing [job-id run-method]
+     ;; job implementation
+     result)"
+  [[job-id run-method] & body]
+  `(with-timing {:start-fn record-job-start!
+                 :success-fn record-job-completion!
+                 :error-fn record-job-failure!}
+     [~job-id ~run-method]
+     (^:once fn* [] ~@body)))
+
+(mu/defn record-python-api-call!
+  "Record metrics about Python API calls."
+  [job-run-id :- pos-int?
+   duration-ms :- int?
+   status :- [:enum :success :error :timeout]]
+  (log/infof "Python API call %s: run-id=%d duration=%dms" (name status) job-run-id duration-ms)
+  (let [labels {:job-run-id (str job-run-id)}]
+    (prometheus/inc! :metabase-transforms/python-api-calls-total
+                     (assoc labels :status (name status)))
+    (prometheus/observe! :metabase-transforms/python-api-call-duration-ms labels duration-ms)))
+
+(defmacro with-python-api-timing
+  "Execute body while timing a Python API call."
+  [[job-run-id] & body]
+  `(with-timing {:success-fn (fn [job-run-id# duration-ms#]
+                               (record-python-api-call! job-run-id# duration-ms# :success))
+                 :error-fn (fn [job-run-id# duration-ms#]
+                             (record-python-api-call! job-run-id# duration-ms# :error))}
+     [~job-run-id]
+     (^:once fn* [] ~@body)))

enterprise/backend/src/metabase_enterprise/transforms/jobs.clj

@@ -5,11 +5,14 @@
    [clojurewerkz.quartzite.schedule.calendar-interval :as calendar-interval]
    [clojurewerkz.quartzite.triggers :as triggers]
    [flatland.ordered.set :as ordered-set]
+   [metabase-enterprise.transforms-python.execute :as transforms-python.execute]
    [metabase-enterprise.transforms.execute :as transforms.execute]
+   [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
    [metabase-enterprise.transforms.models.job-run :as transforms.job-run]
    [metabase-enterprise.transforms.models.transform-run :as transform-run]
    [metabase-enterprise.transforms.ordering :as transforms.ordering]
    [metabase-enterprise.transforms.settings :as transforms.settings]
+   [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.task.core :as task]
    [metabase.util.log :as log]
    [toucan2.core :as t2]))
@@ -64,18 +67,22 @@
         (map transforms-by-id complete)))))
 
 (defn- run-transform! [run-id run-method {transform-id :id :as transform}]
-  (when
-   (transform-run/running-run-for-transform-id transform-id)
-    (log/warn "Transform" (pr-str transform-id) "already running, waiting")
-    (loop []
-      (Thread/sleep 2000)
+  (if-not (transforms.util/check-feature-enabled transform)
+    (log/warnf "Skip running transform %d due to lacking premium features" transform-id)
+    (do
       (when (transform-run/running-run-for-transform-id transform-id)
-        (recur))))
-
-  (log/info "Executing job transform" (pr-str transform-id))
-  (transforms.execute/run-mbql-transform! transform {:run-method run-method})
-
-  (transforms.job-run/add-run-activity! run-id))
+        (log/warn "Transform" (pr-str transform-id) "already running, waiting")
+        (loop []
+          (Thread/sleep 2000)
+          (when (transform-run/running-run-for-transform-id transform-id)
+            (recur))))
+      (log/info "Executing job transform" (pr-str transform-id))
+      (if (transforms.util/python-transform? transform)
+        (transforms.instrumentation/with-stage-timing [run-id :python-execution]
+          (transforms-python.execute/execute-python-transform! transform {:run-method run-method}))
+        (transforms.instrumentation/with-stage-timing [run-id :mbql-query]
+          (transforms.execute/run-mbql-transform! transform {:run-method run-method})))
+      (transforms.job-run/add-run-activity! run-id))))
 
 (defn run-transforms!
   "Run a series of transforms and their dependencies.
@@ -113,12 +120,13 @@
     (let [transforms (job-transform-ids job-id)]
       (log/info "Executing transform job" (pr-str job-id) "with transforms" (pr-str transforms))
       (let [{run-id :id} (transforms.job-run/start-run! job-id run-method)]
-        (try
-          (run-transforms! run-id transforms opts)
-          (transforms.job-run/succeed-started-run! run-id)
-          (catch Throwable t
-            (transforms.job-run/fail-started-run! run-id {:message (.getMessage t)})
-            (throw t)))))))
+        (transforms.instrumentation/with-job-timing [job-id run-method]
+          (try
+            (run-transforms! run-id transforms opts)
+            (transforms.job-run/succeed-started-run! run-id)
+            (catch Throwable t
+              (transforms.job-run/fail-started-run! run-id {:message (.getMessage t)})
+              (throw t))))))))
 
 (def ^:private job-key "metabase-enterprise.transforms.jobs.timeout-job")
 

enterprise/backend/src/metabase_enterprise/transforms/models/transform_run.clj

@@ -228,7 +228,7 @@
         query-options    (merge {:order-by order-by :offset offset :limit limit}
                                 count-options)
         runs             (t2/select :model/TransformRun query-options)]
-    {:data   (t2/hydrate runs :transform)
+    {:data   (t2/hydrate runs [:transform :transform_tag_ids])
      :limit  limit
      :offset offset
      :total  (t2/count :model/TransformRun count-options)}))

enterprise/backend/src/metabase_enterprise/transforms/ordering.clj

@@ -16,23 +16,29 @@
 (set! *warn-on-reflection* true)
 
 (defn- transform-deps [transform]
-  (let [query (-> (get-in transform [:source :query])
-                  transforms.util/massage-sql-query
-                  qp.preprocess/preprocess
-                  ;; legacy usage -- don't do things like this going forward
-                  #_{:clj-kondo/ignore [:discouraged-var]}
-                  lib/->legacy-MBQL)]
-    (case (:type query)
-      :native (driver/native-query-deps (-> (qp.store/metadata-provider)
-                                            lib.metadata/database
-                                            :engine)
-                                        (get-in query [:native :query]))
-      :query (into #{}
-                   (keep #(clojure.core.match/match %
-                            [:source-table source] (when (int? source)
-                                                     {:table source})
-                            _ nil))
-                   (tree-seq coll? seq query)))))
+  (if (transforms.util/python-transform? transform)
+    ;; Python transforms have dependencies via source-tables mapping
+    (into #{}
+          (map #(hash-map :table %))
+          (vals (get-in transform [:source :source-tables])))
+    ;; Query transforms have dependencies via SQL query analysis
+    (let [query (-> (get-in transform [:source :query])
+                    transforms.util/massage-sql-query
+                    qp.preprocess/preprocess
+                    ;; legacy usage -- don't do things like this going forward
+                    #_{:clj-kondo/ignore [:discouraged-var]}
+                    lib/->legacy-MBQL)]
+      (case (:type query)
+        :native (driver/native-query-deps (-> (qp.store/metadata-provider)
+                                              lib.metadata/database
+                                              :engine)
+                                          (get-in query [:native :query]))
+        :query (into #{}
+                     (keep #(clojure.core.match/match %
+                              [:source-table source] (when (int? source)
+                                                       {:table source})
+                              _ nil))
+                     (tree-seq coll? seq query))))))
 
 (defn- dependency-map [transforms]
   (into {}
@@ -56,12 +62,15 @@
 
   The result is a map of transform id -> #{transform ids the transform depends on}. Dependencies are limited to just
   the transforms in the original list -- if a transform depends on some transform not in the list, the 'extra'
-  dependency is ignored."
+  dependency is ignored. Both query and Python transforms can have dependencies on tables produced by other transforms."
   [transforms]
-  (let [transforms-by-db (->> transforms
+  (let [;; Group all transforms by their database
+        transforms-by-db (->> transforms
                               (map (fn [transform]
-                                     {(get-in transform [:source :query :database]) [transform]}))
+                                     (let [db-id (transforms.util/target-database-id transform)]
+                                       {db-id [transform]})))
                               (apply merge-with into))
+
         transform-ids (into #{} (map :id) transforms)
         {:keys [output-tables dependencies]} (->> transforms-by-db
                                                   (map (fn [[db-id db-transforms]]

enterprise/backend/src/metabase_enterprise/transforms/util.clj

@@ -7,12 +7,17 @@
    [metabase-enterprise.transforms.models.transform-run :as transform-run]
    [metabase-enterprise.transforms.settings :as transforms.settings]
    [metabase.driver :as driver]
+   [metabase.driver.common.parameters.dates :as params.dates]
+   [metabase.lib.schema.common :as lib.schema.common]
+   [metabase.premium-features.core :as premium-features :refer [defenterprise]]
    [metabase.query-processor.compile :as qp.compile]
    [metabase.query-processor.pipeline :as qp.pipeline]
    [metabase.sync.core :as sync]
    [metabase.util :as u]
    [metabase.util.date-2 :as u.date]
    [metabase.util.log :as log]
+   [metabase.util.malli :as mu]
+   [metabase.util.malli.registry :as mr]
    [toucan2.core :as t2])
   (:import
    (java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)
@@ -20,6 +25,10 @@
 
 (set! *warn-on-reflection* true)
 
+(def ^:const transform-temp-table-prefix
+  "Prefix used for temporary tables created during transform execution."
+  "mb_transform_temp_table")
+
 (defn qualified-table-name
   "Return the name of the target table of a transform as a possibly qualified symbol."
   [_driver {:keys [schema name]}]
@@ -27,6 +36,24 @@
     (keyword schema name)
     (keyword name)))
 
+(defn query-transform?
+  "Check if this is a query transform: native query / mbql query."
+  [transform]
+  (= :query (-> transform :source :type keyword)))
+
+(defn python-transform?
+  "Check if this is a Python transform."
+  [transform]
+  (= :python (-> transform :source :type keyword)))
+
+(defn check-feature-enabled
+  "Checking whether we have proper feature flags for using a given transform."
+  [transform]
+  (if (python-transform? transform)
+    (and (premium-features/has-feature? :transforms)
+         (premium-features/has-feature? :transforms-python))
+    (premium-features/has-feature? :transforms)))
+
 (defn try-start-unless-already-running
   "Start a transform run, throwing an informative error if already running."
   [id run-method]
@@ -41,8 +68,11 @@
         (throw e)))))
 
 (defn run-cancelable-transform!
-  "Execute a transform with cancellation support and proper error handling."
-  [run-id driver {:keys [db-id conn-spec output-schema]} run-transform!]
+  "Execute a transform with cancellation support and proper error handling.
+
+  Options:
+  - `:ex-message-fn` change how caught exceptions are presented to the user in run logs, by default the same as clojure.core/ex-message"
+  [run-id driver {:keys [db-id conn-spec output-schema]} run-transform! & {:keys [ex-message-fn] :or {ex-message-fn ex-message}}]
   ;; local run is responsible for status, using canceling lifecycle
   (try
     (when-not (driver/schema-exists? driver db-id output-schema)
@@ -55,9 +85,7 @@
       (transform-run/succeed-started-run! run-id)
       ret)
     (catch Throwable t
-      (let [{:keys [transform-run-message]} (ex-data t)
-            message (str/join "\n" (remove str/blank? [transform-run-message (.getMessage t)]))]
-        (transform-run/fail-started-run! run-id {:message message}))
+      (transform-run/fail-started-run! run-id {:message (ex-message-fn t)})
       (throw t))
     (finally
       (canceling/chan-end-run! run-id))))
@@ -76,10 +104,18 @@
    (log/info "Syncing target" (pr-str target) "for transform")
    (activate-table-and-mark-computed! database target)))
 
+;; TODO this and target-database-id can be transforms multimethods?
+(defn target-database-id
+  "Return the target database id of a transform"
+  [transform]
+  (if (python-transform? transform)
+    (-> transform :target :database)
+    (-> transform :source :query :database)))
+
 (defn target-table-exists?
   "Test if the target table of a transform already exists."
-  [{:keys [source target] :as _transform}]
-  (let [db-id (-> source :query :database)
+  [{:keys [target] :as transform}]
+  (let [db-id (target-database-id transform)
         {driver :engine :as database} (t2/select-one :model/Database db-id)]
     (driver/table-exists? driver database target)))
 
@@ -118,10 +154,10 @@
 
 (defn delete-target-table!
   "Delete the target table of a transform and sync it from the app db."
-  [{:keys [id target source], :as _transform}]
+  [{:keys [id target], :as transform}]
   (when target
     (let [target (update target :type keyword)
-          database-id (-> source :query :database)
+          database-id (target-database-id transform)
           {driver :engine :as database} (t2/select-one :model/Database database-id)]
       (driver/drop-transform-target! driver database target)
       (log/info "Deactivating  target " (pr-str target) "for transform" id)
@@ -146,8 +182,10 @@
 (defn required-database-feature
   "Returns the database feature necessary to execute `transform`."
   [transform]
-  (case (-> transform :target :type)
-    "table"             :transforms/table))
+  (if (python-transform? transform)
+    :transforms/python
+    (case (-> transform :target :type)
+      "table"             :transforms/table)))
 
 (defn ->instant
   "Convert a temporal value `t` to an Instant in the system timezone."
@@ -178,9 +216,110 @@
       (u/update-some :start_time utc-timestamp-string)
       (u/update-some :end_time   utc-timestamp-string)))
 
+(mr/def ::column-definition
+  [:map
+   [:name :string]
+   [:type ::lib.schema.common/base-type]
+   [:nullable? {:optional true} :boolean]])
+
+(mr/def ::table-definition
+  [:map
+   [:name :keyword]
+   [:columns [:sequential ::column-definition]]
+   [:primary-key {:optional true} [:sequential :string]]])
+
+(mu/defn create-table-from-schema!
+  "Create a table from a table-schema"
+  [driver :- :keyword
+   database-id :- pos-int?
+   table-schema :- ::table-definition]
+  (let [{:keys [columns] table-name :name} table-schema
+        column-definitions (into {} (map (fn [{:keys [name type database-type]}]
+                                           (let [db-type (if database-type
+                                                           [[:raw database-type]]
+                                                           (try
+                                                             (driver/type->database-type driver type)
+                                                             (catch IllegalArgumentException _
+                                                               (log/warnf "Couldn't determine database type for type %s, fallback to Text" type)
+                                                               (driver/type->database-type driver :type/Text))))]
+                                             [name db-type])))
+                                 columns)
+        primary-key-opts (select-keys table-schema [:primary-key])]
+    (log/infof "Creating table %s with %d columns" table-name (count columns))
+    (driver/create-table! driver database-id table-name column-definitions primary-key-opts)))
+
+(defn drop-table!
+  "Drop a table in the database."
+  [driver database-id table-name]
+  (log/infof "Dropping table %s" table-name)
+  (driver/drop-table! driver database-id table-name))
+
+(defn temp-table-name
+  "Generate a temporary table name with current timestamp in milliseconds.
+  If table name would exceed max table name length for the driver, fallback to using a shorter timestamp"
+  [driver schema]
+  (let [max-len   (max 1 (or (driver/table-name-length-limit driver) Integer/MAX_VALUE))
+        timestamp (str (System/currentTimeMillis))
+        prefix    (str transform-temp-table-prefix "_")
+        available (- max-len (count prefix))
+        ;; If we don't have enough space, take the later digits of the timestamp
+        suffix    (if (>= available (count timestamp))
+                    timestamp
+                    (subs timestamp (- (count timestamp) available)))
+        table-name (str prefix suffix)]
+    (keyword schema table-name)))
+
+(defn rename-tables!
+  "Rename multiple tables atomically within a transaction using the new driver/rename-tables method.
+   This is a simpler, composable operation that only handles renaming."
+  [driver database-id rename-map]
+  (log/infof "Renaming tables: %s" (pr-str rename-map))
+  (driver/rename-tables! driver database-id rename-map))
+
+(defenterprise is-temp-transform-table?
+  "Return true when `table` matches the transform temporary table naming pattern and transforms are enabled."
+  :feature :transforms
+  [table]
+  (when-let [table-name (:name table)]
+    (str/starts-with? (u/lower-case-en table-name) transform-temp-table-prefix)))
+
 (defn db-routing-enabled?
   "Returns whether or not the given database is either a router or destination database"
   [db-or-id]
   (or (t2/exists? :model/DatabaseRouter :database_id (u/the-id db-or-id))
       (some->> (:router-database-id db-or-id)
                (t2/exists? :model/DatabaseRouter :database_id))))
+
+(defn- matching-timestamp?
+  [job field-path {:keys [start end]}]
+  (when-let [field-instant (->instant (get-in job field-path))]
+    (let [start-instant (some-> start u.date/parse ->instant)
+          end-instant (some-> end u.date/parse ->instant)]
+      (and (or (nil? start)
+               (not (.isBefore field-instant start-instant)))
+           (or (nil? end)
+               (.isAfter end-instant field-instant))))))
+
+(defn ->date-field-filter-xf
+  "Returns an xform for a date filter."
+  [field-path filter-value]
+  (let [range (some-> filter-value (params.dates/date-string->range {:inclusive-end? false}))]
+    (if range
+      (filter #(matching-timestamp? % field-path range))
+      identity)))
+
+(defn ->status-filter-xf
+  "Returns an xform for a transform run status filter."
+  [field-path statuses]
+  (let [statuses (->> statuses (map keyword) set not-empty)]
+    (if statuses
+      (filter #(statuses (get-in % field-path)))
+      identity)))
+
+(defn ->tag-filter-xf
+  "Returns an xform for a transform tag filter."
+  [field-path tag-ids]
+  (let [tag-ids (-> tag-ids set not-empty)]
+    (if tag-ids
+      (filter #(some tag-ids (get-in % field-path)))
+      identity)))

enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj

@@ -0,0 +1,335 @@
+(ns metabase-enterprise.transforms-python.execute
+  (:require
+   [clojure.core.async :as a]
+   [clojure.java.io :as io]
+   [clojure.string :as str]
+   [metabase-enterprise.transforms-python.python-runner :as python-runner]
+   [metabase-enterprise.transforms-python.s3 :as s3]
+   [metabase-enterprise.transforms-python.settings :as transforms-python.settings]
+   [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
+   [metabase-enterprise.transforms.util :as transforms.util]
+   [metabase.driver :as driver]
+   [metabase.driver.util :as driver.u]
+   [metabase.util :as u]
+   [metabase.util.format :as u.format]
+   [metabase.util.i18n :as i18n]
+   [metabase.util.jvm :as u.jvm]
+   [metabase.util.log :as log]
+   [toucan2.core :as t2])
+  (:import
+   (java.io Closeable File)
+   (java.net SocketException)
+   (java.time Duration)))
+
+(set! *warn-on-reflection* true)
+
+(defn- empty-message-log
+  "Returns a new message log.
+
+  Python transforms produce output while they run. The message log is a stateful structure that buffers log outputs during the run.
+  The goal is to assist in debugging in the case of python runtime or syntax errors, and to provide some immediate feedback for longer running transforms.
+
+  Important Note: in memory buffering is considered acceptable as we expect /logs output to be truncated or small.
+
+  The intended use of this structure is to produce a string suitable for `transform_run.message`, saving it every few seconds."
+  []
+  (atom {:pre-python  []                                    ; log! outputs previous to the python execution, i.e table read progress
+         :python      nil                                   ; events json structured logs from the /logs endpoint
+         :post-python []}))                                 ; log! outputs after the python execution, i.e. output reads, writes to target
+
+(defn- log!
+  "Appends a string to the message log, the string is user facing and should be suitable for presentation as part of the `transform_run.message` field."
+  [message-log s]
+  (swap! message-log (fn [m] (update m (if (:python m) :post-python :pre-python) conj s)))
+  nil)
+
+(defn- replace-python-logs!
+  "Called when events output is received from a /logs call to copy those events into the message-log."
+  [message-log events]
+  (swap! message-log assoc :python events)
+  nil)
+
+(defn- message-log->transform-run-message
+  "Presents the content of the log as a string suitable for storage in transform_run.message."
+  [message-log]
+  (let [{:keys [pre-python python post-python]} @message-log]
+    (str/join "\n" (concat pre-python
+                           (map :message python)
+                           post-python))))
+
+(defn- save-log-to-transform-run-message!
+  "Saves the content of the log into the transform_run's message field. Lossy, string contains user facing information only."
+  [run-id message-log]
+  (t2/update! :model/TransformRun
+              :id run-id
+              {:message (message-log->transform-run-message message-log)}))
+
+(def ^:private ^Duration python-message-loop-sleep-duration
+  (Duration/ofMillis 1000))
+
+(defn- python-message-update-loop!
+  "Block while relevant log data is replicated from the runner into the message log.
+  When new logs are received, the log data will be flushed to the transform_run.message field as a string.
+
+  The loop will exit on error or when interrupted."
+  [run-id message-log]
+  (try
+    (loop []
+      (if (.isInterrupted (Thread/currentThread))
+        (log/debug "Message update loop interrupted")
+        (do (let [sleep-ms (.toMillis python-message-loop-sleep-duration)]
+              (when (pos? sleep-ms) (Thread/sleep sleep-ms)))
+            (let [{:keys [status body]} (python-runner/get-logs run-id)]
+              (cond
+                (<= 200 status 299)
+                (let [{:keys [execution_id events]} body]
+                  (if-not (= run-id execution_id)
+                    (do (log/debugf "Run id did not match expected: %s actual: %s" run-id execution_id)
+                        (recur))
+                    (do
+                      (replace-python-logs! message-log events)
+                      (save-log-to-transform-run-message! run-id message-log)
+                      (recur))))
+                (= 404 status)
+                (do
+                  (log/debugf "No logs yet (or run finished), run-id: %s" run-id)
+                  (recur))
+                :else
+                (do
+                  (log/warnf "Unexpected status polling for logs %s %s, run-id: %s" status body run-id)
+                  (log/debug "Exiting due to poll error")))))))
+    (catch SocketException se (when-not (= "Closed by interrupt" (ex-message se)) (throw se)))
+    (catch InterruptedException _)
+    (catch Throwable e
+      (log/errorf e "An exception was caught during msg update loop, run-id: %s" run-id))))
+
+(defn- open-python-message-update-future! ^Closeable [run-id message-log]
+  (let [cleanup (fn [fut]
+                  (future-cancel fut)
+                  (if (= ::timeout (try (deref fut 10000 ::timeout) (catch Throwable _)))
+                    (log/fatalf "Log polling task did not respond to interrupt, run-id: %s" run-id)
+                    (log/debugf "Log polling task done, run-id: %s" run-id)))
+        fut     (u.jvm/in-virtual-thread*
+                 (python-message-update-loop! run-id message-log))]
+    (reify Closeable
+      (close [_] (cleanup fut)))))
+
+(defn- maybe-retry-with-backoff
+  [driver thunk]
+  ;; bigquery can have eventual consistency hiccups where the table is not yet available for insertion
+  ;; immediately after creation, so we retry a few times with backoff
+  (if (= driver :bigquery-cloud-sdk)
+    (loop [attempt 0]
+      (let [ret (try
+                  (thunk)
+                  (catch Exception e
+                    (if (and (< attempt 5)
+                             (or (str/includes? (str e) "not found")
+                                 (str/includes? (str e) "Not found")))
+                      (let [delay-ms (* 150 (Math/pow 2 attempt))]
+                        (log/debugf "BigQuery operation failed (attempt %d), retrying in %dms" (inc attempt) delay-ms)
+                        (Thread/sleep (long delay-ms))
+                        ::recur)
+                      (throw e))))]
+        (if (= ::recur ret)
+          (recur (inc attempt))
+          ret)))
+    (thunk)))
+
+(defn- create-table-and-insert-data!
+  "Create a table from metadata and insert data from source."
+  [driver db-id table-name metadata data-source]
+  (let [table-schema {:name (if (keyword? table-name) table-name (keyword table-name))
+                      :columns (mapv (fn [{:keys [name base_type #_database_type]}]
+                                       {:name name
+                                        :type (python-runner/restricted-insert-type base_type)
+                                        ;; :database-type database_type
+                                        :nullable? true})
+                                     (:fields metadata))}
+        data-source (assoc data-source :table-schema table-schema)]
+    (transforms.util/create-table-from-schema! driver db-id table-schema)
+
+    (->> #(driver/insert-from-source! driver db-id table-schema data-source)
+         (maybe-retry-with-backoff driver))))
+
+(defn- transfer-with-rename-tables-strategy!
+  "Transfer data using the rename-tables*! multimethod with atomicity guarantees.
+   Creates new table, then atomically renames target->old and new->target, then drops old."
+  [driver db-id table-name metadata data-source]
+  (let [source-table-name (transforms.util/temp-table-name driver (namespace table-name))
+        temp-table-name (u/poll {:thunk #(transforms.util/temp-table-name driver (namespace table-name))
+                                 :done? #(not= source-table-name %)
+                                 ;; Poll every 1ms to quickly generate a different timestamp-based table name
+                                 :interval-ms 1})]
+    (log/info "Using rename-tables strategy with atomicity guarantees")
+    (try
+
+      (create-table-and-insert-data! driver db-id source-table-name metadata data-source)
+      (transforms.util/rename-tables! driver db-id {table-name temp-table-name
+                                                    source-table-name table-name})
+      (transforms.util/drop-table! driver db-id temp-table-name)
+
+      (catch Exception e
+        (log/error e "Failed to transfer data using rename-tables strategy")
+        (try
+          (transforms.util/drop-table! driver db-id source-table-name)
+          (catch Exception _))
+        (throw e)))))
+
+(defn- transfer-with-create-drop-rename-strategy!
+  "Transfer data using create + drop + rename to minimize time without data.
+   Creates new table, drops old table, then renames new->target."
+  [driver db-id table-name metadata data-source]
+  (let [source-table-name (transforms.util/temp-table-name driver (namespace table-name))]
+    (log/info "Using create-drop-rename strategy to minimize downtime")
+    (try
+
+      (create-table-and-insert-data! driver db-id source-table-name metadata data-source)
+      (transforms.util/drop-table! driver db-id table-name)
+      (driver/rename-table! driver db-id source-table-name table-name)
+
+      (catch Exception e
+        (log/error e "Failed to transfer data using create-drop-rename strategy")
+        (try
+          (transforms.util/drop-table! driver db-id source-table-name)
+          (catch Exception _))
+        (throw e)))))
+
+(defn- transfer-with-drop-create-fallback-strategy!
+  "Transfer data using drop + create fallback strategy.
+   Drops old table, then creates new table directly with target name."
+  [driver db-id table-name metadata data-source]
+  (log/info "Using drop-create fallback strategy")
+  (try
+
+    (transforms.util/drop-table! driver db-id table-name)
+    (create-table-and-insert-data! driver db-id table-name metadata data-source)
+
+    (catch Exception e
+      (log/error e "Failed to transfer data using drop-create fallback strategy")
+      (throw e))))
+
+(defn- transfer-file-to-db [driver {db-id :id :as db} {:keys [target] :as transform} metadata temp-file]
+  (let [table-name (transforms.util/qualified-table-name driver target)
+        table-exists? (transforms.util/target-table-exists? transform)
+        data-source {:type :jsonl-file
+                     :file temp-file}]
+    (cond
+      (not table-exists?)
+      (do
+        (log/info "New table")
+        (create-table-and-insert-data! driver db-id table-name metadata data-source))
+
+      (driver.u/supports? driver :atomic-renames db)
+      (transfer-with-rename-tables-strategy! driver db-id table-name metadata data-source)
+
+      (driver.u/supports? driver :rename db)
+      (transfer-with-create-drop-rename-strategy! driver db-id table-name metadata data-source)
+
+      :else
+      (transfer-with-drop-create-fallback-strategy! driver db-id table-name metadata data-source))))
+
+(defn- start-cancellation-process!
+  "Starts a core.async process that optimistically sends a cancellation request to the python executor if cancel-chan receives a value.
+  Returns a channel that will receive either the async http call j.u.c.FutureTask in the case of cancellation, or nil when the cancel-chan is closed."
+  [server-url run-id cancel-chan]
+  (a/go (when (a/<! cancel-chan)
+          (python-runner/cancel-python-code-http-call! server-url run-id))))
+
+(defn- run-python-transform! [{:keys [source] :as transform} db run-id cancel-chan message-log]
+  ;; TODO restructure things such that s3 can we swapped out for other transfer mechanisms
+  (with-open [log-future-ref     (open-python-message-update-future! run-id message-log)
+              shared-storage-ref (s3/open-shared-storage! (:source-tables source))]
+    (let [driver     (:engine db)
+          server-url (transforms-python.settings/python-runner-url)
+          _          (python-runner/copy-tables-to-s3! {:run-id         run-id
+                                                        :shared-storage @shared-storage-ref
+                                                        :table-name->id (:source-tables source)
+                                                        :cancel-chan    cancel-chan})
+          _          (start-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan
+          {:keys [status body] :as response}
+          (python-runner/execute-python-code-http-call!
+           {:server-url     server-url
+            :code           (:body source)
+            :run-id         run-id
+            :table-name->id (:source-tables source)
+            :shared-storage @shared-storage-ref})
+          ;; TODO temporary to keep more code stable while refactoring
+          ;; no need to materialize these early (i.e output we can stream directly into a tmp file or db if small)
+          {:keys [output output-manifest events]} (python-runner/read-output-objects @shared-storage-ref)]
+      (.close log-future-ref)                               ; early close to force any writes to flush
+      (when (seq events)
+        (replace-python-logs! message-log events))
+      (if (not= 200 status)
+        (throw (ex-info "Python runner call failed"
+                        {:transform-message (i18n/tru "Python execution failure (exit code {0})" (:exit_code body "?"))
+                         :status-code       400
+                         :api-status-code   status
+                         :body              body
+                         :events            events}))
+        (try
+          (let [temp-file (File/createTempFile "transform-output-" ".jsonl")]
+            (when-not (seq (:fields output-manifest))
+              (throw (ex-info "No fields in metadata"
+                              {:metadata               output-manifest
+                               :raw-body               body
+                               :events                 events})))
+            (try
+              (with-open [writer (io/writer temp-file)]
+                (.write writer ^String output))
+              (let [file-size (.length temp-file)]
+                (transforms.instrumentation/with-stage-timing [run-id :file-to-dwh]
+                  (transfer-file-to-db driver db transform output-manifest temp-file))
+                (transforms.instrumentation/record-data-transfer! run-id :file-to-dwh file-size nil))
+              (finally
+                (.delete temp-file))))
+          response
+          (catch Exception e
+            (log/error e "Failed to to create resulting table")
+            (throw (ex-info "Failed to create the resulting table"
+                            {:transform-message (or (:transform-message (ex-data e))
+                                                    ;; TODO keeping messaging the same at this level
+                                                    ;;  should be more specific in underlying calls
+                                                    (i18n/tru "Failed to create the resulting table"))}
+                            e))))))))
+
+(defn- exceptional-run-message [message-log ex]
+  (str/join "\n" (remove str/blank? [(message-log->transform-run-message message-log)
+                                     (or (:transform-message (ex-data ex))
+                                         (if (instance? InterruptedException ex)
+                                           (i18n/tru "Transform interrupted")
+                                           (i18n/tru "Something went wrong")))])))
+
+(defn execute-python-transform!
+  "Execute a Python transform by calling the python runner.
+
+  Blocks until the transform returns."
+  [transform {:keys [run-method start-promise]}]
+  (assert (transforms.util/python-transform? transform) "Transform must be a python transform")
+  (try
+    (let [message-log (empty-message-log)
+          {:keys [target] transform-id :id} transform
+          {driver :engine :as db} (t2/select-one :model/Database (:database target))
+          {run-id :id} (transforms.util/try-start-unless-already-running transform-id run-method)]
+      (some-> start-promise (deliver [:started run-id]))
+      (log! message-log (i18n/tru "Executing Python transform"))
+      (log/info "Executing Python transform" transform-id "with target" (pr-str target))
+      (let [start-ms          (u/start-timer)
+            transform-details {:db-id          (:id db)
+                               :transform-type (keyword (:type target))
+                               :conn-spec      (driver/connection-spec driver db)
+                               :output-schema  (:schema target)
+                               :output-table   (transforms.util/qualified-table-name driver target)}
+            run-fn            (fn [cancel-chan]
+                                (run-python-transform! transform db run-id cancel-chan message-log)
+                                (log! message-log (i18n/tru "Python execution finished successfully in {0}" (u.format/format-milliseconds (u/since-ms start-ms))))
+                                (save-log-to-transform-run-message! run-id message-log))
+            ex-message-fn     #(exceptional-run-message message-log %)
+            result            (transforms.util/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn)]
+        (transforms.instrumentation/with-stage-timing [run-id :table-sync]
+          (transforms.util/sync-target! target db run-id))
+        {:run_id run-id
+         :result result}))
+    (catch Throwable t
+      (log/error t "Error executing Python transform")
+      (throw t))))

enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj

@@ -0,0 +1,291 @@
+(ns metabase-enterprise.transforms-python.python-runner
+  (:require
+   [clj-http.client :as http]
+   [clojure.java.io :as io]
+   [clojure.string :as str]
+   [medley.core :as m]
+   [metabase-enterprise.transforms-python.s3 :as s3]
+   [metabase-enterprise.transforms-python.settings :as transforms-python.settings]
+   [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
+   [metabase.config.core :as config]
+   [metabase.driver :as driver]
+   [metabase.query-processor.compile :as qp.compile]
+   [metabase.query-processor.pipeline :as qp.pipeline]
+   ;; TODO check that querying team are ok with us accessing this directly, otherwise make another plan
+   ^{:clj-kondo/ignore [:discouraged-namespace]}
+   [metabase.query-processor.store :as qp.store]
+   [metabase.util.i18n :as i18n]
+   [metabase.util.json :as json]
+   [metabase.util.log :as log]
+   [toucan2.core :as t2])
+  (:import
+   (clojure.lang PersistentQueue)
+   (java.io BufferedWriter File OutputStream OutputStreamWriter)
+   (java.nio.charset StandardCharsets)))
+
+(set! *warn-on-reflection* true)
+
+(defn- authorization-headers
+  "Returns HTTP headers with Authorization bearer token if configured.
+  Throws configuration error in production if token is not set."
+  []
+  (let [api-token (transforms-python.settings/python-runner-api-token)]
+    (if api-token
+      {"Authorization" (str "Bearer " api-token)}
+      (if config/is-prod?
+        (throw (ex-info "Python runner API token is required in production but not configured"
+                        {:error-type :configuration-error}))
+        {}))))
+
+(defn- python-runner-request
+  "Helper function for making HTTP requests to the python runner service."
+  [server-url method endpoint & [request-options]]
+  (let [url          (str server-url "/v1" endpoint)
+        base-options {:content-type     :json
+                      :accept           :json
+                      :throw-exceptions false
+                      :as               :json
+                      :headers          (authorization-headers)}]
+    (http/request (merge base-options request-options {:method method, :url url}))))
+
+(defn- write-jsonl-to-stream! [^OutputStream os col-names reducible-rows]
+  (let [none? (volatile! true)
+        writer (-> os
+                   (OutputStreamWriter. StandardCharsets/UTF_8)
+                   (BufferedWriter.))]
+
+    (run! (fn [row]
+            (when @none? (vreset! none? false))
+            (let [row-map (zipmap col-names row)]
+              (json/encode-to row-map writer {})
+              (.newLine writer)))
+          reducible-rows)
+
+    ;; Workaround for LocalStack, which doesn't support zero byte files.
+    (when @none?
+      (.write writer " "))
+
+    (doto writer
+      (.flush)
+      (.close))))
+
+(defn- execute-mbql-query
+  [driver db-id query respond cancel-chan]
+  (driver/with-driver driver
+    (let [native (qp.compile/compile {:type :query, :database db-id :query query})
+          query  {:database db-id
+                  :type     :native
+                  :native   native}]
+      (qp.store/with-metadata-provider db-id
+        (binding [qp.pipeline/*canceled-chan* cancel-chan]
+          (driver/execute-reducible-query driver query {:canceled-chan cancel-chan} respond))))))
+
+(defn root-type
+  "Supported type for roundtrip/insertion"
+  [base-type]
+  (when base-type
+    (some #(when (isa? base-type %) %)
+          [:type/Number
+           :type/Date
+           :type/DateTime
+           :type/Instant
+           :type/DateTimeWithTZ
+           :type/Text
+           :type/Boolean])))
+
+(defn restricted-insert-type
+  "Type for insertion restricted to supported"
+  [base_type]
+  (if base_type
+    (let [base-type-kw (keyword "type" base_type)]
+      (if (root-type base-type-kw)
+        base-type-kw
+        :type/Text))
+    :type/Text))
+
+(defn- closest-ancestor [t pred]
+  (loop [remaining (conj PersistentQueue/EMPTY [t])]
+    (when-let [t (first remaining)]
+      (if (pred t)
+        t
+        (recur (into (pop t) (parents t)))))))
+
+(defn- effective-semantic-type-i-think
+  "Kinda sketchy but maybe reasonable way to infer the effective-semantic-type"
+  [{:keys [base_type effective_type semantic_type]}]
+  (or semantic_type (closest-ancestor (or effective_type base_type) #(isa? % :Semantic/*))))
+
+(defn- generate-manifest
+  "Generate a manifest to communicate schema and metadata information around the table."
+  [table-id cols-meta]
+  {:schema_version 1
+   :data_format    "jsonl"
+   :data_version   1
+   :table_metadata {:table_id table-id}
+   :fields         (mapv (fn [col-meta]
+                           {:name           (:name col-meta)
+                            :base_type      (some-> (:base_type col-meta) name)
+                            :database_type  (some-> (:database_type col-meta) name)
+                            :root_type      (some-> (root-type (:base_type col-meta)) name)
+                            :semantic_type  (some-> (effective-semantic-type-i-think col-meta) name)
+                            :effective_type (some-> (or (:effective_type col-meta) (:database_type col-meta)) name)
+                            :field_id       (:id col-meta)})
+                         cols-meta)})
+
+(defn- maybe-fixup-value [col v]
+  (cond
+    (nil? (root-type (:base_type col)))
+    ;; we're not a supported base type, so we just stringify it
+    (when v (json/encode v))
+
+    ;; the clickhouse driver returns bigdecimals for int64 values
+    (and (isa? (:base_type col) :type/Integer)
+         (or (instance? BigDecimal v)
+             (float? v)))
+    (bigint v)
+
+    :else
+    v))
+
+(defn- write-table-data-to-file! [{:keys [db-id driver table-id fields-meta temp-file cancel-chan]}]
+  (let [query    {:source-table table-id}]
+    (execute-mbql-query driver db-id query
+                        (fn [{cols-meta :cols} reducible-rows]
+                          (with-open [os (io/output-stream temp-file)]
+                            (let [filtered-col-meta (m/index-by :name fields-meta)
+                                  col-names         (map :name cols-meta)
+                                  filtered-rows     (eduction (map (fn [row]
+                                                                     (->>
+                                                                      (map vector col-names row)
+                                                                      (filter (fn [[n _]]
+                                                                                (contains? filtered-col-meta n)))
+                                                                      (map (fn [[n v]]
+                                                                             (maybe-fixup-value (filtered-col-meta n) v))))))
+                                                              reducible-rows)]
+                              (write-jsonl-to-stream! os (filter filtered-col-meta col-names) filtered-rows))))
+                        cancel-chan)))
+
+(defn get-logs
+  "Return the logs of the current running python process"
+  [run-id]
+  (let [server-url (transforms-python.settings/python-runner-url)]
+    (python-runner-request server-url :get "/logs" {:query-params {:request_id run-id}})))
+
+(defn execute-python-code-http-call!
+  "Calls the /execute endpoint of the python runner. Blocks until the run either succeeds or fails and returns
+  the response from the server."
+  [{:keys [server-url code run-id table-name->id shared-storage]}]
+  (let [{:keys [objects]} shared-storage
+        {:keys [output output-manifest events]} objects
+
+        url-for-path             (fn [path] (:url (get objects path)))
+        table-name->url          (update-vals table-name->id #(url-for-path [:table % :data]))
+        table-name->manifest-url (update-vals table-name->id #(url-for-path [:table % :manifest]))
+
+        payload                  {:code                code
+                                  :library             (t2/select-fn->fn :path :source :model/PythonLibrary)
+                                  :timeout             (* 30 60) ;; 30 mins harcoded for now
+                                  :request_id          run-id
+                                  :output_url          (:url output)
+                                  :output_manifest_url (:url output-manifest)
+                                  :events_url          (:url events)
+                                  :table_mapping       table-name->url
+                                  :manifest_mapping    table-name->manifest-url}
+
+        response                 (transforms.instrumentation/with-python-api-timing [run-id]
+                                   (python-runner-request server-url :post "/execute" {:body (json/encode payload)}))]
+    ;; when a 500 is returned we observe a string in the body (despite the python returning json)
+    ;; always try to parse the returned string as json before yielding (could tighten this up at some point)
+    (update response :body (fn [string-if-error]
+                             (if (string? string-if-error)
+                               (try
+                                 (json/decode+kw string-if-error)
+                                 (catch Exception _
+                                   {:error string-if-error}))
+                               string-if-error)))))
+
+;; temporary, we should not need to realize data/events files into memory longer term
+(defn read-output-objects
+  "Temporary function that strings/jsons stuff in S3 and returns it for compatibility."
+  [{:keys [s3-client bucket-name objects]}]
+  (let [{:keys [output output-manifest events]} objects
+        output-content          (s3/read-to-string s3-client bucket-name (:path output) nil)
+        output-manifest-content (s3/read-to-string s3-client bucket-name (:path output-manifest) "{}")
+        events-content          (s3/read-to-string s3-client bucket-name (:path events))]
+    {:output          output-content
+     :output-manifest (json/decode+kw output-manifest-content)
+     :events          (mapv json/decode+kw (str/split-lines events-content))}))
+
+(defn cancel-python-code-http-call!
+  "Calls the /cancel endpoint of the python runner. Returns immediately."
+  [server-url run-id]
+  (python-runner-request server-url :post "/cancel" {:body   (json/encode {:request_id run-id})
+                                                     :async? true}
+                         #_success #(log/debug %)
+                         #_failure #(log/error %)))
+
+(defn- safe-delete
+  "Safely delete a file."
+  [^File file]
+  (try (.delete file) (catch Exception _)))
+
+(defn- fields-metadata [_driver table-id]
+  (t2/select [:model/Field :id :name :base_type :effective_type :semantic_type :database_type :database_position]
+             :table_id table-id
+             :active true
+             ;; we are only interested in top-level objects, so filter out nested fields (parent or path)
+             :parent_id nil
+             :nfc_path nil
+             {:order-by [[:database_position :asc]]}))
+
+;; TODO break this up such that s3 can be swapped out for other transfer mechanisms.
+(defn copy-tables-to-s3!
+  "Writes table content to their corresponding objects named in shared-storage, see (open-shared-storage!).
+  Blocks until all tables are fully written and committed to shared storage."
+  [{:keys [run-id
+           shared-storage
+           table-name->id
+           cancel-chan]}]
+  ;; TODO there's scope for some parallelism here, in particular across different databases
+  (doseq [[table-name table-id] table-name->id
+          :let [{:keys [s3-client bucket-name objects]} shared-storage
+                {data-path :path}                       (get objects [:table table-id :data])
+                {manifest-path :path}                   (get objects [:table table-id :manifest])]]
+    (let [tmp-data-file (File/createTempFile data-path "")
+          tmp-meta-file (File/createTempFile manifest-path "")]
+      (try
+        (let [db-id       (t2/select-one-fn :db_id (t2/table-name :model/Table) :id table-id)
+              driver      (t2/select-one-fn :engine :model/Database db-id)
+              fields-meta (fields-metadata driver table-id)
+              manifest    (generate-manifest table-id fields-meta)]
+
+          (transforms.instrumentation/with-stage-timing [run-id :dwh-to-file]
+            (write-table-data-to-file!
+             {:db-id       db-id
+              :driver      driver
+              :table-id    table-id
+              :fields-meta fields-meta
+              :temp-file   tmp-data-file
+              :cancel-chan cancel-chan}))
+
+          (with-open [writer (io/writer tmp-meta-file)]
+            (json/encode-to manifest writer {}))
+          (let [data-size (.length tmp-data-file)
+                meta-size (.length tmp-meta-file)]
+            (transforms.instrumentation/record-data-transfer! run-id :dwh-to-file data-size nil)
+
+            (transforms.instrumentation/with-stage-timing [run-id :file-to-s3]
+              (s3/upload-file s3-client bucket-name data-path tmp-data-file)
+              (s3/upload-file s3-client bucket-name manifest-path tmp-meta-file))
+
+            (transforms.instrumentation/record-data-transfer! run-id :file-to-s3 (+ data-size meta-size) nil)))
+        (catch InterruptedException ie (throw ie))
+        (catch Throwable t
+          (throw (ex-info "An error occurred while copying table data to S3"
+                          {:table-id table-id
+                           :transform-message (or (:transform-message (ex-data t))
+                                                  (i18n/tru "Failed to copy table contents to shared storage {0} ({1})" table-name table-id))}
+                          t)))
+        (finally
+          (safe-delete tmp-data-file)
+          (safe-delete tmp-meta-file))))))

enterprise/backend/src/metabase_enterprise/transforms_python/s3.clj

@@ -0,0 +1,215 @@
+(ns metabase-enterprise.transforms-python.s3
+  (:require
+   [metabase-enterprise.transforms-python.settings :as transforms-python.settings]
+   [metabase.util.log :as log])
+  (:import
+   (clojure.lang IDeref)
+   (java.io Closeable File)
+   (java.net URI)
+   (java.time Duration)
+   (software.amazon.awssdk.auth.credentials AwsBasicCredentials StaticCredentialsProvider)
+   (software.amazon.awssdk.auth.credentials AwsCredentialsProvider DefaultCredentialsProvider)
+   (software.amazon.awssdk.core.sync RequestBody)
+   (software.amazon.awssdk.regions Region)
+   (software.amazon.awssdk.services.s3 S3Client S3ClientBuilder S3Configuration)
+   (software.amazon.awssdk.services.s3.model DeleteObjectRequest GetObjectRequest NoSuchKeyException PutObjectRequest)
+   (software.amazon.awssdk.services.s3.presigner S3Presigner S3Presigner$Builder)
+   (software.amazon.awssdk.services.s3.presigner.model GetObjectPresignRequest PutObjectPresignRequest)))
+
+(set! *warn-on-reflection* true)
+
+;; TODO We should make these durations configurable in future.
+
+;; Longer duration for inputs than for outputs, to compensate for the duration of the code execution itself.
+(def ^:private ^Duration presigned-get-duration (Duration/ofMinutes 30))
+(def ^:private ^Duration presigned-put-duration (Duration/ofHours 5))
+
+;; We should consider namespacing these paths, for example, using the run id to simplify cleanup.
+;; Another idea is to have deterministic paths, so that it's easier to resume incomplete runs.
+(defn- working-dir-for-run
+  "The path within the target bucket within which to store all the files for this run."
+  []
+  (let [shared-prefix (some-> (transforms-python.settings/python-storage-s-3-prefix) (str "/"))]
+    (str shared-prefix "run-" (System/nanoTime) "-" (rand-int 10000))))
+
+(defmacro ^:private maybe-with-endpoint* [builder endpoint]
+  `(let [builder# ~builder]
+     (when-let [region# (transforms-python.settings/python-storage-s-3-region)]
+       (.region builder# (Region/of region#)))
+     (when ~endpoint (.endpointOverride builder# (URI/create ~endpoint)))
+     builder#))
+
+(defn- maybe-with-endpoint-s3-client [^S3ClientBuilder builder endpoint]
+  (maybe-with-endpoint* builder endpoint))
+
+(defn- maybe-with-endpoint-s3-presigner [^S3Presigner$Builder builder endpoint]
+  (maybe-with-endpoint* builder endpoint))
+
+(defn- s3-configuration ^S3Configuration []
+  (-> (S3Configuration/builder)
+      (.pathStyleAccessEnabled (transforms-python.settings/python-storage-s-3-path-style-access))
+      (.build)))
+
+(defn- put-object-request ^PutObjectRequest [^String bucket-name ^String key]
+  (-> (PutObjectRequest/builder) (.bucket bucket-name) (.key key) .build))
+
+(defn- get-object-request ^GetObjectRequest [^String bucket-name ^String key]
+  (-> (GetObjectRequest/builder) (.bucket bucket-name) (.key key) .build))
+
+(defn- delete-object-request ^DeleteObjectRequest [^String bucket-name ^String key]
+  (-> (DeleteObjectRequest/builder) (.bucket bucket-name) (.key key) .build))
+
+(defn- maybe-with-credentials*
+  [credentials-provider]
+  (let [access-key (transforms-python.settings/python-storage-s-3-access-key)
+        secret-key (transforms-python.settings/python-storage-s-3-secret-key)]
+    (if (or access-key secret-key)
+      (if-not (and access-key secret-key)
+        (do (log/warnf "Ignoring %s because %s is not defined"
+                       (if access-key "access-key" "secret-key")
+                       (if (not access-key) "access-key" "secret-key"))
+            (credentials-provider (DefaultCredentialsProvider/create)))
+        (credentials-provider
+         (StaticCredentialsProvider/create
+          (AwsBasicCredentials/create access-key secret-key))))
+      (credentials-provider (DefaultCredentialsProvider/create)))))
+
+(defn- maybe-with-credentials-s3-client [^S3ClientBuilder builder]
+  (maybe-with-credentials* #(.credentialsProvider builder ^AwsCredentialsProvider %)))
+
+(defn- maybe-with-credentials-s3-presigner [^S3Presigner$Builder builder]
+  (maybe-with-credentials* #(.credentialsProvider builder ^AwsCredentialsProvider %)))
+
+;; We just recreate the client every time, to keep things simple if config is changed.
+(defn create-s3-client
+  "Create S3 client for transferring table data and manifests."
+  ^S3Client []
+  (.build
+   (doto (S3Client/builder)
+     (maybe-with-endpoint-s3-client (transforms-python.settings/python-storage-s-3-endpoint))
+     maybe-with-credentials-s3-client
+     (.serviceConfiguration (s3-configuration)))))
+
+(defn- create-s3-presigner-for-container
+  "Create S3 presigner for container operations (presigned URLs). Uses distinct container-endpoint if relevant."
+  ^S3Presigner []
+  (let [container-endpoint (transforms-python.settings/python-storage-s-3-container-endpoint)
+        endpoint           (or container-endpoint (transforms-python.settings/python-storage-s-3-endpoint))]
+    (.build
+     (doto (S3Presigner/builder)
+       (maybe-with-endpoint-s3-presigner endpoint)
+       maybe-with-credentials-s3-presigner
+       (.serviceConfiguration (s3-configuration))))))
+
+(defn- generate-presigned-get-url
+  "Generate GET URL using container presigner"
+  [^S3Presigner presigner ^String bucket-name ^String key]
+  (let [request (-> (GetObjectPresignRequest/builder)
+                    (.signatureDuration presigned-get-duration)
+                    (.getObjectRequest (get-object-request bucket-name key))
+                    (.build))]
+    (.toString (.url (.presignGetObject presigner request)))))
+
+(defn- generate-presigned-put-url
+  "Generate PUT URL using container presigner"
+  [^S3Presigner presigner ^String bucket-name ^String key]
+  (let [request (-> (PutObjectPresignRequest/builder)
+                    (.signatureDuration presigned-put-duration)
+                    (.putObjectRequest (put-object-request bucket-name key))
+                    (.build))]
+    (.toString (.url (.presignPutObject presigner request)))))
+
+(defn- s3-shared-storage [table-name->id]
+  (let [work-dir-name       (working-dir-for-run)
+        container-presigner (create-s3-presigner-for-container)
+        bucket-name         (transforms-python.settings/python-storage-s-3-bucket)
+        ref                 (fn [method relative-path]
+                              (let [path (str work-dir-name "/" relative-path)]
+                                {:path   path
+                                 :method method
+                                 :url    (case method
+                                           :put (generate-presigned-put-url container-presigner bucket-name path)
+                                           :get (generate-presigned-get-url container-presigner bucket-name path))}))]
+    ;; It feels dirty mixing the S3 client object with the other pure values, but it is convenient for the caller
+    {:s3-client   (create-s3-client)
+     :bucket-name bucket-name
+     :objects
+     (into
+      {:output          (ref :put "output.csv")
+       :output-manifest (ref :put "output-manifest.json")
+       :events          (ref :put "events.jsonl")}
+      (for [[table-name id] table-name->id]
+        {[:table id :manifest] (ref :get (str "table-" (name table-name) "-" id ".manifest.json"))
+         [:table id :data]     (ref :get (str "table-" (name table-name) "-" id ".jsonl"))}))}))
+
+(declare delete-many)
+
+;; TODO this feels like it's a hair's width from abstracting away s3 versus other transfer mechanisms - go all the way
+(defn open-shared-storage!
+  "Returns a map wrapped in a deref-able auto-closable, to be used in a with-open.
+
+  The map contains:
+  - :s3-client   - an S3Client that can be used to upload/download files
+  - :bucket-name - the S3 bucket name
+  - :objects     - a map of keys to {:path :method :url} maps, where:
+    - :path   is the S3 key
+    - :method is either :get or :put, indicating whether the python-runner should fetch or upload the file.
+    - :url    is the presigned URL to be used by the python-runner to fetch or upload the file.
+
+  The objects are broken into the following transform inputs:
+
+  - [:table id :manifest] - for each input table, the manifest JSON file
+  - [:table id :data]     - for each input table, the data file, in format specified by the manifest.
+
+  And the following transform outputs:
+  - :output-manifest - the output manifest JSON file
+  - :output          - the output data file, in format specified by the output manifest.
+  - :events          - a JSONL file containing the events logged during python execution.
+
+  When the value is closed, all the relevant keys will be deleted, but this is best-effort only.
+  We rely on the bucket retention policy to ensure any stragglers are eventually deleted."
+  ^Closeable [table-name->id]
+  (let [shared-storage (s3-shared-storage table-name->id)]
+    (reify IDeref
+      (deref [_] shared-storage)
+      Closeable
+      (close [_] (delete-many (:s3-client shared-storage)
+                              (:bucket-name shared-storage)
+                              (map :path (vals (:objects shared-storage))))))))
+
+(defn upload-file
+  "Upload the given file to s3"
+  [^S3Client s3-client ^String bucket-name ^String key ^File file]
+  (let [^PutObjectRequest request (put-object-request bucket-name key)]
+    (.putObject s3-client request (RequestBody/fromFile file))))
+
+;; TODO optimize our ingestion to stream the data. reading a string will not work in any case for binary files.
+(defn read-to-string
+  "Get back the contents of the given key as a string."
+  ([s3-client bucket-name key] (read-to-string s3-client bucket-name key ::throw))
+  ([^S3Client s3-client ^String bucket-name ^String key not-found]
+   (try
+     (let [^GetObjectRequest request (get-object-request bucket-name key)
+           response                  (.getObject s3-client request)]
+       (slurp response))
+     (catch NoSuchKeyException e
+       (if (identical? ::throw not-found)
+         (throw e)
+         not-found)))))
+
+(defn delete
+  ;; TODO better error handling
+  "Delete the given key from s3, ignoring any errors for now"
+  [^S3Client s3-client ^String bucket-name ^String key]
+  (try
+    (.deleteObject s3-client (delete-object-request bucket-name key))
+    (catch Exception e
+      (log/debugf e "Error deleting s3 object %s" key)
+      ;; Ignore deletion errors - object might not exist, or we might not have permissions
+      ;; NOTE: we plan to put general retention on the bucket so that objects will eventually be deleted
+      nil)))
+
+(defn delete-many
+  "Best effort delete the given s3 keys"
+  [^S3Client s3-client bucket-name s3-keys]
+  (run! (partial delete s3-client bucket-name) s3-keys))

enterprise/backend/src/metabase_enterprise/transforms_python/settings.clj

@@ -18,7 +18,7 @@
 (setting/defsetting python-runner-api-token
   (deferred-tru "API token for authorizing with the python-runner service.")
   :type       :string
-  :visibility :internal
+  :visibility :admin
   :default    (when (not config/is-prod?) "dev-token-12345")
   :feature    :transforms-python
   :doc        false
@@ -52,7 +52,7 @@
   (deferred-tru "S3 bucket name for storing Python execution artifacts.")
   :type       :string
   :visibility :admin
-  :default    (if (not config/is-prod?) "metabase-python-runner" "metabase-cloud-python-runner-staging")
+  :default    (when (not config/is-prod?) "metabase-python-runner")
   :feature    :transforms-python
   :doc        false
   :export?    false
@@ -64,7 +64,7 @@
   :type       :string
   :visibility :admin
   :feature    :transforms-python
-  :default    (if (not config/is-prod?) "test-prefix" "hosting-4b9351bb22b84791")
+  :default    (when (not config/is-prod?) "test-prefix")
   :doc        false
   :export?    false
   :encryption :no

enterprise/backend/test/metabase_enterprise/advanced_config/file/users_test.clj

@@ -157,3 +157,21 @@
      :last_name  "Era"
      :email      "cam+config-file-test@metabase.com"}
     (re-pattern (java.util.regex.Pattern/quote "failed: (contains? % :password)"))))
+
+(deftest email-case-irrelevant-test
+  (testing "Using any case in email still works #62894"
+    (try
+      (binding [advanced-config.file/*config* {:version 1
+                                               :config  {:users [{:first_name       "Cam"
+                                                                  :last_name        "Era"
+                                                                  :email            "Cam+config-file-test@metabase.com"
+                                                                  :password         "2cans"
+                                                                  :login_attributes {"a" 1}}]}}]
+        (is (= :ok
+               (advanced-config.file/initialize!)))
+        (testing "It should not fail with 'duplicate'"
+          (is (= :ok
+                 (advanced-config.file/initialize!)))))
+      (finally
+        (t2/delete! :model/User :email "cam+config-file-test@metabase.com")
+        (t2/delete! :model/User :email "Cam+config-file-test@metabase.com")))))

enterprise/backend/test/metabase_enterprise/api/session_test.clj

@@ -47,6 +47,7 @@
                               :sso-ldap
                               :sso-saml
                               :transforms
+                              :transforms-python
                               :upload-management
                               :whitelabel
                               :collection-cleanup
@@ -91,6 +92,7 @@
             :sso_saml                       true
             :table_data_editing             false
             :transforms                     true
+            :transforms-python              true
             :upload_management              true
             :whitelabel                     true
             :collection_cleanup             true

enterprise/backend/test/metabase_enterprise/cache/task/refresh_cache_configs_test.clj

@@ -84,9 +84,9 @@
 (deftest schedule-cache-config->card-ids-test
   (mt/with-temp [:model/Dashboard {dashboard-id :id} {:name "Dashboard"}
                  :model/Card {card-id-1 :id} {:name "Cached card 1"
-                                              :dataset_query {:database 1}}
+                                              :dataset_query {:database (mt/id), :type :native, :native {:query "SELECT 1;"}}}
                  :model/Card {card-id-2 :id} {:name "Cached card 2"
-                                              :dataset_query {:database 1}}
+                                              :dataset_query {:database (mt/id), :type :native, :native {:query "SELECT 1;"}}}
                  :model/DashboardCard {} {:dashboard_id dashboard-id
                                           :card_id      card-id-1}
                  :model/DashboardCard {} {:dashboard_id dashboard-id
@@ -97,7 +97,6 @@
                                           :card_id      card-id-2}]
     (testing "Returns the card ID directly for a question cache config"
       (is (= [card-id-1] (@#'task.cache/schedule-cache-config->card-ids {:model_id card-id-1 :model "question"}))))
-
     (testing "Fetches card IDs associated with a dashboard cache config"
       (is (= [card-id-1 card-id-2] (@#'task.cache/schedule-cache-config->card-ids {:model_id dashboard-id :model "dashboard"}))))))
 
@@ -137,7 +136,7 @@
                 (is (= [nil param-val-2] (map param-vals (to-rerun))))))))))))
 
 (deftest scheduled-base-query-to-rerun-edge-cases-test
-  (let [query {:database 1}]
+  (let [query {:database (mt/id), :type :native, :native {:query "SELECT 1;"}}]
     (mt/with-temp [:model/Card {card-id :id} {:name "Cached card"
                                               :dataset_query query}
                    :model/Query {} {:query_hash (qp.util/query-hash query)
@@ -170,7 +169,7 @@
           (is (= [] (t2/select :model/Query (@#'task.cache/scheduled-base-query-to-rerun-honeysql card-id)))))))))
 
 (deftest scheduled-parameterized-queries-to-rerun-edge-cases-test
-  (let [query {:database 1}]
+  (let [query {:database (mt/id), :type :native, :native {:query "SELECT 1;"}}]
     (mt/with-temp [:model/Card {card-id :id} {:name "Cached card"
                                               :dataset_query query}
                    :model/Query {} {:query_hash (qp.util/query-hash query)
@@ -299,8 +298,8 @@
                 (is (= [nil param-val-2] (map param-vals (to-rerun card-id))))))))))))
 
 (deftest duration-base-queries-to-rerun-edge-cases-test
-  (let [query-1            {:database 1}
-        query-2            {:database 2}
+  (let [query-1            {:database (mt/id), :type "native", :native {:query "SELECT * FROM x"}}
+        query-2            {:database (mt/id), :type "native", :native {:query "SELECT * FROM y"}}
         query-1-cache-hash (qp.util/query-hash query-1)
         query-2-cache-hash (qp.util/query-hash query-2)]
     (mt/with-temp [:model/Dashboard {dashboard-id :id} {:name "Dashboard"}
@@ -390,8 +389,8 @@
                                                [question-cache-config-1] false))))))))))
 
 (deftest duration-parameterized-queries-to-rerun-edge-cases-test
-  (let [query-1            {:database 1}
-        query-2            {:database 2}
+  (let [query-1            {:database (mt/id), :type "native", :native {:query "SELECT * FROM x"}}
+        query-2            {:database (mt/id), :type "native", :native {:query "SELECT * FROM y"}}
         query-1-cache-hash (qp.util/query-hash query-1)
         query-2-cache-hash (qp.util/query-hash query-2)]
     (mt/with-temp [:model/Dashboard {dashboard-id :id} {:name "Dashboard"}

enterprise/backend/test/metabase_enterprise/scim/v2/api_test.clj

@@ -297,64 +297,127 @@
             (is (= ["urn:ietf:params:scim:api:messages:2.0:Error"] (get response :schemas)))
             (is (= "User not found" (get response :detail)))))))))
 
-(deftest list-groups-test
+(deftest patch-user-test-without-path
   (with-scim-setup!
-    (mt/with-temp [:model/PermissionsGroup _group1 {:name "Group 1"}]
-      (testing "Fetch groups with default pagination"
-        (let [response (scim-client :get 200 "ee/scim/v2/Groups")]
-          (is (malli= scim-api/SCIMGroupList response))))
+    (mt/with-temp [:model/User user {:email "testuser@metabase.com"
+                                     :first_name "Test"
+                                     :last_name "User"
+                                     :is_active true
+                                     :locale "en-US"}]
+      (let [entity-id (t2/select-one-fn :entity_id :model/User :id (:id user))]
+        (testing "Deactivate an existing user"
+          (let [patch-body {:schemas ["urn:ietf:params:scim:api:messages:2.0:PatchOp"]
+                            :Operations [{:op "Replace"
+                                          :value {"active" false}}]}
+                response   (scim-client :patch 200 (format "ee/scim/v2/Users/%s" entity-id) patch-body)]
+            (is (malli= scim-api/SCIMUser response))
+            (is (= false (:active response)))))
 
-      (testing "Fetch groups with custom pagination"
-        (let [response (scim-client :get 200 (format "ee/scim/v2/Groups?startIndex=%d&count=%d" 1 2))]
-          (is (= ["urn:ietf:params:scim:api:messages:2.0:ListResponse"] (get response :schemas)))
-          (is (integer? (get response :totalResults)))
-          (is (= 1 (get response :startIndex)))
-          (is (= 2 (get response :itemsPerPage)))
-          (is (= 2 (count (get response :Resources))))))
+        (testing "Reactivate an existing user"
+          (let [patch-body {:schemas ["urn:ietf:params:scim:api:messages:2.0:PatchOp"]
+                            :Operations [{:op "Replace"
+                                          :value {"active" "True"}}]}
+                response   (scim-client :patch 200 (format "ee/scim/v2/Users/%s" entity-id) patch-body)]
+            (is (malli= scim-api/SCIMUser response))
+            (is (true? (:active response)))))
 
-      (testing "Fetch group by name"
-        (let [response (scim-client :get 200 (format "ee/scim/v2/Groups?filter=%s"
-                                                     (codec/url-encode "displayName eq \"Group 1\"")))]
-          (is (malli= scim-api/SCIMGroupList response))
-          (is (= 1 (get response :totalResults)))
-          (is (= 1 (count (get response :Resources))))))
+        (testing "Update family name of an existing user"
+          (let [patch-body {:schemas ["urn:ietf:params:scim:api:messages:2.0:PatchOp"]
+                            :Operations [{:op "Replace"
+                                          :value {"name.familyName" "UpdatedUser"}}]}
+                response   (scim-client :patch 200 (format "ee/scim/v2/Users/%s" entity-id) patch-body)]
+            (is (malli= scim-api/SCIMUser response))
+            (is (= "UpdatedUser" (get-in response [:name :familyName])))))
 
-      (testing "Fetch non-existant group by name"
-        (let [response (scim-client :get 200 (format "ee/scim/v2/Groups?filter=%s"
-                                                     (codec/url-encode "displayName eq \"Fake Group\"")))]
-          (is (malli= scim-api/SCIMUserList response))
-          (is (= 0 (get response :totalResults)))
-          (is (= 0 (count (get response :Resources))))))
+        (testing "Update multiple attributes of an existing user"
+          (let [patch-body {:schemas ["urn:ietf:params:scim:api:messages:2.0:PatchOp"]
+                            :Operations [{:op "Replace"
+                                          :value {"name.givenName" "UpdatedFirstName"}}
+                                         {:op "Replace"
+                                          :value {"name.familyName" "UpdatedLastName"}}
+                                         ;; Unsupported operations are ignored
+                                         {:op "Add"
+                                          :value {"name.active" "False"}}]}
+                response   (scim-client :patch 200 (format "ee/scim/v2/Users/%s" entity-id) patch-body)]
+            (is (malli= scim-api/SCIMUser response))
+            (is (= "UpdatedFirstName" (get-in response [:name :givenName])))
+            (is (= "UpdatedLastName" (get-in response [:name :familyName])))
+            (is (true? (response :active)))))
 
-      (testing "Error if unsupported filter operation is provided"
-        (scim-client :get 400 (format "ee/scim/v2/Users?filter=%s"
-                                      (codec/url-encode "displayName ne \"Group 1\"")))))))
+        (testing "Error when using unsupported path"
+          (let [patch-body {:schemas ["urn:ietf:params:scim:api:messages:2.0:PatchOp"]
+                            :Operations [{:op "replace"
+                                          :value {"name.displayName" "unsupported"}}]}
+                response   (scim-client :patch 400 (format "ee/scim/v2/Users/%s" entity-id) patch-body)]
+            (is (= ["urn:ietf:params:scim:api:messages:2.0:Error"] (get response :schemas)))
+            (is (= "Unsupported path: name.displayName" (get response :detail)))))
 
-(deftest fetch-group-test
-  (with-scim-setup!
-    (testing "A single group can be fetched in the SCIM format by entity ID with its members"
-      (mt/with-temp [:model/PermissionsGroup           group {:name "Test Group"}
-                     :model/PermissionsGroupMembership _     {:user_id (mt/user->id :rasta) :group_id (:id group)}]
-        (let [entity-id (t2/select-one-fn :entity_id :model/PermissionsGroup :id (:id group))
-              response  (scim-client :get 200 (format "ee/scim/v2/Groups/%s" entity-id))]
-          (is (malli= scim-api/SCIMGroup response))
-          (is (=?
-               {:schemas     ["urn:ietf:params:scim:schemas:core:2.0:Group"]
-                :id          entity-id
-                :displayName "Test Group"
-                :members     [{:value (t2/select-one-fn :entity_id :model/User (mt/user->id :rasta))
-                               :display "rasta@metabase.com"}]
-                :meta        {:resourceType "Group"}}
-               response)))))
+        (testing "Error when trying to update a non-existent user"
+          (let [patch-body {:schemas ["urn:ietf:params:scim:api:messages:2.0:PatchOp"]
+                            :Operations [{:op "Replace"
+                                          :value {"name.familyName" "NonexistentUser"}}]}
+                response   (scim-client :patch 404 (format "ee/scim/v2/Users/%s" (random-uuid)) patch-body)]
+            (is (= ["urn:ietf:params:scim:api:messages:2.0:Error"] (get response :schemas)))
+            (is (= "User not found" (get response :detail)))))
 
-    (testing "404 is returned when fetching a non-existant group"
-      (scim-client :get 404 (format "ee/scim/v2/Groups/%s" (random-uuid))))
+        (deftest list-groups-test
+          (with-scim-setup!
+            (mt/with-temp [:model/PermissionsGroup _group1 {:name "Group 1"}]
+              (testing "Fetch groups with default pagination"
+                (let [response (scim-client :get 200 "ee/scim/v2/Groups")]
+                  (is (malli= scim-api/SCIMGroupList response))))
 
-    (testing "404 is returned when fetching the Admin or All Users group"
-      (let [entity-ids (t2/select-fn-set :entity_id :model/PermissionsGroup
-                                         {:where [:in :id #{(:id (perms-group/admin)) (:id (perms-group/all-users))}]})]
-        (doseq [entity-id entity-ids]
-          (scim-client :get 404 (format "ee/scim/v2/Groups/%s" entity-id)))))))
+              (testing "Fetch groups with custom pagination"
+                (let [response (scim-client :get 200 (format "ee/scim/v2/Groups?startIndex=%d&count=%d" 1 2))]
+                  (is (= ["urn:ietf:params:scim:api:messages:2.0:ListResponse"] (get response :schemas)))
+                  (is (integer? (get response :totalResults)))
+                  (is (= 1 (get response :startIndex)))
+                  (is (= 2 (get response :itemsPerPage)))
+                  (is (= 2 (count (get response :Resources))))))
+
+              (testing "Fetch group by name"
+                (let [response (scim-client :get 200 (format "ee/scim/v2/Groups?filter=%s"
+                                                             (codec/url-encode "displayName eq \"Group 1\"")))]
+                  (is (malli= scim-api/SCIMGroupList response))
+                  (is (= 1 (get response :totalResults)))
+                  (is (= 1 (count (get response :Resources))))))
+
+              (testing "Fetch non-existant group by name"
+                (let [response (scim-client :get 200 (format "ee/scim/v2/Groups?filter=%s"
+                                                             (codec/url-encode "displayName eq \"Fake Group\"")))]
+                  (is (malli= scim-api/SCIMUserList response))
+                  (is (= 0 (get response :totalResults)))
+                  (is (= 0 (count (get response :Resources))))))
+
+              (testing "Error if unsupported filter operation is provided"
+                (scim-client :get 400 (format "ee/scim/v2/Users?filter=%s"
+                                              (codec/url-encode "displayName ne \"Group 1\"")))))))))
+
+    (deftest fetch-group-test
+      (with-scim-setup!
+        (testing "A single group can be fetched in the SCIM format by entity ID with its members"
+          (mt/with-temp [:model/PermissionsGroup           group {:name "Test Group"}
+                         :model/PermissionsGroupMembership _     {:user_id (mt/user->id :rasta) :group_id (:id group)}]
+            (let [entity-id (t2/select-one-fn :entity_id :model/PermissionsGroup :id (:id group))
+                  response  (scim-client :get 200 (format "ee/scim/v2/Groups/%s" entity-id))]
+              (is (malli= scim-api/SCIMGroup response))
+              (is (=?
+                   {:schemas     ["urn:ietf:params:scim:schemas:core:2.0:Group"]
+                    :id          entity-id
+                    :displayName "Test Group"
+                    :members     [{:value (t2/select-one-fn :entity_id :model/User (mt/user->id :rasta))
+                                   :display "rasta@metabase.com"}]
+                    :meta        {:resourceType "Group"}}
+                   response)))))
+
+        (testing "404 is returned when fetching a non-existant group"
+          (scim-client :get 404 (format "ee/scim/v2/Groups/%s" (random-uuid))))
+
+        (testing "404 is returned when fetching the Admin or All Users group"
+          (let [entity-ids (t2/select-fn-set :entity_id :model/PermissionsGroup
+                                             {:where [:in :id #{(:id (perms-group/admin)) (:id (perms-group/all-users))}]})]
+            (doseq [entity-id entity-ids]
+              (scim-client :get 404 (format "ee/scim/v2/Groups/%s" entity-id)))))))))
 
 (deftest create-group-test
   (with-scim-setup!

enterprise/backend/test/metabase_enterprise/semantic_search/task/usage_trimmer_test.clj

@@ -2,7 +2,7 @@
   (:require
    [clojure.test :refer :all]
    [metabase-enterprise.semantic-search.task.usage-trimmer :as semantic.task.trimmer]
-   [metabase.test :as mt]
+   [metabase-enterprise.semantic-search.util :as semantic.u]
    [toucan2.core :as t2])
   (:import
    (java.sql Timestamp)
@@ -11,7 +11,7 @@
 (set! *warn-on-reflection* true)
 
 (deftest trim-test
-  (mt/with-premium-features #{:semantic-search}
+  (when (semantic.u/semantic-search-available?)
     (t2/delete! :model/SemanticSearchTokenTracking)
     (let [now (LocalDateTime/now)
           ;; semantic.task.trimmer/storage-months is set to 2

enterprise/backend/test/metabase_enterprise/serialization/api_test.clj

@@ -26,7 +26,7 @@
       (GzipCompressorInputStream.)
       (TarArchiveInputStream.)))
 
-(def FILE-TYPES
+(def ^:private file-types
   [#"([^/]+)?/$"                               :dir
    #"/settings.yaml$"                          :settings
    #"/export.log$"                             :log
@@ -46,7 +46,7 @@
   (some (fn [[re ftype]]
           (when-let [m (re-find re fname)]
             [ftype (when (vector? m) (second m))]))
-        (partition 2 FILE-TYPES)))
+        (partition 2 file-types)))
 
 (defn- log-types
   "Find out entity type by log message"
@@ -74,7 +74,7 @@
       (assoc m k "**ID**")
       m)))
 
-(defn extract-and-sanitize-exception-map [log]
+(defn- extract-and-sanitize-exception-map [log]
   (->> (re-find #"ERROR .* (\{.*\})(\n|$)" log)
        second
        read-string