just tag to clone

* store transform revisions

* Add tests

* remove unneeded arity

* fix whitespace

* include impl.transform in revisions.init

* move transform test to enterprise folder

* use =? and check unneeded keys were removed

* revert =? use

* dissoc source from transform comparison

.clj-kondo/config.edn

@@ -624,10 +624,11 @@
    metabase.test/with-non-admin-groups-no-root-collection-perms                                                              hooks.common/do*
    metabase.test/with-persistence-enabled!                                                                                   hooks.metabase.model-persistence.test-util/with-persistence-enabled!
    metabase.test/with-prometheus-system!                                                                                     hooks.common/with-two-bindings
+   metabase.test/with-random-premium-token!                                                                                  hooks.common/with-one-binding
    metabase.test/with-temp                                                                                                   hooks.toucan2.tools.with-temp/with-temp
    metabase.test/with-temp-file                                                                                              hooks.metabase.test.util/with-temp-file
    metabase.test/with-temporary-setting-values                                                                               hooks.metabase.test.util/with-temporary-setting-values
-   metabase.users.api-test/with-temp-user-email!                                                                              hooks.common/with-one-binding
+   metabase.users.api-test/with-temp-user-email!                                                                             hooks.common/with-one-binding
    metabase.util.i18n/deferred-trs                                                                                           hooks.metabase.util.i18n/deferred-trs
    metabase.util.i18n/deferred-tru                                                                                           hooks.metabase.util.i18n/deferred-tru
    metabase.util.i18n/trs                                                                                                    hooks.metabase.util.i18n/trs

.clj-kondo/config/modules/config.edn

@@ -428,6 +428,11 @@
    :api  #{metabase.database-routing.core}
    :uses #{premium-features}}
 
+  dependencies
+  {:team "Querying"
+   :api  #{metabase.premium-features.core}
+   :uses #{premium-features}}
+
   driver
   {:team "Drivers"
    :api  #{metabase.driver
@@ -556,6 +561,11 @@
            models
            util}}
 
+  graph
+  {:team "Querying"
+   :api  #{metabase.graph.core}
+   :uses #{util}}
+
   indexed-entities
   {:team "UX West"
    :api  #{metabase.indexed-entities.api
@@ -725,6 +735,7 @@
            metabase.native-query-snippets.core}
    :uses #{api
            collections
+           events
            lib
            models
            permissions
@@ -1350,9 +1361,11 @@
   users
   {:team "UX West"
    :api  #{metabase.users.api
+           metabase.users.core
            metabase.users.init
            metabase.users.models.user
-           metabase.users.models.user-parameter-value}
+           metabase.users.models.user-parameter-value
+           metabase.users.schema}
    :uses #{analytics
            api
            appearance
@@ -1716,6 +1729,29 @@
            warehouse-schema
            warehouses}}
 
+  enterprise/dependencies
+  {:team "Querying"
+   :api  #{metabase-enterprise.dependencies.api
+           metabase-enterprise.dependencies.core
+           metabase-enterprise.dependencies.init}
+   :uses #{analyze
+           api
+           app-db
+           collections
+           config
+           driver
+           events
+           graph
+           lib
+           lib-be
+           models
+           native-query-snippets
+           premium-features
+           queries
+           query-processor
+           task
+           util}}
+
   enterprise/documents
   {:team "UX West"
    :api  #{metabase-enterprise.documents.api

.cljfmt.edn

@@ -232,7 +232,6 @@
   metabase.test.data/$ids                                                                [[:inner 0]]
   metabase.test.data/dataset                                                             [[:inner 0]]
   metabase.test.data/mbql-query                                                          [[:inner 0]]
-  metabase.test.data/native-query                                                        [[:block 0]]
   metabase.test.data/query                                                               [[:block 1]]
   metabase.test.data/run-mbql-query                                                      [[:inner 0]]
   metabase.test.data/with-empty-h2-app-db!                                               [[:block 0]]
@@ -253,7 +252,6 @@
   metabase.test/format-rows-by                                                           [[:default]]
   metabase.test/let-url                                                                  [[:block 1]]
   metabase.test/mbql-query                                                               [[:inner 0]]
-  metabase.test/native-query                                                             [[:block 0]]
   metabase.test/query                                                                    [[:block 1]]
   metabase.test/run-mbql-query                                                           [[:inner 0]]
   metabase.test/test-driver                                                              [[:inner 0]]

.github/actions/cleanup-python-runner/action.yml

@@ -0,0 +1,36 @@
+name: Cleanup Python Runner
+description: Cleanup python-runner services and upload logs as artifacts
+
+inputs:
+  upload-logs:
+    description: 'Whether to upload logs (typically on failure)'
+    required: false
+    default: 'false'
+  logs-artifact-suffix:
+    description: 'The unique name of the .zip artifact containing the logs'
+    required: false
+    default: ''
+
+runs:
+  using: composite
+  steps:
+    - name: Cleanup python-runner services
+      if: always()
+      run: |
+        echo "Stopping python-runner services..."
+        if [ -d "python-runner" ]; then
+          cd python-runner
+          make stop || echo "WARNING: Failed to stop python runner"
+        fi
+      shell: bash
+
+    - name: Upload python-runner logs
+      if: ${{inputs.upload-logs == 'true' && inputs.logs-artifact-suffix != ''}}
+      uses: actions/upload-artifact@v4
+      with:
+        name: python-runner-logs-${{inputs.logs-artifact-suffix}}
+        path: |
+          python-runner.out.log
+          python-runner.err.log
+        retention-days: 1
+        if-no-files-found: warn

.github/actions/setup-python-runner/action.yml

@@ -0,0 +1,27 @@
+name: Setup Python Runner
+description: Setup python-runner for tests
+inputs:
+  metabase-automation-token:
+    description: Token for accessing the python-runner repository
+    required: true
+  python-runner-version:
+    description: Git tag or commit of the python-runner repository to checkout
+    required: false
+    default: 'v0.0.1'
+runs:
+  using: composite
+  steps:
+    - name: Clone and setup python-runner
+      run: |
+        echo "Cloning python-runner repository..."
+        git clone --depth 1 --branch ${{ inputs.python-runner-version }} \
+          https://${{ inputs.metabase-automation-token }}@github.com/metabase/python-runner-container.git python-runner
+
+        echo "Starting python-runner..."
+        cd python-runner
+        make run-ci 1> ../python-runner.out.log 2> ../python-runner.err.log
+        make logs 1>> ../python-runner.out.log 2>> ../python-runner.err.log &
+        cd ..
+      shell: bash
+      env:
+        METABASE_AUTOMATION_USER_TOKEN: ${{ inputs.metabase-automation-token }}

.github/scripts/setup-python-runner.sh

@@ -24,6 +24,7 @@ git clone --depth 1 https://${METABASE_AUTOMATION_USER_TOKEN}@github.com/metabas
 if [ "$SKIP_START" = false ]; then
   echo "Starting python-runner..."
   cd python-runner
-  make run-ci
+  make run-ci 1> ../python-runner.out.log 2> ../python-runner.err.log
+  make logs 1>> ../python-runner.out.log 2>> ../python-runner.err.log &
   cd ..
 fi

.github/workflows/drivers.yml

@@ -6,6 +6,10 @@ on:
       skip:
         type: boolean
         default: false
+      python-runner-version:
+        type: string
+        default: 'v0.0.1'
+        description: 'Git tag or commit of python-runner repository to use'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}-drivers
@@ -135,10 +139,12 @@ jobs:
     - uses: actions/checkout@v4
     - name: Setup python-runner
       if: ${{ matrix.job.needs-python-runner == true }}
-      env:
-        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
-      run: ./.github/scripts/setup-python-runner.sh
+      uses: ./.github/actions/setup-python-runner
+      with:
+        metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+        python-runner-version: ${{ inputs.python-runner-version }}
     - name: Test BigQuery Cloud SDK driver
+      id: test-driver
       uses: ./.github/actions/test-driver
       with:
         junit-name: 'be-tests-bigquery-cloud-sdk-ee'
@@ -154,6 +160,12 @@ jobs:
         aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}
         aws-region: ${{ vars.AWS_REGION }}
         trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
+    - name: Cleanup python-runner
+      if: ${{ always() && matrix.job.needs-python-runner == true }}
+      uses: ./.github/actions/cleanup-python-runner
+      with:
+        upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
+        logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}
 
   be-tests-clickhouse-ee:
     if: ${{ !inputs.skip }}
@@ -183,9 +195,10 @@ jobs:
     - uses: actions/checkout@v4
     - name: Setup python-runner
       if: ${{ matrix.job.needs-python-runner == true }}
-      env:
-        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
-      run: ./.github/scripts/setup-python-runner.sh
+      uses: ./.github/actions/setup-python-runner
+      with:
+        metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+        python-runner-version: ${{ inputs.python-runner-version }}
     - name: Add ClickHouse TLS instance to /etc/hosts
       run: |
         sudo echo "127.0.0.1 server.clickhouseconnect.test" | sudo tee -a /etc/hosts
@@ -202,6 +215,7 @@ jobs:
           clickhouse_cluster_node2
           nginx
     - name: Test Clickhouse driver
+      id: test-driver
       uses: ./.github/actions/test-driver
       with:
         junit-name: 'be-tests-clickhouse-ee'
@@ -217,6 +231,12 @@ jobs:
         aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}
         aws-region: ${{ vars.AWS_REGION }}
         trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
+    - name: Cleanup python-runner
+      if: ${{ always() && matrix.job.needs-python-runner == true }}
+      uses: ./.github/actions/cleanup-python-runner
+      with:
+        upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
+        logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}
 
   be-tests-druid-ee:
     needs: determine-driver-skips
@@ -377,10 +397,12 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: Setup python-runner
-      env:
-        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
-      run: ./.github/scripts/setup-python-runner.sh
+      uses: ./.github/actions/setup-python-runner
+      with:
+        metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+        python-runner-version: ${{ inputs.python-runner-version }}
     - name: Test H2 driver
+      id: test-driver
       uses: ./.github/actions/test-driver
       with:
         junit-name: 'be-tests-h2'
@@ -398,6 +420,12 @@ jobs:
         aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}
         aws-region: ${{ vars.AWS_REGION }}
         trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
+    - name: Cleanup python-runner
+      if: always()
+      uses: ./.github/actions/cleanup-python-runner
+      with:
+        upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
+        logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}
 
   be-tests-mysql-mariadb:
     if: ${{ !inputs.skip }}
@@ -431,7 +459,7 @@ jobs:
           - name: Driver Tests
             test-args: >-
               :only-tags [:mb/driver-tests]
-              :exclude-tags [:mb/transforms-python-test]
+            exclude-tag: ':mb/transforms-python-test'
           - name: Transforms Python Tests
             test-args: >-
               :only-tags [:mb/transforms-python-test]
@@ -491,11 +519,13 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Setup python-runner
-      #if: ${{ matrix.job.needs-python-runner == true }}
-        env:
-          METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
-        run: ./.github/scripts/setup-python-runner.sh
+        if: ${{ matrix.job.needs-python-runner == true }}
+        uses: ./.github/actions/setup-python-runner
+        with:
+          metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+          python-runner-version: ${{ inputs.python-runner-version }}
       - name: Test ${{ matrix.version.name }}
+        id: test-driver
         uses: ./.github/actions/test-driver
         with:
           junit-name: ${{ matrix.version.junit-name }}
@@ -513,6 +543,12 @@ jobs:
           aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}
           aws-region: ${{ vars.AWS_REGION }}
           trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
+      - name: Cleanup python-runner
+        if: always()
+        uses: ./.github/actions/cleanup-python-runner
+        with:
+          upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
+          logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}
 
   be-tests-mongo:
     needs: determine-driver-skips
@@ -556,10 +592,12 @@ jobs:
     - uses: actions/checkout@v4
     - name: Setup python-runner
       if: ${{ matrix.job.needs-python-runner == true }}
-      env:
-        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
-      run: ./.github/scripts/setup-python-runner.sh
+      uses: ./.github/actions/setup-python-runner
+      with:
+        metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+        python-runner-version: ${{ inputs.python-runner-version }}
     - name: Test ${{ matrix.version.name }}
+      id: test-driver
       uses: ./.github/actions/test-driver
       with:
         junit-name: ${{ matrix.version.junit-name }}
@@ -575,6 +613,12 @@ jobs:
         aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}
         aws-region: ${{ vars.AWS_REGION }}
         trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
+    - name: Cleanup python-runner
+      if: ${{ always() && matrix.job.needs-python-runner == true }}
+      uses: ./.github/actions/cleanup-python-runner
+      with:
+        upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
+        logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}
 
   be-tests-mongo-ssl:
     needs: determine-driver-skips
@@ -770,7 +814,7 @@ jobs:
           - name: Driver Tests
             test-args: >-
               :only-tags [:mb/driver-tests]
-              :exclude-tags [:mb/transforms-python-test]
+            exclude-tag: ':mb/transforms-python-test'
           - name: Transforms Python Tests
             test-args: >-
               :only-tags [:mb/transforms-python-test]
@@ -817,11 +861,13 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Setup python-runner
-      #if: ${{ matrix.job.needs-python-runner == true }}
-        env:
-          METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
-        run: ./.github/scripts/setup-python-runner.sh
+        if: ${{ matrix.job.needs-python-runner == true }}
+        uses: ./.github/actions/setup-python-runner
+        with:
+          metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+          python-runner-version: ${{ inputs.python-runner-version }}
       - name: Test ${{ matrix.version.name }} (${{ matrix.job.name }})
+        id: test-driver
         uses: ./.github/actions/test-driver
         with:
           junit-name: 'be-tests-${{ matrix.version.junit-name }}'
@@ -839,6 +885,12 @@ jobs:
           aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}
           aws-region: ${{ vars.AWS_REGION }}
           trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
+      - name: Cleanup python-runner
+        if: always()
+        uses: ./.github/actions/cleanup-python-runner
+        with:
+          upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
+          logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}
 
   be-tests-presto-jdbc-ee:
     needs: determine-driver-skips
@@ -949,10 +1001,12 @@ jobs:
     - uses: actions/checkout@v4
     - name: Setup python-runner
       if: ${{ matrix.job.needs-python-runner == true }}
-      env:
-        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
-      run: ./.github/scripts/setup-python-runner.sh
+      uses: ./.github/actions/setup-python-runner
+      with:
+        metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+        python-runner-version: ${{ inputs.python-runner-version }}
     - name: Test ${{ matrix.job.name }}
+      id: test-driver
       if: ${{ !matrix.job.skip }}
       uses: ./.github/actions/test-driver
       with:
@@ -970,6 +1024,12 @@ jobs:
         aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}
         aws-region: ${{ vars.AWS_REGION }}
         trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
+    - name: Cleanup python-runner
+      if: ${{ always() && matrix.job.needs-python-runner == true }}
+      uses: ./.github/actions/cleanup-python-runner
+      with:
+        upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
+        logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}
 
   be-tests-snowflake-ee:
     if: ${{ !inputs.skip }}
@@ -1011,10 +1071,12 @@ jobs:
     - uses: actions/checkout@v4
     - name: Setup python-runner
       if: ${{ matrix.job.needs-python-runner == true }}
-      env:
-        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
-      run: ./.github/scripts/setup-python-runner.sh
+      uses: ./.github/actions/setup-python-runner
+      with:
+        metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+        python-runner-version: ${{ inputs.python-runner-version }}
     - name: Test Snowflake driver
+      id: test-driver
       uses: ./.github/actions/test-driver
       with:
         junit-name: 'be-tests-snowflake-ee'
@@ -1030,6 +1092,12 @@ jobs:
         aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}
         aws-region: ${{ vars.AWS_REGION }}
         trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
+    - name: Cleanup python-runner
+      if: ${{ always() && matrix.job.needs-python-runner == true }}
+      uses: ./.github/actions/cleanup-python-runner
+      with:
+        upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
+        logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}
 
   be-tests-sparksql-ee:
     needs: determine-driver-skips
@@ -1140,10 +1208,12 @@ jobs:
     - uses: actions/checkout@v4
     - name: Setup python-runner
       if: ${{ matrix.job.needs-python-runner == true }}
-      env:
-        METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
-      run: ./.github/scripts/setup-python-runner.sh
+      uses: ./.github/actions/setup-python-runner
+      with:
+        metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+        python-runner-version: ${{ inputs.python-runner-version }}
     - name: Test ${{ matrix.version.name }}
+      id: test-driver
       uses: ./.github/actions/test-driver
       with:
         junit-name: ${{ matrix.version.junit-name }}
@@ -1159,6 +1229,12 @@ jobs:
         aws-secret-access-key: ${{ secrets.AWS_TEST_RESULTS_SECRET_ACCESS_KEY }}
         aws-region: ${{ vars.AWS_REGION }}
         trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
+    - name: Cleanup python-runner
+      if: ${{ always() && matrix.job.needs-python-runner == true }}
+      uses: ./.github/actions/cleanup-python-runner
+      with:
+        upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
+        logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}
 
   # TODO(dpsutton, 2025-08-20): Re-enable starburst when we figure out why it randomly dies in CI
   # be-tests-starburst-ee:

.github/workflows/e2e-test.yml

@@ -28,6 +28,11 @@ on:
       split_index: # index of the chunk
         required: false
         type: number
+      python-runner-version:
+        type: string
+        default: 'v0.0.1'
+        description: 'Git tag or commit of python-runner repository to use'
+        required: false
 
 jobs:
   e2e-tests:
@@ -65,9 +70,10 @@ jobs:
           token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
 
       - name: Setup python-runner
-        env:
-          METABASE_AUTOMATION_USER_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
-        run: ./.github/scripts/setup-python-runner.sh
+        uses: ./.github/actions/setup-python-runner
+        with:
+          metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+          python-runner-version: ${{ inputs.python-runner-version }}
 
       - name: Prepare Docker containers
         uses: ./.github/actions/e2e-prepare-containers
@@ -246,3 +252,9 @@ jobs:
              const summary = formatSummary(results);
 
              await core.summary.addRaw(summary).write();
+
+      - name: Cleanup python-runner
+        if: always()
+        uses: ./.github/actions/cleanup-python-runner
+        with:
+          upload-logs: false

CLAUDE.md

@@ -29,7 +29,8 @@
   - Use the linter as a way to know that you are adhering to conventions in place in the codebase
 - **Lint Changes:** `./bin/mage kondo-updated HEAD`
 - **Format:** `./bin/mage cljfmt-files [path]`
-- **Test file:** `clojure -X:dev:test :only namespace/test-name`
+- **Run a test:** `./bin-mage run-tests namespace/test-name`
+- **Run all tests in a namespace:** `./bin-mage run-tests namespace`
 - **Check Code Readability** `./bin/mage -check-readable` with optional line-number
   - Run this after every change to Clojure code, only accept readable code
 - **Evaluating Clojure Code** `./bin/mage -repl '<code>'`

README.md

@@ -2,7 +2,7 @@
 
 [Metabase](https://www.metabase.com) is the easy, open-source way for everyone in your company to ask questions and learn from data.
 
-![Metabase Product Screenshot](docs/images/metabase-product-screenshot.png)
+![Metabase Product Screenshot](https://www.metabase.com/images/metabase-product-screenshot-updated.png)
 
 [![Latest Release](https://img.shields.io/github/release/metabase/metabase.svg?label=latest%20release)](https://github.com/metabase/metabase/releases)
 [![codecov](https://codecov.io/gh/metabase/metabase/branch/master/graph/badge.svg)](https://codecov.io/gh/metabase/metabase)
@@ -12,28 +12,29 @@
 
 The easiest way to get started with Metabase is to sign up for a free trial of [Metabase Cloud](https://store.metabase.com/checkout). You get support, backups, upgrades, an SMTP server, SSL certificate, SoC2 Type 2 security auditing, and more (plus your money goes toward improving Metabase). Check out our quick overview of [cloud vs self-hosting](https://www.metabase.com/docs/latest/cloud/cloud-vs-self-hosting). If you need to, you can always switch to [self-hosting](https://www.metabase.com/docs/latest/installation-and-operation/installing-metabase) Metabase at any time (or vice versa).
 
-## Features
+## Key Features
 
-- [Set up in five minutes](https://www.metabase.com/docs/latest/setting-up-metabase.html) (we're not kidding).
+- [Set up in five minutes](https://www.metabase.com/docs/latest/configuring-metabase/setting-up-metabase) (we're not kidding).
 - Let anyone on your team [ask questions](https://www.metabase.com/docs/latest/questions/introduction) without knowing SQL.
 - Use the [SQL editor](https://www.metabase.com/docs/latest/questions/native-editor/writing-sql) for more complex queries.
-- Build handsome, interactive [dashboards](https://www.metabase.com/docs/latest/users-guide/07-dashboards.html) with filters, auto-refresh, fullscreen, and custom click behavior.
+- Build handsome, interactive [dashboards](https://www.metabase.com/docs/latest/dashboards/introduction) with filters, auto-refresh, fullscreen, and custom click behavior.
 - Create [models](https://www.metabase.com/learn/metabase-basics/getting-started/models) that clean up, annotate, and/or combine raw tables.
-- Define canonical [segments and metrics](https://www.metabase.com/docs/latest/administration-guide/07-segments-and-metrics.html) for your team to use.
-- Send data to Slack or email on a schedule with [dashboard subscriptions](https://www.metabase.com/docs/latest/users-guide/dashboard-subscriptions).
-- Set up [alerts](https://www.metabase.com/docs/latest/users-guide/15-alerts.html) to have Metabase notify you when your data changes.
-- [Embed charts and dashboards](https://www.metabase.com/docs/latest/administration-guide/13-embedding.html) in your app, or even [your entire Metabase](https://www.metabase.com/docs/latest/enterprise-guide/full-app-embedding.html).
+- Define canonical [segments and metrics](https://www.metabase.com/docs/latest/data-modeling/metrics) for your team to use.
+- Send data to Slack or email on a schedule with [dashboard subscriptions](https://www.metabase.com/docs/latest/dashboards/subscriptions).
+- Set up [alerts](https://www.metabase.com/docs/latest/questions/alerts) to have Metabase notify you when your data changes.
+- [Embed charts and dashboards](https://www.metabase.com/docs/latest/embedding/introduction) in your app, or even [your entire Metabase](https://www.metabase.com/docs/latest/embedding/interactive-embedding).
+- Build with the [Embedded Analytics SDK for React](https://www.metabase.com/docs/latest/embedding/sdk/introduction) to embed standalone components with custom styling that matches your app's design.
 
-Take a [tour of Metabase](https://www.metabase.com/learn/getting-started/tour-of-metabase).
+Take a [tour of Metabase](https://www.metabase.com/learn/metabase-basics/overview/tour-of-metabase).
 
 ## Supported databases
 
 - [Officially supported databases](./docs/databases/connecting.md#connecting-to-supported-databases)
-- [Community drivers](./docs/developers-guide/partner-and-community-drivers.md)
+- [Community drivers](./docs/developers-guide/community-drivers.md)
 
 ## Installation
 
-Metabase can be run just about anywhere. Check out our [Installation Guides](https://www.metabase.com/docs/latest/operations-guide/installing-metabase).
+Metabase can be run just about anywhere. Check out our [Installation Guides](https://www.metabase.com/docs/latest/installation-and-operation/installing-metabase).
 
 ## Contributing
 
@@ -43,7 +44,7 @@ In order to spin up a development environment, you need to start the front end a
 
 ### Frontend quick setup
 
-The following command will install the Javascript dependencies:
+The following command will install the JavaScript dependencies:
 
 ```bash
 yarn install
@@ -63,7 +64,7 @@ yarn build-hot
 
 ### Backend  quick setup
 
-In order to run the backend, you'll need to build the drivers first, and then start the backend:
+To run the backend, you'll need to build the drivers first, and then start the backend:
 
 ```bash
 ./bin/build-drivers.sh
@@ -78,13 +79,13 @@ We want Metabase to be available in as many languages as possible. See which tra
 
 ## Extending Metabase
 
-Hit our Query API from Javascript to integrate analytics. Metabase enables your application to:
+Hit our Query API from JavaScript to integrate analytics. Metabase enables your application to:
 
 - Build moderation interfaces.
 - Export subsets of your users to third party marketing automation software.
 - Provide a custom customer lookup application for the people in your company.
 
-Check out our guide, [Working with the Metabase API](https://www.metabase.com/learn/administration/metabase-api).
+Check out our guide, [Working with the Metabase API](https://www.metabase.com/learn/metabase-basics/administration/administration-and-operation/metabase-api).
 
 ## Security Disclosure
 
@@ -98,6 +99,14 @@ See [LICENSE.txt](./LICENSE.txt) for details.
 
 Unless otherwise noted, all files © 2025 Metabase, Inc.
 
-## [Metabase Experts](https://www.metabase.com/partners/)
+## Community
+
+- [Discourse](https://discourse.metabase.com/)
+- [Twitter](https://x.com/metabase)
+- [LinkedIn](https://www.linkedin.com/company/metabase/)
+- [YouTube](https://www.youtube.com/@metabasedata)
+- [Reddit](https://www.reddit.com/r/Metabase/)
+
+## Metabase Experts
 
 If you’d like more technical resources to set up your data stack with Metabase, connect with a [Metabase Expert](https://www.metabase.com/partners/?utm_source=readme&utm_medium=metabase-expetrs&utm_campaign=readme).

bb.edn

@@ -1,4 +1,4 @@
-{:min-bb-version "1.12.197"
+{:min-bb-version "1.12.208"
  ;; we put path as bin, and everything is in the ./mage subdirectory,
  ;; so the namespaces are mage.cli, mage.format, etc.
  :paths          ["mage" "test" "bin/lint-migrations-file/src"]
@@ -161,7 +161,7 @@
    :task     (stats-repl/connect (cli/parse! (current-task)))}
 
   notify
-  {:doc      "Send notification for a comment (that already exists"
+  {:doc      "Send notification for a comment (that already exists)"
    :requires [[mage.be-dev :refer [nrepl-eval]]]
    :options  [["-c" "--comment-id ID" "Notify about this comment id"]
               ["-p" "--port PORT" "Port to use for the task, defaults to value in .nrepl-port"]]
@@ -172,6 +172,25 @@
                  (nrepl-eval "metabase-enterprise.comments.core" code (:port opts))
                  (nrepl-eval "metabase-enterprise.comments.core" code)))}
 
+  run-tests
+  {:doc        "Run a tests (or a single test) against nrepl server"
+   :requires   [[clojure.string :as str]
+                [mage.be-dev :refer [nrepl-eval]]]
+   :arg-schema [:tuple [:string {:name "NS/TEST" :desc "or just NS"}]]
+   :options    [["-p" "--port PORT" "Port to use for the task, defaults to value in .nrepl-port"]]
+   :task       (let [{:keys  [options]
+                      [test] :arguments} (cli/parse! (current-task))
+                     test                (if (or (str/starts-with? test "test/")
+                                                 (str/starts-with? test "enterprise/"))
+                                           ;; paths should come in as strings
+                                           (format "\"%s\"" test)
+                                           ;; ns or var should be a symbol
+                                           (str "'" test))
+                     code (format
+                           "(do ((requiring-resolve 'dev.reload/reload!)) (find-and-run-tests-repl {:only %s}))"
+                           test)]
+                 (nrepl-eval "mb.hawk.core" (str code) (:port options)))}
+
   ls {:doc      "List all mage public and private tasks"
       :examples [["./bin/mage ls" "List all tasks"]]
       :requires [[mage.util :as u]]

deps.edn

@@ -90,7 +90,7 @@
                                              :git/sha "4998a1fdd9d3713d7034d04509e9212204773bf2"
                                              :git/url "https://github.com/eerohele/pp"}
   ;; TODO: replace back to normal artifact
-  io.github.metabase/macaw                  {:mvn/version "0.2.27"}             ; Parse native SQL queries
+  io.github.metabase/macaw                  {:mvn/version "0.2.28"}             ; Parse native SQL queries
   io.github.resilience4j/resilience4j-retry ^:antq/exclude                      ; 2.x do not support Java 8
                                             {:mvn/version "1.7.1"}              ; Support for retrying operations
   io.prometheus/simpleclient_hotspot        {:mvn/version "0.16.0"}             ; prometheus jvm collector

dev/src/dev/reload.clj

@@ -0,0 +1,151 @@
+(ns dev.reload
+  (:require
+   [clojure.java.shell :as shell]
+   [clojure.string :as str]
+   [metabase.util :as u])
+  (:import
+   (java.io File)
+   (java.nio.file Files LinkOption Paths)))
+
+(set! *warn-on-reflection* true)
+
+#_:clj-kondo/ignore
+(defonce *reload-timestamps (atom {}))
+
+(defn system-classpath
+  "Returns a sequence of File paths from the 'java.class.path' system
+  property."
+  []
+  (into [] (comp (filter #(not (str/ends-with? % ".jar")))
+                 (map #(File. ^String %))
+                 (filter #(.isDirectory ^File %))
+                 (map #(.getCanonicalPath ^File %)))
+        (.split (System/getProperty "java.class.path")
+                (System/getProperty "path.separator"))))
+
+(defn find-classpath-root
+  "Finds the classpath directory that contains the given file path.
+   Returns the portion of the path to strip, or nil if not found."
+  [classpath file-path]
+  (let [canonical-file (try
+                         (.getCanonicalPath (File. ^String file-path))
+                         (catch Exception _ file-path))]
+    (->> classpath
+         (filter #(str/starts-with? canonical-file %))
+         (sort-by count >)              ; Longest match first
+         first)))
+
+(defn git-changed-files
+  "Returns a set of changed/added/untracked files from git status."
+  []
+  (let [{:keys [exit out]} (shell/sh "git" "status" "--porcelain")]
+    (if (and (zero? exit)
+             (not (str/blank? out)))
+      (->> (str/split-lines out)
+           (map #(str/trim (subs % 3))) ; Remove status prefix (e.g., "M ", "A ", "??")
+           (filter #(str/ends-with? % ".clj"))
+           set)
+      #{})))
+
+(defn file-last-modified
+  "Returns the last modified time of a file in milliseconds."
+  [file-path]
+  (try
+    (-> (Paths/get file-path (into-array String []))
+        (Files/getLastModifiedTime (into-array LinkOption []))
+        (.toMillis))
+    (catch Exception _ nil)))
+
+(defn relpath->ns
+  "foo/bar_baz.clj -> foo.bar-baz"
+  [relpath]
+  (-> relpath
+      (str/replace #"\.clj$" "")
+      (str/replace #"/" ".")
+      (str/replace #"_" "-")
+      symbol))
+
+(defn file-path->namespace
+  "Converts a file path to a namespace symbol using classpath information.
+   Example: enterprise/backend/src/foo/bar/baz.clj -> foo.bar.baz"
+  [file-path classpath]
+  (when (and file-path (re-find #".cljc?$" file-path))
+    (when-let [root (find-classpath-root classpath file-path)]
+      (let [canonical-file (try
+                             (.getCanonicalPath (File. ^String file-path))
+                             (catch Exception _ file-path))
+            relative-path (subs canonical-file (inc (count root)))]
+        (relpath->ns relative-path)))))
+
+(defn loaded-namespaces
+  "Returns a set of all currently loaded namespace symbols."
+  []
+  (set (map ns-name (all-ns))))
+
+(defn needs-reload?
+  "Checks if a file needs reloading based on its last modified time."
+  [file-path ns-sym]
+  (let [last-reload   (get @*reload-timestamps ns-sym)
+        last-modified (file-last-modified file-path)]
+    (or (nil? last-reload)
+        (nil? last-modified)
+        (> last-modified last-reload))))
+
+(defn changed-namespaces []
+  (let [changed-files  (git-changed-files)
+        loaded-ns      (loaded-namespaces)
+        classpath-dirs (system-classpath)
+        candidates     (->> changed-files
+                            (map (fn [f] {:file f :ns (file-path->namespace f classpath-dirs)}))
+                            (filter :ns)
+                            (filter #(loaded-ns (:ns %))))
+        to-reload      (filter #(needs-reload? (:file %) (:ns %)) candidates)
+        skipped        (remove #(needs-reload? (:file %) (:ns %)) candidates)]
+    {:to-reload to-reload
+     :skipped   skipped}))
+
+(defn reload-namespaces!
+  [to-reload]
+  (->> (for [{:keys [_file ns]} to-reload]
+         (try
+           (require ns :reload)
+           (swap! *reload-timestamps assoc ns (System/currentTimeMillis))
+           [:reloaded ns]
+           (catch Exception e
+             [:failed {:ns ns :error (.getMessage e)}])))
+       (u/group-by first second)))
+
+(defn reload!
+  "Reloads all changed files from git that are already loaded as namespaces
+   and have been modified since last reload.
+   Returns a map with :reloaded, :skipped, and :failed keys."
+  []
+  (let [{:keys [to-reload
+                skipped]} (changed-namespaces)
+        _                 (println (format "Reloading %s namespaces (%s skipped):" (count to-reload) (count skipped)))
+        {:keys [reloaded
+                failed]
+         :as   res}       (reload-namespaces! to-reload)]
+    (doseq [ns reloaded]
+      (println "  ✓" ns))
+    (doseq [{:keys [ns error]} failed]
+      (println "  ✗" ns ":" error))
+    (assoc res :skipped skipped)))
+
+(defn clear-reload-history!
+  "Clears the reload timestamp history, forcing all files to reload next time."
+  []
+  (reset! *reload-timestamps {})
+  (println "Reload history cleared."))
+
+(defn show-classpath
+  "Shows all classpath directories for debugging."
+  []
+  (println "\n=== Classpath Directories ===")
+  (run! #(println (.getCanonicalPath ^File %)) (sort (system-classpath))))
+
+(comment
+  (show-classpath) ; to see what's in your classpath
+  (clear-reload-history!) ; if you want to force reload everything
+  (changed-namespaces)
+  (reload!))

docs/ai/metabot.md

@@ -53,7 +53,7 @@ Some tips:
 Hover over Metabot's response to:
 
 - Copy the response.
-- Give thumbs-up/thumbs-down feedback on responses. If you downvote a response, you can optionally contribute a bug report to help us improve Metabot.
+- Give thumbs-up/thumbs-down [feedback on responses](#giving-feedback-on-metabot-responses). If you upvote a response, you can optionally add some feedback. If you downvote a response, you can optionally contribute a bug report to help us improve Metabot.
 - Re-run the prompt with Metabot. This is useful if you've updated the chart or just want to have Metabot take another pass (since AI responses aren't deterministic, Metabot may give a different response on another run).
 
 ## Analyze charts with Metabot
@@ -68,7 +68,7 @@ You can also ask Metabot to tell you about specific tables in your database.
 
 ## How Metabot uses the query builder
 
-When you ask Metabot to create a chart from natural language, it first looks for existing questions that might answer your request. If it finds a relevant question, it'll point you to that instead of creating something new. Otherwise, Metabot will use the [query builder](../questions/query-builder/editor.md) to create a new chart for you. 
+When you ask Metabot to create a chart from natural language, it first looks for existing questions that might answer your request. If it finds a relevant question, it'll point you to that instead of creating something new. Otherwise, Metabot will use the [query builder](../questions/query-builder/editor.md) to create a new chart for you.
 
 Keep in mind that Metabot is still learning the query builder. Metabot can only handle basic query builder operations, and it lacks access to the library of [custom expressions](../questions/query-builder/expressions-list.md). Metabot is also limited to single-level aggregation and grouping, so if you need more complex analysis, you can take over and refine the query yourself, or switch to the SQL editor.
 
@@ -98,3 +98,19 @@ When you get an error in a SQL query, you can click the **Have Metabot fix it**
 If Metabot creates a query or takes you to a new item but you want to return to the previous screen, you can navigate using your browser's back button.
 
 You can also save any chart that Metabot creates to a dashboard or collection.
+
+## Giving feedback on Metabot responses
+
+When you hover over Metabot's responses, you'll see options to give feedback.
+
+Thumbs up are just as helpful as thumbs down. If you give a thumbs down, you can optionally provide more details about the issue and set an issue type:
+
+- UI bug
+- Took incorrect actions
+- Overall refusal
+- Did not follow request
+- Not factually correct
+- Incomplete response
+- Other
+
+> When you submit feedback, the form you send may contain sensitive data from your conversation.

docs/ai/settings.md

@@ -28,7 +28,7 @@ When embedding Metabot in your app, you can select a collection for Metabot:
 1. Go to **Settings** > **Admin settings** > **AI**.
 2. Click **Embedded Metabot**.
 3. In the **Collection Embedded Metabot can use** section, click **Pick a collection**.
-3. Select the collection that contains the models and metrics you want Metabot to use.
+4. Select the collection that contains the models and metrics you want Metabot to use.
 
 Metabot will use the models and metrics in that collection to help answer questions and generate queries. You can change this collection at any time. To give Metabot access to all collections, you can set the collection to the root collection, called "Our Analytics" (the default).
 
@@ -91,6 +91,16 @@ Under the hood, Metabase powers Metabot with a variety of generative models. For
 
 To get the best results, we (the Metabase team) use internal benchmarks to determine which AI models Metabot should use for different tasks. And we are constantly iterating on performance, so Metabot will continue to improve over time.
 
-## We don't collect or store the prompts you send to Metabot
+## Unless you submit feedback, we don't collect or store the prompts you send to Metabot
 
-We've intentionally limited what Metabot can do. Metabot lacks access to API keys, and it can't create assets, write data, or send data outside of your Metabase. Your questions and conversations remain private to your Metabase. We do collect some metadata to gauge and improve usage, but we don't train Metabot on your prompts or your data—because we can't see them!
+We've intentionally limited what Metabot can do. Metabot lacks access to API keys, and it can't create assets, write data, or send your data outside of your Metabase. Your questions and conversations remain private to your Metabase (unless you [submit feedback](./metabot.md#giving-feedback-on-metabot-responses)). We do collect some metadata to gauge and improve usage.
+
+### What Metabot can see
+
+Metabot has access to your Metabase metadata and some data values to help answer your questions:
+
+- **Table, Question, Model, Dashboard, and Metric metadata**: Metabot can see the structure and configuration of your content
+- **Sample field values**: When you ask questions like "Filter everyone from Wisconsin," Metabot might check the values in the state field to understand how the data is stored (like "WI" vs "Wisconsin"). See [syncs](../databases/sync-scan.md).
+- **Timeseries data**: For chart analysis, Metabot might see the timeseries data used to draw certain visualizations, depending on the chart type.
+
+When you [submit feedback](./metabot.md#giving-feedback-on-metabot-responses), however, the form you send may contain sensitive data from your conversation.

docs/api.html

@@ -14971,6 +14971,35 @@ redirect_from:
         "tags" : [ "/api/ee/metabot-v3/document" ]
       }
     },
+    "/api/ee/metabot-v3/feedback" : {
+      "post" : {
+        "summary" : "POST /api/ee/metabot-v3/feedback",
+        "description" : "Proxy Metabot feedback to Harbormaster, adding the premium embedding token.",
+        "parameters" : [ ],
+        "responses" : {
+          "2XX" : {
+            "description" : "Successful response"
+          },
+          "4XX" : {
+            "description" : "Client error response"
+          },
+          "5XX" : {
+            "description" : "Server error response"
+          }
+        },
+        "requestBody" : {
+          "content" : {
+            "application/json" : {
+              "schema" : {
+                "type" : "object",
+                "properties" : { }
+              }
+            }
+          }
+        },
+        "tags" : [ "/api/ee/metabot-v3" ]
+      }
+    },
     "/api/ee/metabot-v3/metabot/" : {
       "get" : {
         "summary" : "GET /api/ee/metabot-v3/metabot/",
@@ -15821,6 +15850,16 @@ redirect_from:
                         },
                         "value" : {
                           "anyOf" : [ {
+                            "type" : "object",
+                            "additionalProperties" : {
+                              "anyOf" : [ {
+                                "type" : "string",
+                                "minLength" : 1
+                              }, {
+                                "type" : "boolean"
+                              } ]
+                            }
+                          }, {
                             "type" : "string",
                             "minLength" : 1
                           }, {
@@ -16176,7 +16215,18 @@ redirect_from:
           }
         }, {
           "in" : "query",
-          "name" : "transform_tag_ids",
+          "name" : "last_run_statuses",
+          "required" : false,
+          "schema" : {
+            "type" : "array",
+            "items" : {
+              "type" : "string",
+              "enum" : [ "started", "succeeded", "failed", "timeout" ]
+            }
+          }
+        }, {
+          "in" : "query",
+          "name" : "tag_ids",
           "required" : false,
           "schema" : {
             "type" : "array",
@@ -16498,7 +16548,38 @@ redirect_from:
       "get" : {
         "summary" : "GET /api/ee/transform/",
         "description" : "Get a list of transforms.",
-        "parameters" : [ ],
+        "parameters" : [ {
+          "in" : "query",
+          "name" : "last_run_start_time",
+          "required" : false,
+          "schema" : {
+            "type" : "string",
+            "minLength" : 1
+          }
+        }, {
+          "in" : "query",
+          "name" : "last_run_statuses",
+          "required" : false,
+          "schema" : {
+            "type" : "array",
+            "items" : {
+              "type" : "string",
+              "enum" : [ "started", "succeeded", "failed", "timeout" ]
+            }
+          }
+        }, {
+          "in" : "query",
+          "name" : "tag_ids",
+          "required" : false,
+          "schema" : {
+            "type" : "array",
+            "items" : {
+              "description" : "value must be an integer greater or equal to than zero.",
+              "type" : "integer",
+              "minimum" : 0
+            }
+          }
+        } ],
         "responses" : {
           "2XX" : {
             "description" : "Successful response"

docs/configuring-metabase/caching.md

@@ -122,11 +122,12 @@ If you have databases connected to Metabase that are set to **Use default** poli
 To set a caching policy for a dashboard, you must have [curate access](../permissions/collections.md#curate-access) to the dashboard's collection.
 
 1. Go to your dashboard.
-2. Click on the **info** icon.
-3. Click **Caching policy**.
-4. Select the [caching invalidation policy](#cache-invalidation-policies).
-5. Optional: turn on [refresh cache automatically](#refresh-cache-automatically).
-6. Save your changes.
+2. Click on the **three dots** icon in the top-right corner of the dashboard and choose **Edit settings**.
+3. By default, each question will use the database default caching settings. Click the current caching policy to change it.
+4. Select the new [caching policy](#cache-invalidation-policies).
+5. Optional: turn on [refresh cache automatically](#refresh-cache-automatically)
+6. Optional: to clear the cache for all questions on a dashboard, click **Clear cache for this dashboard** at the bottom of the setting sidebar.
+7. Save your changes.
 
 ### Question caching policy
 

docs/configuring-metabase/config-template.md

@@ -188,6 +188,7 @@ config:
     ldap-security: none
     ldap-sync-user-attributes: true
     ldap-sync-user-attributes-blacklist: userPassword,dn,distinguishedName
+    ldap-timeout-seconds: 15.0
     ldap-user-base: null
     ldap-user-filter: (&(objectClass=inetOrgPerson)(|(uid={login})(mail={login})))
     ldap-user-provisioning-enabled: true

docs/configuring-metabase/environment-variables.md

@@ -1058,6 +1058,14 @@ Should we sync user attributes when someone logs in via LDAP?
 
 Comma-separated list of user attributes to skip syncing for LDAP users.
 
+### `MB_LDAP_TIMEOUT_SECONDS`
+
+- Type: double
+- Default: `15.0`
+- [Configuration file name](./config-file.md): `ldap-timeout-seconds`
+
+Maximum time, in seconds, to wait for LDAP server before falling back to local authentication.
+
 ### `MB_LDAP_USER_BASE`
 
 - Type: string

docs/databases/uploads.md

@@ -19,7 +19,7 @@ To manage upload settings, admins can hit cmd/ctrl + K and search for "Settings
 - [PostgreSQL](../databases/connections/postgresql.md)
 - [MySQL](../databases/connections/mysql.md)
 - [Redshift](../databases/connections/redshift.md)
-- [ClickHouse](../databases/connections/clickhouse.md)
+- [ClickHouse](../databases/connections/clickhouse.md) (only supported on ClickHouse Cloud)
 
 ## Setting up uploads
 

docs/developers-guide/driver-changelog.md

@@ -29,6 +29,8 @@ title: Driver interface changelog
 - Added `metabase.driver/type->database-type` multimethod that returns the database type for a given Metabase
   type (from the type hierarchy) as a HoneySQL spec. This method handles general Metabase base types.
 
+- Added driver multimethods driver/native-result-metadata, driver/validate-native-query-fields, driver.sql/resolve-field, driver.sql.normalize-unquoted-name, driver.sql.normalize/reserved-literal, driver.sql.references/find-used-fields, driver.sql.references/find-returned-fields, and driver.sql.references/field-references-impl for use with the :dependencies/native feature.
+
 - Added `metabase.driver/insert-from-source!` multimethod that abstracts data insertion from various sources
   into existing tables. This multimethod dispatches on both the driver and the data source type
   (`:rows` or `:jsonl-file`). It allows drivers to optimize based on the data source type and returns the number

docs/documents/introduction.md

@@ -42,9 +42,9 @@ If however, you delete the document's copy, that copy is permanently deleted. It
 
 ### Charts generated by Metabot only exist in the document
 
-If Metabot generates a chart, Metabase saves that question to the document itself. If you delete it, it's gone forever. You won't be able to recover the question from the trash.
+If [Metabot](../ai/metabot.md) generates a chart, Metabase saves that question to the document itself. If you delete it, it's gone forever. You won't be able to recover the question from the trash.
 
-### Editing charts
+## Editing charts
 
 To edit a chart, click the **three-dot** menu in the chart's upper right. You can:
 
@@ -54,6 +54,22 @@ To edit a chart, click the **three-dot** menu in the chart's upper right. You ca
 
 Edits only affect the chart in the document. If you added the chart from a collection, Metabase will have created a copy of the chart. Any changes made to the chart in the document won't affect the original question.
 
+## Arranging charts
+
+![Document layout](./images/document-layout.png)
+
+To rearrange charts, simply click and hold on the chart and drag it to where you want it in the document.
+
+You can put up to three charts side by side (more than three charts side-by-side gets crowded). To resize charts, you can click on the edges to adjust their size. 
+
+When there are multiple charts in the same row, you can resize them horizontally by clicking between the charts and sliding left or right to resize.
+
+To resize a chart's height, click the bottom of the chart and resize the chart up or down.
+
+For now, you can only put charts side-by-side (not text elements).
+
+After arranging your charts, save your changes!
+
 ## Formatting text
 
 Documents support Markdown syntax for:
@@ -71,9 +87,13 @@ If you paste a link into a document, Metabase will turn it into a clickable link
 
 ## Commenting on documents
 
-To comment on a document, click the chat bubble next to the section you want to comment on.
+To comment on a text element, click the chat bubble next to the section. You can only comment on whole sections; you can't highlight text to comment on.
 
-You can comment on whole sections like charts, headings, or text blocks. You can't highlight portions of text to comment on.
+To comment on a chart:
+
+1. Hover over the chart.
+2. Click the three-dot menu.
+3. Select **Comment**.
 
 You can:
 
@@ -84,7 +104,9 @@ You can:
 
 ### Comment notifications
 
-If someone mentions you in a thread, or responds to a comment in a thread you're in, Metabase will send you an email to let you know. Currently, there's no central place to view all of your threads and mentions, and no way to disable email notifications for comments.
+If someone mentions you in a thread, or responds to a comment in a thread you're in, Metabase will send you an email to let you know.
+
+Currently, there's no central place to view all of your threads and mentions, and no way to disable email notifications for comments.
 
 ### Deleting comments
 

docs/embedding/embedded-analytics-js.md

@@ -9,6 +9,8 @@ summary: Getting started with Embedded Analytics JS for embedding Metabase entit
 
 Embedded analytics JS lets you embed Metabase entities like questions, dashboards, or even the query builder into your own application using customizable components.
 
+{% include shared/in-page-promo-embedding-workshop.html %}
+
 Embedded Analytics JS is a JavaScript library built on top of Metabase's [Embedded Analytics React SDK](./sdk/introduction.md). But it does not require using React or setting up full SDK embedding.
 Unlike with [interactive embedding](./interactive-embedding.md), where you embed the entire Metabase app in an iframe, Embedded Analytics JS lets you choose from a set of predefined components like a single chart, a dashboard with optional drill-through, or query builder, and customize those components.
 

docs/embedding/interactive-embedding.md

@@ -11,12 +11,10 @@ redirect_from:
 
 {% include shared/in-page-promo-embedding-workshop.html %}
 
-**Interactive embedding** is what you want if you want to offer [multi-tenant, self-service analytics](https://www.metabase.com/learn/metabase-basics/embedding/multi-tenant-self-service-analytics).
+**Interactive embedding** lets you embed the entire Metabase app in an iframe. Interactive embedding integrates your [permissions](../permissions/introduction.md) and [SSO](../people-and-groups/start.md#authentication) to give people the right level of access to [query](../questions/query-builder/editor.md) and [drill-down](https://www.metabase.com/learn/metabase-basics/querying-and-dashboards/questions/drill-through) into your data.
 
 > If you are just starting out with Metabase embedding, consider using [Embedded Analytics JS](./embedded-analytics-js.md) instead of interactive embedding - it's an improved, more customizable option for embedding interactive Metabase elements. Interactive embedding remains fully supported.
 
-Interactive embedding is the only type of embedding that integrates with your [permissions](../permissions/introduction.md) and [SSO](../people-and-groups/start.md#authentication) to give people the right level of access to [query](https://www.metabase.com/glossary/query-builder) and [drill-down](https://www.metabase.com/learn/metabase-basics/querying-and-dashboards/questions/drill-through) into your data.
-
 ## Interactive embedding demo
 
 To get a feel for what you can do with interactive embedding, check out our [interactive embedding demo](https://www.metabase.com/embedding-demo).

docs/embedding/introduction.md

@@ -48,7 +48,6 @@ If you'd like to share your data with the good people of the internet, admins ca
 | [Basic appearance customization](../configuring-metabase/appearance.md)\*\*                                                     | ✅                                 | ✅                               | ✅                                        | ✅                              | ✅                                     |
 | [Advanced theming](./sdk/appearance.md)                                                                                         | ✅                                 | ✅                               | ❌                                        | ❌                              | ❌                                     |
 | View usage of embeds with [usage analytics](../usage-and-performance-tools/usage-analytics.md)                                  | ✅                                 | ✅                               | ✅                                        | ❌                              | ❌                                     |
-| [Actions on dashboards](../dashboards/actions.md)                                                                               | ✅                                 | ✅                               | ✅                                        | ❌                              | ❌                                     |
 | Embed individual Metabase components                                                                                            | ✅                                 | ✅                               | ❌                                        | ❌                              | ❌                                     |
 | Manage access and interactivity per component                                                                                   | ✅                                 | ✅                               | ❌                                        | ❌                              | ❌                                     |
 | Custom layouts                                                                                                                  | ✅                                 | ❌                               | ❌                                        | ❌                              | ❌                                     |
@@ -56,7 +55,7 @@ If you'd like to share your data with the good people of the internet, admins ca
 
 \* Each embedding type allows data downloads by default, but only [Pro and Enterprise](https://www.metabase.com/pricing/) plans can disable data downloads.
 
-\*\* Requires [Pro and Enterprise](https://www.metabase.com/pricing/) for any embedding type.
+\*\* Requires a [Pro and Enterprise](https://www.metabase.com/pricing/) plan for any embedding type.
 
 ### Embedded analytics SDK vs JS
 

docs/embedding/sdk/ai-chat.md

@@ -24,9 +24,11 @@ You can check out a [demo of the AI chat component](https://embedded-analytics-s
 ```typescript
 {% include_file "{{ dirname }}/snippets/questions/ai-question.tsx" %}
 ```
+
 ## API reference
 
 - [Component](./api/MetabotQuestion.html)
+- [Props](./api/MetabotQuestionProps.html)
 
 ## Setting up AI chat
 
@@ -40,3 +42,11 @@ To configure your embedded AI chat in your Metabase:
 When embedding the Metabot component in your app, you should specify a collection that embedded Metabot is allowed to use for creating queries. Embedded Metabot will only have access to that collection.
 
 For tips and more, see [Metabot settings](../../ai/settings.md).
+
+## Layout
+
+Use the `layout` prop to specify which layout to use for the Metabot component:
+
+- `auto` (default): Metabot uses the `stacked` layout on mobile screens, and a `sidebar` layout on larger screens.
+- `stacked`: the question visualization stacks on top of the chat interface.
+- `sidebar`: the question visualization appears to the left of the chat interface, which is on a sidebar on the right.

docs/installation-and-operation/serialization.md

@@ -50,6 +50,8 @@ Metabase will only export the following entities:
 - Collections (but personal collections don't get exported unless explicitly specified them through [export options](#customize-what-gets-exported))
 - Dashboards
 - Saved questions
+- Transforms (including jobs)
+- Documents (without comments)
 - Actions
 - Models
 - Metrics
@@ -61,7 +63,7 @@ Metabase will only export the following entities:
 - Events and timelines
 - Database connection strings (only if specified through [export options](#customize-what-gets-exported))
 
-All other entities—including users, groups, permissions, alerts, subscriptions—won't get exported.
+All other entities—including users, groups, permissions, alerts, subscriptions, document comments—won't get exported.
 
 Metabase will export its artifacts to a directory of YAML files. The export includes:
 

docs/questions/visualizations/map.md

@@ -21,7 +21,7 @@ When you select the **Map** visualization setting, Metabase will automatically t
 
 ## Pin map
 
-Pin maps display pins for individual data points on the map. They work best for displaying unaggregated geographical data.
+Pin maps display markers or tiles for individual data points on the map. They work best for displaying small amounts of unaggregated geographical data.
 
 ### Pin map data shape
 
@@ -30,7 +30,7 @@ To build a pin map, you need a query that returns a result that has latitude and
 ![Pin map data shape](../images/pin-data-shape.png)
 ![Pin map with a tooltip](../images/pin-map-with-tooltip.png)
 
-Note that the tooltips will only be displayed when the map is sufficiently zoomed in using [**Draw box to filter**](#draw-box-to-filter).
+Note that the tooltips will only be displayed when using the [marker pin type](#choose-pin-type).
 
 ### Create a pin map
 
@@ -42,9 +42,22 @@ To create a pin map:
 
    Otherwise, click on the **Gear** icon to go to visualization settings, choose **Map type: Pin map**, and pick columns that contain latitude and longitude coordinates.
 
-### Pin maps display 2000 pins by default
+### Choose pin type
 
-By default, pin maps will display 2000 pins even if there are more rows in your query results (this limit is the same for every chart displaying unaggregated data in Metabase). You can use the environment variable [`MB_UNAGGREGATED_QUERY_ROW_LIMIT`](../../configuring-metabase/environment-variables.md#mb_unaggregated_query_row_limit) to increase the number of data points rendered on charts based on unaggregated queries, but keep in mind that this setting will affect all charts—not just the pin maps—and it might significantly slow down your Metabase and your browser.
+Metabase can display individual data points on a map as markers or tiles (small squares).
+
+![Pin types](../images/pin-type.png)
+
+Maps default to markers for results fewer then 1,000 rows, and to tiles for results with 1,000 rows or more.
+
+To change the pin type:
+
+1. While viewing the map, click the **gear** icon in the bottom left.
+2. In the visualization settings, choose the pin type in the **Pin type** dropdown.
+   - **Markers**. Maps display up to 1,000 markers. If you choose the marker pin type for results with more than 1,000 rows, Metabase will still only display 1,000 markers. Maps default to markers for results fewer then 1,000 rows.
+   - **Tiles**. Maps display up to 2,000 tiles. Maps default to tiles for results greater than 1,000 rows. Tooltips will not be displayed for tiles.
+
+By default, maps can't display more than 2,000 individual unaggregated data points, regardless of the pin type. This limit is the same for every chart displaying unaggregated data in Metabase. To increase the number of data points rendered on charts based on unaggregated queries, you can use the environment variable [`MB_UNAGGREGATED_QUERY_ROW_LIMIT`](../../configuring-metabase/environment-variables.md#mb_unaggregated_query_row_limit). This setting will affect data points on _all_ charts—not just the pin maps—so increasing the limit too much could significantly slow down your Metabase and your browswer.
 
 If you need to display a large number of geographical data points, consider using a grid map to display the distribution instead.
 

e2e/runner/embedding-sdk/sample-apps/constants/sample-app-setup-configs.js

@@ -23,8 +23,8 @@ if (!BASE_ENV.PREMIUM_EMBEDDING_TOKEN) {
 }
 
 const BASE_SETUP_CONFIG = {
-  "docker-up-command": "yarn docker:up",
-  "docker-down-command": "yarn docker:down",
+  "docker-up-command": "yarn docker:local-dist:up",
+  "docker-down-command": "yarn docker:rm",
   "docker-env-example-path": ".env.docker.example",
   "docker-env-path": ".env.docker",
   defaultBranch: BRANCH_NAME,

e2e/runner/embedding-sdk/sample-apps/helpers/copy-shoppy-metabase-app-db-dump.ts

@@ -3,7 +3,7 @@ import * as path from "path";
 
 import { E2E_TMP_FOLDER_PATH } from "../constants/e2e-tmp-folder-path";
 
-const LOCAL_DIST_PATH = "./local-dist";
+const METABASE_FOLDER_PATH = "./metabase";
 
 // See `docs/developers-guide/e2e-tests.md` for more info
 export function copyShoppyMetabaseAppDBDump(rootPath: string) {
@@ -12,11 +12,11 @@ export function copyShoppyMetabaseAppDBDump(rootPath: string) {
     "db_dumps/shoppy_metabase_app_db_dump.sql",
   );
 
-  // The dump must be copied to the Shoppy's `local-dist` directory.
+  // The dump must be copied to the Shoppy's `metabase` directory.
   // After that it will be used by the Shoppy's Docker container when it is built and started.
   const destinationPath = path.join(
     rootPath,
-    LOCAL_DIST_PATH,
+    METABASE_FOLDER_PATH,
     "metabase_dump.sql",
   );
 

e2e/support/helpers/e2e-transform-helpers.ts

@@ -0,0 +1,37 @@
+import type {
+  ListTransformRunsResponse,
+  TransformId,
+  TransformRun,
+} from "metabase-types/api";
+
+export function visitTransform(transformId: TransformId) {
+  cy.visit(`/admin/transforms/${transformId}`);
+}
+
+const WAIT_TIMEOUT = 10000;
+const WAIT_INTERVAL = 100;
+
+export function waitForTransformRuns(
+  filter: (runs: TransformRun[]) => boolean,
+  timeout = WAIT_TIMEOUT,
+): Cypress.Chainable {
+  return cy
+    .request<ListTransformRunsResponse>("GET", "/api/ee/transform/run")
+    .then((response) => {
+      if (filter(response.body.data)) {
+        return cy.wrap(response);
+      } else if (timeout > 0) {
+        cy.wait(WAIT_INTERVAL);
+        return waitForTransformRuns(filter, timeout - WAIT_INTERVAL);
+      } else {
+        throw new Error("Run retry timeout");
+      }
+    });
+}
+
+export function waitForSucceededTransformRuns() {
+  waitForTransformRuns(
+    (runs) =>
+      runs.length > 0 && runs.every((run) => run.status === "succeeded"),
+  );
+}

e2e/support/helpers/embedding-sdk-testing/embedding-sdk-helpers.ts

@@ -11,20 +11,29 @@ import {
 } from "e2e/support/helpers";
 import { enableJwtAuth } from "e2e/support/helpers/e2e-jwt-helpers";
 
-export const getSignedJwtForUser = async (user = USERS.admin) => {
+export const getSignedJwtForUser = async ({
+  user = USERS.admin,
+  expiredInSeconds = 60 * 10, // 10 minutes expiration,
+}: {
+  user?: { email: string; first_name: string; last_name: string };
+  expiredInSeconds?: number;
+}) => {
   const secret = new TextEncoder().encode(JWT_SHARED_SECRET);
 
   return new jose.SignJWT({
     email: user.email,
     first_name: user.first_name,
     last_name: user.last_name,
-    exp: Math.round(Date.now() / 1000) + 60 * 10, // 10 minutes expiration
+    exp: Math.round(Date.now() / 1000) + expiredInSeconds,
   })
     .setProtectedHeader({ alg: "HS256" })
     .sign(secret);
 };
 
-export const mockAuthProviderAndJwtSignIn = (user = USERS.admin) => {
+export const mockAuthProviderAndJwtSignIn = (
+  user = USERS.admin,
+  { jwt }: { jwt?: string } = {},
+) => {
   cy.intercept("GET", `${AUTH_PROVIDER_URL}**`, async (req) => {
     try {
       const url = new URL(req.url);
@@ -39,7 +48,7 @@ export const mockAuthProviderAndJwtSignIn = (user = USERS.admin) => {
         return;
       }
 
-      const jwt = await getSignedJwtForUser(user);
+      jwt = jwt || (await getSignedJwtForUser({ user }));
 
       req.reply({
         statusCode: 200,

e2e/support/helpers/index.ts

@@ -47,6 +47,7 @@ export * from "./e2e-slack-helpers";
 export * from "./e2e-snowplow-helpers";
 export * from "./e2e-table-metadata-helpers";
 export * from "./e2e-token-helpers";
+export * from "./e2e-transform-helpers";
 export * from "./e2e-ui-elements-helpers";
 export * from "./e2e-ui-elements-overflow-helpers";
 export * from "./e2e-upload-helpers";

e2e/test-component/scenarios/embedding-sdk/dashboard-click-behavior.cy.spec.tsx

@@ -1,6 +1,7 @@
 import {
   EditableDashboard,
   InteractiveDashboard,
+  InteractiveQuestion,
 } from "@metabase/embedding-sdk-react";
 
 const { H } = cy;
@@ -26,6 +27,30 @@ describe("scenarios > embedding-sdk > dashboard-click-behavior", () => {
       semantic_type: "type/URL",
     });
 
+    // Create a question with custom column that returns a url as a string
+    H.createQuestion({
+      name: "Orders with URL Expression",
+      query: {
+        "source-table": ORDERS_ID,
+        expressions: {
+          // NOTE: this is a question, so it won't have semantic_type set as url, we'll need a model for that
+          url_column: [
+            "concat",
+            "https://example.org/",
+            ["field", ORDERS.ID, { "base-type": "type/BigInteger" }],
+          ],
+        },
+        fields: [
+          ["field", ORDERS.ID, { "base-type": "type/BigInteger" }],
+          ["expression", "url_column", { "base-type": "type/Text" }],
+          ["field", ORDERS.PRODUCT_ID, null],
+        ],
+        limit: 5,
+      },
+    }).then(({ body: question }) => {
+      cy.wrap(question.id).as("questionId");
+    });
+
     H.createDashboardWithQuestions({
       dashboardName: "Orders in a dashboard",
       questions: [
@@ -60,7 +85,7 @@ describe("scenarios > embedding-sdk > dashboard-click-behavior", () => {
       ],
       cards: [
         {
-          size_x: 12,
+          size_x: 24,
           col: 0,
           visualization_settings: {
             column_settings: {
@@ -77,7 +102,7 @@ describe("scenarios > embedding-sdk > dashboard-click-behavior", () => {
         },
         {
           size_x: 12,
-          col: 12,
+          col: 0,
           visualization_settings: {
             click_behavior: {
               type: "link",
@@ -193,4 +218,123 @@ describe("scenarios > embedding-sdk > dashboard-click-behavior", () => {
       cy.findByTestId("visualization-root").should("be.visible");
     });
   });
+
+  it("columns that return a string url should be rendered as a link", () => {
+    cy.get<string>("@questionId").then((questionId) => {
+      mountSdkContent(<InteractiveQuestion questionId={questionId} />);
+    });
+
+    cy.intercept("GET", "/api/card/*").as("getCard");
+    cy.wait("@getCard");
+
+    cy.findByRole("link", { name: "https://example.org/448" })
+      .should("have.length", 1)
+      .should("have.attr", "href", "https://example.org/448");
+  });
+
+  it("columns that have 'type/URL' semantic type should be rendered as a link", () => {
+    cy.signIn("admin");
+
+    cy.get<string>("@questionId").then((questionId) => {
+      // Convert question to model
+      cy.request("GET", `/api/card/${questionId}`)
+        .then((response) => {
+          const questionData = response.body;
+          return cy.request("PUT", `/api/card/${questionId}`, {
+            ...questionData,
+            type: "model",
+          });
+        })
+        .then(() => {
+          // re-fetch it as model
+          return cy.request("GET", `/api/card/${questionId}`);
+        })
+        .then((modelResponse) => {
+          // Update model column semantic type for url_column to type/URL
+
+          const modelData = modelResponse.body;
+          const updatedMetadata = modelData.result_metadata.map((col: any) => {
+            if (col.name === "url_column") {
+              return { ...col, semantic_type: "type/URL" };
+            }
+            return col;
+          });
+
+          return cy.request("PUT", `/api/card/${questionId}`, {
+            ...modelData,
+            result_metadata: updatedMetadata,
+          });
+        })
+        .then(() => {
+          // Create dashboard with the model
+          H.createDashboardWithQuestions({
+            dashboardName: "URL Dashboard",
+            questions: [
+              {
+                name: "Orders with URLs",
+                query: {
+                  "source-table": `card__${questionId}`,
+                  limit: 5,
+                },
+              },
+            ],
+          }).then(({ dashboard }) => {
+            mountSdkContent(
+              <InteractiveDashboard dashboardId={dashboard.id} />,
+            );
+
+            getSdkRoot().within(() => {
+              H.getDashboardCard(0)
+                .findByRole("link", {
+                  name: "https://example.org/448",
+                })
+                .should("have.attr", "href", "https://example.org/448");
+            });
+          });
+        });
+    });
+  });
+
+  it("columns that have 'Display as Link' should be rendered as a link and use the custom rendering", () => {
+    cy.signIn("admin");
+    cy.intercept("GET", "/api/card/*").as("getCard");
+
+    cy.get<string>("@questionId").then((questionId) => {
+      H.createDashboardWithQuestions({
+        dashboardName: "URL Dashboard",
+        questions: [
+          {
+            name: "Orders with URLs",
+            query: {
+              "source-table": `card__${questionId}`,
+              limit: 5,
+            },
+          },
+        ],
+        cards: [
+          {
+            visualization_settings: {
+              column_settings: {
+                [JSON.stringify(["name", "ID"])]: {
+                  view_as: "link",
+                  link_text: "Link to {{ID}}",
+                  link_url: "https://example.org/{{ID}}",
+                },
+              },
+            },
+          },
+        ],
+      }).then(({ dashboard }) => {
+        mountSdkContent(<EditableDashboard dashboardId={dashboard.id} />);
+
+        getSdkRoot().within(() => {
+          H.getDashboardCard(0)
+            .findByRole("link", {
+              name: "Link to 448",
+            })
+            .should("have.attr", "href", "https://example.org/448");
+        });
+      });
+    });
+  });
 });

e2e/test-component/scenarios/embedding-sdk/interactive-question.cy.spec.tsx

@@ -0,0 +1,50 @@
+const { H } = cy;
+import { InteractiveQuestion } from "@metabase/embedding-sdk-react";
+
+import { getSdkRoot } from "e2e/support/helpers/e2e-embedding-sdk-helpers";
+import { mountSdkContent } from "e2e/support/helpers/embedding-sdk-component-testing/component-embedding-sdk-helpers";
+import { signInAsAdminAndEnableEmbeddingSdk } from "e2e/support/helpers/embedding-sdk-testing";
+import { mockAuthProviderAndJwtSignIn } from "e2e/support/helpers/embedding-sdk-testing/embedding-sdk-helpers";
+
+describe("scenarios > embedding-sdk > interactive-question", () => {
+  beforeEach(() => {
+    signInAsAdminAndEnableEmbeddingSdk();
+
+    H.createNativeQuestion(
+      {
+        name: "SQL Orders",
+        native: { query: "select * from ORDERS limit 5" },
+      },
+      {
+        wrapId: true,
+        idAlias: "sqlQuestionId",
+      },
+    );
+    cy.signOut();
+
+    mockAuthProviderAndJwtSignIn();
+  });
+
+  it("should be able to drill SQL question (EMB-273)", () => {
+    cy.get("@sqlQuestionId").then((sqlQuestionId) => {
+      mountSdkContent(
+        <InteractiveQuestion questionId={sqlQuestionId} withParameters />,
+      );
+    });
+
+    getSdkRoot().within(() => {
+      cy.findByText("SQL Orders").should("be.visible");
+      H.assertTableRowsCount(5);
+
+      // Drill down to "See these Orders"
+      cy.get("[data-dataset-index=0] > [data-column-id='PRODUCT_ID']")
+        .should("have.text", "14")
+        .click();
+      H.popover().within(() => {
+        cy.findByText("Filter by this value").should("be.visible");
+        cy.button(">").click();
+      });
+      H.assertTableRowsCount(4);
+    });
+  });
+});

e2e/test-component/scenarios/embedding-sdk/metabot-question.cy.spec.tsx

@@ -171,4 +171,101 @@ describe("scenarios > embedding-sdk > metabot-question", () => {
       });
     });
   });
+
+  it("should show suggestion buttons when no messages exist", () => {
+    setup(metabotResponse);
+    mockSuggestedPrompts();
+
+    mountSdkContent(<MetabotQuestion />);
+
+    getSdkRoot().within(() => {
+      // Verify suggestion buttons are visible
+      cy.findAllByTestId("metabot-suggestion-button", { timeout: 10_000 })
+        .should("have.length", 3)
+        .should("be.visible");
+
+      // Verify suggestion texts are shown in order
+      cy.findAllByTestId("metabot-suggestion-button")
+        .eq(0)
+        .should("contain.text", "Show me total sales by product");
+
+      cy.findAllByTestId("metabot-suggestion-button")
+        .eq(1)
+        .should("contain.text", "What are the top performing products?");
+
+      cy.findAllByTestId("metabot-suggestion-button")
+        .eq(2)
+        .should("contain.text", "How many orders were placed last month?");
+
+      // Clicking on a suggestion should remove the suggestion buttons and add the user's message
+      cy.findByText("Show me total sales by product").click();
+      cy.findAllByTestId("metabot-suggestion-button").should("not.exist");
+      cy.findAllByTestId("metabot-chat-message").should("have.length", 2);
+
+      cy.findAllByTestId("metabot-chat-message")
+        .first()
+        .should("contain.text", "Show me total sales by product");
+    });
+  });
+
+  it("should switch to stacked layout on small viewport", () => {
+    setup(metabotResponse);
+    mountSdkContent(<MetabotQuestion />);
+
+    getSdkRoot().within(() => {
+      // On desktop, use the sidebar layout
+      cy.findByTestId("metabot-question-container").should(
+        "have.attr",
+        "data-layout",
+        "sidebar",
+      );
+
+      // On mobile, use the stacked layout
+      cy.viewport(400, 600);
+      cy.findByTestId("metabot-overflow-button").should("be.visible");
+      cy.findByTestId("metabot-question-container").should(
+        "have.attr",
+        "data-layout",
+        "stacked",
+      );
+
+      // On desktop, use the sidebar layout
+      cy.viewport(1200, 800);
+      cy.findByTestId("metabot-overflow-button").should("not.be.visible");
+      cy.findByTestId("metabot-question-container").should(
+        "have.attr",
+        "data-layout",
+        "sidebar",
+      );
+    });
+  });
 });
+
+const mockSuggestedPrompts = () => {
+  cy.intercept(
+    "GET",
+    "/api/ee/metabot-v3/metabot/2/prompt-suggestions?limit=3&sample=true",
+    {
+      statusCode: 200,
+      body: {
+        prompts: [
+          {
+            id: 1,
+            prompt: "Show me total sales by product",
+          },
+          {
+            id: 2,
+            prompt: "What are the top performing products?",
+          },
+          {
+            id: 3,
+            prompt: "How many orders were placed last month?",
+          },
+        ],
+        limit: 3,
+        offset: 0,
+        total: 3,
+      },
+    },
+  ).as("suggestedPrompts");
+};

e2e/test-component/scenarios/embedding-sdk/requests.cy.spec.tsx

@@ -1,7 +1,20 @@
-import { CollectionBrowser } from "@metabase/embedding-sdk-react";
+import {
+  CollectionBrowser,
+  InteractiveQuestion,
+  MetabaseProvider,
+} from "@metabase/embedding-sdk-react";
 
-import { mountSdkContent } from "e2e/support/helpers/embedding-sdk-component-testing/component-embedding-sdk-helpers";
-import { signInAsAdminAndEnableEmbeddingSdk } from "e2e/support/helpers/embedding-sdk-testing";
+import { USERS } from "e2e/support/cypress_data";
+import { ORDERS_QUESTION_ID } from "e2e/support/cypress_sample_instance_data";
+import { getSdkRoot } from "e2e/support/helpers/e2e-embedding-sdk-helpers";
+import {
+  DEFAULT_SDK_AUTH_PROVIDER_CONFIG,
+  mountSdkContent,
+} from "e2e/support/helpers/embedding-sdk-component-testing/component-embedding-sdk-helpers";
+import {
+  getSignedJwtForUser,
+  signInAsAdminAndEnableEmbeddingSdk,
+} from "e2e/support/helpers/embedding-sdk-testing";
 import { mockAuthProviderAndJwtSignIn } from "e2e/support/helpers/embedding-sdk-testing/embedding-sdk-helpers";
 
 describe("scenarios > embedding-sdk > requests", () => {
@@ -23,5 +36,39 @@ describe("scenarios > embedding-sdk > requests", () => {
         expect(maxAgeHeader).to.equal("60");
       });
     });
+
+    it("properly performs session token refresh request when multiple data requests are triggered at the same time", () => {
+      const expiredInSeconds = 60;
+      cy.clock(Date.now());
+
+      cy.then(() => getSignedJwtForUser({ expiredInSeconds })).then((jwt) => {
+        mockAuthProviderAndJwtSignIn(USERS.admin, { jwt });
+      });
+
+      cy.mount(
+        <MetabaseProvider authConfig={DEFAULT_SDK_AUTH_PROVIDER_CONFIG}>
+          <InteractiveQuestion questionId={ORDERS_QUESTION_ID} />
+        </MetabaseProvider>,
+      );
+
+      getSdkRoot().within(() => {
+        cy.findByText("Orders").should("exist");
+
+        cy.findByText("Group").click();
+
+        cy.findByRole("dialog").should("be.visible");
+
+        cy.tick(1000 * (expiredInSeconds + 5));
+
+        cy.findByRole("dialog").contains(/^ID$/).click();
+        cy.findByRole("dialog").findByTestId("badge-remove-button").click();
+
+        cy.wait("@jwtProvider");
+
+        cy.intercept("POST", "/api/dataset").as("dataset");
+        // We ensure that both `dataset` requests are made after the token refresh request
+        cy.get("@dataset.all").should("have.length", 2);
+      });
+    });
   });
 });

e2e/test-component/scenarios/embedding-sdk/sdk-question-native.cy.spec.tsx

@@ -3,6 +3,7 @@ import { useState } from "react";
 
 import { SAMPLE_DATABASE } from "e2e/support/cypress_sample_database";
 import {
+  type NativeQuestionDetails,
   createNativeQuestion,
   tableInteractiveBody,
 } from "e2e/support/helpers";
@@ -15,122 +16,281 @@ import { signInAsAdminAndEnableEmbeddingSdk } from "e2e/support/helpers/embeddin
 import { mockAuthProviderAndJwtSignIn } from "e2e/support/helpers/embedding-sdk-testing/embedding-sdk-helpers";
 import { Box, Button } from "metabase/ui";
 import type { DatasetColumn } from "metabase-types/api";
+import { createMockParameter } from "metabase-types/api/mocks";
 
 const { ORDERS, ORDERS_ID } = SAMPLE_DATABASE;
 
-describe("scenarios > embedding-sdk > interactive-question > native", () => {
-  beforeEach(() => {
-    signInAsAdminAndEnableEmbeddingSdk();
+const setup = ({ question }: { question: NativeQuestionDetails }) => {
+  signInAsAdminAndEnableEmbeddingSdk();
 
-    createNativeQuestion(
-      {
-        native: {
-          query: "SELECT * FROM orders WHERE {{ID}}",
-          "template-tags": {
-            ID: {
-              id: "6b8b10ef-0104-1047-1e1b-2492d5954322",
-              name: "ID",
-              "display-name": "ID",
-              type: "dimension",
-              dimension: ["field", ORDERS.ID, null],
-              "widget-type": "category",
-              default: null,
+  createNativeQuestion(question, { wrapId: true });
+
+  cy.signOut();
+  mockAuthProviderAndJwtSignIn();
+};
+
+const ensureParameterColumnValue = ({
+  columnName,
+  columnValue,
+}: {
+  columnName: string;
+  columnValue: string;
+}) => {
+  tableInteractiveBody().within(() => {
+    cy.get(`[data-column-id="${columnName}"]`).each((cell) => {
+      cy.wrap(cell).should("have.text", columnValue);
+    });
+  });
+};
+
+describe("scenarios > embedding-sdk > interactive-question > native", () => {
+  describe("common", () => {
+    beforeEach(() => {
+      setup({
+        question: {
+          native: {
+            query: "SELECT * FROM orders WHERE {{ID}}",
+            "template-tags": {
+              ID: {
+                id: "6b8b10ef-0104-1047-1e1b-2492d5954322",
+                name: "ID",
+                "display-name": "ID",
+                type: "dimension",
+                dimension: ["field", ORDERS.ID, null],
+                "widget-type": "category",
+                default: null,
+              },
             },
           },
         },
-      },
-      { wrapId: true },
-    );
-
-    cy.signOut();
-    mockAuthProviderAndJwtSignIn();
-  });
-
-  it("supports passing sql parameters to native questions", () => {
-    mountInteractiveQuestion({ initialSqlParameters: { ID: ORDERS_ID } });
-
-    cy.wait("@cardQuery").then(({ response }) => {
-      const { body } = response ?? {};
-
-      const rows = tableInteractiveBody().findAllByRole("row");
-
-      // There should be one row in the table
-      rows.should("have.length", 1);
-
-      const idColumnIndex = body.data.cols.findIndex(
-        (column: DatasetColumn) => column.name === "ID",
-      );
-
-      // The first row should have the same ID column value as the initial SQL parameters
-      // eslint-disable-next-line no-unsafe-element-filtering
-      rows
-        .findAllByTestId("cell-data")
-        .eq(idColumnIndex)
-        .should("have.text", String(ORDERS_ID));
-    });
-  });
-
-  it("should not crash when switching from new question to native question (metabase#60160)", () => {
-    const TestComponent = ({
-      nativeQuestionId,
-    }: {
-      nativeQuestionId: string | number;
-    }) => {
-      const [questionId, setQuestionId] = useState<string | number>("new");
-
-      return (
-        <Box>
-          <InteractiveQuestion questionId={questionId} />
-
-          <Button onClick={() => setQuestionId(nativeQuestionId)}>
-            use native question
-          </Button>
-        </Box>
-      );
-    };
-
-    cy.get<string>("@questionId").then((nativeQuestionId) => {
-      mountSdkContent(<TestComponent nativeQuestionId={nativeQuestionId} />);
-
-      getSdkRoot().within(() => {
-        cy.findByText("Pick your starting data").should("be.visible");
       });
+    });
 
-      cy.findByText("use native question").click();
+    it("supports passing sql parameters to native questions", () => {
+      mountInteractiveQuestion({ initialSqlParameters: { ID: ORDERS_ID } });
+
+      cy.wait("@cardQuery").then(({ response }) => {
+        const { body } = response ?? {};
+
+        const rows = tableInteractiveBody().findAllByRole("row");
+
+        // There should be one row in the table
+        rows.should("have.length", 1);
+
+        const idColumnIndex = body.data.cols.findIndex(
+          (column: DatasetColumn) => column.name === "ID",
+        );
+
+        // The first row should have the same ID column value as the initial SQL parameters
+        // eslint-disable-next-line no-unsafe-element-filtering
+        rows
+          .findAllByTestId("cell-data")
+          .eq(idColumnIndex)
+          .should("have.text", String(ORDERS_ID));
+      });
+    });
+
+    it("should not crash when switching from new question to native question (metabase#60160)", () => {
+      const TestComponent = ({
+        nativeQuestionId,
+      }: {
+        nativeQuestionId: string | number;
+      }) => {
+        const [questionId, setQuestionId] = useState<string | number>("new");
+
+        return (
+          <Box>
+            <InteractiveQuestion questionId={questionId} />
+
+            <Button onClick={() => setQuestionId(nativeQuestionId)}>
+              use native question
+            </Button>
+          </Box>
+        );
+      };
+
+      cy.get<string>("@questionId").then((nativeQuestionId) => {
+        mountSdkContent(<TestComponent nativeQuestionId={nativeQuestionId} />);
+
+        getSdkRoot().within(() => {
+          cy.findByText("Pick your starting data").should("be.visible");
+        });
+
+        cy.findByText("use native question").click();
+
+        cy.log("should show native question table data");
+        getSdkRoot().within(() => {
+          cy.findByText("Pick your starting data").should("not.exist");
+          cy.findByText("test question").should("be.visible");
+          cy.findByText("110.93").should("be.visible");
+        });
+
+        cy.log("should not crash due to preview-query error");
+        cy.on("uncaught:exception", (error) => {
+          expect(
+            error.message.includes(
+              "preview-query cannot be called on native queries",
+            ),
+          ).to.be.false;
+        });
+      });
+    });
+
+    it("should not show filter/summarize/breakout buttons for native questions", () => {
+      mountInteractiveQuestion({});
 
-      cy.log("should show native question table data");
       getSdkRoot().within(() => {
-        cy.findByText("Pick your starting data").should("not.exist");
+        cy.log("should show native question data");
         cy.findByText("test question").should("be.visible");
-        cy.findByText("110.93").should("be.visible");
-      });
 
-      cy.log("should not crash due to preview-query error");
-      cy.on("uncaught:exception", (error) => {
-        expect(
-          error.message.includes(
-            "preview-query cannot be called on native queries",
-          ),
-        ).to.be.false;
+        cy.log("should not show Filter button");
+        cy.findByText("Filter").should("not.exist");
+
+        cy.log("should not show Summarize button");
+        cy.findByText("Summarize").should("not.exist");
+
+        cy.log("should not show Group button");
+        cy.findByText("Group").should("not.exist");
       });
     });
   });
 
-  it("should not show filter/summarize/breakout buttons for native questions", () => {
-    mountInteractiveQuestion({});
+  describe("editable parameters for a native question", () => {
+    beforeEach(() => {
+      setup({
+        question: {
+          native: {
+            query:
+              "SELECT * FROM people WHERE state = {{State}} [[ and city = {{City}} ]] [[ and source = {{Source}} ]]",
+            "template-tags": {
+              State: {
+                type: "text",
+                name: "State",
+                id: "1",
+                "display-name": "State",
+              },
+              City: {
+                type: "text",
+                name: "City",
+                id: "2",
+                "display-name": "City",
+              },
+              Source: {
+                type: "text",
+                name: "Source",
+                id: "3",
+                "display-name": "Source",
+              },
+            },
+          },
+          parameters: [
+            createMockParameter({
+              id: "1",
+              slug: "State",
+            }),
+            createMockParameter({
+              id: "2",
+              slug: "City",
+            }),
+            createMockParameter({
+              id: "3",
+              slug: "Source",
+            }),
+          ],
+        },
+      });
+    });
 
-    getSdkRoot().within(() => {
-      cy.log("should show native question data");
-      cy.findByText("test question").should("be.visible");
+    it("does not show editable sql parameters controls for the default view", () => {
+      mountInteractiveQuestion({
+        initialSqlParameters: { State: "NY" },
+        hiddenParameters: ["Source"],
+      });
 
-      cy.log("should not show Filter button");
-      cy.findByText("Filter").should("not.exist");
+      cy.wait("@cardQuery");
 
-      cy.log("should not show Summarize button");
-      cy.findByText("Summarize").should("not.exist");
+      cy.findByTestId("parameter-widget").should("not.exist");
+      cy.findByPlaceholderText("State").should("not.exist");
+      cy.findByPlaceholderText("City").should("not.exist");
+      cy.findByPlaceholderText("Source").should("not.exist");
+    });
 
-      cy.log("should not show Group button");
-      cy.findByText("Group").should("not.exist");
+    describe("when `SqlParametersList` component is defined in a custom question layout", () => {
+      it("allows to edit sql parameters", () => {
+        mountInteractiveQuestion({
+          hiddenParameters: ["Source"],
+          children: (
+            <>
+              <InteractiveQuestion.Title />
+              <InteractiveQuestion.SqlParametersList />
+              <InteractiveQuestion.QuestionVisualization />
+            </>
+          ),
+        });
+
+        cy.wait("@cardQuery");
+
+        cy.findByPlaceholderText("State").should("exist");
+        cy.findByPlaceholderText("City").should("exist");
+        cy.findByPlaceholderText("Source").should("not.exist");
+
+        getSdkRoot().within(() => {
+          cy.findByPlaceholderText("State").clear().type("NY{enter}");
+        });
+
+        ensureParameterColumnValue({
+          columnName: "STATE",
+          columnValue: "NY",
+        });
+
+        getSdkRoot().within(() => {
+          cy.findByPlaceholderText("State").clear().type("AR{enter}");
+        });
+
+        ensureParameterColumnValue({
+          columnName: "STATE",
+          columnValue: "AR",
+        });
+
+        getSdkRoot().within(() => {
+          cy.findByPlaceholderText("City").type("El Paso{enter}");
+        });
+
+        ensureParameterColumnValue({
+          columnName: "CITY",
+          columnValue: "El Paso",
+        });
+      });
+
+      it("initializes sql parameters controls with initial values and keeps an initial value when changing other parameters", () => {
+        mountInteractiveQuestion({
+          initialSqlParameters: { State: "AR" },
+          hiddenParameters: ["Source"],
+          children: (
+            <>
+              <InteractiveQuestion.Title />
+              <InteractiveQuestion.SqlParametersList />
+              <InteractiveQuestion.QuestionVisualization />
+            </>
+          ),
+        });
+
+        cy.wait("@cardQuery");
+
+        ensureParameterColumnValue({
+          columnName: "STATE",
+          columnValue: "AR",
+        });
+
+        getSdkRoot().within(() => {
+          cy.findByPlaceholderText("City").type("El Paso{enter}");
+        });
+
+        ensureParameterColumnValue({
+          columnName: "CITY",
+          columnValue: "El Paso",
+        });
+      });
     });
   });
 });

e2e/test/scenarios/admin/transforms.cy.spec.ts

@@ -6,7 +6,6 @@ import { SAMPLE_DB_ID, WRITABLE_DB_ID } from "e2e/support/cypress_data";
 import { SAMPLE_DATABASE } from "e2e/support/cypress_sample_database";
 import type {
   CardType,
-  ListTransformRunsResponse,
   PythonTransformTableAliases,
   TransformTagId,
 } from "metabase-types/api";
@@ -1944,7 +1943,7 @@ describe("scenarios > admin > transforms > jobs", () => {
           tag_ids: [tag.id],
         });
       });
-      waitForRuns();
+      H.waitForSucceededTransformRuns();
       visitRunListPage();
       getContentTable().within(() => {
         cy.findAllByText("MBQL transform").should("have.length.gte", 1);
@@ -2961,24 +2960,6 @@ function visitTableQuestion({
   );
 }
 
-const WAIT_TIMEOUT = 10000;
-const WAIT_INTERVAL = 100;
-
-function waitForRuns(timeout = WAIT_TIMEOUT): Cypress.Chainable {
-  return cy
-    .request<ListTransformRunsResponse>("GET", "/api/ee/transform/run")
-    .then((response) => {
-      if (response.body.data.length > 0) {
-        return cy.wrap(response);
-      } else if (timeout > 0) {
-        cy.wait(WAIT_INTERVAL);
-        return waitForRuns(timeout - WAIT_INTERVAL);
-      } else {
-        throw new Error("Run retry timeout");
-      }
-    });
-}
-
 function assertTableDoesNotExistError({
   targetTable = TARGET_TABLE,
   targetSchema = TARGET_SCHEMA,

e2e/test/scenarios/collections/permissions.cy.spec.js

@@ -99,14 +99,6 @@ describe("collection permissions", () => {
                 it("should be able to duplicate the dashboard without obstructions from the modal (metabase#15256)", () => {
                   duplicate("Orders in a dashboard");
                 });
-
-                it(
-                  "should be able to duplicate the question (metabase#15255)",
-                  { tags: "@skip" },
-                  () => {
-                    duplicate("Orders");
-                  },
-                );
               });
 
               describe("archive", () => {

e2e/test/scenarios/dashboard-filters/dashboard-filters-auto-apply.cy.spec.js

@@ -33,9 +33,6 @@ function createDashboardDetails({ parameters }) {
 
 const TOAST_TIMEOUT = 16000;
 
-const TOAST_MESSAGE =
-  "You can make this dashboard snappier by turning off auto-applying filters.";
-
 const filterToggleLabel = "Auto-apply filters";
 
 describe(
@@ -288,158 +285,6 @@ describe(
           H.assertTableRowsCount(200);
         });
       });
-
-      it(
-        "should display a toast when a dashboard takes longer than 15s to load even without parameter values (but has parameters with default values)",
-        { tags: "@skip" },
-        () => {
-          cy.clock();
-          openSlowDashboard();
-
-          cy.tick(TOAST_TIMEOUT);
-          cy.wait("@cardQuery");
-          H.undoToast().within(() => {
-            cy.findByText(TOAST_MESSAGE).should("be.visible");
-            cy.button("Turn off").click();
-            cy.wait("@updateDashboard");
-          });
-
-          H.openDashboardSettingsSidebar();
-          H.sidesheet()
-            .findByLabelText(filterToggleLabel)
-            .should("not.be.checked");
-          // Gadget
-          const filterDefaultValue = FILTER_WITH_DEFAULT_VALUE.default[0];
-          H.filterWidget().findByText(filterDefaultValue).should("be.visible");
-          H.getDashboardCard().within(() => {
-            H.assertTableRowsCount(53);
-          });
-        },
-      );
-
-      it(
-        "should not display the toast when we clear out parameter default value",
-        { tags: "@skip" },
-        () => {
-          cy.clock();
-          openSlowDashboard({ [FILTER_WITH_DEFAULT_VALUE.slug]: null });
-
-          cy.tick(TOAST_TIMEOUT);
-          cy.wait("@cardQuery");
-          H.undoToast().should("not.exist");
-          H.getDashboardCard().within(() => {
-            H.assertTableRowsCount(200);
-          });
-        },
-      );
-    });
-
-    describe("auto-apply filter toast", () => {
-      it(
-        "should display a toast when a dashboard takes longer than 15s to load",
-        { tags: "@skip" },
-        () => {
-          cy.clock();
-          createDashboard();
-          openSlowDashboard({ [FILTER.slug]: "Gadget" });
-
-          cy.tick(TOAST_TIMEOUT);
-          cy.wait("@cardQuery");
-          H.undoToast().within(() => {
-            cy.findByText(TOAST_MESSAGE).should("be.visible");
-            cy.button("Turn off").click();
-            cy.wait("@updateDashboard");
-          });
-
-          H.openDashboardSettingsSidebar();
-          H.sidesheet()
-            .findByLabelText(filterToggleLabel)
-            .should("not.be.checked");
-          H.closeDashboardSettingsSidebar();
-          H.filterWidget().findByText("Gadget").should("be.visible");
-          H.getDashboardCard().within(() => {
-            H.assertTableRowsCount(53);
-          });
-        },
-      );
-
-      it(
-        "should display the toast indefinitely unless dismissing manually",
-        { tags: "@skip" },
-        () => {
-          cy.clock();
-          createDashboard();
-          openSlowDashboard({ [FILTER.slug]: "Gadget" });
-
-          cy.tick(TOAST_TIMEOUT);
-          cy.wait("@cardQuery");
-          H.undoToast().findByText(TOAST_MESSAGE).should("be.visible");
-
-          cy.tick(100 * TOAST_TIMEOUT);
-          H.undoToast().findByText(TOAST_MESSAGE).should("be.visible");
-
-          H.undoToast().icon("close").click();
-          H.undoToast().should("not.exist");
-        },
-      );
-
-      it(
-        "should not display the toast when auto applying filters is disabled",
-        { tags: "@skip" },
-        () => {
-          cy.clock();
-          createDashboard({ dashboardDetails: { auto_apply_filters: false } });
-          openSlowDashboard({ [FILTER.slug]: "Gadget" });
-
-          cy.tick(TOAST_TIMEOUT);
-          cy.wait("@cardQuery");
-          H.undoToast().should("not.exist");
-          H.filterWidget().findByText("Gadget").should("be.visible");
-          H.getDashboardCard().within(() => {
-            H.assertTableRowsCount(53);
-          });
-        },
-      );
-
-      it(
-        "should not display the toast if there are no parameter values",
-        { tags: "@skip" },
-        () => {
-          cy.clock();
-          createDashboard();
-          openSlowDashboard();
-
-          cy.tick(TOAST_TIMEOUT);
-          cy.wait("@cardQuery");
-          H.undoToast().should("not.exist");
-        },
-      );
-
-      it(
-        "should not display the same toast twice for a dashboard",
-        { tags: "@skip" },
-        () => {
-          cy.clock();
-          createDashboard();
-          openSlowDashboard({ [FILTER.slug]: "Gadget" });
-
-          cy.tick(TOAST_TIMEOUT);
-          cy.wait("@cardQuery");
-          H.undoToast().within(() => {
-            cy.button("Turn off").should("be.visible");
-            cy.icon("close").click();
-          });
-          H.filterWidget().findByText("Gadget").click();
-          H.popover().within(() => {
-            cy.findByText("Widget").click();
-            cy.findByText("Update filter").click();
-          });
-
-          cy.tick(TOAST_TIMEOUT);
-          cy.wait("@cardQuery");
-          H.undoToast().should("not.exist");
-        },
-      );
     });
 
     describe("no collection curate permission", () => {
@@ -456,19 +301,6 @@ describe(
         H.dashboardHeader().icon("ellipsis").click();
         H.popover().findByText("Edit settings").should("not.exist");
       });
-
-      it(
-        "should not display a toast even when a dashboard takes longer than 15s to load",
-        { tags: "@skip" },
-        () => {
-          cy.clock();
-          openSlowDashboard({ [FILTER.slug]: "Gadget" });
-
-          cy.tick(TOAST_TIMEOUT);
-          cy.wait("@cardQuery");
-          H.undoToast().should("not.exist");
-        },
-      );
     });
 
     describe("embeddings", () => {
@@ -602,71 +434,6 @@ describe(
             H.assertTableRowsCount(54);
           });
         });
-
-        it(
-          "should display a toast when a dashboard takes longer than 15s to load",
-          { tags: "@skip" },
-          () => {
-            createDashboard();
-            // Not sure why I need to pass a date in this case, but it doesn't work without it.
-            cy.clock(Date.now());
-            openSlowFullAppEmbeddingDashboard({ [FILTER.slug]: "Gadget" });
-            cy.tick(TOAST_TIMEOUT);
-            cy.wait("@cardQuery");
-            H.undoToast().within(() => {
-              cy.findByText(TOAST_MESSAGE).should("be.visible");
-              cy.button("Turn off").click();
-              cy.wait("@updateDashboard");
-            });
-
-            // In embedding we'll load bookmark after the dashboard is loaded, it's the opposite in normal app because bookmark is cached from somewhere else.
-            // And somehow, dashboard card query will be completed before the dashboard even start to load, and in entity loader it uses `setTimeout`,
-            // so to make sure callback in `setTimeout` is called, we need to advance the clock using cy.tick().
-            cy.tick();
-
-            H.openDashboardSettingsSidebar();
-            H.sidesheet()
-              .findByLabelText(filterToggleLabel)
-              .should("not.be.checked");
-            H.filterWidget().findByText("Gadget").should("be.visible");
-
-            H.getDashboardCard().within(() => {
-              H.assertTableRowsCount(53);
-            });
-
-            // Card result should be updated after manually updating the filter
-            H.filterWidget().icon("close").click();
-            H.applyFilterButton().should("be.visible").click();
-
-            H.getDashboardCard().within(() => {
-              H.assertTableRowsCount(200);
-            });
-          },
-        );
-
-        it(
-          "should not display a toast when a dashboard takes longer than 15s to load if users have no write access to a dashboard",
-          { tags: "@skip" },
-          () => {
-            createDashboard();
-            cy.signIn("readonly");
-            // Not sure why I need to pass a date in this case, but it doesn't work without it.
-            cy.clock(Date.now());
-            openSlowFullAppEmbeddingDashboard({ [FILTER.slug]: "Gadget" });
-            cy.tick(TOAST_TIMEOUT);
-            cy.wait("@cardQuery");
-            H.undoToast().should("not.exist");
-
-            // In embedding we'll load bookmark after the dashboard is loaded, it's the opposite in normal app because bookmark is cached from somewhere else.
-            // And somehow, dashboard card query will be completed before the dashboard even start to load, and in entity loader it uses `setTimeout`,
-            // so to make sure callback in `setTimeout` is called, we need to advance the clock using cy.tick().
-            cy.tick();
-
-            H.getDashboardCard().within(() => {
-              H.assertTableRowsCount(53);
-            });
-          },
-        );
       });
     });
   },
@@ -753,21 +520,6 @@ const openDashboard = (params = {}) => {
   H.visitDashboard("@dashboardId", { params });
 };
 
-const openSlowDashboard = (params = {}) => {
-  cy.intercept("POST", "/api/dashboard/*/dashcard/*/card/*/query").as(
-    "cardQuery",
-  );
-
-  cy.get("@dashboardId").then((dashboardId) => {
-    return cy.visit({
-      url: `/dashboard/${dashboardId}`,
-      qs: params,
-    });
-  });
-
-  H.getDashboardCard().should("be.visible");
-};
-
 const openSlowPublicDashboard = (params = {}) => {
   cy.intercept("GET", "/api/public/dashboard/*/dashcard/*/card/*").as(
     "cardQuery",
@@ -798,21 +550,6 @@ const openSlowEmbeddingDashboard = (params = {}) => {
   H.getDashboardCard().should("be.visible");
 };
 
-const openSlowFullAppEmbeddingDashboard = (params = {}) => {
-  cy.intercept("POST", "/api/dashboard/*/dashcard/*/card/*/query").as(
-    "cardQuery",
-  );
-
-  cy.get("@dashboardId").then((dashboardId) => {
-    visitFullAppEmbeddingUrl({
-      url: `/dashboard/${dashboardId}`,
-      qs: params,
-    });
-  });
-
-  H.getDashboardCard().should("be.visible");
-};
-
 const visitFullAppEmbeddingUrl = ({ url, qs }) => {
   cy.visit({
     url,

e2e/test/scenarios/dashboard/dashboard-reproductions.cy.spec.js

@@ -2204,3 +2204,60 @@ describe("issue 63176", () => {
     });
   });
 });
+
+describe("issue 64138", () => {
+  const MAP_QUESTION = {
+    query: {
+      "source-table": PEOPLE_ID,
+    },
+
+    display: "map",
+    displayIsLocked: true,
+    visualization_settings: {
+      "map.type": "pin",
+      "map.pin_type": "markers",
+    },
+  };
+
+  beforeEach(() => {
+    H.restore();
+    cy.signInAsNormalUser();
+
+    H.createQuestionAndDashboard({
+      questionDetails: MAP_QUESTION,
+    }).then(({ body: { dashboard_id } }) => {
+      H.visitDashboard(dashboard_id);
+    });
+  });
+
+  it("should hide map controls when editing dashboard (metabase#64138)", () => {
+    H.editDashboard();
+
+    cy.log("hovering the map should not show the zoom controls");
+    H.getDashboardCard(0)
+      .realHover({
+        position: "center",
+      })
+      .within(() => {
+        cy.findByLabelText("Zoom in").should("not.exist");
+        cy.findByText("Set as default view").should("be.visible").click();
+      });
+
+    cy.log("hovering marker icons should not open their tooltips");
+    getMarkerIcon(0).realHover();
+    H.popover({ skipVisibilityCheck: true }).should("not.exist");
+
+    cy.log("clicking marker icons should not navigate to the question");
+    getMarkerIcon(0).click({ force: true });
+    cy.location("pathname").should("match", /^\/dashboard\/[0-9]+$/);
+    H.modal().should("not.exist");
+  });
+
+  function getMarkerIcon(index) {
+    // pick the last one so it will be on top
+    return H.getDashboardCard(index)
+      .get(".leaflet-marker-icon")
+      .should("have.length.gt", 0)
+      .last();
+  }
+});

e2e/test/scenarios/dashboard/x-rays.cy.spec.js

@@ -177,10 +177,16 @@ describe("scenarios > x-rays", { tags: "@slow" }, () => {
 
     cy.button("Save this").click();
 
-    // eslint-disable-next-line no-unscoped-text-selectors -- deprecated usage
-    cy.findByText("Your dashboard was saved");
-    // eslint-disable-next-line no-unscoped-text-selectors -- deprecated usage
-    cy.findByText("See it").click();
+    cy.log(
+      "'See it' link should be displayed both in the header and in the toast",
+    );
+    H.undoToast()
+      .should("contain", "Your dashboard was saved")
+      .and("contain", "See it");
+
+    cy.findByTestId("automatic-dashboard-header").within(() => {
+      cy.findByRole("link", { name: "See it" }).should("be.visible").click();
+    });
 
     cy.url().should("contain", "a-look-at-orders");
 

e2e/test/scenarios/dependencies/dependencies.cy.spec.ts

@@ -0,0 +1,678 @@
+const { H } = cy;
+
+import { WRITABLE_DB_ID } from "e2e/support/cypress_data";
+import { SAMPLE_DATABASE } from "e2e/support/cypress_sample_database";
+import { ORDERS_DASHBOARD_ID } from "e2e/support/cypress_sample_instance_data";
+
+const { PRODUCTS_ID } = SAMPLE_DATABASE;
+
+H.describeWithSnowplowEE("documents", () => {
+  beforeEach(() => {
+    H.restore("postgres-writable");
+    H.resetTestTable({ type: "postgres", table: "many_schemas" });
+    cy.signInAsAdmin();
+    H.activateToken("bleeding-edge");
+    H.resyncDatabase({ dbId: WRITABLE_DB_ID, tableName: "Animals" });
+    H.resetSnowplow();
+
+    cy.intercept("PUT", "/api/card/*").as("updateCard");
+    cy.intercept("PUT", "/api/native-query-snippet/*").as("updateSnippet");
+    cy.intercept("PUT", "/api/ee/transform/*").as("updateTransform");
+  });
+
+  afterEach(() => {
+    H.expectNoBadSnowplowEvents();
+  });
+
+  describe("questions", () => {
+    it("should be able to confirm or cancel breaking changes to a MBQL question", () => {
+      createMbqlQuestionWithDependentMbqlQuestions();
+
+      cy.log("make breaking changes");
+      H.visitQuestion("@questionId");
+      H.openNotebook();
+      H.getNotebookStep("expression").findByText("Expr").icon("close").click();
+
+      cy.log("cancel breaking changes");
+      H.queryBuilderHeader().button("Save").click();
+      H.modal().button("Save").click();
+      H.modal().within(() => {
+        cy.findByText("Question with fields").should("be.visible");
+        cy.findByText("Question without fields").should("not.exist");
+        cy.findByText("Base question").should("not.exist");
+        cy.button("Cancel").click();
+      });
+      cy.get("@updateCard.all").should("have.length", 0);
+
+      cy.log("confirm breaking changes");
+      H.queryBuilderHeader().button("Save").click();
+      H.modal().button("Save").click();
+      H.modal().button("Save anyway").click();
+      cy.wait("@updateCard");
+    });
+
+    it("should be able to navigate to affected questions or their collection", () => {
+      createMbqlQuestionWithDependentMbqlQuestions();
+
+      cy.log("check that we can navigate to the broken question");
+      H.visitQuestion("@questionId");
+      H.openNotebook();
+      H.getNotebookStep("expression").findByText("Expr").icon("close").click();
+      H.queryBuilderHeader().button("Save").click();
+      H.modal().button("Save").click();
+      H.modal().findByText("Question with fields").click();
+      confirmDiscardChanges();
+      H.queryBuilderHeader()
+        .findByDisplayValue("Question with fields")
+        .should("be.visible");
+
+      cy.log("check that we can navigate to the collection of that question");
+      cy.go("back");
+      H.getNotebookStep("expression").findByText("Expr").icon("close").click();
+      H.queryBuilderHeader().button("Save").click();
+      H.modal().button("Save").click();
+      H.modal().within(() => {
+        cy.findByText("Question with fields").should("be.visible");
+        cy.findByText("Our analytics").click();
+      });
+      confirmDiscardChanges();
+      H.collectionTable().should("be.visible");
+
+      cy.log("check that we can navigate to the dashboard of that question");
+      cy.go("back");
+      H.getNotebookStep("expression").findByText("Expr").icon("close").click();
+      H.queryBuilderHeader().button("Save").click();
+      H.modal().button("Save").click();
+      H.modal().within(() => {
+        cy.findByText("Question with fields").should("be.visible");
+        cy.findByText("Orders in a dashboard").click();
+      });
+      confirmDiscardChanges();
+      H.dashboardHeader().should("be.visible");
+    });
+
+    it("should not show a confirmation if there are no breaking changes when updating a MBQL question", () => {
+      createMbqlQuestionWithDependentMbqlQuestions();
+      H.visitQuestion("@questionId");
+      H.openNotebook();
+      H.getNotebookStep("expression").findByText("Expr").click();
+      H.CustomExpressionEditor.clear().type("2 + 2");
+      H.popover().button("Update").click();
+      H.queryBuilderHeader().button("Save").click();
+      H.modal().button("Save").click();
+      cy.wait("@updateCard");
+    });
+  });
+
+  describe("models", () => {
+    it("should be able to confirm or cancel breaking changes to a SQL model", () => {
+      createSqlModelWithDependentSqlQuestions();
+
+      cy.log("make breaking changes");
+      cy.get<number>("@modelId").then(H.visitModel);
+      H.openQuestionActions("Edit query definition");
+      H.NativeEditor.clear().type("SELECT ID, TITLE FROM PRODUCTS");
+      H.runNativeQuery();
+
+      cy.log("cancel breaking changes");
+      H.datasetEditBar().button("Save changes").click();
+      H.modal().within(() => {
+        cy.findByText("Question with fields").should("be.visible");
+        cy.findByText("Question without fields").should("not.exist");
+        cy.findByText("Base model").should("not.exist");
+        cy.button("Cancel").click();
+      });
+      cy.get("@updateCard.all").should("have.length", 0);
+
+      cy.log("confirm breaking changes");
+      H.datasetEditBar().button("Save changes").click();
+      H.modal().button("Save anyway").click();
+      cy.wait("@updateCard");
+    });
+
+    it("should not show a confirmation if there are no breaking changes when updating a SQL model", () => {
+      createSqlModelWithDependentSqlQuestions();
+      cy.get<number>("@modelId").then(H.visitModel);
+      H.openQuestionActions("Edit query definition");
+      H.NativeEditor.clear().type("SELECT ID, CATEGORY FROM PRODUCTS");
+      H.runNativeQuery();
+      H.datasetEditBar().button("Save changes").click();
+      cy.wait("@updateCard");
+    });
+  });
+
+  describe("metrics", () => {
+    it("should be able to confirm or cancel breaking changes to a metric", () => {
+      createMetricWithDependentMbqlQuestionsAndTransforms();
+
+      cy.log("make breaking changes");
+      cy.get<number>("@metricId").then(H.visitMetric);
+      H.openQuestionActions("Edit metric definition");
+      H.getNotebookStep("summarize").findByText("Min of Score").click();
+      H.popover().within(() => {
+        cy.icon("chevronleft").click();
+        cy.findByText("Maximum of ...").click();
+        cy.findByText("Score").click();
+      });
+
+      cy.log("cancel breaking changes");
+      cy.findByTestId("edit-bar").button("Save changes").click();
+      H.modal().within(() => {
+        cy.findByText("Question with 2 stages").should("be.visible");
+        cy.findByText("Transform with 2 stages").should("be.visible");
+        cy.findByText("Question with 1 stage").should("not.exist");
+        cy.findByText("Transform with 1 stage").should("not.exist");
+        cy.findByText("Base metric").should("not.exist");
+        cy.button("Cancel").click();
+      });
+      cy.get("@updateCard.all").should("have.length", 0);
+
+      cy.log("confirm breaking changes");
+      cy.findByTestId("edit-bar").button("Save changes").click();
+      H.modal().button("Save anyway").click();
+      cy.wait("@updateCard");
+    });
+
+    it("should not show a warning when a change to a metric is backward-compatible with existing content", () => {
+      createMetricWithDependentMbqlQuestionsAndTransforms();
+      cy.get<number>("@metricId").then(H.visitMetric);
+      H.openQuestionActions("Edit metric definition");
+      H.getNotebookStep("summarize").findByText("Min of Score").click();
+      H.popover().findByText("Name").click();
+      cy.findByTestId("edit-bar").button("Save changes").click();
+      cy.wait("@updateCard");
+    });
+  });
+
+  describe("transforms", () => {
+    it("should be able to confirm or cancel breaking changes to a SQL transform after it was run", () => {
+      createSqlTransformWithDependentMbqlQuestions();
+
+      cy.log("make breaking changes");
+      cy.get<number>("@transformId").then(H.visitTransform);
+      cy.findByTestId("transform-page").findByText("Edit query").click();
+      H.NativeEditor.clear().type('SELECT name FROM "Schema A"."Animals"');
+
+      cy.log("cancel breaking changes");
+      cy.findByTestId("transform-query-editor").button("Save changes").click();
+      H.modal().within(() => {
+        cy.findByText("Question with fields").should("be.visible");
+        cy.findByText("Question without fields").should("not.exist");
+        cy.findByText("Base transform").should("not.exist");
+        cy.button("Cancel").click();
+      });
+      cy.get("@updateTransform.all").should("have.length", 0);
+
+      cy.log("confirm breaking changes");
+      cy.findByTestId("transform-query-editor").button("Save changes").click();
+      H.modal().within(() => {
+        cy.findByText("Question with fields").should("be.visible");
+        cy.findByText("Question without fields").should("not.exist");
+        cy.findByText("Base transform").should("not.exist");
+        cy.button("Save anyway").click();
+      });
+      cy.wait("@updateTransform");
+    });
+
+    it("should not show a confirmation if there are no breaking changes when updating a SQL transform after it was run", () => {
+      createSqlTransformWithDependentMbqlQuestions();
+      cy.get<number>("@transformId").then(H.visitTransform);
+      cy.findByTestId("transform-page").findByText("Edit query").click();
+      H.NativeEditor.clear().type(
+        'SELECT score, name FROM "Schema A"."Animals"',
+      );
+      cy.findByTestId("transform-query-editor").button("Save changes").click();
+      cy.wait("@updateTransform");
+    });
+
+    it("should be able to confirm or cancel breaking changes to a MBQL transform before it was run", () => {
+      createMbqlTransformWithDependentSqlTransforms();
+
+      cy.log("make breaking changes");
+      cy.get<number>("@transformId").then(H.visitTransform);
+      cy.findByTestId("transform-page").findByText("Edit query").click();
+      H.getNotebookStep("data").findByLabelText("Pick columns").click();
+      H.popover().findByLabelText("Score").click();
+
+      cy.log("cancel breaking changes");
+      cy.findByTestId("transform-query-editor").button("Save changes").click();
+      H.modal().within(() => {
+        cy.findByText("Score transform").should("be.visible");
+        cy.findByText("Name transform").should("not.exist");
+        cy.findByText("Base transform").should("not.exist");
+        cy.button("Cancel").click();
+      });
+      cy.get("@updateTransform.all").should("have.length", 0);
+
+      cy.log("confirm breaking changes");
+      cy.findByTestId("transform-query-editor").button("Save changes").click();
+      H.modal().within(() => {
+        cy.findByText("Score transform").should("be.visible");
+        cy.findByText("Name transform").should("not.exist");
+        cy.button("Save anyway").click();
+      });
+      cy.wait("@updateTransform");
+    });
+
+    it("should not show a confirmation if there are no breaking changes when updating a MBQL transform before it was run", () => {
+      createMbqlTransformWithDependentSqlTransforms();
+      cy.get<number>("@transformId").then(H.visitTransform);
+      cy.findByTestId("transform-page").findByText("Edit query").click();
+      H.getNotebookStep("data").button("Sort").click();
+      H.popover().findByText("Score").click();
+      cy.findByTestId("transform-query-editor").button("Save changes").click();
+      cy.wait("@updateTransform");
+    });
+  });
+
+  describe("snippets", () => {
+    it("should be able to confirm or cancel breaking changes to a snippet", () => {
+      createSnippetWithDependentQuestionsAndTransforms();
+
+      cy.log("make breaking changes");
+      startEditingSnippet();
+      H.modal()
+        .findByLabelText("Enter some SQL here so you can reuse it later")
+        .clear()
+        .type("price = 1");
+
+      cy.log("cancel breaking changes");
+      H.modal().within(() => {
+        cy.button("Save").click();
+        cy.findByText("Question with snippet").should("be.visible");
+        cy.findByText("Transform with snippet").should("be.visible");
+        cy.findByText("ScoreSnippet").should("not.exist");
+        cy.button("Cancel").click();
+      });
+      cy.get("@updateSnippet.all").should("have.length", 0);
+
+      cy.log("confirm breaking changes");
+      H.modal().within(() => {
+        cy.button("Save").click();
+        cy.findByText("Question with snippet").should("be.visible");
+        cy.findByText("Transform with snippet").should("be.visible");
+        cy.findByText("ScoreSnippet").should("not.exist");
+        cy.button("Save anyway").click();
+      });
+      cy.wait("@updateSnippet");
+    });
+
+    it("should not warn if changes to a snippet are backward compatible", () => {
+      createSnippetWithDependentQuestionsAndTransforms();
+      startEditingSnippet();
+      H.modal().within(() => {
+        cy.findByLabelText("Enter some SQL here so you can reuse it later")
+          .clear()
+          .type("score = 2");
+        cy.button("Save").click();
+      });
+      cy.wait("@updateSnippet");
+    });
+  });
+});
+
+function createMbqlQuestionWithDependentMbqlQuestions() {
+  H.createQuestion({
+    name: "Base question",
+    query: {
+      "source-table": PRODUCTS_ID,
+      expressions: {
+        Expr: ["+", 1, 1],
+      },
+    },
+  }).then(({ body: card }) => {
+    cy.wrap(card.id).as("questionId");
+
+    H.createQuestion({
+      name: "Question with fields",
+      query: {
+        "source-table": `card__${card.id}`,
+        filter: [">", ["field", "Expr", { "base-type": "type/Integer" }], 1],
+      },
+      dashboard_id: ORDERS_DASHBOARD_ID,
+    });
+
+    H.createQuestion({
+      name: "Question without fields",
+      query: {
+        "source-table": `card__${card.id}`,
+      },
+    });
+  });
+}
+
+function createSqlModelWithDependentSqlQuestions() {
+  H.createNativeQuestion({
+    name: "Base model",
+    type: "model",
+    native: {
+      query: "SELECT ID, TITLE, CATEGORY FROM PRODUCTS",
+      "template-tags": {},
+    },
+  }).then(({ body: card }) => {
+    cy.wrap(card.id).as("modelId");
+    cy.request("POST", `/api/card/${card.id}/query`);
+
+    const tagName = `#${card.id}`;
+    H.createNativeQuestion({
+      name: "Question with fields",
+      native: {
+        query: `SELECT ID, CATEGORY FROM {{#${card.id}}}`,
+        "template-tags": {
+          [tagName]: {
+            id: "10422a0f-292d-10a3-fd90-407cc9e3e20e",
+            name: tagName,
+            "display-name": tagName,
+            type: "card",
+            "card-id": card.id,
+          },
+        },
+      },
+    });
+
+    H.createNativeQuestion({
+      name: "Question without fields",
+      native: {
+        query: `SELECT COUNT(*) FROM {{#${card.id}}}`,
+        "template-tags": {
+          [tagName]: {
+            id: "10422a0f-292d-10a3-fd90-407cc9e3e20f",
+            name: tagName,
+            "display-name": tagName,
+            type: "card",
+            "card-id": card.id,
+          },
+        },
+      },
+    });
+  });
+}
+
+function createMetricWithDependentMbqlQuestionsAndTransforms() {
+  H.getTableId({ databaseId: WRITABLE_DB_ID, name: "Animals" }).then(
+    (tableId) => {
+      H.getFieldId({ tableId, name: "score" }).then((fieldId) => {
+        H.createQuestion({
+          name: "Base metric",
+          type: "metric",
+          database: WRITABLE_DB_ID,
+          query: {
+            "source-table": tableId,
+            aggregation: [["min", ["field", fieldId, null]]],
+          },
+        }).then(({ body: metric }) => {
+          cy.wrap(metric.id).as("metricId");
+
+          H.createQuestion({
+            name: "Question with 1 stage",
+            database: WRITABLE_DB_ID,
+            query: {
+              "source-table": tableId,
+              aggregation: [["metric", metric.id]],
+            },
+          });
+
+          H.createQuestion({
+            name: "Question with 2 stages",
+            database: WRITABLE_DB_ID,
+            query: {
+              "source-query": {
+                "source-table": tableId,
+                aggregation: [["metric", metric.id]],
+                breakout: [["field", fieldId, null]],
+              },
+              aggregation: [
+                ["avg", ["field", "min", { "base-type": "type/Integer" }]],
+              ],
+            },
+          });
+
+          H.createTransform({
+            name: "Transform with 1 stage",
+            source: {
+              type: "query",
+              query: {
+                database: WRITABLE_DB_ID,
+                type: "query",
+                query: {
+                  "source-table": tableId,
+                  aggregation: [["metric", metric.id]],
+                },
+              },
+            },
+            target: {
+              type: "table",
+              name: "transform_1_stage",
+              schema: "public",
+              database: WRITABLE_DB_ID,
+            },
+          });
+
+          H.createTransform({
+            name: "Transform with 2 stages",
+            source: {
+              type: "query",
+              query: {
+                database: WRITABLE_DB_ID,
+                type: "query",
+                query: {
+                  "source-query": {
+                    "source-table": tableId,
+                    aggregation: [["metric", metric.id]],
+                    breakout: [["field", fieldId, null]],
+                  },
+                  aggregation: [
+                    ["avg", ["field", "min", { "base-type": "type/Integer" }]],
+                  ],
+                },
+              },
+            },
+            target: {
+              type: "table",
+              name: "transform_2_stages",
+              schema: "public",
+              database: WRITABLE_DB_ID,
+            },
+          });
+        });
+      });
+    },
+  );
+}
+
+function createSqlTransformWithDependentMbqlQuestions() {
+  const transformTableName = "base_transform";
+
+  H.createTransform({
+    name: "Base transform",
+    source: {
+      type: "query",
+      query: {
+        database: WRITABLE_DB_ID,
+        type: "native",
+        native: {
+          query: 'SELECT name, score FROM "Schema A"."Animals"',
+          "template-tags": {},
+        },
+      },
+    },
+    target: {
+      type: "table",
+      name: transformTableName,
+      schema: "public",
+      database: WRITABLE_DB_ID,
+    },
+  }).then(({ body: transform }) => {
+    cy.wrap(transform.id).as("transformId");
+    cy.request("POST", `/api/ee/transform/${transform.id}/run`);
+    H.waitForSucceededTransformRuns();
+    H.resyncDatabase({ dbId: WRITABLE_DB_ID, tableName: transformTableName });
+
+    H.getTableId({ databaseId: WRITABLE_DB_ID, name: transformTableName }).then(
+      (tableId) => {
+        H.createQuestion({
+          name: "Question with fields",
+          database: WRITABLE_DB_ID,
+          query: {
+            "source-table": tableId,
+            filter: [
+              ">",
+              ["field", "score", { "base-type": "type/Integer" }],
+              1,
+            ],
+          },
+        });
+        H.createQuestion({
+          name: "Question without fields",
+          database: WRITABLE_DB_ID,
+          query: {
+            "source-table": tableId,
+          },
+        });
+      },
+    );
+  });
+}
+
+function createMbqlTransformWithDependentSqlTransforms() {
+  H.getTableId({ name: "Animals", databaseId: WRITABLE_DB_ID }).then(
+    (tableId) => {
+      H.createTransform(
+        {
+          name: "Base transform",
+          source: {
+            type: "query",
+            query: {
+              database: WRITABLE_DB_ID,
+              type: "query",
+              query: {
+                "source-table": tableId,
+              },
+            },
+          },
+          target: {
+            type: "table",
+            name: "base_transform",
+            schema: "public",
+            database: WRITABLE_DB_ID,
+          },
+        },
+        { wrapId: true },
+      );
+
+      H.createTransform({
+        name: "Name transform",
+        source: {
+          type: "query",
+          query: {
+            database: WRITABLE_DB_ID,
+            type: "native",
+            native: {
+              query: "SELECT name FROM base_transform",
+              "template-tags": {},
+            },
+          },
+        },
+        target: {
+          type: "table",
+          name: "name_transform",
+          schema: "public",
+          database: WRITABLE_DB_ID,
+        },
+      });
+
+      H.createTransform({
+        name: "Score transform",
+        source: {
+          type: "query",
+          query: {
+            database: WRITABLE_DB_ID,
+            type: "native",
+            native: {
+              query: "SELECT score FROM base_transform",
+              "template-tags": {},
+            },
+          },
+        },
+        target: {
+          type: "table",
+          name: "score_transform",
+          schema: "public",
+          database: WRITABLE_DB_ID,
+        },
+      });
+    },
+  );
+}
+
+function createSnippetWithDependentQuestionsAndTransforms() {
+  H.createSnippet({
+    name: "ScoreSnippet",
+    content: "score = 1",
+  }).then(({ body: snippet }) => {
+    cy.wrap(snippet.id).as("snippetId");
+
+    H.createNativeQuestion({
+      name: "Question with snippet",
+      database: WRITABLE_DB_ID,
+      native: {
+        query:
+          'SELECT * FROM "Schema A"."Animals" WHERE {{snippet:ScoreSnippet}}',
+        "template-tags": {
+          "snippet:ScoreSnippet": {
+            id: "4b77cc1f-ea70-4ef6-84db-58432fce6928",
+            name: "snippet:ScoreSnippet",
+            "display-name": "snippet:ScoreSnippet",
+            type: "snippet",
+            "snippet-id": snippet.id,
+            "snippet-name": snippet.name,
+          },
+        },
+      },
+    });
+
+    H.createTransform({
+      name: "Transform with snippet",
+      source: {
+        type: "query",
+        query: {
+          database: WRITABLE_DB_ID,
+          type: "native",
+          native: {
+            query:
+              'SELECT * FROM "Schema A"."Animals" WHERE {{snippet:ScoreSnippet}}',
+            "template-tags": {
+              "snippet:ScoreSnippet": {
+                id: "4b77cc1f-ea70-4ef6-84db-58432fce6929",
+                name: "snippet:ScoreSnippet",
+                "display-name": "snippet:ScoreSnippet",
+                type: "snippet",
+                "snippet-id": snippet.id,
+                "snippet-name": snippet.name,
+              },
+            },
+          },
+        },
+      },
+      target: {
+        type: "table",
+        name: "transform_with_snippet",
+        schema: "public",
+        database: WRITABLE_DB_ID,
+      },
+    });
+  });
+}
+
+function startEditingSnippet() {
+  H.startNewNativeQuestion();
+  cy.findByTestId("native-query-editor-action-buttons").icon("snippet").click();
+  cy.findByTestId("sidebar-content").within(() => {
+    cy.icon("chevrondown").click({ force: true });
+    cy.button(/Edit/).click();
+  });
+}
+
+function confirmDiscardChanges() {
+  H.modal().should("have.length", 2).last().button("Discard changes").click();
+}

e2e/test/scenarios/embedding/embedding-hub/embedding-hub.cy.spec.ts

@@ -23,12 +23,12 @@ describe("scenarios - embedding hub", () => {
         .should("exist");
     });
 
-    it('"Generate a dashboard" card should work correctly', () => {
+    it('"Create a dashboard" card should work correctly', () => {
       cy.visit("/admin/embedding/setup-guide");
 
-      cy.log("Find and click on 'Generate a dashboard' card");
+      cy.log("Find and click on 'Create a dashboard' card");
       cy.findByTestId("admin-layout-content")
-        .findByText("Generate a dashboard")
+        .findByText("Create a dashboard")
         .click();
 
       cy.log("Select a table to generate dashboard from");
@@ -44,11 +44,13 @@ describe("scenarios - embedding hub", () => {
       cy.url().should("include", "/auto/dashboard/table/");
     });
 
-    it('"Add data" card should work correctly', () => {
+    it('"Connect a database" card should work correctly', () => {
       cy.visit("/admin/embedding/setup-guide");
 
-      cy.log("Find and click on 'Add data' card");
-      cy.findByTestId("admin-layout-content").findByText("Add data").click();
+      cy.log("Find and click on 'Connect a database' card");
+      cy.findByTestId("admin-layout-content")
+        .findByText("Connect a database")
+        .click();
 
       cy.log("Add data modal should open");
       cy.findByRole("dialog").within(() => {
@@ -70,14 +72,16 @@ describe("scenarios - embedding hub", () => {
 
       cy.visit("/admin/embedding/setup-guide");
 
-      cy.log("'Add data' should not be marked as done");
+      cy.log("'Connect a database' should not be marked as done");
       cy.findByTestId("admin-layout-content")
-        .findByText("Add data")
+        .findByText("Connect a database")
         .closest("button")
         .findByText("Done")
         .should("not.exist");
 
-      cy.findByTestId("admin-layout-content").findByText("Add data").click();
+      cy.findByTestId("admin-layout-content")
+        .findByText("Connect a database")
+        .click();
 
       H.modal().within(() => {
         cy.findByText("CSV").click();
@@ -101,9 +105,9 @@ describe("scenarios - embedding hub", () => {
 
       cy.wait("@getChecklist");
 
-      cy.log("'Add data' should be marked as done");
+      cy.log("'Connect a database' should be marked as done");
       cy.findByTestId("admin-layout-content")
-        .findByText("Add data")
+        .findByText("Connect a database")
         .closest("button")
         .scrollIntoView()
         .findByText("Done")
@@ -143,11 +147,11 @@ describe("scenarios - embedding hub", () => {
       });
     });
 
-    it('"Embed in your code" card should take you to the embed flow', () => {
+    it('"Get embed snippet" card should take you to the embed flow', () => {
       cy.visit("/admin/embedding/setup-guide");
 
       cy.findByTestId("admin-layout-content")
-        .findByText("Embed in your code")
+        .findByText("Get embed snippet")
         .click();
 
       cy.url().should("include", "/embed-js?auth_method=user_session");
@@ -189,8 +193,8 @@ describe("scenarios - embedding hub", () => {
 
       cy.get("main").within(() => {
         cy.findByText("Create models").should("be.visible");
-        cy.findByText("Generate a dashboard").should("be.visible");
-        cy.findByText("Add data").should("be.visible").click();
+        cy.findByText("Create a dashboard").should("be.visible");
+        cy.findByText("Connect a database").should("be.visible").click();
       });
 
       cy.log("Sanity check: add data modal should open");

e2e/test/scenarios/embedding/sdk-iframe-embedding-setup/embed-parameters.cy.spec.ts

@@ -104,14 +104,6 @@ H.describeWithSnowplow(suiteTitle, () => {
         .findByLabelText("Product ID")
         .should("contain", "456");
 
-      H.expectUnstructuredSnowplowEvent(
-        {
-          event: "embed_wizard_option_changed",
-          event_detail: "initialParameters",
-        },
-        2,
-      );
-
       cy.log("both default values should be in the code snippet");
       getEmbedSidebar().within(() => {
         cy.findByText("Get Code").click();
@@ -147,14 +139,6 @@ H.describeWithSnowplow(suiteTitle, () => {
         .findByTestId("dashboard-parameters-widget-container")
         .should("not.exist");
 
-      H.expectUnstructuredSnowplowEvent(
-        {
-          event: "embed_wizard_option_changed",
-          event_detail: "hiddenParameters",
-        },
-        2,
-      );
-
       cy.log("code snippet should contain the hidden parameters");
       getEmbedSidebar().within(() => {
         cy.findByText("Get Code").click();
@@ -218,11 +202,6 @@ H.describeWithSnowplow(suiteTitle, () => {
         cy.findAllByText("75.41").first().should("be.visible");
       });
 
-      H.expectUnstructuredSnowplowEvent({
-        event: "embed_wizard_option_changed",
-        event_detail: "initialSqlParameters",
-      });
-
       getEmbedSidebar().within(() => {
         cy.findByText("Get Code").click();
         codeBlock().should("contain", "initial-sql-parameters=");

e2e/test/scenarios/embedding/sdk-iframe-embedding-setup/get-code.cy.spec.ts

@@ -115,6 +115,32 @@ H.describeWithSnowplow(suiteTitle, () => {
     getEmbedSidebar().within(() => {
       cy.findByLabelText("Existing Metabase session").should("be.checked");
       codeBlock().should("contain", '"useExistingUserSession": true');
+
+      cy.findByText(/Copy Code/).click();
+
+      H.expectUnstructuredSnowplowEvent({
+        event: "embed_wizard_code_copied",
+        event_detail: "user_session",
+      });
+    });
+  });
+
+  it("should track embed_wizard_code_copied when copy event triggers", () => {
+    navigateToGetCodeStep({
+      experience: "dashboard",
+      resourceName: DASHBOARD_NAME,
+    });
+
+    getEmbedSidebar().within(() => {
+      cy.findByLabelText("Existing Metabase session").should("be.checked");
+
+      codeBlock().should("contain", '"useExistingUserSession": true');
+      codeBlock().trigger("copy");
+
+      H.expectUnstructuredSnowplowEvent({
+        event: "embed_wizard_code_copied",
+        event_detail: "user_session",
+      });
     });
   });
 
@@ -127,13 +153,14 @@ H.describeWithSnowplow(suiteTitle, () => {
 
     getEmbedSidebar().within(() => {
       cy.findByLabelText("Single sign-on (SSO)").click();
+      codeBlock().should("not.contain", "useExistingUserSession");
+
+      cy.findByText(/Copy Code/).click();
 
       H.expectUnstructuredSnowplowEvent({
-        event: "embed_wizard_auth_selected",
+        event: "embed_wizard_code_copied",
         event_detail: "sso",
       });
-
-      codeBlock().should("not.contain", "useExistingUserSession");
     });
   });
 

e2e/test/scenarios/embedding/sdk-iframe-embedding-setup/select-embed-entity.cy.spec.ts

@@ -41,7 +41,25 @@ H.describeWithSnowplow(suiteTitle, () => {
     H.expectNoBadSnowplowEvents();
   });
 
-  it("can select a recent dashboard to embed", () => {
+  it("tracks 'default' when keeping the default dashboard selection", () => {
+    visitNewEmbedPage();
+
+    getEmbedSidebar().within(() => {
+      cy.findByText("Next").click();
+      cy.findByText("Select a dashboard to embed").should("be.visible");
+
+      cy.log("first dashboard should be selected by default");
+      getRecentItemCards().first().should("have.attr", "data-selected", "true");
+      cy.findByText("Next").click();
+    });
+
+    H.expectUnstructuredSnowplowEvent({
+      event: "embed_wizard_resource_selection_completed",
+      event_detail: "default",
+    });
+  });
+
+  it("tracks 'custom' when selecting a different dashboard", () => {
     cy.log("add two dashboards to activity log");
 
     H.createDashboard({ name: SECOND_DASHBOARD_NAME }).then(
@@ -70,14 +88,6 @@ H.describeWithSnowplow(suiteTitle, () => {
       cy.log("second dashboard can be selected");
       cy.findByText(SECOND_DASHBOARD_NAME).click();
 
-      cy.get("@secondDashboardId").then((secondDashboardId) => {
-        H.expectUnstructuredSnowplowEvent({
-          event: "embed_wizard_resource_selected",
-          target_id: secondDashboardId,
-          event_detail: "dashboard",
-        });
-      });
-
       getRecentItemCards().eq(1).should("have.attr", "data-selected", "true");
     });
 
@@ -87,22 +97,11 @@ H.describeWithSnowplow(suiteTitle, () => {
       cy.findByText(SECOND_DASHBOARD_NAME).should("be.visible");
     });
 
-    cy.log("select the same the dashboard again");
-    getEmbedSidebar().within(() => {
-      cy.findByText(SECOND_DASHBOARD_NAME).click();
-      cy.findByText(SECOND_DASHBOARD_NAME).click();
-    });
+    getEmbedSidebar().findByText("Next").click();
 
-    cy.get("@secondDashboardId").then((secondDashboardId) => {
-      cy.log("only a single snowplow event should be sent");
-      H.expectUnstructuredSnowplowEvent(
-        {
-          event: "embed_wizard_resource_selected",
-          target_id: secondDashboardId,
-          event_detail: "dashboard",
-        },
-        1,
-      );
+    H.expectUnstructuredSnowplowEvent({
+      event: "embed_wizard_resource_selection_completed",
+      event_detail: "custom",
     });
   });
 
@@ -131,13 +130,14 @@ H.describeWithSnowplow(suiteTitle, () => {
       cy.log("second question can be selected");
       cy.findByText(SECOND_QUESTION_NAME).click();
 
-      H.expectUnstructuredSnowplowEvent({
-        event: "embed_wizard_resource_selected",
-        target_id: ORDERS_BY_YEAR_QUESTION_ID,
-        event_detail: "chart",
-      });
-
       getRecentItemCards().eq(1).should("have.attr", "data-selected", "true");
+
+      cy.findByText("Next").click();
+
+      H.expectUnstructuredSnowplowEvent({
+        event: "embed_wizard_resource_selection_completed",
+        event_detail: "custom",
+      });
     });
 
     cy.log("selected question should be shown in the preview");
@@ -166,14 +166,6 @@ H.describeWithSnowplow(suiteTitle, () => {
       cy.findByText(SECOND_DASHBOARD_NAME).click();
     });
 
-    cy.get("@secondDashboardId").then((secondDashboardId) => {
-      H.expectUnstructuredSnowplowEvent({
-        event: "embed_wizard_resource_selected",
-        target_id: secondDashboardId,
-        event_detail: "dashboard",
-      });
-    });
-
     cy.log("dashboard is added to the top of recents list and selected");
     getEmbedSidebar().within(() => {
       getRecentItemCards()
@@ -204,12 +196,6 @@ H.describeWithSnowplow(suiteTitle, () => {
       cy.findByText(FIRST_QUESTION_NAME).click();
     });
 
-    H.expectUnstructuredSnowplowEvent({
-      event: "embed_wizard_resource_selected",
-      target_id: ORDERS_COUNT_QUESTION_ID,
-      event_detail: "chart",
-    });
-
     cy.log("question is added to the top of recents list and selected");
     getEmbedSidebar().within(() => {
       getRecentItemCards()
@@ -217,6 +203,13 @@ H.describeWithSnowplow(suiteTitle, () => {
         .first()
         .should("contain", FIRST_QUESTION_NAME)
         .should("have.attr", "data-selected", "true");
+
+      cy.findByText("Next").click();
+    });
+
+    H.expectUnstructuredSnowplowEvent({
+      event: "embed_wizard_resource_selection_completed",
+      event_detail: "custom",
     });
 
     cy.wait("@cardQuery");

e2e/test/scenarios/embedding/sdk-iframe-embedding-setup/select-embed-experience.cy.spec.ts

@@ -39,8 +39,18 @@ H.describeWithSnowplow(suiteTitle, () => {
       visitNewEmbedPage();
       assertRecentItemName("dashboard", dashboardName);
 
+      H.expectUnstructuredSnowplowEvent({ event: "embed_wizard_opened" });
       H.waitForSimpleEmbedIframesToLoad();
 
+      getEmbedSidebar().within(() => {
+        cy.findByText("Next").click();
+      });
+
+      H.expectUnstructuredSnowplowEvent({
+        event: "embed_wizard_experience_completed",
+        event_detail: "default",
+      });
+
       H.getSimpleEmbedIframeContent().within(() => {
         cy.log("dashboard title is visible");
         cy.findByText(dashboardName).should("be.visible");
@@ -59,12 +69,16 @@ H.describeWithSnowplow(suiteTitle, () => {
 
       visitNewEmbedPage();
       assertRecentItemName("card", questionName);
+      H.expectUnstructuredSnowplowEvent({ event: "embed_wizard_opened" });
 
-      getEmbedSidebar().findByText("Chart").click();
+      getEmbedSidebar().within(() => {
+        cy.findByText("Chart").click();
+        cy.findByText("Next").click();
+      });
 
       H.expectUnstructuredSnowplowEvent({
-        event: "embed_wizard_experience_selected",
-        event_detail: "chart",
+        event: "embed_wizard_experience_completed",
+        event_detail: "custom=chart",
       });
 
       cy.wait("@cardQuery");
@@ -77,11 +91,15 @@ H.describeWithSnowplow(suiteTitle, () => {
 
     it("shows exploration template when selected", { tags: "@skip" }, () => {
       visitNewEmbedPage();
-      getEmbedSidebar().findByText("Exploration").click();
+
+      getEmbedSidebar().within(() => {
+        cy.findByText("Exploration").click();
+        cy.findByText("Next").click();
+      });
 
       H.expectUnstructuredSnowplowEvent({
-        event: "embed_wizard_experience_selected",
-        event_detail: "exploration",
+        event: "embed_wizard_experience_completed",
+        event_detail: "custom=exploration",
       });
 
       H.waitForSimpleEmbedIframesToLoad();
@@ -94,11 +112,15 @@ H.describeWithSnowplow(suiteTitle, () => {
 
     it("shows browser template when selected", () => {
       visitNewEmbedPage();
-      getEmbedSidebar().findByText("Browser").click();
+
+      getEmbedSidebar().within(() => {
+        cy.findByText("Browser").click();
+        cy.findByText("Next").click();
+      });
 
       H.expectUnstructuredSnowplowEvent({
-        event: "embed_wizard_experience_selected",
-        event_detail: "browser",
+        event: "embed_wizard_experience_completed",
+        event_detail: "custom=browser",
       });
 
       H.getSimpleEmbedIframeContent().within(() => {
@@ -144,11 +166,14 @@ H.describeWithSnowplow(suiteTitle, () => {
         visitNewEmbedPage();
         cy.wait("@emptyRecentItems");
 
-        getEmbedSidebar().findByText("Chart").click();
+        getEmbedSidebar().within(() => {
+          cy.findByText("Chart").click();
+          cy.findByText("Next").click();
+        });
 
         H.expectUnstructuredSnowplowEvent({
-          event: "embed_wizard_experience_selected",
-          event_detail: "chart",
+          event: "embed_wizard_experience_completed",
+          event_detail: "custom=chart",
         });
 
         H.waitForSimpleEmbedIframesToLoad();

e2e/test/scenarios/embedding/sdk-iframe-embedding-setup/select-embed-options.cy.spec.ts

@@ -55,11 +55,6 @@ H.describeWithSnowplow(suiteTitle, () => {
       .click()
       .should("not.be.checked");
 
-    H.expectUnstructuredSnowplowEvent({
-      event: "embed_wizard_option_changed",
-      event_detail: "drills",
-    });
-
     cy.log("drill-through should be disabled in the preview");
     H.getSimpleEmbedIframeContent().within(() => {
       cy.findByText("110.93").click();
@@ -68,6 +63,13 @@ H.describeWithSnowplow(suiteTitle, () => {
 
     cy.log("snippet should be updated");
     getEmbedSidebar().findByText("Get Code").click();
+
+    H.expectUnstructuredSnowplowEvent({
+      event: "embed_wizard_options_completed",
+      event_detail:
+        "settings=custom,theme=default,auth=user_session,drills=false,withDownloads=false,withTitle=true",
+    });
+
     codeBlock().should("contain", 'drills="false"');
   });
 
@@ -91,17 +93,19 @@ H.describeWithSnowplow(suiteTitle, () => {
       .click()
       .should("be.checked");
 
-    H.expectUnstructuredSnowplowEvent({
-      event: "embed_wizard_option_changed",
-      event_detail: "withDownloads",
-    });
-
     H.getSimpleEmbedIframeContent()
       .findByTestId("export-as-pdf-button")
       .should("be.visible");
 
     cy.log("snippet should be updated");
     getEmbedSidebar().findByText("Get Code").click();
+
+    H.expectUnstructuredSnowplowEvent({
+      event: "embed_wizard_options_completed",
+      event_detail:
+        "settings=custom,theme=default,auth=user_session,drills=true,withDownloads=true,withTitle=true",
+    });
+
     codeBlock().should("contain", 'with-downloads="true"');
   });
 
@@ -125,17 +129,19 @@ H.describeWithSnowplow(suiteTitle, () => {
       .click()
       .should("not.be.checked");
 
-    H.expectUnstructuredSnowplowEvent({
-      event: "embed_wizard_option_changed",
-      event_detail: "withTitle",
-    });
-
     H.getSimpleEmbedIframeContent()
       .findByText("Orders in a dashboard")
       .should("not.exist");
 
     cy.log("snippet should be updated");
     getEmbedSidebar().findByText("Get Code").click();
+
+    H.expectUnstructuredSnowplowEvent({
+      event: "embed_wizard_options_completed",
+      event_detail:
+        "settings=custom,theme=default,auth=user_session,drills=true,withDownloads=false,withTitle=false",
+    });
+
     codeBlock().should("contain", 'with-title="false"');
   });
 
@@ -161,11 +167,6 @@ H.describeWithSnowplow(suiteTitle, () => {
       .click()
       .should("not.be.checked");
 
-    H.expectUnstructuredSnowplowEvent({
-      event: "embed_wizard_option_changed",
-      event_detail: "drills",
-    });
-
     cy.log("drill-through should be disabled in chart preview");
     H.getSimpleEmbedIframeContent().within(() => {
       cy.findByText("18,760").click();
@@ -200,17 +201,19 @@ H.describeWithSnowplow(suiteTitle, () => {
       .click()
       .should("be.checked");
 
-    H.expectUnstructuredSnowplowEvent({
-      event: "embed_wizard_option_changed",
-      event_detail: "withDownloads",
-    });
-
     H.getSimpleEmbedIframeContent()
       .findByTestId("question-download-widget-button")
       .should("be.visible");
 
     cy.log("snippet should be updated");
     getEmbedSidebar().findByText("Get Code").click();
+
+    H.expectUnstructuredSnowplowEvent({
+      event: "embed_wizard_options_completed",
+      event_detail:
+        "settings=custom,theme=default,auth=user_session,drills=true,withDownloads=true,withTitle=true,isSaveEnabled=false",
+    });
+
     codeBlock().should("contain", 'with-downloads="true"');
   });
 
@@ -233,11 +236,6 @@ H.describeWithSnowplow(suiteTitle, () => {
       .click()
       .should("not.be.checked");
 
-    H.expectUnstructuredSnowplowEvent({
-      event: "embed_wizard_option_changed",
-      event_detail: "withTitle",
-    });
-
     H.getSimpleEmbedIframeContent()
       .findByText("Orders, Count")
       .should("not.exist");
@@ -277,22 +275,15 @@ H.describeWithSnowplow(suiteTitle, () => {
     H.getSimpleEmbedIframeContent()
       .findByText("Orders, Count")
       .should("be.visible");
-
-    H.expectUnstructuredSnowplowEvent(
-      {
-        event: "embed_wizard_option_changed",
-        event_detail: "withTitle",
-      },
-      2,
-    );
   });
 
   ["exploration", "chart"].forEach((experience) => {
     it(`toggles save button for ${experience}`, () => {
-      navigateToEmbedOptionsStep({
-        experience,
-        ...(experience === "chart" && { resourceName: QUESTION_NAME }),
-      });
+      navigateToEmbedOptionsStep(
+        experience === "chart"
+          ? { experience: "chart", resourceName: QUESTION_NAME }
+          : { experience: "exploration" },
+      );
 
       if (experience === "exploration") {
         cy.log("visualize a question to enable the save button");
@@ -315,11 +306,6 @@ H.describeWithSnowplow(suiteTitle, () => {
         .click()
         .should("be.checked");
 
-      H.expectUnstructuredSnowplowEvent({
-        event: "embed_wizard_option_changed",
-        event_detail: "isSaveEnabled",
-      });
-
       if (experience === "chart") {
         cy.log("select a different visualization to enable the save button");
         H.getSimpleEmbedIframeContent().within(() => {
@@ -334,6 +320,15 @@ H.describeWithSnowplow(suiteTitle, () => {
 
       cy.log("snippet should be updated");
       getEmbedSidebar().findByText("Get Code").click();
+
+      H.expectUnstructuredSnowplowEvent({
+        event: "embed_wizard_options_completed",
+        event_detail:
+          experience === "chart"
+            ? "settings=custom,theme=default,auth=user_session,drills=true,withDownloads=false,withTitle=true,isSaveEnabled=true"
+            : "settings=custom,theme=default,auth=user_session,isSaveEnabled=true",
+      });
+
       codeBlock().should("contain", 'is-save-enabled="true"');
     });
   });
@@ -358,17 +353,19 @@ H.describeWithSnowplow(suiteTitle, () => {
       .click()
       .should("be.checked");
 
-    H.expectUnstructuredSnowplowEvent({
-      event: "embed_wizard_option_changed",
-      event_detail: "readOnly",
-    });
-
     H.getSimpleEmbedIframeContent().within(() => {
       cy.findByText("New dashboard").should("be.visible");
     });
 
     cy.log("snippet should be updated");
     getEmbedSidebar().findByText("Get Code").click();
+
+    H.expectUnstructuredSnowplowEvent({
+      event: "embed_wizard_options_completed",
+      event_detail:
+        "settings=custom,theme=default,auth=user_session,readOnly=false",
+    });
+
     codeBlock().should("contain", 'read-only="false"');
   });
 
@@ -397,11 +394,6 @@ H.describeWithSnowplow(suiteTitle, () => {
         .type("rgb(255, 0, 0)");
     });
 
-    H.expectUnstructuredSnowplowEvent({
-      event: "embed_wizard_option_changed",
-      event_detail: "theme",
-    });
-
     cy.log("table header cell should now be red");
     H.getSimpleEmbedIframeContent()
       .findAllByTestId("cell-data")
@@ -414,6 +406,12 @@ H.describeWithSnowplow(suiteTitle, () => {
     cy.log("snippet should be updated");
     getEmbedSidebar().findByText("Get Code").click();
 
+    H.expectUnstructuredSnowplowEvent({
+      event: "embed_wizard_options_completed",
+      event_detail:
+        "settings=custom,theme=custom,auth=user_session,drills=true,withDownloads=false,withTitle=true",
+    });
+
     codeBlock().should("contain", '"theme": {');
     codeBlock().should("contain", '"colors": {');
     codeBlock().should("contain", '"brand": "#FF0000"');
@@ -424,11 +422,6 @@ H.describeWithSnowplow(suiteTitle, () => {
     cy.log("click reset button");
     getEmbedSidebar().findByLabelText("Reset colors").click();
 
-    H.expectUnstructuredSnowplowEvent({
-      event: "embed_wizard_option_changed",
-      event_detail: "theme",
-    });
-
     cy.log("table header should be back to default blue");
     H.getSimpleEmbedIframeContent()
       .findAllByTestId("cell-data")
@@ -441,6 +434,11 @@ H.describeWithSnowplow(suiteTitle, () => {
     cy.log("snippet should not contain theme colors");
     getEmbedSidebar().findByText("Get Code").click();
     codeBlock().should("not.contain", '"theme": {');
+
+    H.expectUnstructuredSnowplowEvent({
+      event: "embed_wizard_options_completed",
+      event_detail: "settings=default",
+    });
   });
 
   it("derives colors for dark theme palette", () => {
@@ -475,6 +473,12 @@ H.describeWithSnowplow(suiteTitle, () => {
     cy.log("check that derived colors are applied to snippet");
     getEmbedSidebar().findByText("Get Code").click();
 
+    H.expectUnstructuredSnowplowEvent({
+      event: "embed_wizard_options_completed",
+      event_detail:
+        "settings=custom,theme=custom,auth=user_session,drills=true,withDownloads=false,withTitle=true",
+    });
+
     // derived-colors-for-embed-flow.unit.spec.ts contains the tests for other derived colors.
     cy.log("dark mode colors should be derived");
     codeBlock().should("contain", '"background-hover": "rgb(27, 27, 27)"');

e2e/test/scenarios/embedding/sdk-iframe-embedding/authentication.cy.spec.ts

@@ -134,7 +134,7 @@ describe("scenarios > embedding > sdk iframe embedding > authentication", () =>
         (win as any).metabaseConfig = {
           ...(win as any).metabaseConfig,
           fetchRequestToken: async () => {
-            const jwt = await getSignedJwtForUser(USERS.admin);
+            const jwt = await getSignedJwtForUser({ user: USERS.admin });
 
             return { jwt };
           },

e2e/test/scenarios/metrics/metrics-collection.cy.spec.js

@@ -225,14 +225,14 @@ describe("scenarios > metrics > collection", () => {
     H.createQuestion({
       ...ORDERS_SCALAR_MODEL_METRIC,
       collection_id: FIRST_COLLECTION_ID,
-    }).then(({ body: card }) => {
+    }).then(() => {
       cy.signIn("nocollection");
       H.visitCollection(FIRST_COLLECTION_ID);
     });
     H.getPinnedSection()
-      .findByTestId("scalar-container")
-      .findByText("18,760")
-      .should("be.visible");
+      .findByTestId("scalar-value")
+      .should("not.be.empty")
+      .and("be.visible");
   });
 });
 

e2e/test/scenarios/native/native-reproductions.cy.spec.ts

@@ -818,3 +818,38 @@ describe("issue 59356", () => {
     cy.get("@dataset.all").should("have.length", 2);
   });
 });
+
+describe("issue 63711", () => {
+  beforeEach(() => {
+    H.restore();
+    cy.signInAsNormalUser();
+  });
+
+  it("Completions should be visible when there are a lot of options (metabase#63711)", () => {
+    H.startNewNativeQuestion();
+    H.NativeEditor.type("s");
+
+    cy.log("completions should be scrollable");
+    H.NativeEditor.completions()
+      .findByLabelText("Completions")
+      .then(($el) => {
+        const element = $el[0];
+        cy.wrap(element.scrollHeight).should("be.gt", element.clientHeight);
+      });
+
+    cy.log("completions should not cut off the height of the inner element");
+    H.NativeEditor.completion("SAVEPOINT")
+      .should("be.visible")
+      .then(($outerElement) => {
+        cy.wrap($outerElement)
+          .findByText("AVEPOINT")
+          .should("be.visible")
+          .then(($innerElement) => {
+            cy.wrap($innerElement[0].offsetHeight).should(
+              "be.eq",
+              $outerElement[0].clientHeight,
+            );
+          });
+      });
+  });
+});

e2e/test/scenarios/onboarding/setup/setup.cy.spec.ts

@@ -314,6 +314,19 @@ describe("scenarios > setup", () => {
       .should("be.visible");
   });
 
+  // There are only one step in the setup flow, so there is no need to show step numbers.
+  it("should not show step numbers in cloud embedding use case", () => {
+    H.mockSessionProperty("is-hosted?", true);
+    H.mockSessionProperty("token-features", { hosting: true });
+
+    cy.visit(
+      "/setup?first_name=John&last_name=Doe&email=john@doe.test&site_name=Doe%20Unlimited&use_case=embedding",
+    );
+
+    H.main().findByText("What should we call you?").should("be.visible");
+    cy.findByTestId("step-number").should("not.exist");
+  });
+
   it("should allow localization in the 'embedding' setup flow", () => {
     cy.visit(
       "/setup?first_name=John&last_name=Doe&email=john@doe.test&site_name=Doe%20Unlimited&use_case=embedding",

e2e/test/scenarios/question/notebook-data-source.cy.spec.ts

@@ -436,6 +436,8 @@ describe("issue 28106", () => {
           .findByTestId("scroll-container")
           .as("schemasList");
 
+        H.entityPickerModalLevel(2).should("contain", "Animals");
+
         cy.get("@schemasList").scrollTo("bottom");
 
         // assert scrolling worked and the last item is visible

e2e/test/scenarios/sharing/public-sharing-embed-flow.cy.spec.ts

@@ -7,14 +7,22 @@ import { getEmbedSidebar } from "../embedding/sdk-iframe-embedding-setup/helpers
 
 const { H } = cy;
 
-describe("embed flow pre-selection from sharing modal", () => {
+const suiteTitle = "scenarios > sharing > embed flow pre-selection";
+
+H.describeWithSnowplow(suiteTitle, () => {
   beforeEach(() => {
     H.restore();
+    H.resetSnowplow();
     cy.signInAsAdmin();
-    H.activateToken("pro-self-hosted");
+    H.activateToken("bleeding-edge");
+    H.enableTracking();
     H.updateSetting("enable-embedding-simple", true);
   });
 
+  afterEach(() => {
+    H.expectNoBadSnowplowEvents();
+  });
+
   it("pre-selects dashboard in embed flow when opened from dashboard sharing modal", () => {
     H.visitDashboard(ORDERS_DASHBOARD_ID);
     H.openSharingMenu("Embed");
@@ -23,6 +31,8 @@ describe("embed flow pre-selection from sharing modal", () => {
       cy.findByRole("link", { name: "Embedded Analytics JS" }).click();
     });
 
+    H.expectUnstructuredSnowplowEvent({ event: "embed_wizard_opened" });
+
     cy.location("search").should((search) => {
       const params = new URLSearchParams(search);
 
@@ -30,7 +40,6 @@ describe("embed flow pre-selection from sharing modal", () => {
       expect(params.get("resource_id")).to.equal(String(ORDERS_DASHBOARD_ID));
     });
 
-    cy.log("should navigate to embed options step");
     getEmbedSidebar().within(() => {
       cy.findByText("Behavior").should("be.visible");
       cy.findByText("Appearance").should("be.visible");
@@ -41,15 +50,24 @@ describe("embed flow pre-selection from sharing modal", () => {
     H.getSimpleEmbedIframeContent()
       .findByText("Orders in a dashboard", { timeout: 10_000 })
       .should("be.visible");
+
+    getEmbedSidebar().findByText("Get Code").click();
+
+    H.expectUnstructuredSnowplowEvent({
+      event: "embed_wizard_options_completed",
+      event_detail: "settings=default",
+    });
   });
 
   it("pre-selects question in embed flow when opened from question sharing modal", () => {
     H.visitQuestion(ORDERS_QUESTION_ID);
     H.openSharingMenu("Embed");
 
-    H.getEmbedModalSharingPane().within(() => {
-      cy.findByRole("link", { name: "Embedded Analytics JS" }).click();
-    });
+    H.getEmbedModalSharingPane()
+      .findByRole("link", { name: "Embedded Analytics JS" })
+      .click();
+
+    H.expectUnstructuredSnowplowEvent({ event: "embed_wizard_opened" });
 
     cy.location("search").should((search) => {
       const params = new URLSearchParams(search);
@@ -58,7 +76,6 @@ describe("embed flow pre-selection from sharing modal", () => {
       expect(params.get("resource_id")).to.equal(String(ORDERS_QUESTION_ID));
     });
 
-    cy.log("should navigate to embed options step");
     getEmbedSidebar().within(() => {
       cy.findByText("Behavior").should("be.visible");
       cy.findByText("Appearance").should("be.visible");
@@ -69,5 +86,58 @@ describe("embed flow pre-selection from sharing modal", () => {
     H.getSimpleEmbedIframeContent()
       .findByText("Orders", { timeout: 10_000 })
       .should("be.visible");
+
+    getEmbedSidebar().findByText("Get Code").click();
+
+    H.expectUnstructuredSnowplowEvent({
+      event: "embed_wizard_options_completed",
+      event_detail: "settings=default",
+    });
+  });
+
+  it("tracks default resources for pre-selected dashboard", () => {
+    H.visitDashboard(ORDERS_DASHBOARD_ID);
+    H.openSharingMenu("Embed");
+
+    H.getEmbedModalSharingPane().within(() => {
+      cy.findByRole("link", { name: "Embedded Analytics JS" }).click();
+    });
+
+    H.expectUnstructuredSnowplowEvent({ event: "embed_wizard_opened" });
+
+    cy.log("go back to resource selection step");
+    getEmbedSidebar().within(() => {
+      cy.findByText("Back").click();
+      cy.findByText("Select a dashboard to embed").should("be.visible");
+      cy.findByText("Next").click();
+    });
+
+    H.expectUnstructuredSnowplowEvent({
+      event: "embed_wizard_resource_selection_completed",
+      event_detail: "default",
+    });
+  });
+
+  it("tracks default resources for pre-selected question", () => {
+    H.visitQuestion(ORDERS_QUESTION_ID);
+    H.openSharingMenu("Embed");
+
+    H.getEmbedModalSharingPane().within(() => {
+      cy.findByRole("link", { name: "Embedded Analytics JS" }).click();
+    });
+
+    H.expectUnstructuredSnowplowEvent({ event: "embed_wizard_opened" });
+
+    cy.log("go back to resource selection step");
+    getEmbedSidebar().within(() => {
+      cy.findByText("Back").click();
+      cy.findByText("Select a chart to embed").should("be.visible");
+      cy.findByText("Next").click();
+    });
+
+    H.expectUnstructuredSnowplowEvent({
+      event: "embed_wizard_resource_selection_completed",
+      event_detail: "default",
+    });
   });
 });

e2e/test/scenarios/table-editing/table-editing.cy.spec.ts

@@ -1,6 +1,6 @@
 import dayjs from "dayjs";
 
-import { SAMPLE_DB_ID } from "e2e/support/cypress_data";
+import { SAMPLE_DB_ID, WRITABLE_DB_ID } from "e2e/support/cypress_data";
 import { SAMPLE_DATABASE } from "e2e/support/cypress_sample_database";
 import { resetSnowplow } from "e2e/support/helpers/e2e-snowplow-helpers";
 
@@ -403,35 +403,51 @@ describe("scenarios > table-editing", () => {
     beforeEach(() => {
       resetSnowplow();
 
-      cy.intercept("GET", `/api/table/${ORDERS_ID}/query_metadata`).as(
-        "getOrdersTable",
-      );
+      H.restore("postgres-writable");
+      H.resetTestTable({ type: "postgres", table: "scoreboard_actions" });
+      H.resyncDatabase({
+        dbId: WRITABLE_DB_ID,
+        tableName: "scoreboard_actions",
+      });
 
+      H.activateToken("bleeding-edge");
+      setTableEditingEnabledForDB(WRITABLE_DB_ID);
+
+      cy.intercept("GET", "/api/table/*/query_metadata").as("getTableMetadata");
       cy.intercept("POST", "api/dataset").as("getTableDataQuery");
       cy.intercept("POST", "api/ee/action-v2/execute-bulk").as("executeBulk");
 
-      cy.visit(`/browse/databases/${SAMPLE_DB_ID}/tables/${ORDERS_ID}/edit`);
+      H.getTableId({ name: "scoreboard_actions" }).then((tableId) => {
+        cy.visit(`/browse/databases/${WRITABLE_DB_ID}/tables/${tableId}/edit`);
+      });
 
-      cy.wait("@getOrdersTable");
+      cy.wait("@getTableMetadata");
     });
 
     it("should allow to create a row", () => {
       cy.findByTestId("new-record-button").click();
 
-      H.modal().within(() => {
-        cy.findByText("Create a new record").should("be.visible");
-        cy.findByTestId("Tax-field-input").type("50");
-        cy.findByTestId("Total-field-input").type("100");
-        cy.findByTestId("Discount-field-input").type("10");
-        cy.findByTestId("create-row-form-submit-button").click();
-      });
+      H.modal().findByText("Create a new record").should("be.visible");
+
+      cy.findByTestId("Team Name-field-input").click();
+      H.popover().findByRole("textbox").type("New York Bricks");
+      H.popover()
+        .findByText(/Add option/)
+        .click();
+      cy.findByTestId("Score-field-input").type("987");
+      cy.findByTestId("Status-field-input").click();
+      H.popover().findByText("active").click();
+
+      cy.findByTestId("create-row-form-submit-button").click();
 
       cy.wait("@executeBulk").then(({ response, request }) => {
         expect(request.body.action).to.equal("data-grid.row/create");
         expect(response?.body.outputs[0].op).to.equal("created");
-        expect(response?.body.outputs[0].row.TAX).to.equal(50);
-        expect(response?.body.outputs[0].row.TOTAL).to.equal(100);
-        expect(response?.body.outputs[0].row.DISCOUNT).to.equal(10);
+        expect(response?.body.outputs[0].row.score).to.equal(987);
+        expect(response?.body.outputs[0].row.status).to.equal("active");
+        expect(response?.body.outputs[0].row.team_name).to.equal(
+          "New York Bricks",
+        );
       });
 
       H.undoToast().within(() => {
@@ -439,18 +455,25 @@ describe("scenarios > table-editing", () => {
         cy.findByLabelText("close icon").click();
       });
 
-      H.expectUnstructuredSnowplowEvent({
-        event: "edit_data_record_modified",
-        event_detail: "create",
-        target_id: ORDERS_ID,
-        triggered_from: "modal",
-        result: "success",
+      cy.findByTestId("table-root").within(() => {
+        cy.findByText("New York Bricks").should("be.visible");
+        cy.findByText("987").should("be.visible");
+      });
+
+      H.getTableId({ name: "scoreboard_actions" }).then((tableId) => {
+        H.expectUnstructuredSnowplowEvent({
+          event: "edit_data_record_modified",
+          event_detail: "create",
+          target_id: tableId,
+          triggered_from: "modal",
+          result: "success",
+        });
       });
     });
 
     it("should allow to delete multiple rows (bulk)", () => {
-      cy.findAllByTestId("row-select-checkbox").eq(15).click();
-      cy.findAllByTestId("row-select-checkbox").eq(16).click();
+      cy.findAllByTestId("row-select-checkbox").eq(1).click();
+      cy.findAllByTestId("row-select-checkbox").eq(2).click();
 
       cy.log("should not show edit icon when rows are selected");
       cy.findByTestId("row-edit-icon").should("not.exist");

e2e/test/scenarios/visualizations-charts/visualizations-charts-reproductions.cy.spec.js

@@ -1282,3 +1282,42 @@ describe("issue 54271", () => {
     H.assertQueryBuilderRowCount(1076);
   });
 });
+
+describe("issue 63671", () => {
+  beforeEach(() => {
+    H.restore();
+    cy.signInAsAdmin();
+
+    H.createQuestion(
+      {
+        query: {
+          "source-table": PRODUCTS_ID,
+          aggregation: [["count"]],
+          breakout: [
+            [
+              "field",
+              PRODUCTS.CREATED_AT,
+              {
+                "temporal-unit": "year",
+              },
+            ],
+          ],
+          filter: [
+            "between",
+            ["field", PRODUCTS.CREATED_AT, null],
+            "2025-01-01",
+            "2025-12-31",
+          ],
+        },
+        display: "bar",
+      },
+      { visitQuestion: true },
+    );
+  });
+
+  it("should not show an extra value on bar charts when there is only value on the x axis (metabase#63671)", () => {
+    cy.findByTestId("query-visualization-root")
+      .findByText("2025")
+      .should("have.length", 1);
+  });
+});

e2e/test/scenarios/visualizations-tabular/visualizations-tabular-reproductions.cy.spec.js

@@ -639,8 +639,7 @@ describe("issue 37726", () => {
   });
 });
 
-// unskip once metabase#42049 is addressed
-describe("issue 42049", { tags: "@skip" }, () => {
+describe("issue 42049", () => {
   beforeEach(() => {
     H.restore();
     cy.signInAsAdmin();
@@ -703,11 +702,11 @@ describe("issue 42049", { tags: "@skip" }, () => {
     cy.get("@headerCells").eq(1).should("have.text", "Created At");
     cy.get("@headerCells").eq(2).should("have.text", "Quantity");
 
-    cy.findByRole("button", { name: "Filter" }).click();
+    cy.findByTestId("question-filter-header").click();
 
-    cy.findByRole("dialog").within(() => {
-      cy.findByRole("button", { name: "Last month" }).click();
-      cy.findByRole("button", { name: "Apply filters" }).click();
+    H.popover().within(() => {
+      cy.findByText("Created At").click();
+      cy.button("Previous month").click();
     });
 
     cy.wait("@cardQuery");

enterprise/backend/src/metabase_enterprise/api/routes.clj

@@ -20,6 +20,7 @@
    [metabase-enterprise.content-verification.api.routes]
    [metabase-enterprise.database-replication.api :as database-replication.api]
    [metabase-enterprise.database-routing.api]
+   [metabase-enterprise.dependencies.api]
    [metabase-enterprise.documents.api]
    [metabase-enterprise.email.api]
    [metabase-enterprise.embedding-hub.api]
@@ -52,6 +53,7 @@
    :ai-entity-analysis         (deferred-tru "AI Entity Analysis")
    :collection-cleanup         (deferred-tru "Collection Cleanup")
    :content-translation        (deferred-tru "Content translation")
+   :dependencies               (deferred-tru "Dependency Tracking")
    :documents                  (deferred-tru "Documents")
    :embedding                  (deferred-tru "Embedding")
    :etl-connections            (deferred-tru "ETL Connections")
@@ -104,6 +106,7 @@
                                        (premium-handler :etl-connections)
                                        (premium-handler :etl-connections-pg))
    "/database-routing"             (premium-handler metabase-enterprise.database-routing.api/routes :database-routing)
+   "/dependencies"                 (premium-handler metabase-enterprise.dependencies.api/routes :dependencies)
    "/document"                     (premium-handler metabase-enterprise.documents.api/routes :documents)
    "/email"                        (premium-handler metabase-enterprise.email.api/routes :cloud-custom-smtp)
    "/embedding-hub"                (premium-handler metabase-enterprise.embedding-hub.api/routes :embedding)

enterprise/backend/src/metabase_enterprise/core/init.clj

@@ -12,6 +12,7 @@
    [metabase-enterprise.cache.init]
    [metabase-enterprise.comments.init]
    [metabase-enterprise.database-replication.init]
+   [metabase-enterprise.dependencies.init]
    [metabase-enterprise.documents.init]
    [metabase-enterprise.gsheets.init]
    [metabase-enterprise.llm.init]

enterprise/backend/src/metabase_enterprise/dependencies/api.clj

@@ -0,0 +1,124 @@
+(ns metabase-enterprise.dependencies.api
+  (:require
+   [medley.core :as m]
+   [metabase-enterprise.dependencies.core :as dependencies]
+   [metabase.analyze.core :as analyze]
+   [metabase.api.macros :as api.macros]
+   [metabase.api.routes.common :refer [+auth]]
+   [metabase.api.util.handlers :as handlers]
+   [metabase.collections.models.collection.root :as collection.root]
+   [metabase.lib-be.metadata.jvm :as lib-be.metadata.jvm]
+   [metabase.lib.metadata :as lib.metadata]
+   [metabase.models.interface :as mi]
+   [metabase.native-query-snippets.core :as native-query-snippets]
+   [metabase.queries.schema :as queries.schema]
+   [metabase.util.i18n :refer [tru]]
+   [metabase.util.log :as log]
+   [metabase.util.malli.registry :as mr]
+   [metabase.util.malli.schema :as ms]
+   [toucan2.core :as t2]))
+
+(mr/def ::card-body
+  [:map
+   [:id              {:optional false} ms/PositiveInt]
+   [:dataset_query   {:optional true}  [:maybe ms/Map]]
+   [:type            {:optional true}  [:maybe ::queries.schema/card-type]]
+   [:result_metadata {:optional true}  [:maybe analyze/ResultsMetadata]]])
+
+(defn- broken-cards-response
+  [{:keys [card transform]}]
+  (let [broken-card-ids (keys card)
+        broken-cards (when (seq broken-card-ids)
+                       (-> (t2/select :model/Card :id [:in broken-card-ids])
+                           (t2/hydrate [:collection :effective_ancestors] :dashboard)))
+        broken-transform-ids (keys transform)
+        broken-transforms (when (seq broken-transform-ids)
+                            (t2/select :model/Transform :id [:in broken-transform-ids]))]
+    {:success   (and (empty? broken-card-ids)
+                     (empty? broken-transform-ids))
+     :bad_cards (into [] (comp (filter (fn [card]
+                                         (if (mi/can-read? card)
+                                           card
+                                           (do (log/warnf "Eliding broken card %d - not readable by the user" (:id card))
+                                               nil))))
+                               (map (fn [card]
+                                      (-> card
+                                          collection.root/hydrate-root-collection
+                                          (update :dashboard #(some-> % (select-keys [:id :name])))))))
+                      broken-cards)
+     :bad_transforms (into [] broken-transforms)}))
+
+(api.macros/defendpoint :post "/check_card"
+  "Check a proposed edit to a card, and return the card IDs for those cards this edit will break."
+  [_route-params
+   _query-params
+   body :- ::card-body]
+  (let [database-id    (-> body :dataset_query :database)
+        base-provider  (lib-be.metadata.jvm/application-database-metadata-provider database-id)
+        original       (lib.metadata/card base-provider (:id body))
+        card           (-> original
+                           (assoc :dataset-query (:dataset_query body)
+                                  :type          (:type body (:type original)))
+                           ;; Remove the old `:result-metadata` from the card, it's likely wrong now.
+                           (dissoc :result-metadata)
+                           ;; But if the request includes `:result_metadata`, use that. It may be from a native card
+                           ;; that's been run before saving the card.
+                           (cond-> #_card
+                            (:result_metadata body) (assoc :result-metadata (:result_metadata body))))
+        edits          {:card [card]}
+        breakages      (dependencies/errors-from-proposed-edits base-provider edits)]
+    (broken-cards-response breakages)))
+
+(mr/def ::transform-body
+  [:map
+   [:id     {:optional false} ms/PositiveInt]
+   [:name   {:optional true}  :string]
+   [:source {:optional true}  [:maybe ms/Map]]
+   [:target {:optional true}  [:maybe ms/Map]]])
+
+(api.macros/defendpoint :post "/check_transform"
+  "Check a proposed edit to a transform, and return the card, transform, etc. IDs for things that will break."
+  [_route-params
+   _query-params
+   {:keys [id source target] :as _body} :- ::transform-body]
+  (if (= (keyword (:type source))
+         :query)
+    (let [database-id   (-> source :query :database)
+          base-provider (lib-be.metadata.jvm/application-database-metadata-provider database-id)
+          original      (lib.metadata/transform base-provider id)
+          transform     (-> original
+                            (cond-> #_transform source (assoc :source source))
+                            (cond-> #_transform target (assoc :target target)))
+          edits         {:transform [transform]}
+          breakages     (dependencies/errors-from-proposed-edits base-provider edits)]
+      (broken-cards-response breakages))
+    ;; if this isn't a sql query, just claim it works
+    {:success true}))
+
+(api.macros/defendpoint :post "/check_snippet"
+  "Check a proposed edit to a native snippet, and return the cards, etc. which will be broken."
+  [_route-params
+   _query-params
+   {:keys [id content], snippet-name :name}
+   :- [:map
+       [:id      {:optional false} ms/PositiveInt]
+       [:name    {:optional true}  native-query-snippets/NativeQuerySnippetName]
+       [:content {:optional true}  :string]]]
+  (let [original  (t2/select-one :model/NativeQuerySnippet id)
+        _         (when (and snippet-name
+                             (not= snippet-name (:name original))
+                             (t2/exists? :model/NativeQuerySnippet :name snippet-name))
+                    (throw (ex-info (tru "A snippet with that name already exists. Please pick a different name.")
+                                    {:status-code 400})))
+        snippet   (cond-> (m/assoc-some original
+                                        :lib/type :metadata/native-query-snippet
+                                        :name snippet-name
+                                        :content content)
+                    content native-query-snippets/add-template-tags)
+        breakages (dependencies/errors-from-proposed-edits {:snippet [snippet]})]
+    (broken-cards-response breakages)))
+
+(def ^{:arglists '([request respond raise])} routes
+  "`/api/ee/dependencies` routes."
+  (handlers/routes
+   (api.macros/ns-handler *ns* +auth)))

enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj

@@ -0,0 +1,66 @@
+(ns metabase-enterprise.dependencies.calculation
+  (:require
+   [metabase-enterprise.dependencies.native-validation :as deps.native]
+   [metabase.lib-be.metadata.jvm :as lib-be.metadata.jvm]
+   [metabase.lib.metadata :as lib.metadata]
+   [metabase.lib.schema.id :as lib.schema.id]
+   [metabase.lib.schema.metadata :as lib.schema.metadata]
+   [metabase.lib.util :as lib.util]
+   [metabase.util :as u]
+   [metabase.util.log :as log]
+   [metabase.util.malli :as mu]
+   [metabase.util.malli.registry :as mr]))
+
+(defn- upstream-deps:mbql-query [legacy-query]
+  (lib.util/source-tables-and-cards [legacy-query]))
+
+(defn- upstream-deps:native-query [metadata-provider query]
+  (let [engine (:engine (lib.metadata/database metadata-provider))
+        deps   (deps.native/native-query-deps engine metadata-provider query)]
+    ;; The deps are in #{{:table 7} ...} form and need conversion to ::upstream-deps form.
+    (u/group-by ffirst (comp second first) conj #{} deps)))
+
+(mr/def ::upstream-deps
+  [:map
+   [:card      {:optional true} [:set ::lib.schema.id/card]]
+   [:table     {:optional true} [:set ::lib.schema.id/table]]
+   [:snippet   {:optional true} [:set ::lib.schema.id/snippet]]
+   [:transform {:optional true} [:set ::lib.schema.id/transform]]])
+
+(defn- upstream-deps:query [metadata-provider legacy-query]
+  (case (:type legacy-query)
+    (:query  "query")  (upstream-deps:mbql-query legacy-query)
+    (:native "native") (upstream-deps:native-query metadata-provider legacy-query)
+    (throw (ex-info "Unhandled kind of query" {:query legacy-query}))))
+
+(mu/defn upstream-deps:card :- ::upstream-deps
+  "Given a Toucan `:model/Card`, return its upstream dependencies as a map from the kind to a set of IDs."
+  ([card] (upstream-deps:card (lib-be.metadata.jvm/application-database-metadata-provider (:database_id card))
+                              card))
+  ([metadata-provider                      :- ::lib.schema.metadata/metadata-provider
+    {query :dataset_query :as _toucan-card}]
+   (upstream-deps:query metadata-provider query)))
+
+(mu/defn upstream-deps:transform :- ::upstream-deps
+  "Given a Transform (in Toucan form), return its upstream dependencies."
+  ([transform] (upstream-deps:transform nil transform))
+  ([metadata-provider :- [:maybe ::lib.schema.metadata/metadata-provider]
+    {{:keys [query type]} :source :as transform}]
+   (if (#{"query" :query} type)
+     (upstream-deps:query (or metadata-provider
+                              (lib-be.metadata.jvm/application-database-metadata-provider (:database query)))
+                          query)
+     (do (log/warnf "Don't know how to analyze the deps of Transform %d with source type '%s'" (:id transform) type)
+         {}))))
+
+(mu/defn upstream-deps:snippet :- ::upstream-deps
+  "Given a native query snippet, return its upstream dependencies in the usual `{entity-type #{1 2 3}}` format."
+  [{:keys [template_tags] :as _snippet}]
+  (let [type->id-key {:card :card-id, :snippet :snippet-id}
+        dependencies (keep (fn [tag]
+                             (let [entity-type (:type tag)]
+                               (when-let [id-key (type->id-key entity-type)]
+                                 (when-let [entity-id (id-key tag)]
+                                   [entity-type entity-id]))))
+                           (vals template_tags))]
+    (u/group-by first second conj #{} dependencies)))

enterprise/backend/src/metabase_enterprise/dependencies/core.clj

@@ -0,0 +1,205 @@
+(ns metabase-enterprise.dependencies.core
+  "API namespace for the `metabase-enterprise.dependencies` module.
+
+  Call [[errors-from-proposed-edits]] to find out what things will break downstream of a set of new/updated entities."
+  (:require
+   [metabase-enterprise.dependencies.metadata-provider :as deps.provider]
+   [metabase-enterprise.dependencies.models.dependency :as deps.graph]
+   [metabase-enterprise.dependencies.native-validation :as deps.native]
+   [metabase.graph.core :as graph]
+   [metabase.lib-be.metadata.jvm :as lib-be.metadata.jvm]
+   [metabase.lib.core :as lib]
+   [metabase.lib.metadata :as lib.metadata]
+   [metabase.lib.schema.metadata :as lib.schema.metadata]
+   [metabase.util :as u]
+   [metabase.util.log :as log]
+   [metabase.util.malli :as mu]
+   [metabase.util.malli.registry :as mr]
+   [toucan2.core :as t2]))
+
+(mr/def ::entity-type
+  [:enum :card :transform :snippet :table])
+
+(mr/def ::updates-map
+  ;; TODO: Make this more specific.
+  [:map-of ::entity-type [:sequential [:map [:id {:optional true} :int]]]])
+
+(mu/defn- metadata-provider :- ::lib.schema.metadata/metadata-provider
+  "Constructs a `MetadataProvider` with some pending edits applied.
+
+  The edits are *transitive*! Since a card's output columns can depend on its (possibly updated) inputs, all dependents
+  of editing things are considered edited too.
+
+  Note that if `:result-metadata` is present on an updated Card, it **will be used!** The only case where that is
+  actually useful is with a native query which has been executed, so the caller has driver metadata to give us. Any
+  old, pre-update `:result-metadata` should be dropped from any other cards in `updated-entities`."
+  ([base-provider    :- ::lib.schema.metadata/metadata-provider
+    updated-entities :- ::updates-map
+    & {:keys [graph dependents]}]
+   (let [dependents (or dependents (deps.graph/transitive-dependents graph updated-entities))]
+     (deps.provider/override-metadata-provider base-provider updated-entities dependents))))
+
+(defmulti check-query-inner
+  "Given a `MetadataProvider` and a lib query, find any bad refs in that query."
+  {:arglists '([metadata-provider driver query])}
+  (fn [_mp _driver query]
+    (-> (lib/query-stage query 0)
+        :lib/type)))
+
+(defmethod check-query-inner :mbql.stage/mbql
+  [_mp _driver query]
+  (lib/find-bad-refs query))
+
+(defmethod check-query-inner :mbql.stage/native
+  [mp driver query]
+  (deps.native/validate-native-query driver mp query))
+
+(defn- check-query
+  "Given a `MetadataProvider` and a query, find any bad refs in that query."
+  [mp driver query]
+  (check-query-inner mp driver (lib/query mp query)))
+
+(defmulti check-entity
+  "Given a `MetadataProvider`, and entity type, and an entity id, find any bad refs in that entity."
+  {:arglists '([metadata-provider entity-type entity-id])}
+  (fn [_mp entity-type _entity-id]
+    entity-type))
+
+(defmethod check-entity :default
+  [_mp _entity-type _entity-id]
+  nil)
+
+(defmethod check-entity :card
+  [mp _entity-type entity-id]
+  (let [query (:dataset-query (lib.metadata/card mp entity-id))
+        driver (:engine (lib.metadata/database mp))]
+    (check-query mp driver query)))
+
+(defmethod check-entity :transform
+  [mp _entity-type entity-id]
+  (let [{{target-schema :schema target-name :name} :target
+         {:keys [query]} :source
+         :as _transform} (lib.metadata/transform mp entity-id)
+        driver (:engine (lib.metadata/database mp))
+        output-table (some #(when (and (= (:schema %) target-schema)
+                                       (= (:name %) target-name))
+                              %)
+                           (lib.metadata/tables mp))
+        output-fields (lib.metadata/active-fields mp (:id output-table))
+        {:keys [duplicate-fields]} (reduce (fn [{:keys [seen duplicate-fields]}
+                                                {name :name :as field}]
+                                             (if (seen name)
+                                               {:seen seen
+                                                :duplicate-fields (conj duplicate-fields field)}
+                                               {:seen (conj seen name)
+                                                :duplicate-fields duplicate-fields}))
+                                           {:seen #{}}
+                                           output-fields)]
+    (into duplicate-fields (check-query mp driver query))))
+
+(mu/defn- check-query-soundness ;; :- [:map-of ::lib.schema.id/card [:sequential ::lib.schema.mbql-clause/clause]]
+  "Given a `MetadataProvider` as returned by [[metadata-provider]], scan all its updated entities and their dependents
+  to check that everything is still sound.
+
+  In particular, we're looking for bad clauses with refs to columns which no longer exist, since those queries fail
+  to compile.
+
+  May return false positives: if a card already has a bad ref in it, then it will be returned even if that bad ref has
+  nothing to do with the updates in the `provider`.
+
+  Returns a map `{:card {card-id [bad-ref ...]}, :transform {...}}`. It will be empty, if there are no bad refs
+  detected."
+  [provider :- ::lib.schema.metadata/metadata-provider]
+  (let [overrides (deps.provider/all-overrides provider)
+        errors    (volatile! {})]
+    (doseq [[entity-type ids] overrides
+            id ids
+            :let [bad-refs (check-entity provider entity-type id)]]
+      (when (seq bad-refs)
+        (vswap! errors assoc-in [entity-type id] bad-refs)))
+    @errors))
+
+(defn- group-by-db [deps]
+  (let [by-db (volatile! {})]
+    (when (seq (:card deps))
+      (doseq [[db-id card-ids] (->> (t2/select [:model/Card :id :database_id :card_schema] :id [:in (:card deps)])
+                                    (u/group-by :database_id :id conj #{}))]
+        (vswap! by-db assoc-in [db-id :card] card-ids)))
+    (when (seq (:table deps))
+      (doseq [[db-id table-ids] (->> (t2/select [:model/Table :id :db_id] :id [:in (:table deps)])
+                                     (u/group-by :db_id :id conj #{}))]
+        (vswap! by-db assoc-in [db-id :table] table-ids)))
+    (when (seq (:transform deps))
+      (doseq [[db-id transform-ids] (->> (t2/select [:model/Transform :id :source] :id [:in (:transform deps)])
+                                         (u/group-by #(get-in % [:source :query :database]) :id conj #{}))]
+        (if db-id
+          (vswap! by-db assoc-in [db-id :transform] transform-ids)
+          (log/warnf "Unable to infer database from transforms %s" transform-ids))))
+    (when (seq (:snippet deps))
+      ;; Copy any snippet deps to each database, since they span them all.
+      (vswap! by-db update-vals #(assoc % :snippet (:snippet deps))))
+
+    @by-db))
+
+;; TODO: (Braden 09/22/2025) More precise schemas for the errors this function returns. Currently they're pretty
+;; opaque, since the consumers of this function really care about "working/broken" and not the details of what's wrong
+;; with any particular card.
+(mu/defn errors-from-proposed-edits :- [:map-of ::entity-type [:map-of :int [:or :boolean [:sequential :any]]]]
+  "Given a regular `MetadataProvider`, and a map of entity types (`:card`, `:transform`, `:snippet`) to lists of
+  updated entities, this returns a map of `{entity-type {entity-id [bad-ref ...]}}`.
+
+  The 1-arity groups all the dependents by which Database they are part of, and runs the analysis for each of them.
+
+  The output is a map: `{entity-type {id [errors...]}}`; an empty map is returned when there are no errors
+  detected."
+  ([edits :- ::updates-map]
+   (let [all-deps (deps.graph/transitive-dependents edits)
+         by-db    (group-by-db all-deps)]
+     (reduce (fn [errors [db-id deps]]
+               (-> (lib-be.metadata.jvm/application-database-metadata-provider db-id)
+                   (metadata-provider edits :dependents deps)
+                   check-query-soundness
+                   (merge errors)))
+             {} by-db)))
+
+  ([base-provider :- ::lib.schema.metadata/metadata-provider
+    edits         :- ::updates-map]
+   (errors-from-proposed-edits base-provider nil edits))
+
+  ([base-provider :- ::lib.schema.metadata/metadata-provider
+    graph         :- [:maybe ::graph/graph]
+    edits         :- ::updates-map]
+   (-> base-provider
+       (metadata-provider edits :graph graph)
+       check-query-soundness)))
+
+#_{:clj-kondo/ignore [:unresolved-namespace]}
+(comment
+  ;; This should work on any fresh-ish Metabase instance; these are the built-in example questions.
+  (let [base-mp (lib-be.metadata.jvm/application-database-metadata-provider 39)
+        transform (lib.metadata/transform base-mp 1)
+        transform (-> transform
+                      (update-in [:source :query :query :expressions]
+                                 ;; Replacing the expression Age with Duration - this breaks downstream uses!
+                                 update-keys (constantly "Fair Die"))
+                      #_(dissoc :result-metadata))
+        mp       (metadata-provider base-mp {:transform [transform]})]
+    #_(deps.provider/all-overrides mp)
+    (check-query-soundness mp))
+
+  ;; Checking that we can properly group everything by database. Surprisingly fast even fetching everything.
+  (let [deps {:card      (t2/select-fn-set :id :model/Card)
+              :transform (t2/select-fn-set :id :model/Transform)
+              :table     (t2/select-fn-set :id :model/Table)}]
+    (group-by-db deps))
+
+  (t2/select-fn-vec (juxt :id :name :active :table_id) :model/Field :id [:in [1065 1066 1067 1068 1069]])
+
+  (t2/select-one :model/Card :id 124)
+  (deps.graph/transitive-dependents {:transform [{:id 1}]})
+  (deps.graph/transitive-dependents {:table [{:id 155}]})
+  (metabase.premium-features.core/token-features)
+
+  (let [card (t2/select-one :model/Card :id 121)
+        mp   (lib-be.metadata.jvm/application-database-metadata-provider (:database_id card))]
+    (upstream-deps:card mp card)))

enterprise/backend/src/metabase_enterprise/dependencies/init.clj

@@ -0,0 +1,4 @@
+(ns metabase-enterprise.dependencies.init
+  (:require
+   [metabase-enterprise.dependencies.models.dependency]
+   [metabase-enterprise.dependencies.task.backfill]))

enterprise/backend/src/metabase_enterprise/dependencies/metadata_provider.clj

@@ -0,0 +1,271 @@
+(ns metabase-enterprise.dependencies.metadata-provider
+  "A `MetadataProvider` wrapper that understands how to update entities, and to lazily compute the new metadata for
+  updated entities on demand. This is useful when analyzing a tree of dependents, which might depend on each other in
+  complex ways.
+
+  Just to illustrate, imagine I change a card A which is used by a transform B whose output table C is used by a card
+  D. If I remove a column from card A, and transform B selects all columns, that field needs to disappear from C
+  before we can correctly analyze card D.
+
+  By representing the updated entities as lazy `delay`s, this `MetadataProvider` can support analysis of the
+  downstream entities in any order."
+  (:require
+   [medley.core :as m]
+   [metabase-enterprise.dependencies.native-validation :as deps.native]
+   [metabase.lib.core :as lib]
+   [metabase.lib.metadata :as lib.metadata]
+   [metabase.lib.metadata.protocols :as lib.metadata.protocols]
+   [metabase.util :as u]
+   [metabase.util.log :as log]
+   [pretty.core :as pretty]))
+
+(set! *warn-on-reflection* true)
+
+(defn- metadatas [delegate overrides {metadata-type :lib/type
+                                      search-name :name
+                                      :keys [id card-id table-id]
+                                      :as metadata-spec}]
+  (let [type-overrides  (get overrides metadata-type)
+        metadata-keys   (set (or id search-name))]
+    (log/tracef "OverridingMetadataProvider request for %s" metadata-spec)
+    (cond
+      ;; For tables, cards, snippets, transforms, and columns, return overrides if present and looking for specific
+      ;; keys and delegate if not.
+      (and (#{:metadata/table :metadata/card :metadata/transform :metadata/native-query-snippet :metadata/column} metadata-type)
+           (seq metadata-keys))
+      (let [overrides       (if (seq id)
+                              (select-keys type-overrides id)
+
+                              (into {} (keep #(when-let [x (deref %)]
+                                                (when (search-name (:name x))
+                                                  [(:name x) %])))
+                                    (vals type-overrides)))
+            override-key    (if (seq id) :id :name)
+            overridden-keys (set (keys overrides))
+            missing-keys    (remove overridden-keys metadata-keys)]
+        (log/tracef "Overridden keys %s; missing keys %s" (sort overridden-keys) (sort missing-keys))
+        (cond-> (map deref (vals overrides))
+          (seq missing-keys)
+          (concat (lib.metadata.protocols/metadatas delegate {:lib/type    metadata-type
+                                                              override-key (set missing-keys)}))))
+
+      ;; if fetching all tables or transforms, get every override and everything from the delegate and then dedupe
+      (and (empty? metadata-keys)
+           (#{:metadata/table :metadata/transform} metadata-type))
+      (as-> (vals type-overrides) entities
+        (into [] (map force) entities)
+        (into entities (lib.metadata.protocols/metadatas delegate metadata-spec))
+        (m/distinct-by :id entities))
+
+      ;; For columns, return overrides if they're present.
+      (= metadata-type :metadata/column)
+      (if-let [overridden-columns (cond
+                                    card-id  (get-in overrides [::card-columns  card-id])
+                                    table-id (get-in overrides [::table-columns table-id]))]
+        @overridden-columns
+        (lib.metadata.protocols/metadatas delegate metadata-spec))
+
+      :else (throw (ex-info "Unknown :lib/metadata type for OverridingMetadataProvider" metadata-spec)))))
+
+(declare all-overrides setup-transform! setup-transforms!)
+
+(deftype OverridingMetadataProvider [delegate *overrides]
+  lib.metadata.protocols/MetadataProvider
+  (database [_this] (lib.metadata.protocols/database delegate))
+  (metadatas [this metadata-spec]
+    (setup-transforms! this)
+    (metadatas delegate @*overrides metadata-spec))
+  (setting [_this setting-key]
+    (lib.metadata.protocols/setting delegate setting-key))
+
+  ;; Cached things are direct delegation.
+  lib.metadata.protocols/CachedMetadataProvider
+  (cached-metadatas [_this metadata-type metadata-ids]
+    (lib.metadata.protocols/cached-metadatas delegate metadata-type metadata-ids))
+  (store-metadata! [_this a-metadata]
+    (lib.metadata.protocols/store-metadata! delegate a-metadata))
+  (cached-value [_this k not-found]
+    (lib.metadata.protocols/cached-value delegate k not-found))
+  (cache-value! [_this k v]
+    (lib.metadata.protocols/cache-value! delegate k v))
+  (has-cache? [_this]
+    (lib.metadata.protocols/has-cache? delegate))
+
+  pretty/PrettyPrintable
+  (pretty [this]
+    (list `overriding-metadata-provider (all-overrides this) delegate)))
+
+(defmulti add-override
+  "Given an `OverridingMetadataProvider`, an `entity-type` and that entity, add this entity as an override."
+  {:arglists '([metadata-provider entity-type id updates-or-nil])}
+  (fn [_mp entity-type _id _updates]
+    entity-type))
+
+(defn- inner-mp [^OverridingMetadataProvider mp]
+  (.delegate mp))
+
+(defn- with-overrides [^OverridingMetadataProvider mp kvs]
+  (swap! (.*overrides mp)
+         #(reduce (fn [m [ks v]]
+                    (assoc-in m ks v))
+                  % kvs)))
+
+(defn- get-returned-columns [mp queryable]
+  (let [query (lib/query mp queryable)]
+    (if (= (:lib/type (lib/query-stage query 0))
+           :mbql.stage/mbql)
+      (lib/returned-columns query)
+      (deps.native/native-result-metadata (:engine (lib.metadata/database mp))
+                                          mp query))))
+
+(defmethod add-override :card [^OverridingMetadataProvider mp _entity-type id updates]
+  (with-overrides mp
+    ;; If the `updates` contain `:result-metadata`, we want to use that. However, any `:result-metadata` from the
+    ;; inner-mp should be ignored.
+    {[:metadata/card id] (delay (merge (when id
+                                         (-> (inner-mp mp)
+                                             (lib.metadata/card id)
+                                             (dissoc :result-metadata)))
+                                       updates))
+     ;; This uses the outer OMP and so the overrides are visible!
+     [::card-columns id] (delay (get-returned-columns mp (lib.metadata/card mp id)))}))
+
+(defonce ^:private last-fake-id (atom 2000000000))
+
+(defn- fake-id []
+  (swap! last-fake-id inc))
+
+(defn- setup-transform!
+  [^OverridingMetadataProvider mp s+n->table {:keys [source target] :as _transform}]
+  ;; See if the output table exists in the underlying MetadataProvider.
+  ;; - If the table doesn't exist, invent an ID for it.
+  ;; - Always add an override for the table, even if it already existed.
+  ;; - Compare the transform's output columns to the fields that exist in the metadata.
+  ;;   - For any newly added fields, invent an ID and add an override for the field.
+  ;;   - For any removed fields that used to exist, add an override for their ID that is nil!
+  ;;   - Add an internal [::table-columns table-id] with the list of output columns.
+  ;; Note that this assumes that all other overrides are already loaded!
+  (let [existing-table (get s+n->table [(:schema target) (:name target)])
+        output-table   (or existing-table
+                           (merge (select-keys target [:schema :name])
+                                  {:lib/type        :metadata/table
+                                   :id              (fake-id)
+                                   :db-id           (:id (lib.metadata/database (inner-mp mp)))
+                                   :display-name    (:name target)
+                                   :visibility-type nil}))
+        existing-cols  (when existing-table
+                         (lib/returned-columns (lib/query (inner-mp mp) existing-table)))
+        output-cols    (delay
+                         ;; Note that this will analyze the query with any upstream changes included!
+                         (let [new-cols (get-returned-columns mp (:query source))
+                               by-name  (m/index-by :lib/desired-column-alias existing-cols)]
+                           (into [] (for [col new-cols
+                                          :let [old-col (by-name (:lib/desired-column-alias col))]]
+                                      (merge (select-keys col [:display-name
+                                                               :base-type :effective-type :semantic-type])
+                                             {:name     (:lib/desired-column-alias col)
+                                              :lib/type :metadata/column
+                                              :id       (or (:id old-col) (fake-id))
+                                              :table-id (:id output-table)
+                                              :active   true})))))
+        outputs-by-id   (delay (m/index-by :id @output-cols))]
+    ;; NOTE: Any newly added output columns can't be looked up directly by ID, with this version.
+    ;; The `output-cols` code above can be adapted to add them to the `*overrides` atom, but YAGNI for now.
+    (with-overrides mp
+      (into {[:metadata/table (:id output-table)] (delay output-table)
+             [::table-columns (:id output-table)] output-cols}
+            ;; For each pre-existing output column, we need to add an override for a direct read by ID.
+            ;; If the column does not exist in the output, return the original with :active false
+            (map (fn [existing-col]
+                   [[:metadata/column (:id existing-col)]
+                    (delay (or (get @outputs-by-id (:id existing-col))
+                               (assoc existing-col :active false)))]))
+            existing-cols))))
+
+(defn- setup-transforms! [^OverridingMetadataProvider mp]
+  (locking mp
+    (when-not (-> (.*overrides mp) deref ::setup-complete)
+      (swap! (.*overrides mp) assoc ::setup-complete true)
+      (let [;; Fetching all the pre-existing tables.
+            ;; TODO: Once Cam's new by-name logic in the `MetadataProvider` lands, we can use that here.
+            tables     (lib.metadata/tables (inner-mp mp))
+            s+n->table (m/index-by (juxt :schema :name) tables)]
+        (doseq [transform (-> (.*overrides mp) deref :metadata/transform vals)]
+          (setup-transform! mp s+n->table (deref transform)))))))
+
+(defmethod add-override :transform [^OverridingMetadataProvider mp _entity-type id updates]
+  ;; Transforms are complicated:
+  ;; 1. Treated as a card on the input side.
+  ;; 2. The returned-columns of that card become the columns of the output table.
+  ;; 3. The output table is constructed from the transform.
+  ;; 4. Its fields are adapted from the transform's input card.
+  ;; All of these can be wrapped in delays and therefore computed lazily on demand.
+  ;;
+  ;; The trouble is that a transform does not know the output table's ID, nor its field IDs.
+  ;; A modified transform might produce a new table, or have new fields or removed fields.
+  ;; The OverridingMetadataProvider has to account for all of this when it returns tables or fields by ID, or gets
+  ;; the column list for a table.
+  ;;
+  ;; This is achieved by putting `[:metadata/transform id]` into overrides for all transforms, and then one-time
+  ;; logic that runs before the first use of the OverridingMetadataProvider as a MetadataProvider will walk all the
+  ;; transforms and make sure their tables etc. are overridden properly.
+  (let [transform (merge (when id
+                           (lib.metadata/transform (inner-mp mp) id))
+                         updates)]
+    (with-overrides mp
+      {;; The transform itself, merged with the incoming updates.
+       [:metadata/transform id] (delay transform)
+       #_#_[::transform-columns id] (delay (->> (lib.metadata/transform mp id)
+                                                :source
+                                                :query
+                                                (lib/query mp)
+                                                lib/returned-columns))})))
+
+(defmethod add-override :table [^OverridingMetadataProvider mp _entity-type id updates]
+  (when (seq updates)
+    (throw (ex-info "Updating a table directly is not supported" {:table-id id, :edited updates})))
+  (with-overrides mp
+    {[:metadata/table id] (delay (log/warnf "Can't happen: Table %d dep should be replaced by its transform" id)
+                                 (lib.metadata/table (inner-mp mp) id))
+     [::table-columns id] (delay
+                            (log/warnf "Can't happen: Table %d ::table-columns should be replaced by its transform" id)
+                            (lib.metadata/fields (inner-mp mp) id))}))
+
+(defmethod add-override :snippet [^OverridingMetadataProvider mp _entity-type id updates]
+  (with-overrides mp
+    {[:metadata/native-query-snippet id] (delay (merge (when id
+                                                         (lib.metadata/native-query-snippet (inner-mp mp) id))
+                                                       (u/normalize-map updates)))}))
+
+(defn all-overrides
+  "Returns all the overrides by ID, in the same form as the map input to [[with-deps]]:
+  `{:card [1 2 3], :transform [45 99]}."
+  [^OverridingMetadataProvider override-metadata-provider]
+  (-> (.*overrides override-metadata-provider)
+      deref
+      (dissoc ::setup-complete)
+      (update-vals keys)
+      (update-keys {:metadata/card                 :card
+                    :metadata/transform            :transform
+                    :metadata/native-query-snippet :snippet})
+      (select-keys [:card :transform :snippet])))
+
+(defn override-metadata-provider
+  "Given an underlying `MetadataProvider`, wraps it to support in-memory overrides of cards, fields, etc.
+
+  Important note: all overrides must be added first, before this is used as a `MetadataProvider`.
+  The easiest way to do that is to call the 3-arity, which provides the overrides up front."
+  ([metadata-provider]
+   (->OverridingMetadataProvider metadata-provider (atom {})))
+
+  ([metadata-provider updated-entities dependent-ids]
+   (let [^OverridingMetadataProvider omp (override-metadata-provider metadata-provider)]
+     (doseq [[entity-type updates] updated-entities
+             updated-entity        updates]
+       (add-override omp entity-type (:id updated-entity) updated-entity))
+     (doseq [[entity-type dependents] dependent-ids
+             :let  [updated (into #{} (map :id) (get updated-entities entity-type))]
+             id    dependents
+             :when (not (updated id))]
+       (add-override omp entity-type id nil))
+     omp)))

enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj

@@ -0,0 +1,212 @@
+(ns metabase-enterprise.dependencies.models.dependency
+  (:require
+   [clojure.set :as set]
+   [metabase-enterprise.dependencies.calculation :as deps.calculation]
+   [metabase.events.core :as events]
+   [metabase.graph.core :as graph]
+   [metabase.models.interface :as mi]
+   [metabase.premium-features.core :as premium-features]
+   [metabase.util :as u]
+   [methodical.core :as methodical]
+   [potemkin :as p]
+   [toucan2.core :as t2]))
+
+(def current-dependency-analysis-version
+  "Current version of the dependency analysis logic.
+  This should be incremented when the dependency analysis logic changes."
+  1)
+
+(methodical/defmethod t2/table-name :model/Dependency [_model] :dependency)
+
+(derive :model/Dependency :metabase/model)
+
+(t2/deftransforms :model/Dependency
+  {:from_entity_type mi/transform-keyword
+   :to_entity_type   mi/transform-keyword})
+
+(defn- deps-children [src-type src-id dst-type dst-id key-seq]
+  ;; Group all keys with the same type together, so we make O(types) indexed [[t2/select]] calls, not O(n).
+  (transduce (map (fn [[entity-type entity-keys]]
+                    (let [deps (t2/select :model/Dependency
+                                          src-type entity-type
+                                          src-id   [:in entity-keys])]
+                      (u/group-by (juxt src-type src-id)
+                                  (juxt dst-type dst-id)
+                                  conj #{}
+                                  deps))))
+             merge {}
+             (u/group-by first second key-seq)))
+
+(defn- key-dependents
+  "Get the dependent entity keys for the entity keys in `entity-keys`.
+  Entity keys are [entity-type, entity-id] pairs. See [[entity-type->model]]."
+  [key-seq]
+  (deps-children :to_entity_type :to_entity_id :from_entity_type :from_entity_id key-seq))
+
+(p/deftype+ DependencyGraph [children-fn]
+  graph/Graph
+  (children-of [_this key-seq]
+    (children-fn key-seq)))
+
+;; NOTE: We can easily construct a graph of upstream dependencies too, if it's useful.
+(defn- graph-dependents []
+  (->DependencyGraph key-dependents))
+
+(defn transitive-dependents
+  "Given a map of updated entities `{entity-type [{:id 1, ...} ...]}`, return a map of its transitive dependents
+  as `{entity-type #{4 5 6}}` - that is, a map from downstream entity type to a set of IDs.
+
+  Uses the provided `graph`, or defaults to the `:model/Dependency` table in AppDB.
+
+  The inputs must be maps containing `:id`; anything without an `:id` is skipped. They could be Toucan entities,
+  `MetadataProvider` entities, user input, etc.
+
+  **Excludes** the input entities from the list of dependents!"
+  ([updated-entities] (transitive-dependents nil updated-entities))
+  ([graph updated-entities]
+   (let [graph    (or graph (graph-dependents))
+         starters (for [[entity-type updates] updated-entities
+                        entity                updates
+                        :when (:id entity)]
+                    [entity-type (:id entity)])]
+     (->> (graph/transitive graph starters) ; This returns a flat list.
+          (u/group-by first second conj #{})))))
+
+(defn replace-dependencies
+  "Replace the dependencies of the entity of type `entity-type` with id `entity-id` with
+  the ones specified in `dependencies-by-type`. "
+  [entity-type entity-id dependencies-by-type]
+  (let [current-dependencies (t2/select [:model/Dependency :id :to_entity_type :to_entity_id]
+                                        :from_entity_type entity-type
+                                        :from_entity_id entity-id)
+        to-remove (keep (fn [{:keys [id to_entity_type to_entity_id]}]
+                          (when-not (get-in dependencies-by-type [to_entity_type to_entity_id])
+                            id))
+                        current-dependencies)
+        current-by-type (-> (group-by :to_entity_type current-dependencies)
+                            (update-vals #(into #{} (map :to_entity_id) %)))
+        to-add (for [[to-entity-type ids] dependencies-by-type
+                     to-entity-id (set/difference ids (current-by-type to-entity-type))]
+                 {:from_entity_type entity-type
+                  :from_entity_id   entity-id
+                  :to_entity_type   to-entity-type
+                  :to_entity_id     to-entity-id})]
+    (t2/with-transaction [_conn]
+      (when (seq to-remove)
+        (t2/delete! :model/Dependency :id [:in to-remove]))
+      (when (seq to-add)
+        (t2/insert! :model/Dependency to-add)))))
+
+;; ## Maintaining the dependency graph
+;; The below listens for inserts, updates and deletes of cards, snippets and transforms in order to keep the
+;; dependency graph up to date. Transform *runs* are also a trigger, since the transform's output table may be created
+;; or changed at that point.
+
+;; ### Cards
+(derive ::card-deps :metabase/event)
+(derive :event/card-create ::card-deps)
+(derive :event/card-update ::card-deps)
+
+(methodical/defmethod events/publish-event! ::card-deps
+  [_ {:keys [object]}]
+  (when (premium-features/has-feature? :dependencies)
+    (t2/with-transaction [_conn]
+      (replace-dependencies :card (:id object) (deps.calculation/upstream-deps:card object))
+      (when (not= (:dependency_analysis_version object) current-dependency-analysis-version)
+        (t2/update! :model/Card (:id object)
+                    {:dependency_analysis_version current-dependency-analysis-version})))))
+
+(derive ::card-delete :metabase/event)
+(derive :event/card-delete ::card-delete)
+
+(methodical/defmethod events/publish-event! ::card-delete
+  [_ {:keys [object]}]
+  (when (premium-features/has-feature? :dependencies)
+    (t2/delete! :model/Dependency :from_entity_type :card :from_entity_id (:id object))))
+
+;; ### Snippets
+(derive ::snippet-deps :metabase/event)
+(derive :event/snippet-create ::snippet-deps)
+(derive :event/snippet-update ::snippet-deps)
+
+(methodical/defmethod events/publish-event! ::snippet-deps
+  [_ {:keys [object]}]
+  (when (premium-features/has-feature? :dependencies)
+    (t2/with-transaction [_conn]
+      (replace-dependencies :snippet (:id object) (deps.calculation/upstream-deps:snippet object))
+      (when (not= (:dependency_analysis_version object) current-dependency-analysis-version)
+        (t2/update! :model/NativeQuerySnippet (:id object)
+                    {:dependency_analysis_version current-dependency-analysis-version})))))
+
+(derive ::snippet-delete :metabase/event)
+(derive :event/snippet-delete ::snippet-delete)
+
+(methodical/defmethod events/publish-event! ::snippet-delete
+  [_ {:keys [object]}]
+  (when (premium-features/has-feature? :dependencies)
+    (t2/delete! :model/Dependency :from_entity_type :snippet :from_entity_id (:id object))))
+
+;; ### Transforms
+(derive ::transform-deps :metabase/event)
+(derive :event/create-transform ::transform-deps)
+(derive :event/update-transform ::transform-deps)
+
+;; On *saving* a transform, the upstream deps of its query are computed and saved.
+(defn- drop-outdated-target-dep! [{:keys [id source target] :as _transform}]
+  (let [db-id                (some-> source :query :database)
+        downstream-table-ids (t2/select-fn-set :from_entity_id :model/Dependency
+                                               :from_entity_type :table
+                                               :to_entity_type   :transform
+                                               :to_entity_id     id)
+        downstream-tables    (when (seq downstream-table-ids)
+                               (t2/select :model/Table :id [:in downstream-table-ids]))
+        outdated-tables      (remove (fn [table]
+                                       (and (= (:schema table) (:schema target))
+                                            (= (:name   table) (:name   target))
+                                            (or (not db-id)
+                                                (= db-id (:db_id table)))))
+                                     downstream-tables)
+        not-found-table-ids  (remove (into #{} (map :id) downstream-tables)
+                                     downstream-table-ids)]
+    (when-let [outdated-downstream-table-ids (seq (into (set not-found-table-ids)
+                                                        (map :id) outdated-tables))]
+      (t2/delete! :model/Dependency
+                  :from_entity_type :table
+                  :from_entity_id   [:in outdated-downstream-table-ids]
+                  :to_entity_type   :transform
+                  :to_entity_id     id))))
+
+(methodical/defmethod events/publish-event! ::transform-deps
+  [_ {:keys [object]}]
+  (when (premium-features/has-feature? :dependencies)
+    (t2/with-transaction [_conn]
+      (replace-dependencies :transform (:id object) (deps.calculation/upstream-deps:transform object))
+      (when (not= (:dependency_analysis_version object) current-dependency-analysis-version)
+        (t2/update! :model/Transform (:id object) {:dependency_analysis_version current-dependency-analysis-version}))
+      (drop-outdated-target-dep! object))))
+
+(derive ::transform-delete :metabase/event)
+(derive :event/delete-transform ::transform-delete)
+
+(methodical/defmethod events/publish-event! ::transform-delete
+  [_ {:keys [id]}]
+  (when (premium-features/has-feature? :dependencies)
+    ;; TODO: (Braden 09/18/2025) Shouldn't we be deleting the downstream deps for dead edges as well as upstream?
+    (t2/delete! :model/Dependency :from_entity_type :transform :from_entity_id id)))
+
+;; On *executing* a transform, its (freshly synced) output table is made to depend on the transform.
+;; (And if the target has changed, the old table's dep on the transform is dropped.)
+;; The upstream deps of the transform are not touched - those change only when the transform is edited.
+(derive ::transform-run :metabase/event)
+(derive :event/transform-run-complete ::transform-run)
+
+(defn- transform-table-deps! [{:keys [db-id output-schema output-table transform-id] :as _details}]
+  (let [;; output-table is a keyword like :my_schema/my_table
+        table-name (name output-table)]
+    (when-let [table-id (t2/select-one-fn :id :model/Table :db_id db-id :schema output-schema :name table-name)]
+      (replace-dependencies :table table-id {:transform #{transform-id}}))))
+
+(methodical/defmethod events/publish-event! ::transform-run
+  [_ {:keys [object]}]
+  (when (premium-features/has-feature? :dependencies)
+    (transform-table-deps! object)))

enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj

@@ -0,0 +1,38 @@
+(ns metabase-enterprise.dependencies.native-validation
+  (:require
+   [metabase.driver :as driver]
+   [metabase.lib.core :as lib]
+   [metabase.query-processor :as qp]))
+
+(defn- compile-query [metadata-provider query]
+  (->> (lib/query metadata-provider query)
+       lib/add-parameters-for-template-tags
+       (qp/compile-query-with-metadata-provider metadata-provider)
+       :query))
+
+(defn validate-native-query
+  "Compiles a (native) query and validates that the fields and tables it refers to really exist.
+
+   Returns either nil or a list of errors."
+  [driver metadata-provider query]
+  (->> (compile-query metadata-provider query)
+       (driver/validate-native-query-fields driver metadata-provider)))
+
+(defn native-result-metadata
+  "Compiles a (native) query and calculates its result metadata"
+  [driver metadata-provider query]
+  (->> (compile-query metadata-provider query)
+       (driver/native-result-metadata driver metadata-provider)))
+
+(defn native-query-deps
+  "Returns the upstream dependencies of a native query, as a set of `{:kind id}` pairs."
+  [driver metadata-provider query]
+  (let [compiled (compile-query metadata-provider query)]
+    (into (driver/native-query-deps driver compiled metadata-provider)
+          (keep #(case (:type %)
+                   "snippet" {:snippet (:snippet-id %)}
+                   :snippet  {:snippet (:snippet-id %)}
+                   "card"    {:card (:card-id %)}
+                   :card     {:card (:card-id %)}
+                   nil))
+          (-> query :native :template-tags vals))))

enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj

@@ -0,0 +1,191 @@
+(ns metabase-enterprise.dependencies.task.backfill
+  "Implements a task that brings all entities with dependencies tracked in the dependency table
+  (see [[metabase-enterprise.dependencies.models.dependency]]) up to date, that is, makes sure
+  the dependency table contains fresh entries.
+
+  This is done by finding entities whose dependency_analysis_version is less than
+  [[metabase-enterprise.dependencies.models.dependency/current-dependency-analysis-version]] and
+  updates this field. In most cases this triggers an event handler that calculates the dependencies
+  and populates the dependency table. For cards, the event is emitted by the job handler itself,
+  because the update of the record doesn't trigger the event handler. "
+  (:require
+   [clojurewerkz.quartzite.jobs :as jobs]
+   [clojurewerkz.quartzite.scheduler :as qs]
+   [clojurewerkz.quartzite.triggers :as triggers]
+   [environ.core :as env]
+   [java-time.api :as t]
+   [metabase-enterprise.dependencies.models.dependency :as models.dependency]
+   [metabase.config.core :as config]
+   [metabase.events.core :as events]
+   [metabase.premium-features.core :as premium-features]
+   [metabase.task.core :as task]
+   [metabase.util.log :as log]
+   [toucan2.core :as t2])
+  (:import
+   (java.util Map Set)
+   (java.util.concurrent ConcurrentHashMap)))
+
+(set! *warn-on-reflection* true)
+
+(defn- current-millis
+  "Returns the current epoch millis. Uses java-time so that clock settings can be used in tests."
+  []
+  (t/to-millis-from-epoch (t/instant)))
+
+(def ^:private entities [:model/Card :model/Transform :model/NativeQuerySnippet])
+
+;; In-memory state for tracking failed entities
+;; Stores {:model/Type {id {:fail-count N :next-retry-timestamp M}}}
+(def ^:private retry-state
+  (zipmap entities (repeatedly #(ConcurrentHashMap.))))
+
+;; Stores {:model/Type #{id1 id2 ...}} for entities that have exceeded MAX_RETRIES
+(def ^:private terminally-broken
+  (zipmap entities (repeatedly #(ConcurrentHashMap/newKeySet))))
+
+(def ^:private MAX_RETRIES 5)
+
+(defn- long-env-value
+  [env-var-kw default]
+  (or (parse-long (env/env env-var-kw "")) default))
+
+(defn- get-batch-size []
+  (long-env-value :mb-dependency-backfill-batch-size 20))
+
+(defn- get-delay-minutes []
+  (long-env-value :mb-dependency-backfill-delay-minutes 60))
+
+(defn- get-variance-minutes []
+  (long-env-value :mb-dependency-backfill-variance-minutes 10))
+
+(defn- get-processable-ids [model-kw batch-size]
+  (let [target-version models.dependency/current-dependency-analysis-version
+        terminally-broken-ids ^Set (terminally-broken model-kw)
+        retry-state-map ^Map (retry-state model-kw)
+        current-time (current-millis)]
+    (into []
+          (comp
+           (map :id)
+           (filter (fn [id]
+                     (and (not (.contains terminally-broken-ids id))
+                          (let [entity-retry-info (.get retry-state-map id)]
+                            (or (nil? entity-retry-info)
+                                (>= current-time (:next-retry-timestamp entity-retry-info)))))))
+           (take batch-size))
+          (t2/reducible-select [model-kw :id] :dependency_analysis_version [:< target-version]))))
+
+(defn- backfill-card!
+  [id target-version]
+  ;; We don't want to change the card at all, we just want to update the dependency data and
+  ;; mark the card as processed for this dependency analysis version.
+  (let [update-count (t2/update! :model/Card id :dependency_analysis_version [:< target-version]
+                                 {:dependency_analysis_version target-version})]
+    (when-let [card (and (pos? update-count)
+                         (t2/select-one :model/Card id))]
+      (events/publish-event! :event/card-update {:object card :user-id config/internal-mb-user-id}))
+    update-count))
+
+(defn- backfill-entity!
+  [model-kw id target-version]
+  (t2/with-transaction [_]
+    (case model-kw
+      :model/Card (backfill-card! id target-version)
+      (t2/update! model-kw id :dependency_analysis_version [:< target-version]
+                  {:dependency_analysis_version target-version}))))
+
+(defn- backfill-entity-batch!
+  [model-kw batch-size]
+  (let [model-name (name model-kw)
+        target-version models.dependency/current-dependency-analysis-version
+        retry-state-map ^Map (retry-state model-kw)
+        terminally-broken-set ^Set (terminally-broken model-kw)
+        ids (get-processable-ids model-kw batch-size)] ; Use the new get-processable-ids
+    (when (seq ids)
+      (log/infof "Processing a batch of %s %s(s)..." (count ids) model-name))
+    (reduce (fn [total id]
+              (+ total
+                 (try
+                   ;; this should update the dependency table via a toucan2 update hook
+                   (let [update-count (backfill-entity! model-kw id target-version)]
+                     (.remove retry-state-map id)
+                     update-count)
+                   (catch Exception e
+                     (let [current-time (current-millis)
+                           entity-retry-info (.get retry-state-map id)
+                           failure-count (inc (:fail-count entity-retry-info 0))
+                           retry-minutes (* failure-count (get-delay-minutes))
+                           new-next-retry-timestamp (+ current-time (* retry-minutes 60 1000))]
+                       (if (> failure-count MAX_RETRIES)
+                         (do (log/errorf e "Entity %s %s failed %d times, marking as terminally broken."
+                                         model-name id failure-count)
+                             (.add terminally-broken-set id)
+                             (.remove retry-state-map id)) ; Remove from retry map
+                         (do (log/warnf e "Entity %s %s failed, failure count: %d, next retry no sooner than %d minutes."
+                                        model-name id failure-count retry-minutes)
+                             (.put retry-state-map id {:fail-count failure-count
+                                                       :next-retry-timestamp new-next-retry-timestamp}))))
+                     0))))
+            0
+            ids)))
+
+(defn- backfill-dependencies
+  "Job to backfill dependencies for all entities.
+  Returns true if a full batch has been selected, nil or false otherwise."
+  []
+  (when (premium-features/has-feature? :dependencies)
+    (-> (reduce (fn [batch-size model-kw]
+                  (if (< batch-size 1)
+                    (reduced 0)
+                    (let [processed (backfill-entity-batch! model-kw batch-size)]
+                      (when (pos? processed)
+                        (log/info "Updated" processed "entities."))
+                      (- batch-size processed))))
+                (get-batch-size)
+                entities)
+        (< 1))))
+
+(defn- has-pending-retries? []
+  (some (fn [^Map model-retry-state]
+          (not (.isEmpty model-retry-state)))
+        (vals retry-state)))
+
+(declare schedule-next-run!)
+
+(task/defjob
+  ^{:doc "Backfill the dependency table."
+    org.quartz.DisallowConcurrentExecution true}
+  BackfillDependencies [ctx]
+  (log/info "Executing BackfillDependencies job...")
+  (let [full-batch-selected? (backfill-dependencies)
+        retries? (has-pending-retries?)]
+    (if (or full-batch-selected?
+            retries?)
+      (let [delay-seconds    (* (get-delay-minutes) 60)
+            variance-seconds (* (get-variance-minutes) 60)
+            delay-in-seconds (max 0 (+ (- delay-seconds variance-seconds) (rand-int (* 2 variance-seconds))))]
+        (schedule-next-run! delay-in-seconds (.getScheduler ctx)))
+      (log/info "No more entities to backfill for, stopping."))))
+
+(def ^:private job-key     "metabase.task.dependency-backfill.job")
+(def ^:private trigger-key "metabase.task.dependency-backfill.trigger")
+
+(defn- schedule-next-run!
+  ([delay-in-seconds] (schedule-next-run! delay-in-seconds nil))
+  ([delay-in-seconds scheduler]
+   (let [start-at (java.util.Date. (long (+ (current-millis) (* delay-in-seconds 1000))))
+         trigger  (triggers/build
+                   (triggers/with-identity (triggers/key (str trigger-key \. (random-uuid))))
+                   (triggers/for-job job-key)
+                   (triggers/start-at start-at))]
+     (log/info "Scheduling next run at" start-at)
+     (if scheduler
+       ;; re-scheduling from the job
+       (qs/add-trigger scheduler trigger)
+       ;; first schedule
+       (let [job (jobs/build (jobs/of-type BackfillDependencies) (jobs/with-identity job-key))]
+         (task/schedule-task! job trigger))))))
+
+(defmethod task/init! ::DependencyBackfill [_]
+  (if (pos? (get-batch-size))
+    (schedule-next-run! (rand-int (* (get-variance-minutes) 60)))
+    (log/info "Not starting dependency backfill job because the batch size is not positive")))

enterprise/backend/src/metabase_enterprise/metabot_v3/api.clj

@@ -94,8 +94,8 @@
    feedback :- :map]
   (let [token (premium-features/premium-embedding-token)
         base-url (store-api/store-api-url)]
-    (when (or (str/blank? token) (str/blank? base-url))
-      (throw (ex-info "Cannot build a request. The license token and/or Store api url are missing!" {})))
+    (api/check-400 (not (or (str/blank? token) (str/blank? base-url)))
+                   "Cannot build a request. The license token and/or Store api url are missing!")
     (try
       (http/post (str base-url "/api/v2/metabot/feedback/" token)
                  {:content-type :json

enterprise/backend/src/metabase_enterprise/metabot_v3/dummy_tools.clj

@@ -264,9 +264,10 @@
     (let [options (cond-> arguments
                     (= (:with-field-values? arguments) false) (assoc :field-values-fn identity))
           details (cond
-                    (int? model-id)    (u/prog1 (card-details model-id (assoc options :only-model true))
-                                         (api/check (= :model (:type <>)) 404
-                                                    (format "ID %s is not a valid model id, it's a question" model-id)))
+                    (int? model-id)    (let [card (card-details model-id (assoc options :only-model true))]
+                                         (if (= :model (:type card))
+                                           card
+                                           (format "ID %s is not a valid model id, it's a question" model-id)))
                     (int? table-id)    (table-details table-id options)
                     (string? table-id) (if-let [[_ card-id] (re-matches #"card__(\d+)" table-id)]
                                          (card-details (parse-long card-id) options)

enterprise/backend/src/metabase_enterprise/metabot_v3/tools/search.clj

@@ -107,6 +107,7 @@
         use-verified-content? (if metabot-id
                                 (:use_verified_content metabot)
                                 false)
+        limit (or limit 50)
         search-fn (fn [query]
                     (let [search-context (search/search-context
                                           (merge
@@ -122,7 +123,7 @@
                                             :current-user-perms @api/*current-user-permissions-set*
                                             :context :metabot
                                             :archived false
-                                            :limit (or limit 50)
+                                            :limit limit
                                             :offset 0}
                                            (when use-verified-content?
                                              {:verified true})))
@@ -131,5 +132,5 @@
         ;; Create futures for parallel execution
         futures (mapv #(future (search-fn %)) all-queries)
         result-lists (mapv deref futures)
-        fused-results (reciprocal-rank-fusion result-lists)]
+        fused-results (take limit (reciprocal-rank-fusion result-lists))]
     (map transform-search-result fused-results)))

enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/column_level_perms_check.clj

@@ -1,47 +0,0 @@
-(ns metabase-enterprise.sandbox.query-processor.middleware.column-level-perms-check
-  (:require
-   [medley.core :as m]
-   [metabase.api.common :refer [*current-user-id*]]
-   ;; legacy usage -- don't use Legacy MBQL utils in QP code going forward, prefer Lib. This will be updated to use
-   ;; Lib only soon
-   ^{:clj-kondo/ignore [:discouraged-namespace]}
-   [metabase.legacy-mbql.schema :as mbql.s]
-   [metabase.lib.core :as lib]
-   [metabase.lib.schema :as lib.schema]
-   [metabase.lib.util.match :as lib.util.match]
-   [metabase.premium-features.core :refer [defenterprise-schema]]
-   [metabase.query-processor.schema :as qp.schema]
-   [metabase.util.i18n :refer [tru]]
-   [metabase.util.log :as log]
-   [metabase.util.malli :as mu]))
-
-(defn- field-ids [form]
-  (set (lib.util.match/match form
-         [:field (id :guard integer?) _]
-         id)))
-
-(def ^:private ^:dynamic *gtap-perms*
-  "TODO: these used to be passed as a `:gtap-perms` key as part of QP context... not sure where this used to get passed
-  in."
-  nil)
-
-(mu/defn- maybe-apply-column-level-perms-check*
-  [{{{source-query-fields :fields} :source-query} :query, :as query} :- ::mbql.s/Query]
-  (let [restricted-field-ids (and *gtap-perms*
-                                  (field-ids source-query-fields))]
-    (when (seq restricted-field-ids)
-      (let [fields-ids-in-query (field-ids (m/dissoc-in query [:query :source-query]))]
-        (when-not (every? restricted-field-ids fields-ids-in-query)
-          (log/warnf "User '%s' attempted to access an inaccessible field. Accessible fields %s, fields in query %s"
-                     *current-user-id* (pr-str restricted-field-ids) (pr-str fields-ids-in-query))
-          (throw (ex-info (str (tru "User not able to query field")) {:status 403})))))))
-
-(defenterprise-schema maybe-apply-column-level-perms-check :- ::qp.schema/qp
-  "Check column-level permissions if applicable."
-  :feature :sandboxes
-  [qp :- ::qp.schema/qp]
-  (mu/fn [query :- ::lib.schema/query
-          rff   :- ::qp.schema/rff]
-    ;; TODO (Cam 9/11/25) -- update this middleware to use Lib/MBQL 5
-    (maybe-apply-column-level-perms-check* (lib/->legacy-MBQL query))
-    (qp query rff)))

enterprise/backend/src/metabase_enterprise/scim/v2/api.clj

@@ -10,6 +10,7 @@
    [metabase.models.interface :as mi]
    [metabase.permissions.core :as perms]
    [metabase.users.models.user :as user]
+   [metabase.users.schema :as users.schema]
    [metabase.util :as u]
    [metabase.util.i18n :as i18n]
    [metabase.util.malli :as mu]
@@ -171,7 +172,7 @@
    :active   (:is_active user)
    :meta     {:resourceType "User"}})
 
-(mu/defn ^:private scim-user->mb :- user/NewUser
+(mu/defn ^:private scim-user->mb :- users.schema/NewUser
   "Given a SCIM user, returns a Metabase user."
   [user]
   (let [{email :userName name-obj :name locale :locale is-active? :active} user

enterprise/backend/src/metabase_enterprise/serialization/api.clj

@@ -87,8 +87,8 @@
                      (catch Exception e
                        (reset! err e)
                        (if full-stacktrace
-                         (log/error e "Error during serialization")
-                         (log/error (u/strip-error e "Error during serialization"))))))]
+                         (log/error e "Error during serialization export")
+                         (log/error (u/strip-error e "Error during serialization export"))))))]
     {:archive       (when (.exists dst)
                       dst)
      :log-file      (when (.exists log-file)
@@ -142,8 +142,8 @@
                      (catch Exception e
                        (reset! err e)
                        (if full-stacktrace
-                         (log/error e "Error during serialization")
-                         (log/error (u/strip-error e "Error during serialization"))))))]
+                         (log/error e "Error during serialization import")
+                         (log/error (u/strip-error e "Error during serialization import"))))))]
     {:log-file      log-file
      :status        (:status (ex-data @err))
      :error-message (when @err

enterprise/backend/src/metabase_enterprise/serialization/cmd.clj

@@ -270,8 +270,8 @@
                                                 (u/strip-error @err nil))})
     (when @err
       (if (:full-stacktrace opts)
-        (log/error @err "Error during serialization")
-        (log/error (u/strip-error @err "Error during deserialization")))
+        (log/error @err "Error during serialization export")
+        (log/error (u/strip-error @err "Error during serialization export")))
       (throw (ex-info (ex-message @err) {:cmd/exit true})))
     (log/info (format "Export to '%s' complete!" path) (u/emoji "🚛💨 📦"))
     report))

enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj

@@ -65,6 +65,7 @@
    "DashboardBookmark"
    "DataPermissions"
    "DatabaseRouter"
+   "Dependency"
    "DocumentBookmark"
    "HTTPAction"
    "ImplicitAction"

enterprise/backend/src/metabase_enterprise/sso/integrations/ldap.clj

@@ -5,7 +5,6 @@
    [metabase-enterprise.sso.settings :as ee.sso.settings]
    [metabase.premium-features.core :refer [defenterprise-schema]]
    [metabase.sso.core :as sso]
-   [metabase.users.models.user :as user]
    [metabase.util :as u]
    [metabase.util.malli.schema :as ms]
    [toucan2.core :as t2])
@@ -63,10 +62,10 @@
    {:keys [sync-groups?], :as settings}                                  :- sso/LDAPSettings]
   (let [user (or (attribute-synced-user user-info)
                  (sso-utils/check-user-provisioning :ldap)
-                 (-> (user/create-new-ldap-auth-user! {:first_name       first-name
-                                                       :last_name        last-name
-                                                       :email            email
-                                                       :login_attributes attributes})
+                 (-> (sso/create-new-ldap-auth-user! {:first_name       first-name
+                                                      :last_name        last-name
+                                                      :email            email
+                                                      :login_attributes attributes})
                      (assoc :is_active true)))]
     (u/prog1 user
       (when sync-groups?

enterprise/backend/src/metabase_enterprise/transforms/api.clj

@@ -15,6 +15,7 @@
    [metabase.api.routes.common :refer [+auth]]
    [metabase.api.util.handlers :as handlers]
    [metabase.driver.util :as driver.u]
+   [metabase.events.core :as events]
    [metabase.request.core :as request]
    [metabase.util :as u]
    [metabase.util.i18n :refer [deferred-tru]]
@@ -119,15 +120,17 @@
   (api/check (not (transforms.util/target-table-exists? body))
              403
              (deferred-tru "A table with that name already exists."))
-  (t2/with-transaction [_]
-    (let [tag-ids (:tag_ids body)
-          transform (t2/insert-returning-instance!
-                     :model/Transform (select-keys body [:name :description :source :target :run_trigger]))]
-      ;; Add tag associations if provided
-      (when (seq tag-ids)
-        (transform.model/update-transform-tags! (:id transform) tag-ids))
-      ;; Return with hydrated tag_ids
-      (t2/hydrate transform :transform_tag_ids))))
+  (let [transform (t2/with-transaction [_]
+                    (let [tag-ids (:tag_ids body)
+                          transform (t2/insert-returning-instance!
+                                     :model/Transform (select-keys body [:name :description :source :target :run_trigger]))]
+                      ;; Add tag associations if provided
+                      (when (seq tag-ids)
+                        (transform.model/update-transform-tags! (:id transform) tag-ids))
+                      ;; Return with hydrated tag_ids
+                      (t2/hydrate transform :transform_tag_ids)))]
+    (events/publish-event! :event/transform-create {:object transform :user-id api/*current-user-id*})
+    transform))
 
 (api.macros/defendpoint :get "/:id"
   "Get a specific transform."
@@ -188,28 +191,29 @@
             [:tag_ids {:optional true} [:sequential ms/PositiveInt]]]]
   (log/info "put transform" id)
   (api/check-superuser)
-  (t2/with-transaction [_]
-    ;; Cycle detection should occur within the transaction to avoid race
-    (let [old (t2/select-one :model/Transform id)
-          new (merge old body)
-          target-fields #(-> % :target (select-keys [:schema :name]))]
-      ;; we must validate on a full transform object
-      (check-feature-enabled! new)
-      (check-database-feature new)
-      (when (transforms.util/query-transform? old)
-        (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]
-          (throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)
-                          {:status-code 400}))))
-      (api/check (not (and (not= (target-fields old) (target-fields new))
-                           (transforms.util/target-table-exists? new)))
-                 403
-                 (deferred-tru "A table with that name already exists.")))
-
-    (t2/update! :model/Transform id (dissoc body :tag_ids))
-    ;; Update tag associations if provided
-    (when (contains? body :tag_ids)
-      (transform.model/update-transform-tags! id (:tag_ids body)))
-    (t2/hydrate (t2/select-one :model/Transform id) :transform_tag_ids)))
+  (let [transform (t2/with-transaction [_]
+                    ;; Cycle detection should occur within the transaction to avoid race
+                    (let [old (t2/select-one :model/Transform id)
+                          new (merge old body)
+                          target-fields #(-> % :target (select-keys [:schema :name]))]
+                      ;; we must validate on a full transform object
+                      (check-feature-enabled! new)
+                      (check-database-feature new)
+                      (when (transforms.util/query-transform? old)
+                        (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]
+                          (throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)
+                                          {:status-code 400}))))
+                      (api/check (not (and (not= (target-fields old) (target-fields new))
+                                           (transforms.util/target-table-exists? new)))
+                                 403
+                                 (deferred-tru "A table with that name already exists.")))
+                    (t2/update! :model/Transform id (dissoc body :tag_ids))
+                    ;; Update tag associations if provided
+                    (when (contains? body :tag_ids)
+                      (transform.model/update-transform-tags! id (:tag_ids body)))
+                    (t2/hydrate (t2/select-one :model/Transform id) :transform_tag_ids))]
+    (events/publish-event! :event/transform-update {:object transform :user-id api/*current-user-id*})
+    transform))
 
 (api.macros/defendpoint :delete "/:id"
   "Delete a transform."

enterprise/backend/src/metabase_enterprise/transforms/core.clj

@@ -1,9 +1,12 @@
 (ns metabase-enterprise.transforms.core
   "API namespace for the `metabase-enterprise.transform` module."
   (:require
+   [metabase-enterprise.transforms.models.transform-run]
    [metabase-enterprise.transforms.settings]
    [potemkin :as p]))
 
 (p/import-vars
  [metabase-enterprise.transforms.settings
-  transform-timeout])
+  transform-timeout]
+ [metabase-enterprise.transforms.models.transform-run
+  timeout-run!])

enterprise/backend/src/metabase_enterprise/transforms/execute.clj

@@ -4,6 +4,7 @@
    [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.driver :as driver]
    [metabase.driver.util :as driver.u]
+   [metabase.events.core :as events]
    [metabase.lib.schema.common :as schema.common]
    [metabase.util.log :as log]
    [metabase.util.malli.registry :as mr]
@@ -35,6 +36,7 @@
            {driver :engine :as database} (t2/select-one :model/Database db)
            feature (transforms.util/required-database-feature transform)
            transform-details {:db-id db
+                              :transform-id   id
                               :transform-type (keyword (:type target))
                               :conn-spec (driver/connection-spec driver database)
                               :query (transforms.util/compile-source source)
@@ -56,7 +58,9 @@
           run-id driver transform-details
           (fn [_cancel-chan] (driver/run-transform! driver transform-details opts)))
          (transforms.instrumentation/with-stage-timing [run-id :table-sync]
-           (transforms.util/sync-target! target database run-id))))
+           (transforms.util/sync-target! target database run-id)
+         ;; This event must be published only after the sync is complete - the new table needs to be in AppDB.
+           (events/publish-event! :event/transform-run-complete {:object transform-details}))))
      (catch Throwable t
        (log/error t "Error executing transform")
        (when start-promise

enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj

@@ -3,6 +3,7 @@
    [clojure.set :as set]
    [medley.core :as m]
    [metabase-enterprise.transforms.models.transform-run :as transform-run]
+   [metabase.events.core :as events]
    [metabase.models.interface :as mi]
    [metabase.models.serialization :as serdes]
    [metabase.util :as u]
@@ -17,7 +18,7 @@
   (derive :model/Transform trait))
 
 (t2/deftransforms :model/Transform
-  {:source      mi/transform-json
+  {:source      mi/transform-transform-source
    :target      mi/transform-json
    :run_trigger mi/transform-keyword})
 
@@ -61,6 +62,18 @@
       (for [transform transforms]
         (assoc transform :tag_ids (vec (get transform-id->tag-ids (:id transform) [])))))))
 
+(t2/define-after-insert :model/Transform [transform]
+  (events/publish-event! :event/create-transform {:object transform})
+  transform)
+
+(t2/define-after-update :model/Transform [transform]
+  (events/publish-event! :event/update-transform {:object transform})
+  transform)
+
+(t2/define-before-delete :model/Transform [transform]
+  (events/publish-event! :event/delete-transform {:id (:id transform)})
+  transform)
+
 (defn update-transform-tags!
   "Update the tags associated with a transform using smart diff logic.
    Only modifies what has changed: deletes removed tags, updates positions for moved tags,
@@ -134,7 +147,7 @@
 (defmethod serdes/make-spec "Transform"
   [_model-name opts]
   {:copy [:name :description :entity_id]
-   :skip []
+   :skip [:dependency_analysis_version]
    :transform {:created_at (serdes/date)
                :updated_at (serdes/date)
                :source {:export serdes/export-mbql :import serdes/import-mbql}

enterprise/backend/src/metabase_enterprise/transforms/models/transform_run_cancelation.clj

@@ -14,16 +14,12 @@
   [run-id]
   (t2/query-one
    [(str "INSERT INTO transform_run_cancelation (run_id) "
-         "SELECT ? "
-         "WHERE "
-         "      EXISTS (SELECT 1 "
-         "              FROM transform_run "
-         "              WHERE (transform_run.id = ?) "
-         "                AND transform_run.is_active)"
-         "      AND NOT EXISTS (SELECT 1 "
-         "                      FROM transform_run_cancelation "
-         "                      WHERE run_id = ?)")
-    run-id run-id run-id])
+         "SELECT transform_run.id "
+         "FROM transform_run "
+         "WHERE transform_run.id = ? "
+         "AND transform_run.is_active "
+         "AND NOT EXISTS (SELECT 1 FROM transform_run_cancelation WHERE run_id = ?)")
+    run-id run-id])
   (t2/update! :model/TransformRun
               :id run-id
               {:status "canceling"})

enterprise/backend/src/metabase_enterprise/transforms/util.clj

@@ -176,8 +176,8 @@
 (defn compile-source
   "Compile the source query of a transform."
   [{query-type :type :as source}]
-  (case query-type
-    "query" (:query (qp.compile/compile-with-inline-parameters (massage-sql-query (:query source))))))
+  (case (keyword query-type)
+    :query (:query (qp.compile/compile-with-inline-parameters (massage-sql-query (:query source))))))
 
 (defn required-database-feature
   "Returns the database feature necessary to execute `transform`."

enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj

@@ -6,6 +6,7 @@
    [metabase-enterprise.transforms-python.python-runner :as python-runner]
    [metabase-enterprise.transforms-python.s3 :as s3]
    [metabase-enterprise.transforms-python.settings :as transforms-python.settings]
+   [metabase-enterprise.transforms.core :as transforms]
    [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
    [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.driver :as driver]
@@ -17,8 +18,10 @@
    [metabase.util.log :as log]
    [toucan2.core :as t2])
   (:import
-   (java.io Closeable File)
+   (java.io Closeable)
    (java.net SocketException)
+   (java.nio.file Files)
+   (java.nio.file.attribute FileAttribute)
    (java.time Duration)))
 
 (set! *warn-on-reflection* true)
@@ -240,13 +243,14 @@
   ;; TODO restructure things such that s3 can we swapped out for other transfer mechanisms
   (with-open [log-future-ref     (open-python-message-update-future! run-id message-log)
               shared-storage-ref (s3/open-shared-storage! (:source-tables source))]
-    (let [driver     (:engine db)
-          server-url (transforms-python.settings/python-runner-url)
-          _          (python-runner/copy-tables-to-s3! {:run-id         run-id
-                                                        :shared-storage @shared-storage-ref
-                                                        :table-name->id (:source-tables source)
-                                                        :cancel-chan    cancel-chan})
-          _          (start-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan
+    (let [driver          (:engine db)
+          server-url      (transforms-python.settings/python-runner-url)
+          _               (python-runner/copy-tables-to-s3! {:run-id         run-id
+                                                             :shared-storage @shared-storage-ref
+                                                             :table-name->id (:source-tables source)
+                                                             :cancel-chan    cancel-chan})
+          _               (start-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan
+
           {:keys [status body] :as response}
           (python-runner/execute-python-code-http-call!
            {:server-url     server-url
@@ -254,29 +258,33 @@
             :run-id         run-id
             :table-name->id (:source-tables source)
             :shared-storage @shared-storage-ref})
-          ;; TODO temporary to keep more code stable while refactoring
-          ;; no need to materialize these early (i.e output we can stream directly into a tmp file or db if small)
-          {:keys [output output-manifest events]} (python-runner/read-output-objects @shared-storage-ref)]
+
+          output-manifest (python-runner/read-output-manifest @shared-storage-ref)
+          events          (python-runner/read-events @shared-storage-ref)]
       (.close log-future-ref)                               ; early close to force any writes to flush
       (when (seq events)
         (replace-python-logs! message-log events))
       (if (not= 200 status)
-        (throw (ex-info "Python runner call failed"
-                        {:transform-message (i18n/tru "Python execution failure (exit code {0})" (:exit_code body "?"))
-                         :status-code       400
-                         :api-status-code   status
-                         :body              body
-                         :events            events}))
+        (do
+          (when (:timeout body)
+            (transforms/timeout-run! run-id))
+          (throw (ex-info "Python runner call failed"
+                          {:transform-message (i18n/tru "Python execution failure (exit code {0})" (:exit_code body "?"))
+                           :status-code       400
+                           :api-status-code   status
+                           :body              body
+                           :events            events})))
         (try
-          (let [temp-file (File/createTempFile "transform-output-" ".jsonl")]
+          (let [temp-path (Files/createTempFile "transform-output-" ".jsonl" (u/varargs FileAttribute))
+                temp-file (.toFile temp-path)]
             (when-not (seq (:fields output-manifest))
               (throw (ex-info "No fields in metadata"
                               {:metadata               output-manifest
                                :raw-body               body
                                :events                 events})))
             (try
-              (with-open [writer (io/writer temp-file)]
-                (.write writer ^String output))
+              (with-open [in (python-runner/open-output @shared-storage-ref)]
+                (io/copy in temp-file))
               (let [file-size (.length temp-file)]
                 (transforms.instrumentation/with-stage-timing [run-id :file-to-dwh]
                   (transfer-file-to-db driver db transform output-manifest temp-file))

enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj

@@ -2,7 +2,6 @@
   (:require
    [clj-http.client :as http]
    [clojure.java.io :as io]
-   [clojure.string :as str]
    [medley.core :as m]
    [metabase-enterprise.transforms-python.s3 :as s3]
    [metabase-enterprise.transforms-python.settings :as transforms-python.settings]
@@ -20,7 +19,7 @@
    [toucan2.core :as t2])
   (:import
    (clojure.lang PersistentQueue)
-   (java.io BufferedWriter File OutputStream OutputStreamWriter)
+   (java.io BufferedWriter File InputStream OutputStream OutputStreamWriter)
    (java.nio.charset StandardCharsets)))
 
 (set! *warn-on-reflection* true)
@@ -184,7 +183,7 @@
 
         payload                  {:code                code
                                   :library             (t2/select-fn->fn :path :source :model/PythonLibrary)
-                                  :timeout             (* 30 60) ;; 30 mins harcoded for now
+                                  :timeout             (transforms-python.settings/python-runner-timeout-seconds)
                                   :request_id          run-id
                                   :output_url          (:url output)
                                   :output_manifest_url (:url output-manifest)
@@ -204,17 +203,24 @@
                                    {:error string-if-error}))
                                string-if-error)))))
 
-;; temporary, we should not need to realize data/events files into memory longer term
-(defn read-output-objects
-  "Temporary function that strings/jsons stuff in S3 and returns it for compatibility."
+(defn read-events
+  "Returns a vector of event contents (or nil if the event file does not exist)"
   [{:keys [s3-client bucket-name objects]}]
-  (let [{:keys [output output-manifest events]} objects
-        output-content          (s3/read-to-string s3-client bucket-name (:path output) nil)
-        output-manifest-content (s3/read-to-string s3-client bucket-name (:path output-manifest) "{}")
-        events-content          (s3/read-to-string s3-client bucket-name (:path events))]
-    {:output          output-content
-     :output-manifest (json/decode+kw output-manifest-content)
-     :events          (mapv json/decode+kw (str/split-lines events-content))}))
+  ;; note we expect :events to be a small file, limits ought to be enforced by the runner
+  (when-some [in (s3/open-object s3-client bucket-name (:path (:events objects)))]
+    (with-open [in in
+                rdr (io/reader in)]
+      (mapv json/decode+kw (line-seq rdr)))))
+
+(defn read-output-manifest
+  "Return the output manifest map. Returns nil if it does not exist."
+  [{:keys [s3-client bucket-name objects]}]
+  (json/decode+kw (s3/read-to-string s3-client bucket-name (:path (:output-manifest objects)) "{}")))
+
+(defn open-output
+  "Return an InputStream with the output jsonl contents. Close with .close. Returns nil if the object does not exist."
+  ^InputStream [{:keys [s3-client bucket-name objects]}]
+  (s3/open-object s3-client bucket-name (:path (:output objects))))
 
 (defn cancel-python-code-http-call!
   "Calls the /cancel endpoint of the python runner. Returns immediately."

enterprise/backend/src/metabase_enterprise/transforms_python/s3.clj

@@ -4,7 +4,7 @@
    [metabase.util.log :as log])
   (:import
    (clojure.lang IDeref)
-   (java.io Closeable File)
+   (java.io Closeable File InputStream)
    (java.net URI)
    (java.time Duration)
    (software.amazon.awssdk.auth.credentials AwsBasicCredentials StaticCredentialsProvider)
@@ -183,19 +183,22 @@
   (let [^PutObjectRequest request (put-object-request bucket-name key)]
     (.putObject s3-client request (RequestBody/fromFile file))))
 
-;; TODO optimize our ingestion to stream the data. reading a string will not work in any case for binary files.
+(defn open-object
+  "Get back the contents of the given key as a InputStream."
+  ^InputStream [^S3Client s3-client ^String bucket-name ^String key]
+  (try
+    (let [^GetObjectRequest request (get-object-request bucket-name key)]
+      (.getObject s3-client request))
+    (catch NoSuchKeyException _ nil)))
+
 (defn read-to-string
   "Get back the contents of the given key as a string."
-  ([s3-client bucket-name key] (read-to-string s3-client bucket-name key ::throw))
+  ([s3-client bucket-name key] (read-to-string s3-client bucket-name key nil))
   ([^S3Client s3-client ^String bucket-name ^String key not-found]
-   (try
-     (let [^GetObjectRequest request (get-object-request bucket-name key)
-           response                  (.getObject s3-client request)]
-       (slurp response))
-     (catch NoSuchKeyException e
-       (if (identical? ::throw not-found)
-         (throw e)
-         not-found)))))
+   (if-some [in (open-object s3-client bucket-name key)]
+     (with-open [ret in]
+       (slurp ret))
+     not-found)))
 
 (defn delete
   ;; TODO better error handling

enterprise/backend/src/metabase_enterprise/transforms_python/settings.clj

@@ -113,3 +113,14 @@
   :export?    false
   :encryption :no
   :audit      :getter)
+
+(setting/defsetting python-runner-timeout-seconds
+  (deferred-tru "Timeout in seconds for Python script execution. Defaults to 30 minutes (1800 seconds).")
+  :type       :integer
+  :visibility :admin
+  :default    1800 ;; 30 minutes
+  :feature    :transforms-python
+  :doc        false
+  :export?    false
+  :encryption :no
+  :audit      :getter)

enterprise/backend/test/metabase_enterprise/advanced_permissions/models/permissions/block_permissions_test.clj

@@ -572,11 +572,11 @@
                 (mt/with-temp [:model/Card {card-1-id :id, :as card-1} {:collection_id (u/the-id collection)
                                                                         :dataset_query card-1-query}]
                   (let [card-2-query (mt/native-query
-                                       {:query         "SELECT * FROM {{card}}"
-                                        :template-tags {"card" {:name         "card"
-                                                                :display-name "card"
-                                                                :type         :card
-                                                                :card-id      card-1-id}}})]
+                                      {:query         "SELECT * FROM {{card}}"
+                                       :template-tags {"card" {:name         "card"
+                                                               :display-name "card"
+                                                               :type         :card
+                                                               :card-id      card-1-id}}})]
                     (mt/with-temp [:model/Card card-2 {:collection_id (u/the-id collection)
                                                        :dataset_query card-2-query}]
                       (testing "should be able to read nested-nested Card if we have Collection permissions"
@@ -625,10 +625,10 @@
             (mt/with-temp [:model/Collection collection]
               (perms/grant-collection-read-permissions! (perms/all-users-group) collection)
               (let [card-1-query (mt/native-query
-                                   {:query (str "SELECT id, name, category_id, latitude, longitude, price "
-                                                "FROM venues "
-                                                "ORDER BY id ASC "
-                                                "LIMIT 2")})]
+                                  {:query (str "SELECT id, name, category_id, latitude, longitude, price "
+                                               "FROM venues "
+                                               "ORDER BY id ASC "
+                                               "LIMIT 2")})]
                 (mt/with-temp [:model/Card {card-1-id :id, :as card-1} {:collection_id (u/the-id collection)
                                                                         :dataset_query card-1-query}]
                   (let [card-2-query (mt/mbql-query nil

enterprise/backend/test/metabase_enterprise/api/session_test.clj

@@ -99,5 +99,6 @@
             :database_routing               true
             :cloud_custom_smtp              true
             :etl_connections                false
-            :etl_connections_pg             false}
+            :etl_connections_pg             false
+            :dependencies                   false}
            (:token-features (mt/user-http-request :crowberto :get 200 "session/properties"))))))

enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj

@@ -0,0 +1,189 @@
+(ns metabase-enterprise.dependencies.core-test
+  (:require
+   [clojure.test :refer [deftest is testing]]
+   [medley.core :as m]
+   [metabase-enterprise.dependencies.core :as dependencies]
+   [metabase-enterprise.dependencies.test-util :as deps.tu]
+   [metabase.graph.core :as graph]
+   [metabase.lib.core :as lib]
+   [metabase.lib.metadata :as lib.metadata]
+   [metabase.lib.test-metadata :as meta]
+   [metabase.lib.test-util :as lib.tu]))
+
+(defn- testbed
+  "A `MetadataProvider` with a chain of MBQL cards and transforms for testing."
+  []
+  (let [mp            (deps.tu/default-metadata-provider)
+        query1        (-> (lib/query mp (meta/table-metadata :orders))
+                          (lib/expression "Tax Rate" (lib// (meta/field-metadata :orders :tax)
+                                                            (meta/field-metadata :orders :subtotal))))
+        mp            (lib.tu/metadata-provider-with-card-from-query mp 101 query1)
+        card1         (lib.metadata/card mp 101)
+        card1-query   (lib/query mp card1)
+        card1-cols    (lib/returned-columns card1-query)
+        tax-rate      (m/find-first #(= (:lib/desired-column-alias %) "Tax Rate") card1-cols)
+        query2        (-> card1-query
+                          (lib/filter (lib/> tax-rate 0.06)))
+        mp            (lib.tu/metadata-provider-with-card-from-query mp 102 query2)
+
+        ;; A transform that depends on card1
+        tf1-query     (-> (lib/query mp card1)
+                          lib/->legacy-MBQL)
+        transform1    {:id     30
+                       :name   "MBQL Transform"
+                       :source {:query tf1-query}
+                       :target {:schema "Transformed"
+                                :name   "output_tf30"}}
+        tf1-output    (-> (meta/table-metadata :orders)
+                          (assoc :id           1234567
+                                 :schema       "Transformed"
+                                 :name         "output_tf30"
+                                 :display-name "Transform 30 Output"))
+        tf1-cols      (map-indexed (fn [i col]
+                                     (assoc col
+                                            :id         (+ 123456700 i)
+                                            :table-id   1234567
+                                            :lib/source :source/table-defaults))
+                                   card1-cols)
+        mp            (lib.tu/mock-metadata-provider mp {:tables     [tf1-output]
+                                                         :fields     tf1-cols
+                                                         :transforms [transform1]})
+
+        ;; An MBQL card that consumes the output table of transform1.
+        ;; References some fields but does not change the output columns.
+        tf1-consumer  (-> (lib/query mp (lib.metadata/table mp 1234567))
+                          (lib/filter (lib/= (m/find-first #(= (:name %) "Tax Rate") tf1-cols)
+                                             101)))
+        mp            (lib.tu/metadata-provider-with-card-from-query mp 301 tf1-consumer)
+
+        ;; SQL cards and snippets
+        ;; Card 7 from [[deps.tu/default-metadata-provider]] uses the "outer" snippet (2).
+        ;; This adds a SQL transform that consumes card 7, and a card that selects from that transform.
+        tf2-query     (lib/native-query mp "SELECT * FROM {{#7}}")
+        transform2    {:id      31
+                       :name    "SQL transform"
+                       :source {:query (lib/->legacy-MBQL tf2-query)}
+                       :target {:schema "Transformed"
+                                :name   "output_tf31"}}
+        tf2-output    (-> (meta/table-metadata :orders)
+                          (assoc :id           1234568
+                                 :schema       "Transformed"
+                                 :name         "output_tf31"
+                                 :display-name "Transform 31 Output"))
+        ;; Output columns for transform2 are derived from card 7. That's "SELECT * FROM {{snippet: outer}}";
+        ;; the outer snippet references {{snippet: inner}}; the inner snippet is {{#card-ref}} to card 1;
+        ;; card 1 is a basic MBQL query of products.
+        card7         (-> (lib.metadata/card mp 7)
+                          (assoc :result-metadata (:result-metadata (:products/native (lib.tu/mock-cards)))))
+        mp            (lib.tu/mock-metadata-provider mp {:cards [card7]})
+        tf2-cols      (->> card7
+                           (lib/query mp)
+                           lib/returned-columns
+                           (map-indexed (fn [i col]
+                                          (assoc col
+                                                 :id         (+ 123456800 i)
+                                                 :table-id   1234568
+                                                 :lib/source :source/table-defaults))))
+        mp            (lib.tu/mock-metadata-provider mp {:tables     [tf2-output]
+                                                         :fields     tf2-cols
+                                                         :transforms [transform2]})
+
+        ;; MBQL card consuming tf2-output.
+        query3        (-> (lib/query mp tf2-output)
+                          (lib/filter (lib/< (m/find-first #(= (:name %) "RATING") tf2-cols)
+                                             3)))
+        ;; SQL card consuming tf2-output.
+        query4        (lib/native-query mp "SELECT * FROM Transformed.output_tf31")
+        mp            (-> mp
+                          (lib.tu/metadata-provider-with-card-from-query 303 query3)
+                          (lib.tu/metadata-provider-with-card-from-query 304 query4))]
+    {:provider                mp
+     ;; NOTE: This is a *downstream* graph! So it's a map of keys to those entities which depend on them.
+     :graph                   (graph/in-memory {[:card 101]      #{[:card 102]
+                                                                   [:transform 30]}
+                                                [:transform 30]  #{[:table 1234567]}
+                                                [:table 1234567] #{[:card 301]}
+
+                                                [:snippet 1]     #{[:snippet   2]}
+                                                [:snippet 2]     #{[:card      7]}
+                                                [:card 7]        #{[:transform 31]}
+                                                [:transform 31]  #{[:table     1234568]}
+                                                [:table 1234568] #{[:card      303]
+                                                                   [:card      304]}})
+     :mbql-base                   card1
+     :mbql-dependent              (lib.metadata/card mp 102)
+     :mbql-transform              transform1
+     :mbql-transform-output       tf1-output
+     :mbql-transform-cols         (m/index-by :name tf1-cols)
+     :mbql-transform-consumer     (lib.metadata/card mp 301)
+
+     :snippet-inner               (lib.metadata/native-query-snippet mp 1)
+     :snippet-outer               (lib.metadata/native-query-snippet mp 2)
+     :sql-base                    (lib.metadata/card mp 7)
+     :sql-transform               transform2
+     :sql-transform-output        tf2-output
+     :sql-transform-cols          (m/index-by :name tf2-cols)
+     :sql-transform-mbql-consumer (lib.metadata/card mp 303)
+     :sql-transform-sql-consumer  (lib.metadata/card mp 304)}))
+
+(comment
+  *e
+  (:mbql-dependent (testbed))
+  (:mbql-transform-consumer (testbed)))
+
+(deftest ^:parallel basic-mbql-card-test
+  (testing "when changing an MBQL card with dependents"
+    (let [{:keys [provider graph mbql-base]
+           {tax-rate "Tax Rate"}     :mbql-transform-cols
+           {downstream-card-id  :id} :mbql-dependent
+           {transformed-card-id :id} :mbql-transform-consumer} (testbed)]
+      (testing "a column that no longer exists will cause errors when referenced"
+        (let [card'  (-> mbql-base
+                         (update-in [:dataset-query :query :expressions]
+                                    update-keys (constantly "Sales Taxes"))
+                         (dissoc :result-metadata))
+              errors (dependencies/errors-from-proposed-edits provider graph {:card [card']})]
+          (is (=? {:card {downstream-card-id  [[:field {} "Tax Rate"]]
+                          transformed-card-id [[:field {} (:id tax-rate)]]}}
+                  errors))
+          (is (= [:card] (keys errors)))
+          (is (= #{downstream-card-id transformed-card-id} (set (keys (:card errors)))))))
+
+      (testing "changing something unrelated will cause no errors"
+        (let [card' (-> mbql-base
+                        (assoc-in [:dataset-query :query :filter]
+                                  [:> [:field (meta/id :orders :quantity) nil] 100])
+                        (dissoc :result-metadata))]
+          (is (= {} (dependencies/errors-from-proposed-edits provider graph {:card [card']}))))))))
+
+(deftest ^:parallel sql-snippet->card->transform->cards-test
+  (testing "changing a snippet correctly finds downstream errors"
+    (let [{:keys [provider graph sql-transform snippet-inner sql-transform-cols]
+           {direct-sql-card-id       :id} :sql-base
+           {transformed-sql-card-id  :id} :sql-transform-sql-consumer
+           {transformed-mbql-card-id :id} :sql-transform-mbql-consumer} (testbed)]
+      (testing "when breaking the inner snippet with a nonexistent table"
+        (let [snippet' (assoc snippet-inner
+                              :content       "nonexistent_table"
+                              :template-tags {})
+              rating   (get sql-transform-cols "RATING")
+              errors   (dependencies/errors-from-proposed-edits provider graph {:snippet [snippet']})]
+          ;; That breaks (1) the SQL card which uses the snippets, (2) the transforms, (3) both the MBQL and (4) SQL
+          ;; queries that consume the transform's table.
+          (is (=? {:card      {direct-sql-card-id       [{:table {:table "NONEXISTENT_TABLE"},
+                                                          :type :all-columns,
+                                                          :metabase.driver.sql/bad-reference true}]
+                               transformed-sql-card-id  [{:table {:schema "TRANSFORMED", :table "OUTPUT_TF31"},
+                                                          :type :all-columns,
+                                                          :metabase.driver.sql/bad-reference true}]
+                               transformed-mbql-card-id [[:field {} (:id rating)]]}
+                   :transform {(:id sql-transform)      [{:table {:table "NONEXISTENT_TABLE"},
+                                                          :type :all-columns,
+                                                          :metabase.driver.sql/bad-reference true}]}}
+                  errors))
+          (is (= #{:card :transform}
+                 (set (keys errors))))
+          (is (= #{direct-sql-card-id transformed-sql-card-id transformed-mbql-card-id}
+                 (set (keys (:card errors)))))
+          (is (= #{(:id sql-transform)}
+                 (set (keys (:transform errors))))))))))

enterprise/backend/test/metabase_enterprise/dependencies/metadata_provider_test.clj

@@ -0,0 +1,131 @@
+(ns metabase-enterprise.dependencies.metadata-provider-test
+  (:require
+   [clojure.test :refer [deftest is testing]]
+   [medley.core :as m]
+   [metabase-enterprise.dependencies.metadata-provider :as deps.mp]
+   [metabase.lib.core :as lib]
+   [metabase.lib.metadata :as lib.metadata]
+   [metabase.lib.test-metadata :as meta]
+   [metabase.lib.test-util :as lib.tu]
+   [metabase.lib.test-util.macros :as lib.tu.macros]))
+
+(deftest basic-cards-test
+  (testing "overriding a single card, it is returned correctly"
+    (let [card    (:orders (lib.tu/mock-cards))
+          mp      (-> card
+                      lib.tu/metadata-provider-with-mock-card
+                      deps.mp/override-metadata-provider)]
+      (deps.mp/add-override mp :card (:id card)
+                            (-> card
+                                (assoc-in [:dataset-query :query :fields]
+                                          (lib.tu.macros/$ids orders
+                                            [$subtotal $created-at]))
+                                (dissoc :result-metadata)))
+      (is (=? [(meta/field-metadata :orders :subtotal)
+               (meta/field-metadata :orders :created-at)]
+              (->> (lib.metadata/card mp (:id card))
+                   (lib/query mp)
+                   lib/returned-columns))))))
+
+(defn- replace-query [new-query card]
+  (-> card
+      (assoc :dataset-query (lib/->legacy-MBQL new-query))
+      (dissoc :result-metadata)))
+
+(deftest upstream-card-test
+  (testing "overriding an upstream card that's consumed by a downstream card"
+    (let [upstream   (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
+                         (lib/breakout (meta/field-metadata :products :category))
+                         (lib/aggregate (lib/count)))
+          mp1        (lib.tu/metadata-provider-with-card-from-query 1 upstream)
+          card1      (lib.metadata/card mp1 1)
+          by-name    (m/index-by :name (lib/returned-columns (lib/query mp1 card1)))
+          downstream (-> (lib/query mp1 card1)
+                         (lib/filter (lib/= (get by-name "count") 1)))
+          mp         (-> (lib.tu/metadata-provider-with-card-from-query mp1 2 downstream)
+                         deps.mp/override-metadata-provider)]
+      (deps.mp/add-override mp :card 1
+                            (-> upstream
+                                (lib/breakout (lib/with-temporal-bucket (meta/field-metadata :orders :created-at) :day))
+                                (replace-query card1)))
+      ;; NOTE: This line is important: this is what would happen to all the dependents returned by the graph.
+      ;; There's no update (`nil`) but it still includes the downstream stuff in the scope of the
+      ;; OverrideMetadataProvider.
+      (deps.mp/add-override mp :card 2 nil)
+      (testing "upstream card's columns are correct"
+        (is (=? [{:name "CATEGORY"}
+                 {:name "CREATED_AT"}
+                 {:name "count"}]
+                (->> (lib.metadata/card mp 1)
+                     (lib/query mp)
+                     lib/returned-columns))))
+      (testing "downstream card (which selects all) is also correct"
+        (is (=? [{:name "CATEGORY"}
+                 {:name "CREATED_AT"}
+                 {:name "count"}]
+                (->> (lib.metadata/card mp 2)
+                     (lib/query mp)
+                     lib/returned-columns)))))))
+
+(deftest card->transform->card-test
+  (testing "changing the columns of an upstream card that is transformed and then used by a card"
+    (let [upstream   (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
+                         (lib/breakout (meta/field-metadata :products :category))
+                         (lib/aggregate (lib/count)))
+          mp1        (lib.tu/metadata-provider-with-card-from-query 1 upstream)
+          card1      (lib.metadata/card mp1 1)
+          card1-cols (-> (lib/query mp1 card1)
+                         lib/returned-columns)
+
+          transform  {:lib/type :metadata/transform
+                      :id     123
+                      :name   "The Transform"
+                      :source {:type  :query
+                               :query (-> (lib/query mp1 (lib.metadata/card mp1 1))
+                                          lib/->legacy-MBQL)}
+                      :target {:type   "table"
+                               :schema "transform_schema_123"
+                               :name   "autobots"}}
+          mp2        (lib.tu/mock-metadata-provider
+                      mp1 {:tables [{:id              123000000
+                                     :lib/type        :metadata/table
+                                     :name            "autobots"
+                                     :schema          "transform_schema_123"
+                                     :db-id           (meta/id)
+                                     :visibility-type nil
+                                     :active          true}]
+                           :fields (map-indexed (fn [i col]
+                                                  (assoc col
+                                                         :id       (+ 123000001 i)
+                                                         :table-id 123000000))
+                                                card1-cols)})
+
+          xform-cols (m/index-by :name (lib.metadata/fields mp2 123000000))
+          downstream (-> (lib/query mp2 (lib.metadata/table mp2 123000000))
+                         (lib/filter (lib/= (get xform-cols "count") 1)))
+          mp         (-> (lib.tu/metadata-provider-with-card-from-query mp2 2 downstream)
+                         deps.mp/override-metadata-provider)]
+      (deps.mp/add-override mp :card 1
+                            (-> upstream
+                                (lib/remove-clause (first (lib/breakouts upstream)))
+                                (lib/breakout (lib/with-temporal-bucket (meta/field-metadata :orders :created-at) :day))
+                                (replace-query card1)))
+      ;; NOTE: Theses lines are important: this is what would happen to all the dependents returned by the graph.
+      ;; There's no update (`nil`) but it still includes the downstream stuff in the scope of the
+      ;; OverrideMetadataProvider.
+      (deps.mp/add-override mp :card 2 nil)
+      (deps.mp/add-override mp :transform 123 transform)
+
+      (testing "upstream card's columns are correct"
+        (is (=? [{:name "CREATED_AT"}
+                 {:name "count"}]
+                (->> (lib.metadata/card mp 1)
+                     (lib/query mp)
+                     lib/returned-columns))))
+      (testing "downstream card (which selects all) is also correct"
+        (is (=? [{:name "CREATED_AT"}
+                 {:name "count"}]
+                (->> (lib.metadata/card mp 2)
+                     (lib/query mp)
+                     lib/returned-columns
+                     (sort-by :name))))))))

enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj

@@ -0,0 +1,216 @@
+(ns metabase-enterprise.dependencies.models.dependency-test
+  (:require
+   [clojure.set :as set]
+   [clojure.test :refer [deftest is testing]]
+   [metabase-enterprise.dependencies.models.dependency :as deps.graph]
+   [metabase.lib.core :as lib]
+   [metabase.queries.models.card :as card]
+   [metabase.test :as mt]
+   [toucan2.core :as t2]))
+
+(defn- depends-on-> [from-type from-id to-type to-id]
+  {:from_entity_type from-type
+   :from_entity_id   from-id
+   :to_entity_type   to-type
+   :to_entity_id     to-id})
+
+(defn- upstream-of [from-type from-id]
+  (t2/select-fn-set #(select-keys % [:from_entity_type :from_entity_id :to_entity_type :to_entity_id])
+                    :model/Dependency :from_entity_type from-type :from_entity_id from-id))
+
+(defn- basic-orders []
+  {:name                   "Test card"
+   :database_id            (mt/id)
+   :table_id               (mt/id :orders)
+   :display                :table
+   :query_type             :query
+   :type                   :question
+   :dataset_query          (mt/mbql-query orders)
+   :visualization_settings {}})
+
+(defn- wrap-card [inner-card]
+  {:name                   "Downstream card"
+   :database_id            (mt/id)
+   :display                :table
+   :query_type             :query
+   :type                   :question
+   :dataset_query          (mt/mbql-query nil
+                             {:source-table (str "card__" (:id inner-card))})
+   :visualization_settings {}})
+
+(deftest ^:sequential card-deps-maintenance-test-1-new-card
+  (testing "upstream deps of a card are updated correctly"
+    (mt/dataset test-data
+      (mt/with-temp [:model/User user {:email "me@wherever.com"}]
+        (mt/with-premium-features #{:dependencies}
+          (let [card1 (card/create-card! (basic-orders) user)]
+            (is (integer? (:id card1)))
+            (testing "when creating a new card"
+              (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :orders))}
+                      (upstream-of :card (:id card1))))
+
+              (testing "that depends on another card"
+                (let [card2 (card/create-card! (wrap-card card1) user)]
+                  (is (=? #{(depends-on-> :card (:id card2) :card (:id card1))}
+                          (upstream-of :card (:id card2))))
+                  (testing "but that doesn't affect the upstream deps of the inner card"
+                    (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :orders))}
+                            (upstream-of :card (:id card1))))))))
+
+            (testing "when updating an existing card"
+              (testing "to add a new table dep"
+                (card/update-card! {:card-before-update card1
+                                    :card-updates
+                                    {:dataset_query (mt/mbql-query orders
+                                                      {:joins [{:alias "Products"
+                                                                :source-table (mt/id :products)
+                                                                :condition [:= $id &Products.$products.id]}]})}})
+                (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :orders))
+                          (depends-on-> :card (:id card1) :table (mt/id :products))}
+                        (upstream-of :card (:id card1)))))
+              (testing "to remove a table dep"
+                (card/update-card! {:card-before-update (t2/select-one :model/Card :id (:id card1))
+                                    :card-updates
+                                    {:dataset_query (mt/mbql-query products)}})
+                (is (=? #{(depends-on-> :card (:id card1) :table (mt/id :products))}
+                        (upstream-of :card (:id card1))))))))))))
+
+(deftest ^:sequential card-deps-graph-test-1-mbql-card-chain
+  (testing "deps graph is connected properly for a chain of MBQL cards"
+    (mt/dataset test-data
+      (mt/with-temp [:model/User user {:email "me@wherever.com"}]
+        (mt/with-premium-features #{:dependencies}
+          (let [{id1 :id :as card1} (card/create-card! (basic-orders) user)
+                {id2 :id :as card2} (card/create-card! (wrap-card card1) user)
+                {id3 :id :as card3} (card/create-card! (wrap-card card2) user)]
+            (testing "raw deps are recorded correctly"
+              (is (=? #{(depends-on-> :card id1 :table (mt/id :orders))} (upstream-of :card id1)))
+              (is (=? #{(depends-on-> :card id2 :card  id1)}             (upstream-of :card id2)))
+              (is (=? #{(depends-on-> :card id3 :card  id2)}             (upstream-of :card id3))))
+
+            (testing "transitive deps are computed correctly"
+              (testing "for each card"
+                (is (=? {:card #{id2 id3}}
+                        (deps.graph/transitive-dependents {:card [card1]})))
+                (is (=? {:card #{id3}}
+                        (deps.graph/transitive-dependents {:card [card2]})))
+                (is (=? {}
+                        (deps.graph/transitive-dependents {:card [card3]}))))
+              (testing "for the table"
+                (is (set/subset? #{id1 id2 id3}
+                                 (-> {:table [{:id (mt/id :orders)}]}
+                                     deps.graph/transitive-dependents
+                                     :card)))))))))))
+
+(defn- sql-card [metadata-provider sql]
+  {:name                   "SQL card"
+   :database_id            (mt/id)
+   :display                :table
+   :query_type             :query
+   :type                   :question
+   :dataset_query          (-> (lib/native-query metadata-provider sql)
+                               lib/->legacy-MBQL)
+   :visualization_settings {}})
+
+(deftest ^:sequential card-deps-graph-test-2-native-card-chain
+  (testing "deps graph is connected properly for a chain of native cards"
+    (mt/dataset test-data
+      (mt/with-temp [:model/User user {:email "me@wherever.com"}]
+        (mt/with-premium-features #{:dependencies}
+          (let [mp                  (mt/metadata-provider)
+                {id1 :id :as card1} (card/create-card! (sql-card mp "SELECT * FROM orders;") user)
+                {id2 :id :as card2} (card/create-card! (sql-card mp (str "SELECT * FROM {{#" id1 "}}")) user)
+                {id3 :id :as card3} (card/create-card! (sql-card mp (str "SELECT * FROM {{#" id2 "}}")) user)]
+            (testing "raw deps are recorded correctly"
+              (is (=? #{(depends-on-> :card id1 :table (mt/id :orders))} (upstream-of :card id1)))
+              (is (=? #{(depends-on-> :card id2 :table (mt/id :orders))
+                        (depends-on-> :card id2 :card  id1)}             (upstream-of :card id2)))
+              (is (=? #{(depends-on-> :card id3 :table (mt/id :orders))
+                        (depends-on-> :card id3 :card  id2)}             (upstream-of :card id3))))
+            (testing "transitive deps are computed correctly"
+              (testing "for each card"
+                (is (=? {:card #{id2 id3}}
+                        (deps.graph/transitive-dependents {:card [card1]})))
+                (is (=? {:card #{id3}}
+                        (deps.graph/transitive-dependents {:card [card2]})))
+                (is (=? {}
+                        (deps.graph/transitive-dependents {:card [card3]}))))
+              (testing "for the table"
+                (is (set/subset? #{id1 id2 id3}
+                                 (-> {:table [{:id (mt/id :orders)}]}
+                                     deps.graph/transitive-dependents
+                                     :card)))))))))))
+
+(deftest ^:sequential card-deps-graph-metric-test
+  (testing "deps graph is connected properly for a question using a metric"
+    (mt/dataset test-data
+      (mt/with-temp [:model/User user {:email "me@wherever.com"}]
+        (mt/with-premium-features #{:dependencies}
+          (let [metric-card (card/create-card! {:name        "Test Metric"
+                                                :database_id (mt/id)
+                                                :table_id    (mt/id :orders)
+                                                :display     :scalar
+                                                :query_type  :query
+                                                :type        :metric
+                                                :dataset_query (mt/mbql-query orders {:aggregation [[:count]]})
+                                                :visualization_settings {}}
+                                               user)
+                question-card (card/create-card! {:name        "Question using Metric"
+                                                  :database_id (mt/id)
+                                                  :display     :table
+                                                  :query_type  :query
+                                                  :type        :question
+                                                  :dataset_query (mt/mbql-query orders
+                                                                   {:aggregation [[:metric (:id metric-card)]]})
+                                                  :visualization_settings {}}
+                                                 user)]
+            (testing "raw deps are recorded correctly for question using metric"
+              (is (=? #{(depends-on-> :card (:id question-card) :table (mt/id :orders))
+                        (depends-on-> :card (:id question-card) :card (:id metric-card))}
+                      (upstream-of :card (:id question-card)))))))))))
+
+(deftest dependency-analysis-version-test
+  (testing "dependency_analysis_version is updated when entities are created or updated"
+    (testing "cards"
+      (mt/with-temp [:model/Card card {:dependency_analysis_version 0}]
+        (is (zero? (t2/select-one-fn :dependency_analysis_version :model/Card (:id card))))
+        (mt/with-premium-features #{:dependencies}
+          (card/update-card! {:card-before-update card
+                              :card-updates       {:dataset_query (mt/mbql-query orders)}}))
+        (is (= deps.graph/current-dependency-analysis-version
+               (t2/select-one-fn :dependency_analysis_version :model/Card (:id card))))))
+    (testing "transforms create"
+      (mt/with-premium-features #{:dependencies}
+        (mt/with-temp [:model/Transform transform]
+          (is (= deps.graph/current-dependency-analysis-version
+                 (t2/select-one-fn :dependency_analysis_version :model/Transform (:id transform)))))))
+    (testing "transforms update"
+      (mt/with-premium-features #{}
+        (mt/with-temp [:model/Transform transform]
+          (is (zero? (t2/select-one-fn :dependency_analysis_version :model/Transform (:id transform))))
+          (mt/with-premium-features #{:dependencies}
+            (t2/update! :model/Transform (:id transform) {:source {:type "query" :query (mt/mbql-query products)}}))
+          (is (= deps.graph/current-dependency-analysis-version
+                 (t2/select-one-fn :dependency_analysis_version :model/Transform (:id transform)))))))
+    (testing "snippets create"
+      (mt/with-premium-features #{:dependencies}
+        (mt/with-temp [:model/NativeQuerySnippet snippet]
+          (is (= deps.graph/current-dependency-analysis-version
+                 (t2/select-one-fn :dependency_analysis_version :model/NativeQuerySnippet (:id snippet)))))))
+    (testing "snippets update"
+      (mt/with-premium-features #{}
+        (mt/with-temp [:model/NativeQuerySnippet snippet]
+          (is (zero? (t2/select-one-fn :dependency_analysis_version :model/NativeQuerySnippet (:id snippet))))
+          (mt/with-premium-features #{:dependencies}
+            (t2/update! :model/NativeQuerySnippet (:id snippet) {:content "new content"}))
+          (is (= deps.graph/current-dependency-analysis-version
+                 (t2/select-one-fn :dependency_analysis_version :model/NativeQuerySnippet (:id snippet)))))))))
+
+(deftest dependency-analysis-version-create-card-test
+  (testing "dependency_analysis_version is updated when a card is created"
+    (mt/with-premium-features #{:dependencies}
+      (mt/with-model-cleanup [:model/Card]
+        (mt/with-temp [:model/User user]
+          (let [card (card/create-card! (basic-orders) user)]
+            (is (= deps.graph/current-dependency-analysis-version
+                   (t2/select-one-fn :dependency_analysis_version :model/Card :id (:id card))))))))))

enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj

@@ -0,0 +1,241 @@
+(ns metabase-enterprise.dependencies.native-validation-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase-enterprise.dependencies.native-validation :as deps.native-validation]
+   [metabase-enterprise.dependencies.test-util :as deps.tu]
+   [metabase.lib.metadata :as lib.metadata]
+   [metabase.lib.test-metadata :as meta]))
+
+(defn- fake-query
+  ([mp query]
+   (fake-query mp query {}))
+  ([mp query template-tags]
+   {:database (:id (lib.metadata/database mp))
+    :type     :native
+    :native   {:query query
+               :template-tags template-tags}}))
+
+(defn- validates?
+  [mp driver card-id expected]
+  (is (=? expected
+          (->> (lib.metadata/card mp card-id)
+               :dataset-query
+               (deps.native-validation/validate-native-query driver mp)))))
+
+(deftest basic-deps-test
+  (let [mp     (deps.tu/default-metadata-provider)
+        driver (:engine (lib.metadata/database mp))]
+    (is (= #{{:table (meta/id :products)}}
+           (->> (lib.metadata/card mp 4)
+                :dataset-query
+                (deps.native-validation/native-query-deps driver mp))))
+    (is (= #{{:table (meta/id :products)}
+             {:card 1}}
+           (->> (lib.metadata/card mp 5)
+                :dataset-query
+                (deps.native-validation/native-query-deps driver mp))))
+    (is (= #{{:table (meta/id :products)}
+             {:card 1}
+             {:snippet 1}}
+           (->> (lib.metadata/card mp 6)
+                :dataset-query
+                (deps.native-validation/native-query-deps driver mp))))
+    (is (= #{{:table (meta/id :products)}
+             {:card 1}
+             {:snippet 1}
+             {:snippet 2}}
+           (->> (lib.metadata/card mp 7)
+                :dataset-query
+                (deps.native-validation/native-query-deps driver mp))))
+    (is (= #{{:table (meta/id :products)}
+             {:table (meta/id :orders)}
+             {:card 1}
+             {:card 2}}
+           (->> (lib.metadata/card mp 9)
+                :dataset-query
+                (deps.native-validation/native-query-deps driver mp))))))
+
+(deftest validate-bad-queries-test
+  (testing "validate-native-query handles nonsense queries"
+    (let [mp (deps.tu/default-metadata-provider)
+          driver (:engine (lib.metadata/database mp))]
+      (testing "complete nonsense query"
+        (is (= [{:error :bad-sql}]
+               (deps.native-validation/validate-native-query
+                driver mp
+                (fake-query mp "this is not a query")))))
+      (testing "bad table wildcard"
+        (is (= [{:type :invalid-table-wildcard,
+                 :table "products",
+                 :metabase.driver.sql/bad-reference true}]
+               (deps.native-validation/validate-native-query
+                driver mp
+                (fake-query mp "select products.* from orders")))))
+      (testing "bad col reference"
+        (is (= [{:column "BAD",
+                 :alias nil,
+                 :type :single-column,
+                 :source-columns [[{:type :all-columns, :table {:table "PRODUCTS"}}]],
+                 :metabase.driver.sql/bad-reference true}]
+               (deps.native-validation/validate-native-query
+                driver mp
+                (fake-query mp "select bad from products"))))))))
+
+(deftest validate-native-query-with-subquery-columns-test
+  (testing "validate-native-query should detect invalid columns in subqueries"
+    (let [mp (deps.tu/default-metadata-provider)
+          driver (:engine (lib.metadata/database mp))]
+
+      (testing "Valid query - selecting existing columns from subquery"
+        (validates? mp driver 10 empty?))
+
+      (testing "Invalid query - selecting non-existent column from subquery"
+        (validates? mp driver 11 [{:column "CATEGORY",
+                                   :alias nil,
+                                   :type :single-column,
+                                   :source-columns [[]],
+                                   :metabase.driver.sql/bad-reference true}])
+        (validates? mp driver 12 [{:column "CATEGORY",
+                                   :alias nil,
+                                   :type :single-column,
+                                   :source-columns [[]],
+                                   :metabase.driver.sql/bad-reference true}]))
+
+      (testing "Nested subqueries"
+        (validates? mp driver 13 empty?)
+        (validates? mp driver 14 [{:column "CATEGORY",
+                                   :alias nil,
+                                   :type :single-column,
+                                   :source-columns [[]],
+                                   :metabase.driver.sql/bad-reference true}]))
+
+      (testing "SELECT * from subquery expands to subquery columns"
+        (validates? mp driver 15 empty?)
+        (validates? mp driver 16 empty?)
+        (validates? mp driver 17 [{:column "EMAIL",
+                                   :alias nil,
+                                   :type :single-column,
+                                   :source-columns [[]],
+                                   :metabase.driver.sql/bad-reference true}])))))
+
+(deftest validate-card-reference-after-expansion-test
+  (testing "Validation of queries after card references have been expanded"
+    (let [mp (deps.tu/default-metadata-provider)
+          driver (:engine (lib.metadata/database mp))]
+
+      (testing "Card reference expanded to subquery - valid columns"
+        (validates? mp driver 18 empty?))
+
+      (testing "Card reference expanded to subquery - invalid column"
+        (validates? mp driver 19
+                    [{:column "DESCRIPTION",
+                      :alias nil,
+                      :type :single-column,
+                      :source-columns [[]],
+                      :metabase.driver.sql/bad-reference true}]))
+
+      (testing "Card reference with alias - valid column"
+        (validates? mp driver 20 empty?))
+
+      (testing "Card reference with alias - invalid column"
+        (validates? mp driver 21
+                    [{:column "PASSWORD",
+                      :alias nil,
+                      :type :single-column,
+                      :source-columns [[]],
+                      :metabase.driver.sql/bad-reference true}]))
+
+      (testing "Wildcard selection from card reference"
+        (validates? mp driver 22 empty?))
+
+      (testing "Invalid column from aliased card"
+        (validates? mp driver 23
+                    [{:column "LATITUDE",
+                      :alias nil,
+                      :type :single-column,
+                      :source-columns [[]],
+                      :metabase.driver.sql/bad-reference true}])))))
+
+(defn- check-result-metadata [driver mp query expected]
+  (is (=? expected
+          (->> query
+               (fake-query mp)
+               (deps.native-validation/native-result-metadata driver mp)))))
+
+(defn- add-desired-column-alias [fields]
+  (map #(assoc % :lib/desired-column-alias (:name %)) fields))
+
+(deftest result-metadata-test
+  (testing "Calculates result metadata"
+    (let [mp (deps.tu/default-metadata-provider)
+          driver (:engine (lib.metadata/database mp))]
+      (testing "Selecting a wildcard"
+        (check-result-metadata
+         driver mp
+         "select * from orders"
+         (add-desired-column-alias (lib.metadata/fields mp (meta/id :orders)))))
+      (testing "Selecting a table wildcard"
+        (check-result-metadata
+         driver mp
+         "select orders.* from orders"
+         (add-desired-column-alias (lib.metadata/fields mp (meta/id :orders)))))
+      (testing "Selecting a single col"
+        (check-result-metadata
+         driver mp
+         "select total from orders"
+         (add-desired-column-alias [(lib.metadata/field mp (meta/id :orders :total))])))
+      (testing "Selecting a nonexistent col"
+        (check-result-metadata
+         driver mp
+         "select bad from orders"
+         []))
+      (testing "Selecting a col with an alias"
+        (check-result-metadata
+         driver mp
+         "select subtotal as new_subtotal from orders"
+         [{:lib/type :metadata/column,
+           :base-type :type/Float,
+           :semantic-type nil,
+           :name "SUBTOTAL",
+           :lib/desired-column-alias "NEW_SUBTOTAL"
+           :database-type "DOUBLE PRECISION",
+           :display-name "Subtotal"}]))
+      (testing "Selecting a custom col"
+        (check-result-metadata
+         driver mp
+         "select subtotal + tax as sum from orders"
+         [{:base-type :type/*,
+           :name "SUM",
+           :display-name "Sum",
+           :effective-type :type/*,
+           :semantic-type :Semantic/*}]))
+      (testing "Selecting a union"
+        (check-result-metadata
+         driver mp
+         "select total from orders union select subtotal from orders"
+         [{:name "TOTAL",
+           :lib/desired-column-alias "TOTAL",
+           :display-name "Total",
+           :base-type :type/Float,
+           :effective-type :type/Float,
+           :semantic-type :Semantic/*}]))
+      (testing "Selecting a union with different types"
+        (check-result-metadata
+         driver mp
+         "select category from products union select title from products"
+         [{:name "CATEGORY",
+           :lib/desired-column-alias "CATEGORY",
+           :display-name "Category",
+           :base-type :type/Text,
+           :effective-type :type/Text,
+           :semantic-type :type/Category}]))
+      (testing "Selecting a bad table wildcard"
+        (check-result-metadata
+         driver mp
+         "select orders.* from products"
+         []))
+      (testing "Using a nonsense query"
+        (check-result-metadata
+         driver mp
+         "this is not a query"
+         [])))))

enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj

@@ -0,0 +1,231 @@
+(ns metabase-enterprise.dependencies.task.backfill-test
+  (:require
+   [clojure.test :refer :all]
+   [environ.core :as env]
+   [java-time.api :as t]
+   [metabase-enterprise.dependencies.models.dependency :as dependencies.model]
+   [metabase-enterprise.dependencies.task.backfill :as dependencies.backfill]
+   [metabase.task.core :as task]
+   [metabase.test :as mt]
+   [metabase.test.fixtures :as fixtures]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(use-fixtures :once (fixtures/initialize :db))
+
+(defn- backfill-all-existing-entities
+  []
+  (mt/with-premium-features #{:dependencies}
+    (while (#'dependencies.backfill/backfill-dependencies))))
+
+(defn- backfill-dependencies-single-trigger
+  []
+  (mt/with-premium-features #{:dependencies}
+    (#'dependencies.backfill/backfill-dependencies)))
+
+(deftest ^:sequential backfill-dependency-analysis-test
+  (testing "Test that the backfill job correctly updates the dependency_analysis_version"
+    (backfill-all-existing-entities)
+    (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")]
+      (let [query (mt/mbql-query orders)]
+        (mt/with-premium-features #{}
+          (mt/with-temp [:model/Card {card1-id :id} {:dependency_analysis_version 0, :dataset_query query}
+                         :model/Card {card2-id :id} {:dependency_analysis_version 0, :dataset_query query}
+                         :model/Card {card3-id :id} {:dependency_analysis_version 0, :dataset_query query}]
+            (let [card-count (fn [& args]
+                               (apply t2/count :model/Card :id [:in [card1-id card2-id card3-id]] args))
+                  current-version dependencies.model/current-dependency-analysis-version]
+              (is (= 3 (card-count)))
+              ;; first run, should process 2 cards
+              (is (true? (backfill-dependencies-single-trigger)))
+              (is (= 2 (card-count :dependency_analysis_version current-version)))
+              ;; second run, should process the last card
+              (is (false? (backfill-dependencies-single-trigger)))
+              (is (= 3 (card-count :dependency_analysis_version current-version)))
+              ;; third run, should not process anything
+              (is (false? (backfill-dependencies-single-trigger)))
+              (is (= 3 (card-count :dependency_analysis_version current-version))))))))))
+
+(deftest ^:sequential backfill-transform-test
+  (testing "Test that transforms are correctly backfilled"
+    (backfill-all-existing-entities)
+    (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")]
+      (mt/with-premium-features #{}
+        (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)
+                                                  :dependency_analysis_version 0}
+                       :model/Transform {transform-id :id} {:name "Test Transform"
+                                                            :source {:type "query"
+                                                                     :query (mt/mbql-query nil
+                                                                              {:source-table (str "card__" card-id)})}
+                                                            :target {:type "table" :name "test_table"}
+                                                            :dependency_analysis_version 0}]
+          (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))
+          (is (t2/exists? :model/Transform :id transform-id :dependency_analysis_version 0))
+          (is (false? (t2/exists? :model/Dependency :from_entity_type :card      :from_entity_id card-id)))
+          (is (false? (t2/exists? :model/Dependency :from_entity_type :transform :from_entity_id transform-id)))
+          (backfill-dependencies-single-trigger)
+          (is (t2/exists? :model/Card :id card-id
+                          :dependency_analysis_version dependencies.model/current-dependency-analysis-version))
+          (is (t2/exists? :model/Transform :id transform-id
+                          :dependency_analysis_version dependencies.model/current-dependency-analysis-version))
+          (is (t2/exists? :model/Dependency
+                          :from_entity_type :card :from_entity_id card-id
+                          :to_entity_type :table :to_entity_id (mt/id :orders)))
+          (is (t2/exists? :model/Dependency
+                          :from_entity_type :transform :from_entity_id transform-id
+                          :to_entity_type :card :to_entity_id card-id)))))))
+
+(deftest ^:sequential backfill-snippet-test
+  (testing "Test that cards with snippets are correctly backfilled"
+    (backfill-all-existing-entities)
+    (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")]
+      (mt/with-premium-features #{}
+        (mt/with-temp [:model/NativeQuerySnippet {snippet-id :id} {:name "my_snippet"
+                                                                   :content "SELECT 1"
+                                                                   :dependency_analysis_version 0}
+                       :model/Card {card-id :id} {:dataset_query {:database (mt/id)
+                                                                  :type :native
+                                                                  :native {:query "SELECT * FROM {{my_snippet}}"
+                                                                           :template-tags
+                                                                           {:my_snippet
+                                                                            {:id (str (random-uuid))
+                                                                             :name "my_snippet"
+                                                                             :display-name "my_snippet"
+                                                                             :type :snippet
+                                                                             :snippet-name "my_snippet"
+                                                                             :snippet-id snippet-id}}}}
+                                                  :dependency_analysis_version 0}]
+          (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))
+          (is (t2/exists? :model/NativeQuerySnippet :id snippet-id :dependency_analysis_version 0))
+          (is (false? (t2/exists? :model/Dependency
+                                  :from_entity_type :card :from_entity_id card-id
+                                  :to_entity_type :snippet :to_entity_id snippet-id)))
+          (backfill-dependencies-single-trigger)
+          (is (t2/exists? :model/Card :id card-id
+                          :dependency_analysis_version dependencies.model/current-dependency-analysis-version))
+          (is (t2/exists? :model/NativeQuerySnippet :id snippet-id
+                          :dependency_analysis_version dependencies.model/current-dependency-analysis-version))
+          (is (t2/exists? :model/Dependency
+                          :from_entity_type :card :from_entity_id card-id
+                          :to_entity_type :snippet :to_entity_id snippet-id)))))))
+
+(deftest ^:sequential backfill-idempotency-test
+  (testing "Running the backfill multiple times should be idempotent"
+    (backfill-all-existing-entities)
+    (with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "1")]
+      (mt/with-premium-features #{}
+        (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)
+                                                  :dependency_analysis_version 0}]
+          (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))
+          (is (false? (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id
+                                  :to_entity_type :table :to_entity_id (mt/id :orders))))
+          ;; First run
+          (backfill-dependencies-single-trigger)
+          (is (t2/exists? :model/Card :id card-id
+                          :dependency_analysis_version dependencies.model/current-dependency-analysis-version))
+          ;; Second run - should not change anything
+          (backfill-dependencies-single-trigger)
+          (is (t2/exists? :model/Card :id card-id
+                          :dependency_analysis_version dependencies.model/current-dependency-analysis-version))
+          (is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id
+                          :to_entity_type :table :to_entity_id (mt/id :orders))))))))
+
+(defn- wait-for-condition
+  [predicate timeout-ms]
+  (let [limit (+ (System/currentTimeMillis) timeout-ms)]
+    (loop []
+      (Thread/sleep 100)
+      (or (predicate)
+          (when (< (System/currentTimeMillis) limit)
+            (recur))))))
+
+(deftest ^:sequential backfill-scheduling-test
+  (testing "Test that the backfill job schedules and reschedules itself correctly"
+    (backfill-all-existing-entities)
+    (mt/with-temp-scheduler!
+      (with-redefs [env/env (assoc env/env
+                                   :mb-dependency-backfill-batch-size "1"
+                                   :mb-dependency-backfill-delay-minutes "0"
+                                   :mb-dependency-backfill-variance-minutes "0")]
+        (let [card-data {:dependency_analysis_version 0, :dataset_query (mt/mbql-query orders)}]
+          (mt/test-helpers-set-global-values!
+            (mt/with-premium-features #{}
+              (mt/with-temp [:model/Card {card1-id :id} card-data
+                             :model/Card {card2-id :id} card-data
+                             :model/Card {card3-id :id} card-data]
+                (let [card-count (fn [& args]
+                                   (apply t2/count :model/Card :id [:in [card1-id card2-id card3-id]] args))
+                      current-version dependencies.model/current-dependency-analysis-version]
+                  (is (= 3 (card-count :dependency_analysis_version 0)))
+                  (mt/with-premium-features #{:dependencies}
+                    ;; Initialize the task, which should schedule the first run
+                    (task/init! ::dependencies.backfill/DependencyBackfill)
+                    (wait-for-condition #(= 3 (card-count :dependency_analysis_version current-version)) 10000)
+                    (is (= 3 (card-count :dependency_analysis_version current-version)))))))))))))
+
+(deftest ^:sequential backfill-terminal-failure-test
+  (testing "Entities should be marked as terminally broken after MAX_RETRIES failures"
+    (backfill-all-existing-entities)
+    (mt/with-premium-features #{}
+      (mt/with-temp [:model/Card {card-id :id} {:dependency_analysis_version 0, :dataset_query (mt/mbql-query orders)}]
+        (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))
+
+        (let [update-attempts (volatile! 0)
+              failures (inc @#'dependencies.backfill/MAX_RETRIES)]
+          (with-redefs [env/env (assoc env/env
+                                       :mb-dependency-backfill-delay-minutes "0"
+                                       :mb-dependency-backfill-variance-minutes "0")
+                        t2/update! (fn [model-kw id & args]
+                                     (if (and (= model-kw :model/Card)
+                                              (= id card-id)
+                                              (< @update-attempts failures))
+                                       (do
+                                         (vswap! update-attempts inc)
+                                         (throw (ex-info "Simulated DB error" {:id id})))
+                                       (apply t2/update! model-kw id args)))]
+            ;; fail MAX_RETRIES + 1 times
+            (while (< @update-attempts failures)
+              (backfill-dependencies-single-trigger))))
+
+        ;; verify card is not processed and is terminally broken
+        (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))
+
+        ;; verify subsequent runs don't process it
+        (backfill-dependencies-single-trigger)
+        (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))))))
+
+(deftest ^:sequential backfill-delayed-retry-test
+  (testing "Failed entities should be retried after their delay period expires"
+    (backfill-all-existing-entities)
+    (let [current-version dependencies.model/current-dependency-analysis-version
+          update-attempts (volatile! 0)]
+      (mt/with-premium-features #{}
+        (mt/with-temp [:model/Card {card-id :id} {:dependency_analysis_version 0, :dataset_query (mt/mbql-query orders)}]
+          (with-redefs [env/env (assoc env/env
+                                       :mb-dependency-backfill-delay-minutes "1") ; 1 minute delay for retries
+                        t2/update! (fn [model-kw id & args]
+                                     (if (and (= model-kw :model/Card)
+                                              (= id card-id)
+                                              (zero? @update-attempts))
+                                       (do
+                                         (vswap! update-attempts inc)
+                                         (throw (ex-info "Simulated DB error" {:id id})))
+                                       (apply t2/update! model-kw id args)))]
+            (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))
+
+            ;; first failure - should be put into retry state
+            ;; fail MAX_RETRIES + 1 times
+            (while (zero? @update-attempts)
+              (backfill-dependencies-single-trigger))
+            (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0)))
+
+          ;; advance time by less than retry delay - should NOT be processed
+          (mt/with-clock (t/plus (t/zoned-date-time) (t/duration 10 :seconds))
+            (backfill-dependencies-single-trigger))
+          (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))
+
+          ;; advance time by more than retry delay - should be processed
+          (mt/with-clock (t/plus (t/zoned-date-time) (t/duration 2 :minutes))
+            (backfill-dependencies-single-trigger))
+          (is (t2/exists? :model/Card :id card-id :dependency_analysis_version current-version)))))))

enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj

@@ -0,0 +1,125 @@
+(ns metabase-enterprise.dependencies.test-util
+  (:require
+   [medley.core :as m]
+   [metabase.lib.convert :as lib.convert]
+   [metabase.lib.core :as lib]
+   [metabase.lib.metadata :as lib.metadata]
+   [metabase.lib.test-metadata :as meta]
+   [metabase.lib.test-util.metadata-providers.mock :as providers.mock]))
+
+(defn mock-card [metadata-provider {:keys [id query details]}]
+  (merge {:lib/type        :metadata/card
+          :id              id
+          :database-id     (:id (lib.metadata/database metadata-provider))
+          :dataset-query   (if (string? query)
+                             {:database (:id (lib.metadata/database metadata-provider))
+                              :type     :native
+                              :native   {:query         query
+                                         :template-tags (lib/extract-template-tags metadata-provider query)}}
+                             (lib.convert/->legacy-MBQL query))
+          :name            (str (gensym))
+          :result-metadata (when (not (string? query))
+                             (->> (lib/returned-columns query)
+                                  (sort-by :id)))}
+         details))
+
+(defn mock-snippet [{:keys [id name content template-tags]}]
+  {:description   nil,
+   :archived      false,
+   :collection-id nil,
+   :id            id
+   :name          name
+   :content       content
+   :template-tags template-tags
+   :lib/type      :metadata/native-query-snippet})
+
+(defn mock-metadata-provider [{:keys [cards snippets]}]
+  (lib/composed-metadata-provider
+   meta/metadata-provider
+   (providers.mock/mock-metadata-provider
+    {:cards cards
+     :native-query-snippets snippets})))
+
+(defn default-metadata-provider []
+  (let [snippets        (map mock-snippet
+                             [{:id            1
+                               :name          "inner"
+                               :content       "{{#card-ref}}"
+                               :template-tags {"#card-ref" {:card-id      1
+                                                            :type         :card
+                                                            :name         "#card-ref"
+                                                            :display-name "#Card Ref"
+                                                            :id           (str (random-uuid))}}}
+                              {:id            2
+                               :name          "outer"
+                               :content       "{{snippet: inner}}"
+                               :template-tags {"snippet: inner" {:snippet-id   1
+                                                                 :snippet-name "inner"
+                                                                 :type         :snippet
+                                                                 :name         "snippet: inner"
+                                                                 :display-name "Snippet: Inner"
+                                                                 :id           (str (random-uuid))}}}])
+        initial-mock-mp (mock-metadata-provider {:snippets snippets})
+        initial-card-a  (mock-card initial-mock-mp
+                                   {:id    1
+                                    :query (lib/query initial-mock-mp (meta/table-metadata :products))})
+        initial-card-b  (mock-card initial-mock-mp
+                                   {:id    2
+                                    :query (lib/query initial-mock-mp (meta/table-metadata :orders))})]
+    (mock-metadata-provider
+     {:cards    (into [initial-card-a
+                       initial-card-b]
+                      (map #(mock-card initial-mock-mp %))
+                      [{:id    3
+                        :query (lib/query initial-mock-mp initial-card-a)}
+                       {:id    4
+                        :query "select * from products"}
+                       {:id    5
+                        :query "select * from {{#1}}"}
+                       {:id    6
+                        :query "select * from {{snippet: inner}}"}
+                       {:id    7
+                        :query "select * from {{snippet: outer}}"}
+                       {:id    8
+                        :query (as-> (lib/query initial-mock-mp initial-card-b) q
+                                 (lib/join q (-> (lib/join-clause initial-card-a)
+                                                 (lib/with-join-conditions
+                                                  [(lib/=
+                                                    (m/find-first #(= (:name %) "PRODUCT_ID")
+                                                                  (lib/join-condition-lhs-columns
+                                                                   q initial-card-b nil nil))
+                                                    (m/find-first #(= (:name %) "ID")
+                                                                  (lib/join-condition-lhs-columns
+                                                                   q initial-card-a nil nil)))])
+                                                 (lib/with-join-fields :all))))}
+                       {:id    9
+                        :query "select * from {{#1}} inner join {{#2}}"}
+                       {:id    10
+                        :query "SELECT id, name FROM (SELECT id, name, email FROM PEOPLE)"}
+                       {:id    11
+                        :query "SELECT category FROM (SELECT id, name FROM PEOPLE)"}
+                       {:id    12
+                        :query "SELECT id, category FROM (SELECT id, name FROM PEOPLE)"}
+                       {:id    13
+                        :query "SELECT id FROM (SELECT id, name FROM (SELECT id, name, email FROM PEOPLE))"}
+                       {:id    14
+                        :query "SELECT category FROM (SELECT id, name FROM (SELECT id, name, email FROM PEOPLE))"}
+                       {:id    15
+                        :query "SELECT * FROM (SELECT id, name FROM PEOPLE)"}
+                       {:id    16
+                        :query "SELECT name FROM (SELECT * FROM (SELECT id, name FROM PEOPLE))"}
+                       {:id    17
+                        :query "SELECT email FROM (SELECT * FROM (SELECT id, name FROM PEOPLE))"}
+                       {:id    18
+                        :query "SELECT ID, TITLE FROM {{#1}}"}
+                       {:id    19
+                        :query "SELECT DESCRIPTION FROM {{#1}}"}
+                       {:id    20
+                        :query "SELECT ID FROM {{#1}} AS p"}
+                       {:id    21
+                        :query "SELECT PASSWORD FROM {{#1}} AS p"}
+                       {:id    22
+                        :query "SELECT * FROM {{#1}}"}
+                       {:id    23
+                        :query "SELECT p.LATITUDE FROM {{#1}} AS p"}])
+      :snippets snippets})))

enterprise/backend/test/metabase_enterprise/metabot_v3/api_test.clj

@@ -6,8 +6,6 @@
    [medley.core :as m]
    [metabase-enterprise.metabot-v3.client-test :as client-test]
    [metabase-enterprise.metabot-v3.util :as metabot.u]
-   [metabase.premium-features.token-check :as token-check]
-   [metabase.premium-features.token-check-test :as token-check-test]
    [metabase.test :as mt]
    [metabase.util.json :as json]
    [toucan2.core :as t2]))
@@ -63,29 +61,23 @@
 
 (deftest feedback-endpoint-test
   (mt/with-premium-features #{:metabot-v3}
-    (with-redefs [token-check/fetch-token-status (fn [_x]
-                                                   {:valid    true
-                                                    :status   "fake"
-                                                    :features ["test" "fixture"]
-                                                    :trial    false})]
-      (let [premium-token (token-check-test/random-token)
-            store-url     "http://hm.example"]
+    (mt/with-random-premium-token! [premium-token]
+      (let [store-url "http://hm.example"]
         (testing "Submits feedback to Harbormaster with token and base URL"
-          (mt/with-temporary-setting-values [premium-embedding-token premium-token
-                                             store-api-url           store-url]
-            (let [captured (atom nil)
-                  feedback {:metabot_id 1
-                            :feedback {:positive true
-                                       :message_id "m-1"
-                                       :freeform_feedback "ok"}
-                            :conversation_data {}
-                            :version "v0.0.0"
-                            :submission_time "2025-01-01T00:00:00Z"
-                            :is_admin false}
+          (mt/with-temporary-setting-values [store-api-url store-url]
+            (let [captured     (atom nil)
+                  feedback     {:metabot_id        1
+                                :feedback          {:positive          true
+                                                    :message_id        "m-1"
+                                                    :freeform_feedback "ok"}
+                                :conversation_data {}
+                                :version           "v0.0.0"
+                                :submission_time   "2025-01-01T00:00:00Z"
+                                :is_admin          false}
                   expected-url (str store-url "/api/v2/metabot/feedback/" premium-token)]
               (mt/with-dynamic-fn-redefs
                 [http/post (fn [url opts]
-                             (reset! captured {:url url
+                             (reset! captured {:url  url
                                                :body (json/decode+kw (:body opts))}))]
                 (let [_resp (mt/user-http-request :rasta :post 204 "ee/metabot-v3/feedback" feedback)]
                   (is (= {:url expected-url :body feedback}
@@ -95,11 +87,11 @@
           (mt/with-temporary-setting-values [premium-embedding-token premium-token]
             (mt/with-dynamic-fn-redefs
               [http/post (fn [_url _opts]
-                           (throw (ex-info "boom" {})))]
+                           (throw (ex-info "boom" {:status 404})))]
               (mt/user-http-request :rasta :post 500 "ee/metabot-v3/feedback" {:any "payload"}))))
 
         ;; We're not testing the branch where the store-api-url is missing because that defsetting
         ;; has the default value. It doesn't work well with `with-temporary-setting-values` helper.
         (testing "Throws when premium token is missing"
           (mt/with-temporary-setting-values [premium-embedding-token nil]
-            (mt/user-http-request :rasta :post 500 "ee/metabot-v3/feedback" {:foo "bar"})))))))
+            (mt/user-http-request :rasta :post 400 "ee/metabot-v3/feedback" {:foo "bar"})))))))