Parametrise PythonLibraryEditor

- python-runner no longer simulates a single http api that does
everything (incl s3 interaction)
- Separates file lifetimes (shared storage) from execution
- Easier to see how to avoid reading into strings (e.g. for piping into
tmp files, follow up)

Earlier on we had a service that managed much of the I/O, metabase now
coordinates it so it makes little sense pretending there is still
an overall http api.

.clj-kondo/config.edn

@@ -1,7 +1,7 @@
 ;; -*- comment-column: 100; -*-
 {:config-paths ["macros" "src" "config"]
  :linters
- { ;;
+ {;;
   ;; enabled optional linters (these are disabled by default)
   ;;
 
@@ -186,7 +186,7 @@
   :metabase/disallow-hardcoded-driver-names-in-tests
   ;; don't include `:h2` since that will probably lead to a mountain of tests where we are specifically testing
   ;; middleware or compilation behavior with just H2 since it's the default.
-  {:drivers #{ ;; core drivers
+  {:drivers #{;; core drivers
               #_:h2
               :postgres
               :mysql
@@ -418,6 +418,7 @@
   metabase-enterprise.transforms.api-test/with-transform-tags                           clojure.core/let
   metabase-enterprise.transforms.api-test/with-worker-runs                              clojure.core/let
   metabase-enterprise.transforms.test-util/with-transform-cleanup!                      clojure.core/let
+  metabase-enterprise.transforms.execute/with-transform-lifecycle                       clojure.core/let
   metabase.actions.test-util/with-actions                                               clojure.core/let
   metabase.api.common/let-404                                                           clojure.core/let
   metabase.app-db.custom-migrations.util/with-temp-scheduler!                           clojure.core/fn
@@ -734,7 +735,9 @@
   ;; `metabase-enterprise.sandbox.query-processor*-test`
   ;;
   {:pattern "^metabase(?:((?:-enterprise\\.sandbox)?\\.query-processor)|(?:\\.driver\\.sql(?!server)(?!ite))).*-test$"
-   :name    general-qp-and-driver-test-namespaces}]
+   :name    general-qp-and-driver-test-namespaces}
+  {:pattern "^metabase\\.python-runner\\..*"
+   :name    python-runner-namespaces}]
 
  :config-in-ns
  {test-namespaces

.clj-kondo/config/modules/config.edn

@@ -574,6 +574,7 @@
    :api  #{metabase.legacy-mbql.normalize
            metabase.legacy-mbql.predicates
            metabase.legacy-mbql.schema
+           metabase.legacy-mbql.schema.helpers
            metabase.legacy-mbql.util}
    :uses #{lib
            models
@@ -590,7 +591,8 @@
            metabase.lib.field
            metabase.lib.field.resolution
            metabase.lib.filter
-           metabase.lib.filter.desugar.jvm ; only so [[metabase.legacy-mbql.jvm-util]] can use defined regexes
+           metabase.lib.filter.desugar
+           metabase.lib.filter.desugar.jvm
            metabase.lib.init
            metabase.lib.join.util
            metabase.lib.metadata
@@ -885,6 +887,19 @@
            enterprise/advanced-permissions
            enterprise/sandbox}}
 
+  python-runner
+  {:team "Workflows"
+   :api  #{metabase.python-runner.api
+           metabase.python-runner.core
+           metabase.python-runner.init}
+   :uses #{api
+           driver
+           models
+           query-processor
+           request
+           util
+           enterprise/transforms}}
+
   queries
   {:team "Querying"
    :api  #{metabase.queries.api
@@ -1612,17 +1627,6 @@
            util
            enterprise/harbormaster}}
 
-  enterprise/comments
-  {:team "Workflows"
-   :api  #{metabase-enterprise.comments.api
-           metabase-enterprise.comments.init}
-   :uses #{api
-           events
-           models
-           request
-           users
-           util}}
-
   enterprise/content-translation
   {:team "UX West"
    :api  #{metabase-enterprise.content-translation.dictionary
@@ -1788,13 +1792,11 @@
            lib
            lib-be
            models
-           native-query-snippets
            permissions
            premium-features
            pulse
            query-processor
            request
-           search
            server
            settings
            sync
@@ -1803,8 +1805,7 @@
            util
            warehouse-schema
            warehouses
-           enterprise/documents
-           enterprise/transforms}}
+           enterprise/documents}}
 
   enterprise/permission-debug
   {:team "UX West"
@@ -1960,8 +1961,11 @@
   {:team "Querying"
    :api  #{metabase-enterprise.transforms.api
            metabase-enterprise.transforms.core
-           metabase-enterprise.transforms.init}
-   :uses #{api
+           metabase-enterprise.transforms.init
+           ;; TODO: Remove this from API once we remove the python-runner callback to metabase
+           metabase-enterprise.transforms.settings}
+   :uses #{analytics
+           api
            app-db
            config
            driver
@@ -1977,6 +1981,8 @@
            sync
            task
            task-history
+           ;; TODO: Remove this once we're using a better file format than CSV
+           upload
            util}}
 
   enterprise/upload-management

.clj-kondo/src/hooks/clojure/core/require.clj

@@ -52,13 +52,7 @@
 
 (defn- lint-require* [node current-ns config]
   (let [[_require & args]   (:children node)
-        required-namespaces (->> (keep unwrap-require args)
-                                 ;; `require` runs through `cc/load-libs` which allows
-                                 ;; #{:as :reload :reload-all :require :use :verbose :refer :as-alias}
-                                 ;; as args. If you use a REPL tool that applies clj-kondo linting (e.g., Clojure MCP)
-                                 ;; you may wish to `:reload` or `:reload-all` a namespace. So we shouldn't assume all
-                                 ;; args are keywords:
-                                 (remove keyword?))]
+        required-namespaces (keep unwrap-require args)]
     (lint* node current-ns config required-namespaces)))
 
 (defn- lint-requiring-resolve* [node current-ns config]

.github/actions/await-port/action.yml

@@ -11,6 +11,18 @@ runs:
     - name: Wait for port ${{ inputs.port }} to be ready
       shell: bash
       run: |
+        echo "Waiting for port ${{ inputs.port }} to be ready..."
+        timeout=300  # 5 minutes timeout
+        elapsed=0
         while ! nc -z localhost ${{ inputs.port }}; do
-          sleep 0.1;
+          if [ $elapsed -ge $timeout ]; then
+            echo "❌ Timeout waiting for port ${{ inputs.port }} after ${timeout} seconds"
+            exit 1
+          fi
+          sleep 1
+          elapsed=$((elapsed + 1))
+          if [ $((elapsed % 10)) -eq 0 ]; then
+            echo "  Still waiting for port ${{ inputs.port }}... ($elapsed seconds)"
+          fi
         done
+        echo "✅ Port ${{ inputs.port }} is ready!"

.github/file-paths.yaml

@@ -41,17 +41,10 @@ frontend_sources: &frontend_sources
   - "enterprise/frontend/!(test)**/**/{!(*.spec),}.{js,jsx,ts,tsx,css}"
   - "frontend/!(test)**/**/{!(*.spec),}.{js,jsx,ts,tsx,css}"
 
-frontend_embedding_sources:
-  - "enterprise/frontend/src/{embedding,embedding-*}/**/{!(*.spec),}.{js,jsx,ts,tsx,css}"
-  - "enterprise/frontend/src/metabase-enterprise/{embedding,embedding-*}**/{!(*.spec),}.{js,jsx,ts,tsx,css}"
-
 frontend_embedding_sdk_sources:
   - *frontend_configs
   - "enterprise/frontend/!(test)**/**/{!(*.spec),}.{js,jsx,ts,tsx,css}"
 
-frontend_embedding_sdk_package_sources:
-  - "enterprise/frontend/src/embedding-sdk-package/**/!(*.spec).{js,jsx,ts,tsx,css}"
-
 frontend_specs: &frontend_specs
   - *shared_specs
   - "frontend/test/**"

.github/scripts/conventional-commit-pr-title-reminder/utils.js

@@ -1,66 +1,24 @@
-const TYPES = new Set([
-  "feat",
-  "fix",
-  "perf",
-  "docs",
-  "style",
-  "refactor",
-  "test",
-  "build",
-  "ci",
-]);
-
-/**
- * @param title {string}
- */
-function isConventionalTitle(title) {
-  const match = /^(?<type>\w+)\((?<rawScopes>[^)]+)\):/.exec(title);
-
-  if (!match) {
-    return false;
-  }
-
-  const [, type, rawScopes] = match;
-  const scopes = rawScopes
-    .split(",")
-    .map((scope) => scope.trim())
-    .filter(Boolean);
-
-  return TYPES.has(type) && scopes.length > 0;
-}
+exports.updateComment = updateComment;
 
 /** @param {import('@types/github-script').AsyncFunctionArguments} AsyncFunctionArguments */
-async function updateCommentForEmbedding({ github, context }) {
+async function updateComment({ github, context }) {
   const githubUsername = context.payload.sender.login;
-
-  const comment = `${CONVENTIONAL_COMMENT_IDENTIFIER}
-@${githubUsername} You have modified embedding code. Please make sure the PR title follows [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) style.
-Here are all the supported types:
+  const comment = `${CONVENTIONAL_COMMIT_REMINDER_COMMENT_IDENTIFIER}
+@${githubUsername} You have modified embedding SDK code. Please make sure the PR title follows [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) style.
+Here are all the supported types that will be shown in the changelog:
 - \`feat\`
 - \`fix\`
 - \`perf\`
-- \`docs\`
-- \`style\`
-- \`refactor\`
-- \`test\`
-- \`build\`
-- \`ci\`
 
-Please also make sure to include a scope.
+These types \`docs\`, \`style\`, \`refactor\`, \`test\`, \`build\`, and \`ci\` are also encouraged for non-changelog related tasks.
+
+Please also make sure to include \`sdk\` scope, otherwise, the changelog will not include this task.
 
 For example, these are valid PR titles:
-- \`feat(sdk-bundle): Add interactive dashboard component\`
-- \`fix(sdk-package): Fixed MetabaseProviderPropsStore logic\`
-- \`fix(sdk-package, embed-js): Fix API\``;
+- \`feat(sdk): Add interactive dashboard component\`
+- \`feat(sdk): Support theming pie chart\`
+- \`fix(sdk): Fix static dashboard crash\``;
 
-  return updateComment(comment, { github, context });
-}
-
-/**
- * @param comment {string}
- * @param {import('@types/github-script').AsyncFunctionArguments} AsyncFunctionArguments
- */
-async function updateComment(comment, { github, context }) {
   const owner = context.repo.owner;
   const repo = context.repo.repo;
   const pullRequestNumber = context.issue.number;
@@ -91,19 +49,20 @@ async function updateComment(comment, { github, context }) {
   }
 }
 
-const CONVENTIONAL_COMMENT_IDENTIFIER =
-  "<!---conventional comment identifier-->";
-
+const CONVENTIONAL_COMMIT_REMINDER_COMMENT_IDENTIFIER =
+  "<!---conventional commit reminder comment ID-->";
 /**
- * @param comments {{body: string}[]}
+ * @typedef Comment
+ * @property {string} body
+ *
+ * @param {Comment[]} comments
  */
 function getExistingConventionalCommitReminderComment(comments) {
   for (const comment of comments) {
-    if (comment.body.startsWith(CONVENTIONAL_COMMENT_IDENTIFIER)) {
+    if (
+      comment.body.startsWith(CONVENTIONAL_COMMIT_REMINDER_COMMENT_IDENTIFIER)
+    ) {
       return comment;
     }
   }
 }
-
-exports.isConventionalTitle = isConventionalTitle;
-exports.updateCommentForEmbedding = updateCommentForEmbedding;

.github/scripts/conventional-commit-pr-title-reminder/utils.unit.spec.js

@@ -1,45 +0,0 @@
-const { isConventionalTitle } = require("./utils");
-
-describe("isConventionalTitle", () => {
-  it("returns true for a simple conventional commit", () => {
-    expect(isConventionalTitle("feat: add new feature")).toBe(false);
-  });
-
-  it("returns true for a commit with a scope", () => {
-    expect(isConventionalTitle("fix(api): correct API bug")).toBe(true);
-  });
-
-  it("returns true for a commit with multiple scopes, comma separated", () => {
-    expect(isConventionalTitle("feat(build,deps): update dependencies")).toBe(
-      true,
-    );
-  });
-
-  it("returns true for a commit with multiple scopes, comma separated with space", () => {
-    expect(isConventionalTitle("feat(build, deps): update dependencies")).toBe(
-      true,
-    );
-  });
-
-  it("returns false for a commit with round brackets in a message", () => {
-    expect(isConventionalTitle("feat: (something) update dependencies")).toBe(
-      false,
-    );
-  });
-
-  it("returns false for missing colon", () => {
-    expect(isConventionalTitle("feat add feature")).toBe(false);
-  });
-
-  it("returns false for invalid type", () => {
-    expect(isConventionalTitle("invalidType: some message")).toBe(false);
-  });
-
-  it("returns false for empty message", () => {
-    expect(isConventionalTitle("feat:")).toBe(false);
-  });
-
-  it("returns false for empty input", () => {
-    expect(isConventionalTitle("")).toBe(false);
-  });
-});

.github/workflows/backend.yml

@@ -195,6 +195,7 @@ jobs:
       - name: Run cljfmt
         run: ./bin/mage cljfmt-all --force-check
 
+
   be-tests-result:
     needs:
       - be-linter-eastwood

.github/workflows/conventional-commit-pr-title-reminder.yml

@@ -4,53 +4,39 @@ on:
   pull_request:
     branches:
       - master
+    paths:
+      - enterprise/frontend/src/embedding-sdk-bundle/**
 
 jobs:
-  files-changed:
-    name: Check which files changed
-    runs-on: ubuntu-22.04
-    timeout-minutes: 3
-    outputs:
-      frontend_embedding_sources: ${{ steps.changes.outputs.frontend_embedding_sources }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Test which files changed
-        uses: dorny/paths-filter@v3.0.0
-        id: changes
-        with:
-          token: ${{ github.token }}
-          filters: .github/file-paths.yaml
-
   pr-title-reminder:
     runs-on: ubuntu-22.04
     timeout-minutes: 5
     permissions:
       pull-requests: write
-    needs: [ files-changed ]
-    if: |
-      !cancelled() &&
-      needs.files-changed.outputs.frontend_embedding_sources == 'true'
     steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.head_ref }}
-
       - name: Check if the PR title is already following Conventional Commits style
         uses: actions/github-script@v7
         id: is-conventional-commit
         with:
           result-encoding: string
           script: | # js
-            const { isConventionalTitle } = require(".github/scripts/conventional-commit-pr-title-reminder/utils.js")
             const pullRequestTitle = context.payload.pull_request.title
+            const conventionalCommitRegex = /^(?<type>\w+)\((?<scope>\w+)\):/
+            const match = pullRequestTitle.match(conventionalCommitRegex)
+            if (match) {
+              const { type, scope } = match.groups
+              return ['feat', 'fix', 'perf', 'docs', 'style', 'refactor', 'test', 'build', 'ci'].includes(type) && scope === 'sdk'
+            }
 
-            return isConventionalTitle(pullRequestTitle)
+      - name: Checkout repository so `gh` CLI works
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
 
       - name: Remind PR authors to use conventional commit in the title
         if: ${{ steps.is-conventional-commit.outputs.result != 'true' }}
         uses: actions/github-script@v7
         with:
-          script: | #js
-            const { updateCommentForEmbedding } = require(".github/scripts/conventional-commit-pr-title-reminder/utils.js")
-
-            await updateCommentForEmbedding({github, context})
+          script: |
+            const { updateComment } = require(".github/scripts/conventional-commit-pr-title-reminder/utils.js")
+            await updateComment({github, context})

.github/workflows/drivers.yml

@@ -338,6 +338,12 @@ jobs:
     name: H2
     steps:
     - uses: actions/checkout@v4
+      with:
+        submodules: true
+        token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+    - name: Start python-runner
+      working-directory: python-runner
+      run: make run-all
     - name: Test H2 driver
       uses: ./.github/actions/test-driver
       with:
@@ -442,6 +448,12 @@ jobs:
     name: "${{ matrix.version.name }} ${{ matrix.job.name }}"
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
+          token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      - name: Start python-runner
+        working-directory: python-runner
+        run: make run-all
       - name: Test ${{ matrix.version.name }}
         uses: ./.github/actions/test-driver
         with:
@@ -745,6 +757,12 @@ jobs:
           POSTGRES_HOST_AUTH_METHOD: trust
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
+          token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+      - name: Start python-runner
+        working-directory: python-runner
+        run: make run-all
       - name: Test ${{ matrix.version.name }} (${{ matrix.job.name }})
         uses: ./.github/actions/test-driver
         with:

.github/workflows/e2e-component-tests-embedding-sdk-backward-compatibility.yml

@@ -140,10 +140,7 @@ jobs:
         run: node e2e/runner/run_cypress_ci.js snapshot --browser ${{ steps.cypress-prep.outputs.chrome-path }}
 
       - name: Run component tests for Embedding SDK
-        run: |
-          node e2e/runner/run_cypress_ci.js component \
-            --browser ${{ steps.cypress-prep.outputs.chrome-path }} \
-            --env grepTags="-@skip-backward-compatibility",grepOmitFiltered=true
+        run: node e2e/runner/run_cypress_ci.js component --browser ${{ steps.cypress-prep.outputs.chrome-path }}
 
       - name: Upload Test Results
         uses: ./.github/actions/upload-test-results

.github/workflows/embedding-sdk-labels-reminder.yml

@@ -1,49 +0,0 @@
-name: Embedding SDK Labels Reminder
-
-on:
-  pull_request:
-    types: [auto_merge_enabled, review_requested, labeled, unlabeled, synchronize]
-    branches:
-      - "master"
-  merge_group:
-
-jobs:
-  files-changed:
-    name: Check which files changed
-    runs-on: ubuntu-22.04
-    timeout-minutes: 3
-    outputs:
-      frontend_embedding_sdk_package_sources: ${{ steps.changes.outputs.frontend_embedding_sdk_package_sources }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Test which files changed
-        uses: dorny/paths-filter@v3.0.0
-        id: changes
-        with:
-          token: ${{ github.token }}
-          filters: .github/file-paths.yaml
-
-  embedding-sdk-package-no-release-notes-reminder:
-    needs: [ files-changed ]
-    if: |
-      !cancelled() &&
-      (needs.files-changed.outputs.frontend_embedding_sdk_package_sources == 'true')
-    name: Embedding SDK package release-notes/no-release-notes label reminder
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    steps:
-      - uses: actions/checkout@v4
-      - uses: jesusvasquez333/verify-pr-label-action@v1.4.0
-        id: verify-release-notes-labels
-        with:
-          github-token: '${{ secrets.GITHUB_TOKEN }}'
-          valid-labels: 'release-notes, no-release-notes'
-          disable-reviews: true
-      - name: Comment on PR if label should be added
-        if: ${{ always() }}
-        uses: ./.github/actions/notify-pull-request
-        with:
-          remove-previous-comment: true
-          add-new-comment: ${{ steps.verify-release-notes-labels.outcome == 'failure' }}
-          message: "⚠️ Changes were made to the <b>Embedding SDK Package</b>. Review this PR and apply one of the following labels.<br><br>- <code>release-notes</code> — select if this PR must be included in the main Metabase changelog release notes.<br>- <code>no-release-notes</code> — select if this PR should not be included in the main Metabase changelog release notes."
-          include-log: true

.github/workflows/perf-test-labeled.yml

@@ -9,13 +9,9 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  uberjar:
-    if: ${{ github.event.label.name == 'Run-perf' }}
-    uses: ./.github/workflows/uberjar.yml
-    secrets: inherit
-
   run-perf-test:
-    needs: [uberjar]
     if: ${{ github.event.label.name == 'Run-perf' }}
     uses: ./.github/workflows/perf-test.yml
+    with:
+      wait_for_uberjar: true
     secrets: inherit

.github/workflows/perf-test.yml

@@ -3,6 +3,11 @@ run-name: Run perf test for ${{ github.ref_name }} by @${{ github.actor }}
 
 on:
   workflow_call:
+    inputs:
+      wait_for_uberjar:
+        description: "Wait for uberjar build"
+        required: false
+        type: boolean
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -26,6 +31,63 @@ jobs:
         uses: aws-actions/amazon-ecr-login@v2
         with:
           registries: "${{ secrets.PR_ENV_AWS_ACCOUNT_ID }}"
+      - name: Wait for uberjar
+        id: wait_for_uberjar
+        if: ${{ inputs.wait_for_uberjar == true }}
+        run: |
+          ## Get workflow run id for uberjar build
+          curl -Ls --output e2e-tests.json \
+          -H "Accept: application/vnd.github+json" \
+          -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+          -H "X-GitHub-Api-Version: 2022-11-28" \
+          https://api.github.com/repos/${{ github.repository }}/actions/workflows/run-tests.yml/runs?head_sha=${{ github.event.pull_request.head.sha || github.sha }}
+          ID=$(jq -r '.workflow_runs[0].id' e2e-tests.json)
+          echo "Run ID: ${ID}"
+
+          ## Get workflow run id for uberjar build
+          while [ -z "$JOB_ID" ]; do
+            NEXT_URL="https://api.github.com/repos/${{ github.repository }}/actions/runs/${ID}/jobs?filter=latest"
+
+            # Search for build job
+            echo "Searching for build job..."
+            while [ -z "$JOB_ID" ] && [ -n "$NEXT_URL" ]; do
+              # Get one page of jobs
+              RESPONSE=$(curl -s -i \
+                -H "Accept: application/vnd.github+json" \
+                -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+                -H "X-GitHub-Api-Version: 2022-11-28" \
+                "$NEXT_URL")
+
+              # Extract headers and body
+              HEADERS=$(echo "$RESPONSE" | sed -n '1,/^\r$/p')
+              BODY=$(echo "$RESPONSE" | sed '1,/^\r$/d')
+              LINK_HEADER=$(echo "$HEADERS" | grep -i '^link:' | sed 's/^link: //')
+
+              # Check for next page in Link header
+              if echo "$LINK_HEADER" | grep -q 'rel="next"'; then
+                NEXT_URL=$(echo "$LINK_HEADER" | sed -E 's/.*<([^>]+)>; rel="next".*/\1/')
+              else
+                NEXT_URL=""
+              fi
+
+              # Check for build job
+              JOB_ID=$(echo "$BODY" | jq -r '.jobs[] | select(.name | contains("build (ee)")) | .id')
+            done
+          done
+          echo "Job ID: ${JOB_ID}"
+
+          ## Wait for uberjar build to complete
+          while [ true ]; do
+            curl -Ls --output uberjar.json \
+            -H "Accept: application/vnd.github+json" \
+            -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            https://api.github.com/repos/${{ github.repository }}/actions/jobs/${JOB_ID}
+            jq -r '.steps[] | select(.name == "Prepare uberjar artifact") | .status' uberjar.json | grep -q "completed" && break
+            echo "Waiting for uberjar build..."
+            sleep 10
+          done
+          echo "run_id=$(jq -r '.workflow_runs[0].id' e2e-tests.json)" >> $GITHUB_OUTPUT
       - name: Retrieve uberjar artifact for ee
         uses: ./.github/actions/fetch-artifact
         with:

.github/workflows/release-embedding-sdk.yml

@@ -285,6 +285,10 @@ jobs:
         run: |
           ./bin/embedding-sdk/release_utils.bash update_package_json_template ${{ needs.determine-sdk-version.outputs.sdk_version }}
 
+      - name: Update changelog
+        run: |
+          yarn embedding-sdk:generate-changelog -o changelog-diff
+
       - name: Build SDK package
         run: yarn run build-embedding-sdk-package
 
@@ -297,6 +301,12 @@ jobs:
           name: metabase-sdk
           path: ./resources/embedding-sdk
 
+      - name: Upload changelog diff
+        uses: actions/upload-artifact@v4
+        with:
+          name: sdk-changelog-diff
+          path: ./changelog-diff
+
   publish-npm:
     needs: [build-sdk, determine-sdk-version]
     runs-on: ubuntu-22.04
@@ -325,7 +335,19 @@ jobs:
         run: |
           bash ./bin/embedding-sdk/release_utils.bash update_package_json_template ${{ env.sdk_version }}
 
-      - name: Create a PR updating readme + published version (using GitHub Metabase Automation account)
+      - name: Retrieve SDK changelog diff
+        if: inputs.branch != 'master'
+        uses: actions/download-artifact@v4
+        with:
+          name: sdk-changelog-diff
+
+      - name: Update changelog
+        if: inputs.branch != 'master'
+        run: |
+          cat changelog-diff enterprise/frontend/src/embedding-sdk-bundle/CHANGELOG.md > new-changelog
+          mv new-changelog enterprise/frontend/src/embedding-sdk-bundle/CHANGELOG.md
+
+      - name: Create a PR updating readme + published version, and changelog (using GitHub Metabase Automation account)
         run: |
           git checkout -b update-sdk-version-${{ env.sdk_version }}
 

.gitignore

@@ -163,3 +163,4 @@ CLAUDE.local.md
 
 /logs
 /.clojure-mcp
+__pycache__/

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "python-runner"]
+	path = python-runner
+	url = https://github.com/metabase/python-runner-container.git
+	branch = prototype

bin/embedding-sdk/generate-sdk-package-files.js

@@ -89,6 +89,6 @@ generateSdkPackage();
 copyFileToOutput("LICENSE.txt");
 copyFileToOutput("frontend/src/embedding-sdk-package/README.md", "README.md");
 copyFileToOutput(
-  "frontend/src/embedding-sdk-package/CHANGELOG.md",
+  "frontend/src/embedding-sdk-bundle/CHANGELOG.md",
   "CHANGELOG.md",
 );

deps.edn

@@ -36,7 +36,7 @@
   com.clearspring.analytics/stream          {:mvn/version "2.9.8"               ; Various sketching algorithms
                                              :exclusions  [it.unimi.dsi/fastutil
                                                            org.slf4j/slf4j-api]}
-  com.clojure-goes-fast/jvm-alloc-rate-meter{:mvn/version "0.1.4"}              ; Monitor heap allocation rate.
+  com.clojure-goes-fast/jvm-alloc-rate-meter {:mvn/version "0.1.4"}              ; Monitor heap allocation rate.
   com.clojure-goes-fast/jvm-hiccup-meter    {:mvn/version "0.1.1"}              ; Monitor GC pauses and other system-induced pauses.
   com.draines/postal                        {:mvn/version "2.0.5"               ; SMTP library
                                              :exclusions [com.sun.mail/jakarta.mail]}
@@ -45,7 +45,7 @@
   com.github.albfernandez/juniversalchardet {:mvn/version "2.5.0"}
   com.github.blagerweij/liquibase-sessionlock {:mvn/version "1.6.9"}            ; session lock for liquibase migrations
   com.github.jknack/handlebars              ^:antq/exclude                      ; 4.4.0 requires java 17+
-                                            {:mvn/version "4.3.1"}              ; templating engine
+  {:mvn/version "4.3.1"}              ; templating engine
   com.github.oliyh/martian-clj-http         {:mvn/version "0.1.31"}             ; openapi client
   com.github.pangloss/transducers           {:git/sha "c91d39046c8c64dc31b84ebb27154a8084fcda4c" ; like medley for transducers
                                              :git/url "https://github.com/pangloss/transducers"}
@@ -55,14 +55,14 @@
   com.github.vertical-blank/sql-formatter   {:mvn/version "2.0.5"}              ; Java SQL formatting library https://github.com/vertical-blank/sql-formatter
   com.google.guava/guava                    {:mvn/version "33.4.8-jre"}         ; dep for BigQuery, Spark, and GA. Require here rather than letting different dep versions stomp on each other — see comments on #9697
   com.h2database/h2                         ^:antq/exclude                      ; https://metaboat.slack.com/archives/CKZEMT1MJ/p1727191010259979
-                                            {:mvn/version "2.1.214"}            ; embedded SQL database
+  {:mvn/version "2.1.214"}            ; embedded SQL database
   com.ibm.icu/icu4j                         {:mvn/version "77.1"}               ; Used to transliterate Unicode characters into Latin characters
   com.knuddels/jtokkit                      {:mvn/version "1.1.0"}              ; Java tokenizer library for OpenAI models, used for token counting in semantic search
   com.mchange/c3p0                          {:mvn/version "0.10.1"}             ; DB connection pool
   com.snowplowanalytics/snowplow-java-tracker ^:antq/exclude                    ; 2+ depend on apache httpclient 5.x, but saml and clj-http depend on 4.x
-                                            {:mvn/version "1.0.1"}              ; Snowplow analytics
+  {:mvn/version "1.0.1"}              ; Snowplow analytics
   com.sun.mail/jakarta.mail                 ^:antq/exclude                      ; 2+ changes imports from javax.mail to jakarta.mail
-                                            {:mvn/version "1.6.8"}              ; SMTP lib, required by postal
+  {:mvn/version "1.6.8"}              ; SMTP lib, required by postal
   com.taoensso/nippy                        {:mvn/version "3.6.0"}              ; Fast serialization (i.e., GZIP) library for Clojure
   com.vladsch.flexmark/flexmark             {:mvn/version "0.64.8"}             ; Markdown parsing
   com.vladsch.flexmark/flexmark-ext-autolink
@@ -92,7 +92,7 @@
   ;; TODO: replace back to normal artifact
   io.github.metabase/macaw                  {:mvn/version "0.2.27"}             ; Parse native SQL queries
   io.github.resilience4j/resilience4j-retry ^:antq/exclude                      ; 2.x do not support Java 8
-                                            {:mvn/version "1.7.1"}              ; Support for retrying operations
+  {:mvn/version "1.7.1"}              ; Support for retrying operations
   io.prometheus/simpleclient_hotspot        {:mvn/version "0.16.0"}             ; prometheus jvm collector
   junegunn/grouper                          {:mvn/version "0.1.1"}              ; Batch processing helper
   kixi/stats                                {:mvn/version "0.5.7"               ; Various statistic measures implemented as transducers
@@ -198,6 +198,8 @@
                                                            org.eclipse.jetty.ee9/jetty-ee9-servlet
                                                            org.eclipse.jetty.ee9.websocket/jetty-ee9-websocket-jetty-server]}
   slingshot/slingshot                       {:mvn/version "0.12.2"}             ; enhanced throw/catch, used by other deps
+  software.amazon.awssdk/s3                 {:mvn/version "2.33.0"}
+  software.amazon.awssdk/sts                {:mvn/version "2.33.0"}           ; AWS SDK v2 for S3
   stencil/stencil                           {:mvn/version "0.5.0"}              ; Mustache templates for Clojure
   user-agent/user-agent                     {:mvn/version "0.1.1"}              ; User-Agent string parser, for Login History page & elsewhere
   weavejester/dependency                    {:mvn/version "1.0.0"}              ; Dependency graphs and topological sorting
@@ -836,8 +838,13 @@
   :test/lib
   {:exec-args {:only ["test/metabase/lib"
                       "test/metabase/lib_be"]}}
+  ;; test the python transforms
+  ;;    clj -X:dev:ee:ee-dev:driver:test:test/python-transform
+  :test/python-transform
+  {:exec-args {:only [metabase-enterprise.transforms.api-test
+                      "enterprise/backend/test/metabase_enterprise/transforms"]}}
 
-  ;; test the Query Processor
+;; test the Query Processor
   ;;
   ;;    clj -X:dev:ee:ee-dev:test:test/qp
   :test/qp

docs/configuring-metabase/localization.md

@@ -136,7 +136,3 @@ This setting doesn't affect [SQL queries](../questions/native-editor/writing-sql
 - **Unit of currency:** if you do most of your business in a particular currency, you can specify that here.
 - **Currency label style:** whether you want to have your currencies labeled with a symbol, a code (like "USD"), or its full name.
 - **Where to display the unit of currency:** this pertains specifically to tables, and lets you choose whether you want the currency labels to appear only in the column heading, or next to each value in the column.
-
-## Localizing embedded Metabase
-
-You can tranlate both Metabase UI elements (like button labels) and content (like dashboard names) in static embeds. See [Setting the language for static embeds](../embedding/static-embedding-parameters.md#setting-the-language-for-a-static-embed) and [Translating embedded dashboards and questions](../embedding/translations.md).

docs/developers-guide/driver-changelog.md

@@ -19,6 +19,12 @@ title: Driver interface changelog
 
 - Added the driver multi-methods `metabase.driver/schema-exists?` and `metabase.driver/create-schema-if-needed!` which should be implemented by drivers that support `:schemas` and `:transforms/table`.
 
+- Added `metabase.driver/type->database-type` multimethod that returns the database type for a given Metabase type (from the type hierarchy) as a HoneySQL spec. Unlike `upload-type->database-type` which is specific to upload types, this method handles general Metabase base types.
+
+- Added `metabase.driver/rename-table!` multimethod to rename a table from one name to another. Both table names may be qualified by schema (e.g., `schema.table`). This is currently used for transform operations and uploads.
+
+- Added `metabase.driver/insert-from-source!` multimethod that abstracts data insertion from various sources into existing tables. This multimethod dispatches on both the driver and the data source type (`:rows` or `:csv-file`). It allows drivers to optimize based on the data source type and returns the number of rows inserted. Default implementations are provided for both `:rows` and `:csv-file` source types.
+
 - `metabase.driver/humanize-connection-error-message` now takes a list of all messages in the exception chain,
   instead of just the top-level message as a string.
 

docs/embedding/sdk/next-js.md

@@ -14,11 +14,11 @@ See a [sample Next.js app that uses the SDK](https://github.com/metabase/metabas
 
 ## SDK components with Server Side Rendering (SSR) or React Server Components
 
-As of Embedded Analytics SDK v57, SDK components automatically skip server-side rendering (SSR) and render only on the client.
+As of Embedding Analytics SDK v57, SDK components automatically skip server-side rendering (SSR) and render only on the client.
 
 ### Compatibility layer for Server Side Rendering (SSR) (DEPRECATED)
 
-As of Embedded Analytics SDK 57, the compatibility layer for server-side rendering (SSR) is deprecated and no longer required. If you use the compatibility layer, change your imports from `@metabase/embedding-sdk-react/next` to `@metabase/embedding-sdk-react`.
+As of Embedding analytics SDK 57, the compatibility layer for server-side rendering (SSR) is deprecated and no longer required. If you use the compatibility layer, change your imports from `@metabase/embedding-sdk-react/next` to `@metabase/embedding-sdk-react`.
 
 ## Handling authentication
 

docs/embedding/sdk/upgrade.md

@@ -10,8 +10,8 @@ Here's a basic overview of the steps you'll want to take when upgrading your SDK
 ## 1. Read the release post and changelog for Metabase and the Embedded analytics SDK
 
 - [Release posts](https://www.metabase.com/releases) give a good overview of what's in each release, and call out breaking changes (which are rare).
-- [Metabase + Embedding analytics SDK changelogs](https://www.metabase.com/changelog) list Metabase and Embedding analytics SDK changes.
-- [Embedded analytics SDK package changelog](https://github.com/metabase/metabase/blob/release-x.56.x/enterprise/frontend/src/embedding-sdk-bundle/CHANGELOG.md) list changes specific to the SDK lightweight `@metabase/embedding-sdk-react` package.
+- [Metabase changelogs](https://www.metabase.com/changelog) list Metabase changes.
+- [Embedded analytics SDK changelogs](https://www.metabase.com/changelog/56) list changes specific to the SDK Package.
 
 Check for any relevant changes, especially deprecations or breaking changes that require you to update your application's code. If there are deprecation changes, we'll have docs that'll walk you through what changes you'll need to make and why.
 

docs/embedding/translations.md

@@ -5,7 +5,7 @@ summary: Upload a translation dictionary to translate questions and dashboards i
 
 # Translate embedded dashboards and questions
 
-{% include plans-blockquote.html feature="Translation of embedded content" convert_pro_link_to_embbedding=true %}
+{% include plans-blockquote.html feature="Content translation for static embeds" convert_pro_link_to_embbedding=true %}
 
 For now, translations are only available for [static embeds](./static-embedding.md), not Interactive embedding or the Embedded analytics SDK.
 
@@ -60,16 +60,6 @@ For example, if you have a dashboard called "Monthly Sales", it's not sufficient
 
 Exact translations also apply to strings that use punctuation and special characters. For example, if you have a question title "How many Widgets did we sell this week?", you must include that exact string (with "?") into the translation dictionary. Metabase would treat "How many Widgets did we sell this week" as a different string. Essentially, the strings are keys in a table Metabase looks up, so they must match exactly.
 
-## Translations are case-insensitive
-
-Translations are case-insensitive. For example, if your translation dictionary has a translation:
-
-| Language | String      | Translation  |
-| -------- | ----------- | ------------ |
-| pt-BR    | Examples    | Exemplos     |
-
-`Examples`, `examples`, `EXAMPLES` would all be translated as `Exemplos`. That is, Metabase won't preserve the original case; it will output the exact translation string in the dictionary.
-
 ## Include markdown formatting in translation dictionaries
 
 If the strings you want to translate include markdown formatting, you'll need to include the formatting in the dictionary. For example:

e2e/runner/cypress-runner-run-tests.js

@@ -127,10 +127,7 @@ const configs = {
       openMode: args["--open"] || process.env.OPEN_UI === "true",
     };
 
-    const userArgs = await parseArguments(args);
-
-    const finalConfig = Object.assign({}, sdkComponentConfig, userArgs);
-    return finalConfig;
+    return sdkComponentConfig;
   },
 };
 

e2e/support/helpers/api/createComment.ts

@@ -1,19 +0,0 @@
-import type { Comment, CreateCommentRequest } from "metabase-types/api";
-
-export const createComment = ({
-  target_id,
-  target_type,
-  child_target_id,
-  parent_comment_id = null,
-  content,
-}: Partial<CreateCommentRequest>): Cypress.Chainable<
-  Cypress.Response<Comment>
-> => {
-  return cy.request<Comment>("POST", "/api/ee/comment", {
-    target_id,
-    target_type,
-    child_target_id,
-    parent_comment_id,
-    content,
-  });
-};

e2e/support/helpers/api/createReaction.ts

@@ -1,13 +0,0 @@
-import type { Comment, CommentId } from "metabase-types/api";
-
-export const createReaction = ({
-  comment_id,
-  emoji,
-}: {
-  comment_id: CommentId;
-  emoji: string;
-}): Cypress.Chainable<Cypress.Response<Comment>> => {
-  return cy.request<Comment>("POST", `/api/ee/comment/${comment_id}/reaction`, {
-    emoji,
-  });
-};

e2e/support/helpers/api/deleteComment.ts

@@ -1,7 +0,0 @@
-import type { CommentId } from "metabase-types/api";
-
-export const deleteComment = (
-  commentId: CommentId,
-): Cypress.Chainable<Cypress.Response<void>> => {
-  return cy.request<void>("DELETE", `/api/ee/comment/${commentId}`);
-};

e2e/support/helpers/api/index.ts

@@ -3,9 +3,9 @@ export { addQuestionToDashboard } from "./addQuestionToDashboard";
 export { archiveCollection } from "./archiveCollection";
 export { archiveDashboard } from "./archiveDashboard";
 export { archiveQuestion } from "./archiveQuestion";
+export { createQuestionAlert } from "./createNotification";
 export { createApiKey } from "./createApiKey";
 export { createCollection } from "./createCollection";
-export { createComment } from "./createComment";
 export { createDashboard } from "./createDashboard";
 export type { DashboardDetails } from "./createDashboard";
 export { createDashboardWithQuestions } from "./createDashboardWithQuestions";
@@ -15,7 +15,6 @@ export { createModerationReview } from "./createModerationReview";
 export { createNativeQuestion } from "./createNativeQuestion";
 export type { NativeQuestionDetails } from "./createNativeQuestion";
 export { createNativeQuestionAndDashboard } from "./createNativeQuestionAndDashboard";
-export { createQuestionAlert } from "./createNotification";
 export { createPulse } from "./createPulse";
 export { createQuestion } from "./createQuestion";
 export type {
@@ -24,7 +23,6 @@ export type {
 } from "./createQuestion";
 export { createQuestionAndAddToDashboard } from "./createQuestionAndAddToDashboard";
 export { createQuestionAndDashboard } from "./createQuestionAndDashboard";
-export { createReaction } from "./createReaction";
 export { createSnippet } from "./createSnippet";
 export { createTimeline } from "./createTimeline";
 export { createTimelineEvent } from "./createTimelineEvent";
@@ -32,10 +30,8 @@ export { createTimelineWithEvents } from "./createTimelineWithEvents";
 export { createTransform } from "./createTransform";
 export { createTransformJob } from "./createTransformJob";
 export { createTransformTag } from "./createTransformTag";
-export { deleteComment } from "./deleteComment";
 export { editDashboardCard } from "./editDashboardCard";
 export { getCurrentUser } from "./getCurrentUser";
 export { remapDisplayValueToFK } from "./remapDisplayValueToFK";
-export { updateComment } from "./updateComment";
 export { updateDashboardCards } from "./updateDashboardCards";
 export { updateSetting } from "./updateSetting";

e2e/support/helpers/api/updateComment.ts

@@ -1,14 +0,0 @@
-import type { Comment, UpdateCommentRequest } from "metabase-types/api";
-
-export const updateComment = ({
-  id,
-  content,
-  is_resolved,
-}: Partial<UpdateCommentRequest>): Cypress.Chainable<
-  Cypress.Response<Comment>
-> => {
-  return cy.request<Comment>("PUT", `/api/ee/comment/${id}`, {
-    content,
-    is_resolved,
-  });
-};

e2e/support/helpers/e2e-browser-helpers.ts

@@ -21,9 +21,3 @@ export function grantClipboardPermissions() {
     },
   });
 }
-
-export function readClipboard() {
-  return cy.window().then((win) => {
-    return win.navigator.clipboard.readText();
-  });
-}

e2e/support/helpers/e2e-comment-helpers.ts

@@ -1,103 +0,0 @@
-import type { DocumentId } from "metabase-types/api";
-
-import { modal } from "./e2e-ui-elements-helpers";
-
-export const Comments = {
-  getDocumentNodeButton,
-  getDocumentNodeButtons,
-  getNewThreadInput,
-  getCommentInput,
-  getCommentInputs,
-  getCommentByText,
-  getPlaceholder,
-  getEmojiPicker,
-  resolveCommentByText,
-  reopenCommentByText,
-  openAllComments,
-  reactToComment,
-};
-
-function getDocumentNodeButton({
-  targetId,
-  childTargetId,
-  hasComments,
-}: {
-  targetId: DocumentId;
-  childTargetId: string;
-  hasComments?: boolean;
-}) {
-  const threadUrl = `/document/${targetId}/comments/${childTargetId}${hasComments ? "" : "?new=true"}`;
-
-  return getDocumentNodeButtons().filter(
-    (_, element) => element.getAttribute("href") === threadUrl,
-  );
-}
-
-function getDocumentNodeButtons() {
-  return cy.findAllByRole("link", { name: "Comments" });
-}
-
-function getNewThreadInput() {
-  return cy.findByTestId("new-thread-editor");
-}
-
-function getCommentInput() {
-  return cy.findByTestId("comment-editor");
-}
-
-function getCommentInputs() {
-  return cy.findAllByTestId("comment-editor");
-}
-
-function getPlaceholder() {
-  return cy.get("[data-placeholder]");
-}
-
-function getCommentByText(text: string | RegExp) {
-  return cy.findByText(text).closest("[data-testid='discussion-comment']");
-}
-
-function openAllComments() {
-  cy.findByRole("link", { name: "Show all comments" }).click();
-  cy.findByRole("dialog").should("contain.text", "All comments");
-}
-
-function getEmojiPicker() {
-  return cy.findByTestId("emoji-picker");
-}
-
-function resolveCommentByText(text: string | RegExp) {
-  return getCommentByText(text)
-    .realHover()
-    .within(() => {
-      cy.findByTestId("comment-action-panel").should("be.visible");
-      cy.findByTestId("comment-action-panel-resolve")
-        .should("be.visible")
-        .click();
-    });
-}
-
-function reopenCommentByText(text: string | RegExp) {
-  return getCommentByText(text)
-    .realHover()
-    .within(() => {
-      cy.findByTestId("comment-action-panel").should("be.visible");
-      cy.findByTestId("comment-action-panel-reopen")
-        .should("be.visible")
-        .click();
-    });
-}
-
-function reactToComment(comment: string | RegExp, emoji: string) {
-  modal().within(() => {
-    getCommentByText(comment)
-      .realHover()
-      .within(() => {
-        cy.findByTestId("comment-action-panel").should("be.visible");
-        cy.findByRole("button", { name: "Add reaction" }).click();
-      });
-  });
-  getEmojiPicker().within(() => {
-    cy.findByText(emoji).click();
-  });
-}

e2e/support/helpers/e2e-document-helpers.ts

@@ -1,5 +1,3 @@
-import type { CommentId, DocumentId } from "metabase-types/api";
-
 export const documentContent = () => cy.findByTestId("document-content");
 
 export const documentSaveButton = () =>
@@ -53,52 +51,3 @@ export const openDocumentCardMenu = (name: string) => {
     .findByRole("button", { name: /ellipsis/ })
     .click();
 };
-
-export function visitDocument(documentIdOrAlias: DocumentId | string) {
-  if (typeof documentIdOrAlias === "number") {
-    visitDocumentById(documentIdOrAlias);
-  }
-
-  if (typeof documentIdOrAlias === "string") {
-    cy.get(documentIdOrAlias).then((id) => visitDocumentById(Number(id)));
-  }
-}
-
-export function visitDocumentComment(
-  documentIdOrAlias: DocumentId | string,
-  nodeId: string,
-  commentId?: CommentId,
-) {
-  if (typeof documentIdOrAlias === "number") {
-    visitDocumentCommentById(documentIdOrAlias, nodeId, commentId);
-  }
-
-  if (typeof documentIdOrAlias === "string") {
-    cy.get(documentIdOrAlias).then((id) =>
-      visitDocumentCommentById(Number(id), nodeId, commentId),
-    );
-  }
-}
-
-function visitDocumentCommentById(
-  documentId: DocumentId | string,
-  nodeId: string,
-  commentId?: CommentId,
-) {
-  const alias = `documentQuery-${documentId}`;
-  cy.intercept("GET", `/api/ee/document/${documentId}`).as(alias);
-
-  const hash = commentId == null ? "" : `#comment-${commentId}`;
-  cy.visit(`/document/${documentId}/comments/${nodeId}${hash}`);
-
-  cy.wait(`@${alias}`);
-}
-
-const visitDocumentById = (id: DocumentId) => {
-  const alias = `documentQuery-${id}`;
-  cy.intercept("GET", `/api/ee/document/${id}`).as(alias);
-
-  cy.visit(`/document/${id}`);
-
-  cy.wait(`@${alias}`);
-};

e2e/support/helpers/e2e-email-helpers.js

@@ -31,22 +31,19 @@ export const setupSMTP = () => {
   clearInbox();
 };
 
-export const getInbox = (emailsCount) => {
-  return getInboxWithRetry(emailsCount);
+export const getInbox = () => {
+  return getInboxWithRetry();
 };
 
-const getInboxWithRetry = (emailsCount, timeout = INBOX_TIMEOUT) => {
+const getInboxWithRetry = (timeout = INBOX_TIMEOUT) => {
   return cy
     .request("GET", `http://localhost:${WEB_PORT}/email`)
     .then((response) => {
-      if (
-        response.body.length &&
-        (emailsCount == null || emailsCount === response.body.length)
-      ) {
+      if (response.body.length) {
         return cy.wrap(response);
       } else if (timeout > 0) {
         cy.wait(INBOX_INTERVAL);
-        return getInboxWithRetry(emailsCount, timeout - INBOX_INTERVAL);
+        return getInboxWithRetry(timeout - INBOX_INTERVAL);
       } else {
         throw new Error("Inbox retry timeout");
       }

e2e/support/helpers/index.ts

@@ -8,7 +8,6 @@ export * from "./e2e-browser-helpers";
 export * from "./e2e-cloud-helpers";
 export * from "./e2e-collection-helpers";
 export * from "./e2e-command-palette-helpers";
-export * from "./e2e-comment-helpers";
 export * from "./e2e-custom-column-helpers";
 export * from "./e2e-dashboard-helpers";
 export * from "./e2e-dashboard-visualizer-helpers";

e2e/test-component/scenarios/embedding-sdk/metabot-question.cy.spec.tsx

@@ -115,32 +115,6 @@ describe("scenarios > embedding-sdk > metabot-question", () => {
     });
   });
 
-  it("should start a new conversation when clicking a `start new conversation` button", () => {
-    setup(metabotResponse);
-
-    mountSdkContent(<MetabotQuestion />);
-
-    getSdkRoot().within(() => {
-      cy.findByTestId("metabot-chat-input").type("Show orders {enter}");
-
-      H.mockMetabotResponse({
-        statusCode: 200,
-        body: metabotRetryResponse,
-      });
-
-      cy.findAllByTestId("metabot-chat-message").should("have.length", 2);
-
-      cy.findByTestId("metabot-new-conversation").click();
-
-      cy.findAllByTestId("metabot-chat-message").should("not.exist");
-      cy.findByTestId("metabot-chat-input").should("have.value", "");
-
-      cy.findByTestId("metabot-chat-input").type("Show orders {enter}");
-
-      cy.findAllByTestId("metabot-chat-message").should("have.length", 2);
-    });
-  });
-
   it("should set correct metabot_id both for the a new message and when retrying", () => {
     setup(metabotResponse);
 

e2e/test-component/scenarios/embedding-sdk/sdk-bundle.cy.spec.tsx

@@ -26,13 +26,14 @@ const sdkBundleCleanup = () => {
 
 describe(
   "scenarios > embedding-sdk > sdk-bundle",
-  {
-    tags: ["@skip-backward-compatibility"],
-    // These test in some cases load a new SDK Bundle that in combination with the Component Testing is memory-consuming
-    numTestsKeptInMemory: 1,
-  },
+  // These test in some cases load a new SDK Bundle that in combination with the Component Testing is memory-consuming
+  { numTestsKeptInMemory: 1 },
   () => {
     beforeEach(() => {
+      cy.intercept("POST", "/api/dashboard/*/dashcard/*/card/*/query").as(
+        "dashcardQuery",
+      );
+
       signInAsAdminAndEnableEmbeddingSdk();
 
       cy.signOut();
@@ -43,6 +44,10 @@ describe(
     [{ strictMode: false }, { strictMode: true }].forEach(({ strictMode }) => {
       describe(`Common cases ${strictMode ? "with" : "without"} strict mode`, () => {
         it("should display an SDK question", () => {
+          cy.window().then((win) => {
+            cy.spy(win.console, "warn").as("consoleWarn");
+          });
+
           mountSdkContent(
             <InteractiveQuestion questionId={ORDERS_QUESTION_ID} />,
             { strictMode },
@@ -194,13 +199,6 @@ describe(
         it("should show a custom loader when the SDK bundle is loading", () => {
           sdkBundleCleanup();
 
-          cy.intercept("GET", "/api/card/*", (request) => {
-            // Delay request for 500ms to avoid flakiness
-            request.continue(
-              () => new Promise((resolve) => setTimeout(resolve, 500)),
-            );
-          });
-
           mountSdkContent(
             <InteractiveQuestion questionId={ORDERS_QUESTION_ID} />,
             {
@@ -287,7 +285,7 @@ describe(
     describe("Components", () => {
       it("should display an SDK question with custom layout components", () => {
         cy.window().then((win) => {
-          cy.spy(win.console, "error").as("consoleError");
+          cy.spy(win.console, "warn").as("consoleWarn");
         });
 
         mountSdkContent(
@@ -305,13 +303,15 @@ describe(
 
           cy.findByTestId("visualization-root").should("be.visible");
         });
-
-        cy.get("@consoleError").should("not.be.called");
       });
 
       it("should show an error on a component level if SDK components are not wrapped within the MetabaseProvider", () => {
         sdkBundleCleanup();
 
+        cy.window().then((win) => {
+          cy.spy(win.console, "warn").as("consoleWarn");
+        });
+
         cy.mount(<InteractiveQuestion questionId={ORDERS_QUESTION_ID} />);
 
         getSdkRoot().within(() => {
@@ -320,41 +320,6 @@ describe(
           ).should("exist");
         });
       });
-
-      it("should show a console error if SDK Package component uses a prop that is not yet available in SDK bundle", () => {
-        sdkBundleCleanup();
-
-        cy.window().then((win) => {
-          cy.spy(win.console, "error").as("consoleError");
-        });
-
-        mountSdkContent(
-          <InteractiveQuestion
-            questionId={ORDERS_QUESTION_ID}
-            {...{ foo: "bar" }}
-          />,
-        );
-
-        cy.get("@consoleError").should(
-          "be.calledWithMatch",
-          "this property is not recognized by the component",
-        );
-      });
-
-      it("should show a console error if SDK Package component does not use a prop that is still expected by SDK bundle", () => {
-        sdkBundleCleanup();
-
-        cy.window().then((win) => {
-          cy.spy(win.console, "error").as("consoleError");
-        });
-
-        mountSdkContent(<InteractiveQuestion />);
-
-        cy.get("@consoleError").should(
-          "be.calledWithMatch",
-          "this property is required by the component",
-        );
-      });
     });
 
     describe("Error handling", { retries: 3 }, () => {
@@ -379,7 +344,7 @@ describe(
 
           cy.findByTestId("sdk-error-container").should(
             "contain.text",
-            "Error loading the Embedded Analytics SDK",
+            "Error loading the Embedding Analytics SDK",
           );
         });
 
@@ -398,7 +363,7 @@ describe(
 
           cy.findByTestId("sdk-error-container").should(
             "contain.text",
-            "Custom error: Error loading the Embedded Analytics SDK",
+            "Custom error: Error loading the Embedding Analytics SDK",
           );
         });
       });

e2e/test/scenarios/admin/i18n/content-translation/dashboards.cy.spec.ts

@@ -7,10 +7,6 @@ import type {
   DashboardDetails,
   StructuredQuestionDetails,
 } from "e2e/support/helpers";
-import {
-  ORDERS_COUNT_BY_CREATED_AT_AND_PRODUCT_CATEGORY,
-  PRODUCTS_COUNT_BY_CATEGORY_PIE,
-} from "e2e/support/test-visualizer-data";
 import type { DictionaryArray } from "metabase-types/api";
 const { ORDERS, ORDERS_ID } = SAMPLE_DATABASE;
 
@@ -27,169 +23,6 @@ import { uploadTranslationDictionaryViaAPI } from "./helpers/e2e-content-transla
 const { H } = cy;
 
 describe("scenarios > content translation > static embedding > dashboards", () => {
-  describe("values translation", () => {
-    beforeEach(() => {
-      H.restore();
-      cy.signInAsAdmin();
-      H.activateToken("bleeding-edge");
-
-      uploadTranslationDictionaryViaAPI([
-        { locale: "fr", msgid: "Gadget", msgstr: "Le gadget" },
-        { locale: "fr", msgid: "Doohickey", msgstr: "Le doohickey" },
-        { locale: "fr", msgid: "Gizmo", msgstr: "Le gizmo" },
-        { locale: "fr", msgid: "Widget", msgstr: "Le widget" },
-      ]);
-
-      cy.intercept("POST", "/api/card/*/query").as("cardQuery");
-      cy.intercept("GET", "/api/embed/dashboard/*").as("dashboard");
-
-      cy.signInAsAdmin();
-
-      H.createQuestion(ORDERS_COUNT_BY_CREATED_AT_AND_PRODUCT_CATEGORY, {
-        idAlias: "productsCountByCreatedAtQuestionId",
-        wrapId: true,
-      });
-
-      H.createQuestion(
-        {
-          ...PRODUCTS_COUNT_BY_CATEGORY_PIE,
-          visualization_settings: {
-            "pie.rows": [
-              ["Widget", "#69C8C8"],
-              ["Gadget", "#C7EAEA"],
-              ["Gizmo", "#98D9D9"],
-              ["Doohickey", "#F3F3F4"],
-            ].map(([key, color]) => ({
-              key,
-              name: key,
-              originalName: key,
-              color,
-              defaultColor: false,
-              enabled: true,
-              hidden: false,
-              isOther: false,
-            })),
-          },
-        },
-        {
-          idAlias: "productsCountByCategoryPieQuestionId",
-          wrapId: true,
-        },
-      );
-    });
-
-    it("should assign the proper colors to a pie", () => {
-      H.createDashboard({
-        name: "the_dashboard",
-      }).then(({ body: { id: dashboardId } }) => {
-        H.visitDashboard(dashboardId);
-        H.editDashboard();
-        H.openQuestionsSidebar();
-
-        H.sidebar().findByText(PRODUCTS_COUNT_BY_CATEGORY_PIE.name).click();
-        H.saveDashboard();
-
-        H.openStaticEmbeddingModal({
-          acceptTerms: false,
-        });
-        H.publishChanges("dashboard", () => {});
-
-        H.visitEmbeddedPage(
-          {
-            resource: { dashboard: dashboardId as number },
-            params: {},
-          },
-          {
-            additionalHashOptions: {
-              locale: "fr",
-            },
-          },
-        );
-
-        cy.wait("@dashboard");
-        cy.wait("@cardQuery");
-
-        H.getDashboardCard(0).within(() => {
-          cy.findAllByText("Le gadget").should("exist");
-          cy.findAllByText("Le doohickey").should("exist");
-          cy.findAllByText("Le gizmo").should("exist");
-          cy.findAllByText("Le widget").should("exist");
-
-          // Verify colors
-          cy.findByTestId("chart-legend").within(() => {
-            cy.get("button [color]").then(($elements) => {
-              const actualColors = Array.from($elements).map((el) =>
-                el.getAttribute("color"),
-              );
-              expect(actualColors).to.deep.equal([
-                "#69C8C8",
-                "#C7EAEA",
-                "#98D9D9",
-                "#F3F3F4",
-              ]);
-            });
-          });
-        });
-      });
-    });
-
-    it("should translate static embedding dashboard values on visualizer cards (metabase#62373)", () => {
-      H.visitDashboard(ORDERS_DASHBOARD_ID);
-
-      H.editDashboard();
-      H.removeDashboardCard(0);
-      H.openQuestionsSidebar();
-
-      // Add the regular question
-      H.sidebar()
-        .findByText(ORDERS_COUNT_BY_CREATED_AT_AND_PRODUCT_CATEGORY.name)
-        .click();
-
-      // Add the visualizer question
-      H.clickVisualizeAnotherWay(
-        ORDERS_COUNT_BY_CREATED_AT_AND_PRODUCT_CATEGORY.name,
-      );
-      H.selectVisualization("bar");
-      H.saveDashcardVisualizerModal({ mode: "create" });
-      H.saveDashboard();
-
-      H.openStaticEmbeddingModal({
-        acceptTerms: false,
-      });
-      H.publishChanges("dashboard", () => {});
-
-      H.visitEmbeddedPage(
-        {
-          resource: { dashboard: ORDERS_DASHBOARD_ID as number },
-          params: {},
-        },
-        {
-          additionalHashOptions: {
-            locale: "fr",
-          },
-        },
-      );
-
-      cy.wait("@dashboard");
-      cy.wait("@cardQuery");
-      cy.wait("@cardQuery");
-
-      H.getDashboardCard(0).within(() => {
-        cy.findAllByText("Le gadget").should("exist");
-        cy.findAllByText("Le doohickey").should("exist");
-        cy.findAllByText("Le gizmo").should("exist");
-        cy.findAllByText("Le widget").should("exist");
-      });
-
-      H.getDashboardCard(1).within(() => {
-        cy.findAllByText("Le gadget").should("exist");
-        cy.findAllByText("Le doohickey").should("exist");
-        cy.findAllByText("Le gizmo").should("exist");
-        cy.findAllByText("Le widget").should("exist");
-      });
-    });
-  });
-
   describe("measure names", () => {
     before(() => {
       cy.intercept("POST", "/api/ee/content-translation/upload-dictionary").as(

e2e/test/scenarios/admin/transforms.cy.spec.ts

@@ -1327,7 +1327,7 @@ describe("scenarios > admin > transforms > jobs", () => {
   });
 
   describe("tags", () => {
-    it("should be able to add and remove tags", { tags: "@flaky" }, () => {
+    it("should be able to add and remove tags", () => {
       H.createTransformJob({ name: "New job" }, { visitTransformJob: true });
       getTagsInput().click();
 

e2e/test/scenarios/custom-column/custom-column-reproductions.cy.spec.js

@@ -2495,67 +2495,3 @@ describe("issue 62987", () => {
     });
   });
 });
-
-describe("issue 63180", () => {
-  beforeEach(() => {
-    H.restore();
-    cy.signInAsNormalUser();
-    H.createQuestion(
-      {
-        query: {
-          "source-table": ORDERS_ID,
-          expressions: {
-            Foo: ["+", 1, 2],
-          },
-        },
-      },
-      { visitQuestion: true },
-    );
-
-    H.openNotebook();
-  });
-
-  it("should not be possible to close the custom expression editor when creating a new expression from a combine or extract shortcut (metabase#63180)", () => {
-    function testCombineColumns() {
-      H.getNotebookStep("expression").icon("add").click();
-      H.expressionEditorWidget().within(() => {
-        cy.findByText("Combine columns").click();
-        cy.button("Done").scrollIntoView().click();
-      });
-
-      cy.log("clicking outside the editor should not close it");
-      H.getNotebookStep("data").click();
-      H.expressionEditorWidget().should("be.visible");
-      H.modal().should("not.exist");
-
-      cy.log("clearing the expression should allow clicking outside to work");
-      H.CustomExpressionEditor.clear();
-      H.getNotebookStep("data").click();
-      H.expressionEditorWidget().should("not.exist");
-      H.modal().should("not.exist");
-    }
-
-    function testExtractColumns() {
-      H.getNotebookStep("expression").icon("add").click();
-      H.expressionEditorWidget().within(() => {
-        cy.findByText("Extract columns").click();
-        cy.findByText("Email").click();
-        cy.findByText("Domain").click();
-      });
-
-      cy.log("clicking outside the editor should not close it");
-      H.getNotebookStep("data").click();
-      H.expressionEditorWidget().should("be.visible");
-      H.modal().should("not.exist");
-
-      cy.log("clearing the expression should allow clicking outside to work");
-      H.CustomExpressionEditor.clear();
-      H.getNotebookStep("data").click();
-      H.expressionEditorWidget().should("not.exist");
-      H.modal().should("not.exist");
-    }
-
-    testCombineColumns();
-    testExtractColumns();
-  });
-});

e2e/test/scenarios/dashboard/dashboard-reproductions.cy.spec.js

@@ -2180,28 +2180,3 @@ describe("issue 53370", () => {
     //
   });
 });
-
-describe("issue 63176", () => {
-  beforeEach(() => {
-    H.restore();
-    cy.signInAsNormalUser();
-  });
-
-  it("should not be possible to save a dashboard with an empty name and the correct error should be displayed (metabase#63176)", () => {
-    cy.visit("/");
-    H.newButton().click();
-    H.popover().findByText("Dashboard").click();
-    H.modal().within(() => {
-      cy.findByPlaceholderText("What is the name of your dashboard?").type(" ");
-      cy.button("Create").click();
-
-      cy.findByText("value must be a non-blank string.").should("be.visible");
-      cy.findByPlaceholderText("What is the name of your dashboard?").should(
-        "have.attr",
-        "aria-invalid",
-        "true",
-      );
-      cy.button("Failed").should("be.visible");
-    });
-  });
-});

e2e/test/scenarios/documents/documents.cy.spec.ts

@@ -144,23 +144,16 @@ H.describeWithSnowplowEE("documents", () => {
                     text: "Lorem Ipsum and some more words",
                   },
                 ],
-                attrs: {
-                  _id: "1",
-                },
               },
               {
                 type: "cardEmbed",
                 attrs: {
                   id: ORDERS_QUESTION_ID,
                   name: null,
-                  _id: "2",
                 },
               },
               {
                 type: "paragraph",
-                attrs: {
-                  _id: "3",
-                },
               },
             ],
             type: "doc",

e2e/test/scenarios/table-editing/table-editing.cy.spec.ts

@@ -452,10 +452,6 @@ describe("scenarios > table-editing", () => {
       cy.findAllByTestId("row-select-checkbox").eq(15).click();
       cy.findAllByTestId("row-select-checkbox").eq(16).click();
 
-      cy.log("should not show edit icon when rows are selected");
-      cy.findByTestId("row-edit-icon").should("not.exist");
-
-      cy.log("should bulk delete rows");
       cy.findByTestId("toast-card").findByText("Delete").click();
 
       H.modal().within(() => {
@@ -471,9 +467,6 @@ describe("scenarios > table-editing", () => {
       cy.findByTestId("toast-card").should("not.exist");
 
       H.undoToast().findByText("Successfully deleted").should("be.visible");
-
-      cy.log("should show edit icon when no rows are selected");
-      cy.findAllByTestId("row-edit-icon").should("exist");
     });
   });
 });

e2e/test/scenarios/visualizations-charts/visualizations-charts-reproductions.cy.spec.ts

@@ -495,39 +495,3 @@ describe("issue 54755", () => {
     cy.findByTestId("visualization-placeholder").should("be.visible");
   });
 });
-
-describe("issue 63026", () => {
-  beforeEach(() => {
-    H.restore();
-    cy.signInAsNormalUser();
-  });
-
-  it("should show tooltips with reasonable width for pie charts with long text labels (metabase#63026)", () => {
-    const query = `select '${"a".repeat(1000)}' as category, 45 as count
-union all select 'Short name', 25 as count
-union all select 'Medium length category', 30 as count`;
-
-    H.visitQuestionAdhoc({
-      display: "pie",
-      dataset_query: {
-        type: "native",
-        native: {
-          query,
-        },
-        database: SAMPLE_DB_ID,
-      },
-      visualization_settings: {
-        "pie.show_labels": true,
-      },
-    });
-
-    H.chartPathWithFillColor("#88BF4D").trigger("mousemove");
-
-    cy.get("[data-testid='echarts-tooltip']")
-      .should("be.visible")
-      .then(($tooltip) => {
-        const width = $tooltip.width();
-        expect(width).to.be.lte(550);
-      });
-  });
-});

e2e/test/scenarios/visualizations-tabular/object_detail.cy.spec.js

@@ -624,7 +624,9 @@ describe("scenarios > question > object details", { tags: "@slow" }, () => {
         const expectedUrl = `http://localhost:4000/table/${PEOPLE_ID}/detail/1`;
 
         cy.findByLabelText("Copy link to this record").click();
-        H.readClipboard().should("equal", expectedUrl);
+        cy.window()
+          .then((window) => window.navigator.clipboard.readText())
+          .should("equal", expectedUrl);
 
         cy.findByLabelText("Open in full page").click();
         cy.location("href").should("eq", expectedUrl);
@@ -722,7 +724,9 @@ describe("scenarios > question > object details", { tags: "@slow" }, () => {
           const expectedUrl = `http://localhost:4000/model/${slug}/detail/1`;
 
           cy.findByLabelText("Copy link to this record").click();
-          H.readClipboard().should("equal", expectedUrl);
+          cy.window()
+            .then((window) => window.navigator.clipboard.readText())
+            .should("equal", expectedUrl);
 
           cy.findByLabelText("Open in full page").click();
           cy.location("href").should("eq", expectedUrl);

e2e/test/scenarios/visualizations-tabular/table.cy.spec.js

@@ -126,14 +126,15 @@ describe("scenarios > visualizations > table", () => {
 
     // Copy formatted content with Cmd+C
     cy.realPress(["Meta", "c"]);
-    H.readClipboard().should("equal", "39.72		February 11, 2025, 9:40 PM");
+    cy.window()
+      .then((win) => win.navigator.clipboard.readText())
+      .should("equal", "39.72		February 11, 2025, 9:40 PM");
 
     // Copy unformatted content with Shift+Cmd+C
     cy.realPress(["Shift", "Meta", "c"]);
-    H.readClipboard().should(
-      "equal",
-      "39.718145389078366	null	2025-02-11T21:40:27.892-08:00",
-    );
+    cy.window()
+      .then((win) => win.navigator.clipboard.readText())
+      .should("equal", "39.718145389078366	null	2025-02-11T21:40:27.892-08:00");
 
     // Escape to clear selection
     cy.realPress("Escape");

enterprise/backend/src/metabase_enterprise/api/routes.clj

@@ -15,7 +15,6 @@
    [metabase-enterprise.audit-app.api.routes]
    [metabase-enterprise.billing.api.routes]
    [metabase-enterprise.cloud-add-ons.api]
-   [metabase-enterprise.comments.api]
    [metabase-enterprise.content-translation.routes]
    [metabase-enterprise.content-verification.api.routes]
    [metabase-enterprise.database-replication.api :as database-replication.api]
@@ -94,7 +93,6 @@
    "/billing"                      metabase-enterprise.billing.api.routes/routes
    "/content-translation"          (premium-handler metabase-enterprise.content-translation.routes/routes :content-translation)
    "/cloud-add-ons"                metabase-enterprise.cloud-add-ons.api/routes
-   "/comment"                      (premium-handler metabase-enterprise.comments.api/routes :documents)
    "/database-replication"         (-> database-replication.api/routes ;; database-replication requires all these features.
                                        (premium-handler :attached-dwh)
                                        (premium-handler :etl-connections)

enterprise/backend/src/metabase_enterprise/comments/api.clj

@@ -1,242 +0,0 @@
-(ns metabase-enterprise.comments.api
-  "`/api/ee/comment/` routes"
-  (:require
-   [honey.sql.helpers :as sql.helpers]
-   [metabase-enterprise.comments.models.comment :as comment]
-   [metabase-enterprise.comments.models.comment-reaction :as comment-reaction]
-   [metabase.api.common :as api]
-   [metabase.api.macros :as api.macros]
-   [metabase.api.routes.common :refer [+auth]]
-   [metabase.events.core :as events]
-   [metabase.request.core :as request]
-   [metabase.users.api :as api.user]
-   [metabase.util :as u]
-   [metabase.util.i18n :refer [deferred-tru]]
-   [metabase.util.malli :as mu]
-   [metabase.util.malli.schema :as ms]
-   [toucan2.core :as t2]))
-
-(def ^:private TYPE->MODEL
-  {"document" :model/Document})
-
-(defn- entity-archived?
-  "Check if the target entity is archived"
-  [entity]
-  (case (t2/model entity)
-    :model/Document (:archived entity)))
-
-(defn- urlpath-for
-  "Generate an URL to an entity"
-  [entity]
-  (case (t2/model entity)
-    :model/Document (str "/document/" (:id entity))))
-
-(defn- content->str [content]
-  (when content
-    (or (:text content)
-        (pr-str content))))
-
-;;; schemas
-
-(def CommentContent
-  "Validation for comment content - expects JSON"
-  (mu/with-api-error-message
-   [:and
-    {:error/message "Comment content must be valid JSON"
-     :json-schema   {:type "object"}}
-    [:map]]
-   (deferred-tru "Comment content must be valid JSON.")))
-
-(def CreateComment
-  "Schema for creating a new comment"
-  [:map
-   [:target_type [:enum "document"]]
-   [:target_id   ms/PositiveInt]
-   [:content     CommentContent]
-   [:child_target_id {:optional true} [:maybe :string]]
-   [:parent_comment_id {:optional true} [:maybe ms/PositiveInt]]])
-
-(def UpdateComment
-  "Schema for updating a comment"
-  [:map
-   [:content {:optional true} CommentContent]
-   [:is_resolved {:optional true} :boolean]])
-
-;;; routes
-
-(defn- render-comments
-  "Process comments to prepare them for the world:
-  - remove/scrub deleted comments
-  - render reactions with limited amount of users"
-  [comments]
-  (let [has-replies?     (into #{} (map :parent_comment_id comments))
-        delete-comment   (fn [c]
-                           (cond
-                             (not (:deleted_at c))        c
-                             (not (has-replies? (:id c))) nil
-                             ;; if comment has replies, we scrub it
-                             :else                        (assoc c :content {})))
-        render-reactions (fn [c]
-                           (update c :reactions (fn [emoji-map]
-                                                  (vec (for [[emoji users] emoji-map]
-                                                         {:emoji emoji
-                                                          :count (count users)
-                                                          :users (take 10 users)})))))]
-    (into [] (comp (map render-reactions) (keep delete-comment)) comments)))
-
-(api.macros/defendpoint :get "/"
-  "Get comments for an entity"
-  [_route-params
-   {:keys [target_type target_id]} :- [:map
-                                       [:target_type [:enum "document"]]
-                                       [:target_id ms/PositiveInt]]]
-  (let [_entity  (api/read-check (TYPE->MODEL target_type) target_id)
-        comments (-> (t2/select :model/Comment
-                                {:where    [:and
-                                            [:= :target_type target_type]
-                                            [:= :target_id target_id]]
-                                 :order-by [[:created_at :asc]]})
-                     (t2/hydrate :creator :reactions))]
-    {:comments (render-comments comments)}))
-
-(api.macros/defendpoint :post "/"
-  "Create a new comment"
-  [_route-params
-   _query-params
-   {:keys [target_type target_id child_target_id parent_comment_id content]} :- CreateComment]
-  (let [entity     (-> (api/read-check (TYPE->MODEL target_type) target_id)
-                       (u/prog1 (api/check-400 (not (entity-archived? <>))
-                                               "Cannot comment on archived entities")))
-        ;; If this is a reply, validate the parent comment exists and belongs to same entity
-        parent     (when parent_comment_id
-                     (-> (api/check-404 (t2/select-one :model/Comment :id parent_comment_id))
-                         (u/prog1 (api/check-400 (and (= (:target_type <>) target_type)
-                                                      (= (:target_id <>) target_id)
-                                                      (= (:child_target_id <>) child_target_id))
-                                                 "Parent comment doesn't belong to the same entity"))
-                         (t2/hydrate :creator)))
-        comment    (-> (t2/insert-returning-instance! :model/Comment
-                                                      {:target_type       target_type
-                                                       :target_id         target_id
-                                                       :child_target_id   child_target_id
-                                                       :parent_comment_id parent_comment_id
-                                                       :content           content
-                                                       :creator_id        api/*current-user-id*})
-                       (t2/hydrate :creator)
-                       ;; New comments always have empty reactions map
-                       (assoc :reactions []))
-        clause     (if parent_comment_id
-                     {:where [:in :id {:from   [:comment]
-                                       :select [:creator_id]
-                                       :where  [:or
-                                                [:= :id parent_comment_id]
-                                                [:= :parent_comment_id parent_comment_id]]}]}
-                     ;; FIXME: add dispatch on different entity types
-                     {:where [:= :id (:creator_id entity)]})
-        mentions   (comment/mentions content)
-        recipients (-> (t2/select-fn-set :email [:model/User :email]
-                                         (cond-> clause
-                                           (seq mentions) (sql.helpers/where :or [:in :id mentions])))
-                       (disj (:email @api/*current-user*)))
-        payload    {:entity_type    target_type
-                    :entity_title   (:name entity)
-                    :comment_href   (comment/url entity comment)
-                    :document_href  (urlpath-for entity)
-                    :created_at     (:created_at comment)
-                    :author         (:common_name (:creator comment))
-                    :comment        (content->str (:content comment))
-                    :parent_author  (:common_name (:creator parent))
-                    :parent_comment (content->str (:content parent))}]
-    (doseq [email recipients]
-      (events/publish-event! :event/comment-created (assoc payload :email email)))
-    comment))
-
-(api.macros/defendpoint :put "/:comment-id"
-  "Update a comment"
-  [{:keys [comment-id]} :- [:map [:comment-id ms/PositiveInt]]
-   _query-params
-   {:keys [content is_resolved]} :- UpdateComment]
-  (let [comment (api/check-404 (t2/select-one :model/Comment :id comment-id))
-        entity  (-> (api/read-check (TYPE->MODEL (:target_type comment)) (:target_id comment))
-                    (u/prog1 (api/check-400 (not (entity-archived? <>))
-                                            "Cannot edit comments on archived entities")))]
-
-    (when content
-      ;; Cannot edit content of deleted comments
-      (api/check-400 (not (:deleted_at comment))
-                     "Cannot edit content of a deleted comment")
-      ;; Only creator or admin can edit comment content
-      (api/check-403 (or (= (:creator_id comment) api/*current-user-id*)
-                         (:is_superuser @api/*current-user*))))
-
-    (when (some? is_resolved)
-      ;; Anyone with write permission to target entity can resolve/unresolve
-      (api/write-check entity))
-
-    (when-let [updates (-> {:content content :is_resolved is_resolved}
-                           u/remove-nils
-                           not-empty)]
-      (t2/update! :model/Comment comment-id updates))
-
-    (-> (t2/select-one :model/Comment :id comment-id)
-        (t2/hydrate :creator :reactions))))
-
-(api.macros/defendpoint :delete "/:comment-id"
-  "Soft delete a comment"
-  [{:keys [comment-id]} :- [:map [:comment-id ms/PositiveInt]]
-   _query-params]
-  (let [comment (api/check-404 (t2/select-one :model/Comment :id comment-id))]
-
-    (-> (api/read-check (TYPE->MODEL (:target_type comment)) (:target_id comment))
-        (u/prog1 (api/check-400 (not (entity-archived? <>))
-                                "Cannot delete comments on archived entities")))
-
-    ;; Only creator or admin can delete comments
-    (api/check-403 (or (= (:creator_id comment) api/*current-user-id*)
-                       (:is_superuser @api/*current-user*)))
-    (api/check-400 (not (:deleted_at comment)) "Comment is already deleted")
-
-    ;; Soft delete the comment
-    (t2/update! :model/Comment comment-id {:deleted_at [:now]})
-
-    ;; Return 204 No Content
-    api/generic-204-no-content))
-
-(api.macros/defendpoint :post "/:comment-id/reaction"
-  "Toggle a reaction on a comment"
-  [{:keys [comment-id]} :- [:map [:comment-id ms/PositiveInt]]
-   _query-params
-   {:keys [emoji]} :- [:map [:emoji [:string {:min 1 :max 10}]]]]
-  (let [comment (api/check-404 (t2/select-one :model/Comment :id comment-id))]
-    (api/check-400 (not (:deleted_at comment))
-                   "Cannot react to deleted comments")
-
-    (-> (api/read-check (TYPE->MODEL (:target_type comment)) (:target_id comment))
-        (u/prog1 (api/check-400 (not (entity-archived? <>))
-                                "Cannot react to comments on archived entities")))
-
-    (comment-reaction/toggle-reaction comment-id api/*current-user-id* emoji)))
-
-(api.macros/defendpoint :get "/mentions"
-  "Get a list of entities suitable for mentions. NOTE: only users for now."
-  [_route-params _query-params]
-  (let [clauses (api.user/user-clauses nil nil nil nil)]
-    ;; returns nothing while we're trying to figure out how do we deal with sandboxes and tenants etc
-    ;; do not forget to uncomment tests (both api and e2e)
-    {:data   []
-     #_(->> (t2/select [:model/User :id :first_name :last_name]
-                       (-> clauses
-                           (sql.helpers/order-by [:%lower.first_name :asc]
-                                                 [:%lower.last_name :asc]
-                                                 [:id :asc])))
-            (mapv #(assoc % :model "user")))
-     :total  (:count (t2/query-one
-                      (merge {:select [[[:count [:distinct :core_user.id]] :count]]
-                              :from   :core_user}
-                             (api.user/filter-clauses-without-paging clauses))))
-     :limit  (request/limit)
-     :offset (request/offset)}))
-
-(def ^{:arglists '([request respond raise])} routes
-  "`/api/ee/comment/` routes."
-  (api.macros/ns-handler *ns* +auth))

enterprise/backend/src/metabase_enterprise/comments/init.clj

@@ -1,4 +0,0 @@
-(ns metabase-enterprise.comments.init
-  (:require
-   [metabase-enterprise.comments.models.comment]
-   [metabase-enterprise.comments.models.comment-reaction]))

enterprise/backend/src/metabase_enterprise/comments/models/comment.clj

@@ -1,63 +0,0 @@
-(ns metabase-enterprise.comments.models.comment
-  (:require
-   [metabase-enterprise.comments.models.comment-reaction :as comment-reaction]
-   [metabase.api.common :as api]
-   [metabase.models.interface :as mi]
-   [methodical.core :as methodical]
-   [toucan2.core :as t2]))
-
-(methodical/defmethod t2/table-name :model/Comment [_model] :comment)
-
-(doto :model/Comment
-  (derive :metabase/model)
-  (derive :hook/timestamped?))
-
-(t2/deftransforms :model/Comment
-  {:content mi/transform-json})
-
-(methodical/defmethod t2/batched-hydrate [:model/Comment :creator]
-  "Hydrate the creator (user) of a comment based on the creator_id."
-  [_model k comments]
-  (mi/instances-with-hydrated-data
-   comments k
-   #(t2/select-pk->fn identity [:model/User :id :email :first_name :last_name]
-                      :id (keep :creator_id comments))
-   :creator_id
-   {:default {}}))
-
-(methodical/defmethod t2/batched-hydrate [:model/Comment :reactions]
-  "Hydrate the creator (user) of a comment based on the creator_id."
-  [_model k comments]
-  (mi/instances-with-hydrated-data
-   comments k
-   #(->> comments
-         (remove :deleted_at)
-         (map :id)
-         (comment-reaction/reactions-for-comments api/*current-user-id*))
-   :id
-   {:default {}}))
-
-;;;
-
-(defn url
-  "Generate an URL to an entity anchored to a comment"
-  [entity comment]
-  (case (t2/model entity)
-    :model/Document (if (:child_target_id comment)
-                      (format "/document/%s/comments/%s#comment-%s"
-                              (:id entity)
-                              (:child_target_id comment)
-                              (:id comment))
-                      ;; NOTE: not used at the time of writing the code, but given that child_target_id is optional I
-                      ;; feel the need to have this clause
-                      (format "/document/%s#comment-%s"
-                              (:id entity)
-                              (:id comment)))))
-
-(defn mentions
-  "Find mentioned users inside of a comment content"
-  [content]
-  (->> (tree-seq :content :content content)
-       (filter #(and (= "smartLink" (-> % :type))
-                     (= "user" (-> % :attrs :model))))
-       (mapv #(-> % :attrs :entityId))))

enterprise/backend/src/metabase_enterprise/comments/models/comment_reaction.clj

@@ -1,84 +0,0 @@
-(ns metabase-enterprise.comments.models.comment-reaction
-  "Model for comment reactions (emoji reactions on comments)"
-  (:require
-   [metabase.models.interface :as mi]
-   [methodical.core :as methodical]
-   [toucan2.core :as t2]))
-
-(methodical/defmethod t2/table-name :model/CommentReaction [_model] :comment_reaction)
-
-(doto :model/CommentReaction
-  (derive :metabase/model))
-
-(methodical/defmethod t2/batched-hydrate [:model/CommentReaction :user]
-  [_model k reactions]
-  (mi/instances-with-hydrated-data
-   reactions k
-   #(t2/select-pk->fn identity [:model/User :id :email :first_name :last_name]
-                      :id (map :user_id reactions))
-   :user_id {:default {}}))
-
-;;; Helpers
-
-(defn reaction-exists?
-  "Check if a reaction already exists for a given comment, user, and emoji"
-  [comment-id user-id emoji]
-  (t2/exists? :model/CommentReaction
-              :comment_id comment-id
-              :user_id user-id
-              :emoji emoji))
-
-(defn create-reaction!
-  "Create a new reaction"
-  [comment-id user-id emoji]
-  (t2/insert! :model/CommentReaction
-              {:comment_id comment-id
-               :user_id user-id
-               :emoji emoji}))
-
-(defn delete-reaction!
-  "Delete a specific reaction"
-  [comment-id user-id emoji]
-  (t2/delete! :model/CommentReaction
-              :comment_id comment-id
-              :user_id user-id
-              :emoji emoji))
-
-(defn toggle-reaction
-  "Toggle a reaction - add it if it doesn't exist, remove it if it does"
-  [comment-id user-id emoji]
-  (if (reaction-exists? comment-id user-id emoji)
-    (do
-      (delete-reaction! comment-id user-id emoji)
-      {:reacted false})
-    (do
-      (create-reaction! comment-id user-id emoji)
-      {:reacted true})))
-
-(defn- format-user [{:keys [id first_name last_name]}]
-  {:id   id
-   :name (if (and first_name last_name)
-           (str first_name " " last_name)
-           (or first_name last_name "Unknown User"))})
-
-(defn reactions-for-comments
-  "Get all reactions for a list of comment IDs, grouped and formatted for API response.
-
-   Returns a map of `{comment-id {emoji [user1 user2...]}}."
-  [current-user-id comment-ids]
-  (when (seq comment-ids)
-    (let [reactions   (-> (t2/select :model/CommentReaction
-                                     {:where    [:in :comment_id comment-ids]
-                                      :order-by [[:comment_id :asc] [:created_at :asc] [:emoji :asc]]})
-                          (t2/hydrate :user))
-
-          ;; first user comes first if they reacted
-          first-or-last (fn [acc user]
-                          (if (= (:id user) current-user-id)
-                            (into [user] acc)
-                            (conj (or acc []) user)))]
-
-      (reduce (fn [acc reaction]
-                (let [user (format-user (:user reaction))]
-                  (update-in acc [(:comment_id reaction) (:emoji reaction)] first-or-last user)))
-              {} reactions))))

enterprise/backend/src/metabase_enterprise/core/init.clj

@@ -10,7 +10,6 @@
    [metabase-enterprise.advanced-config.init]
    [metabase-enterprise.audit-app.init]
    [metabase-enterprise.cache.init]
-   [metabase-enterprise.comments.init]
    [metabase-enterprise.database-replication.init]
    [metabase-enterprise.documents.init]
    [metabase-enterprise.gsheets.init]

enterprise/backend/src/metabase_enterprise/metabot_v3/api.clj

@@ -81,6 +81,7 @@
   "Send a chat message to the LLM via the AI Proxy."
   [_route-params
    _query-params
+
    body :- [:map
             [:profile_id {:optional true} :string]
             [:metabot_id {:optional true} :string]

enterprise/backend/src/metabase_enterprise/metabot_v3/api/metabot.clj

@@ -3,19 +3,66 @@
   (:require
    [medley.core :as m]
    [metabase-enterprise.metabot-v3.client :as metabot-v3.client]
-   [metabase-enterprise.metabot-v3.config :as metabot-v3.config]
    [metabase-enterprise.metabot-v3.dummy-tools :as metabot-v3.dummy-tools]
    [metabase-enterprise.metabot-v3.tools.util :as metabot-v3.tools.u]
    [metabase.api.common :as api]
    [metabase.api.macros :as api.macros]
    [metabase.api.routes.common :refer [+auth]]
    [metabase.app-db.core :as mdb]
+   [metabase.collections.models.collection :as collection]
    [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]
-   [metabase.premium-features.core :as premium-features]
    [metabase.request.core :as request]
-   [metabase.util.i18n :refer [tru]]
    [toucan2.core :as t2]))
 
+;; TODO: Eventually this should be paged but since we are just going to hardcode two models for now
+;; lets not
+(api.macros/defendpoint :get "/"
+  "List configured metabot instances"
+  []
+  (api/check-superuser)
+  {:items (t2/select :model/Metabot {:order-by [[:name :asc]]})})
+
+(api.macros/defendpoint :get "/:id"
+  "Retrieve one metabot instance"
+  [{:keys [id]} :- [:map [:id pos-int?]]]
+  (api/check-superuser)
+  (api/check-404 (t2/select-one :model/Metabot :id id)))
+
+(def ^:private default-entities-page-size 200)
+
+(api.macros/defendpoint :get "/:id/entities"
+  "List the entities this metabot has access to"
+  [{:keys [id]} :- [:map [:id pos-int?]]]
+  (api/check-superuser)
+  (api/check-404 (t2/exists? :model/Metabot :id id))
+  (let [limit (or (request/limit) default-entities-page-size)
+        offset (or (request/offset) 0)
+        root-collection (collection/root-collection-with-ui-details nil)
+        entities (t2/select [:model/MetabotEntity
+                             :model_id
+                             :model
+                             [[:coalesce :card.name :collection.name] :name]
+                             :created_at
+                             [:parent_collection.id :collection_id]
+                             [:parent_collection.name :collection_name]]
+                            {:left-join [[:report_card :card] [:and [:in :model [[:inline "dataset"] [:inline "metric"]]]  [:= :model_id :card.id]]
+                                         [:collection :collection] [:and [:= :model [:inline "collection"]]  [:= :model_id :collection.id]]
+                                         ;; TODO: How to join parent collections from the location column?
+                                         [:collection :parent_collection] [:= :card.collection_id :parent_collection.id]]
+                             :where [:= :metabot_id id]
+                             :limit limit
+                             :offset offset
+                             :order-by [[:name :asc]]})
+        total (t2/count :model/MetabotEntity :metabot_id id)]
+    {:items (for [{:keys [collection_id model] :as entity} entities]
+              (cond-> entity
+                (and (nil? collection_id)
+                     (not= :collection model)) (assoc :collection_id (:id root-collection)
+                                                      :collection_name (:name root-collection))))
+     :total total
+     :limit limit
+     :offset offset}))
+
 (defn- column-input
   [answer-source-column]
   (some-> answer-source-column (select-keys [:name :type :description :table-reference])))
@@ -38,14 +85,14 @@
       (update :fields #(map column-input %))))
 
 (defn- generate-sample-prompts
-  [metabot-id]
+  [metabot-entity-ids]
   (lib.metadata.jvm/with-metadata-provider-cache
-    (let [cards (metabot-v3.tools.u/get-metrics-and-models metabot-id)
+    (let [scope (metabot-v3.tools.u/metabot-scope metabot-entity-ids)
           {metrics :metric, models :model}
-          (->> (for [[[card-type database-id] group-cards] (group-by (juxt :type :database_id) cards)
+          (->> (for [[[card-type database-id] cards] (group-by (juxt :type :database_id) (keys scope))
                      detail (map (fn [detail card] (assoc detail ::origin card))
-                                 (metabot-v3.dummy-tools/cards-details card-type database-id group-cards nil)
-                                 group-cards)]
+                                 (metabot-v3.dummy-tools/cards-details card-type database-id cards nil)
+                                 cards)]
                  detail)
                (group-by :type))
           metric-inputs (map metric-input metrics)
@@ -53,9 +100,9 @@
           {:keys [table_questions metric_questions]}
           (metabot-v3.client/generate-example-questions {:metrics metric-inputs, :tables model-inputs})
           ->prompt (fn [{:keys [questions]} {::keys [origin]}]
-                     (let [base {:metabot_id metabot-id
-                                 :model      (:type origin)
-                                 :card_id    (:id origin)}]
+                     (let [base {:metabot_entity_id (scope origin)
+                                 :model             (:type origin)
+                                 :card_id           (:id origin)}]
                        (map #(assoc base :prompt %) questions)))
           metric-prompts (mapcat ->prompt metric_questions metrics)
           model-prompts (mapcat ->prompt table_questions models)]
@@ -64,54 +111,52 @@
       (when (seq model-prompts)
         (t2/insert! :model/MetabotPrompt model-prompts)))))
 
-(defn- delete-all-metabot-prompts
-  [metabot-id]
-  (t2/delete! :model/MetabotPrompt {:where [:= :metabot_id metabot-id]}))
-
-;; TODO: Eventually this should be paged but since we are just going to hardcode two models for now
-;; lets not
-(api.macros/defendpoint :get "/"
-  "List configured metabot instances"
-  []
-  (api/check-superuser)
-  {:items (t2/select :model/Metabot {:order-by [[:name :asc]]})})
-
-(api.macros/defendpoint :get "/:id"
-  "Retrieve one metabot instance"
-  [{:keys [id]} :- [:map [:id pos-int?]]]
-  (api/check-superuser)
-  (api/check-404 (t2/select-one :model/Metabot :id id)))
-
-(api.macros/defendpoint :put "/:id"
-  "Update a metabot instance"
+(api.macros/defendpoint :put "/:id/entities"
+  "Update the entities this metabot has access to"
   [{:keys [id]} :- [:map [:id pos-int?]]
    _query-params
-   metabot-updates :- [:map
-                       [:use_verified_content {:optional true} :boolean]
-                       [:collection_id {:optional true} [:maybe pos-int?]]]]
+   {:keys [items]} :- [:map
+                       [:items [:sequential [:map
+                                             [:id pos-int?]
+                                             [:model [:enum "dataset" "metric" "collection"]]]]]]]
+  (api/check-superuser)
+  (t2/with-transaction [_conn]
+    (api/check-404 (t2/exists? :model/Metabot :id id))
+    (let [new-entity-ids (into []
+                               (keep (fn [{model-id :id model :model}]
+                                       (when-not (t2/exists? :model/MetabotEntity
+                                                             :metabot_id id
+                                                             :model model
+                                                             :model_id model-id)
+                                         (t2/insert-returning-pk! :model/MetabotEntity
+                                                                  {:metabot_id id
+                                                                   :model model
+                                                                   :model_id model-id}))))
+                               items)]
+      (generate-sample-prompts new-entity-ids)))
+  api/generic-204-no-content)
+
+(api.macros/defendpoint :delete ["/:id/entities/:model/:model-id" :model #"dataset|metric|collection"]
+  "Remove an entity from this metabot's access list"
+  [{:keys [id model model-id]} :- [:map
+                                   [:id pos-int?]
+                                   [:model [:enum "dataset" "metric" "collection"]]
+                                   [:model-id pos-int?]]]
   (api/check-superuser)
   (api/check-404 (t2/exists? :model/Metabot :id id))
-  (let [old-metabot (t2/select-one :model/Metabot :id id)]
-    ;; Prevent updating collection_id on the primary metabot instance
-    (when (and (contains? metabot-updates :collection_id)
-               (= (:entity_id old-metabot)
-                  (get-in metabot-v3.config/metabot-config [metabot-v3.config/internal-metabot-id :entity-id])))
-      (api/check-400 false "Cannot update collection_id for the primary metabot instance."))
-    ;; Prevent enabling verified content without the premium feature
-    (when (and (contains? metabot-updates :use_verified_content)
-               (:use_verified_content metabot-updates))
-      (premium-features/assert-has-feature :content-verification (tru "Content verification")))
-    (let [verified-content-changed? (and (contains? metabot-updates :use_verified_content)
-                                         (not= (:use_verified_content old-metabot)
-                                               (:use_verified_content metabot-updates)))
-          collection-changed? (and (contains? metabot-updates :collection_id)
-                                   (not= (:collection_id old-metabot)
-                                         (:collection_id metabot-updates)))]
-      (t2/update! :model/Metabot id metabot-updates)
-      (when (or verified-content-changed? collection-changed?)
-        (delete-all-metabot-prompts id)
-        (generate-sample-prompts id))
-      (t2/select-one :model/Metabot :id id))))
+  (t2/delete! :model/MetabotEntity
+              :metabot_id id
+              :model model
+              :model_id model-id)
+  api/generic-204-no-content)
+
+(defn- delete-all-metabot-prompts
+  [metabot-id]
+  (t2/delete! :model/MetabotPrompt {:where [:exists {:select [:*]
+                                                     :from   [[:metabot_entity :mbe]]
+                                                     :where  [:and
+                                                              [:= :mbe.id :metabot_prompt.metabot_entity_id]
+                                                              [:= :mbe.metabot_id metabot-id]]}]}))
 
 (api.macros/defendpoint :post "/:id/prompt-suggestions/regenerate"
   "Remove any existing prompt suggestions for the Metabot instance with `id` and generate new ones."
@@ -119,8 +164,9 @@
   (api/check-superuser)
   (t2/with-transaction [_conn]
     (api/check-404 (t2/exists? :model/Metabot :id id))
-    (delete-all-metabot-prompts id)
-    (generate-sample-prompts id))
+    (when-let [entity-ids (not-empty (t2/select-pks-vec :model/MetabotEntity :metabot_id id))]
+      (delete-all-metabot-prompts id)
+      (generate-sample-prompts entity-ids)))
   api/generic-204-no-content)
 
 (api.macros/defendpoint :get "/:id/prompt-suggestions"
@@ -134,14 +180,15 @@
         rand-fn (case (mdb/db-type)
                   :postgres :random
                   :rand)
-        base-query (cond-> {:join  [[{:select [:id :name :type]
-                                      :from   [[(metabot-v3.tools.u/metabot-metrics-and-models-query id)
+        base-query (cond-> {:join  [[{:select [:id :name :type :metabot_entity_id]
+                                      :from   [[(metabot-v3.tools.u/metabot-scope-query
+                                                 [:= :mbe.metabot_id id])
                                                 :scope]]}
                                      :card]
                                     [:and
-                                     [:= :card.id :metabot_prompt.card_id]]]
-                            :where [:and
-                                    [:= :metabot_prompt.metabot_id id]]}
+                                     [:= :card.id                :metabot_prompt.card_id]
+                                     [:= :card.metabot_entity_id :metabot_prompt.metabot_entity_id]]]
+                            :where [:and]}
                      model    (update :where conj [:= :card.type model])
                      model_id (update :where conj [:= :card.id model_id]))
         total (t2/count :model/MetabotPrompt base-query)
@@ -181,7 +228,11 @@
   (api/check-superuser)
   (t2/delete! :model/MetabotPrompt {:where [:and
                                             [:= :id prompt-id]
-                                            [:= :metabot_id id]]})
+                                            [:exists {:select [:*]
+                                                      :from   [[:metabot_entity :mbe]]
+                                                      :where  [:and
+                                                               [:= :mbe.id :metabot_prompt.metabot_entity_id]
+                                                               [:= :mbe.metabot_id id]]}]]})
   api/generic-204-no-content)
 
 (def ^{:arglists '([request respond raise])} routes

enterprise/backend/src/metabase_enterprise/metabot_v3/config.clj

@@ -14,9 +14,9 @@
 
 (def metabot-config
   "The name of the collection exposed by the answer-sources tool."
-  {internal-metabot-id {:profile-id "internal"
+  {internal-metabot-id {:profile-id "experimental"
                         :entity-id "metabotmetabotmetabot"}
-   embedded-metabot-id {:profile-id "embedding"
+   embedded-metabot-id {:profile-id "default"
                         :entity-id "embeddedmetabotmetabo"}})
 
 (defn metabot-id->profile-id

enterprise/backend/src/metabase_enterprise/metabot_v3/context.clj

@@ -53,50 +53,42 @@
                                           :api/endpoints)]
     (str "backend:/api/ee/metabot-tools" url)))
 
-(defn- query-for-sql-parsing
-  "Given an item in context, return the query if it is a native query or SQL transform that can have table usage parsed
-  from it, otherwise nil."
-  [item]
-  (when-let [query (case (:type item)
-                     "transform" (-> item :source :query)
-                     "adhoc" (-> item :query)
-                     nil)]
-    (when (and (#{:native "native"} (:type query))
-               (:database query))
-      query)))
+(def ^:private max-database-tables
+  "If the number of tables in the database doesn't exceed this number, we send them all to the agent."
+  100)
 
 (defn- database-tables-for-context
-  "Get database tables formatted for metabot context. Only includes tables used in the query, formatted for API output.
-   Removes duplicate tables by id while preserving first occurrence order."
-  [{:keys [query]}]
-  (try
-    (if query
-      (let [used-tables (table-utils/used-tables query)
-            tables (table-utils/enhanced-database-tables (:database query)
-                                                         {:priority-tables used-tables
-                                                          :all-tables-limit (count used-tables)})]
-        ;; Ensure no duplicate tables (by id) are returned, preserve first occurrence order
-        (reduce (fn [acc t]
-                  (if (some #(= (:id %) (:id t)) acc)
-                    acc
-                    (conj acc t)))
-                []
-                tables))
-      [])
-    (catch Exception e
-      (log/error e "Error getting database tables for context")
-      [])))
+  "Get database tables formatted for metabot context, prioritizing tables used in the query (if provided), then filling up to the limit with most viewed tables."
+  ([database-id] (database-tables-for-context database-id nil))
+  ([database-id {:keys [all-tables-limit query] :or {all-tables-limit max-database-tables}}]
+   (when database-id
+     (try
+       (let [used-tables (if query
+                           (table-utils/used-tables query)
+                           [])
+             used-table-ids (set (map :id used-tables))]
+         (table-utils/database-tables database-id
+                                      {:all-tables-limit all-tables-limit
+                                       :priority-tables used-tables
+                                       :exclude-table-ids used-table-ids}))
+       (catch Exception e
+         (log/error e "Error getting database tables for context")
+         ;; If we can't get table info, just return empty - don't break the context
+         [])))))
 
 (defn- enhance-context-with-schema
-  "Enhance context by adding table schema information for native queries & SQL transforms"
+  "Enhance context by adding table schema information for native queries"
   [context]
   (if-let [user-viewing (get context :user_is_viewing)]
     (let [enhanced-viewing
           (mapv (fn [item]
-                  (if-let [query (query-for-sql-parsing item)]
-                    (if-let [tables (seq (database-tables-for-context {:query query}))]
-                      (assoc item :used_tables tables)
-                      item)
+                  (if (and (#{:native "native"} (get-in item [:query :type]))
+                           (get-in item [:query :database]))
+                    (let [database-id (get-in item [:query :database])
+                          tables (database-tables-for-context database-id {:query (:query item)})]
+                      (if (seq tables)
+                        (assoc item :database_schema tables)
+                        item))
                     item))
                 user-viewing)]
       (assoc context :user_is_viewing enhanced-viewing))

enterprise/backend/src/metabase_enterprise/metabot_v3/dummy_tools.clj

@@ -9,6 +9,7 @@
    [metabase.lib.core :as lib]
    [metabase.lib.metadata :as lib.metadata]
    [metabase.lib.types.isa :as lib.types.isa]
+   [metabase.lib.util :as lib.util]
    [metabase.util :as u]
    [metabase.util.humanization :as u.humanization]
    [metabase.warehouse-schema.models.field-values :as field-values]
@@ -16,16 +17,6 @@
 
 (set! *warn-on-reflection* true)
 
-(defn- verified-review?
-  "Return true if the most recent ModerationReview for the given item id/type is verified."
-  [id item-type]
-  (let [review (t2/select-one [:model/ModerationReview :status]
-                              :moderated_item_id id
-                              :moderated_item_type item-type
-                              :most_recent true
-                              {:order-by [[:id :desc]]})]
-    (= (:status review) "verified")))
-
 (defn get-current-user
   "Get information about the current user."
   []
@@ -43,11 +34,7 @@
   [{:keys [dashboard-id]}]
   (if-let [dashboard (t2/select-one [:model/Dashboard :id :description :name :collection_id] dashboard-id)]
     (do (api/read-check dashboard)
-        {:structured-output
-         (-> dashboard
-             (dissoc :collection_id)
-             (assoc :type :dashboard
-                    :verified (verified-review? dashboard-id "dashboard")))})
+        {:structured-output (-> dashboard (dissoc :collection_id) (assoc :type :dashboard))})
     {:output "dashboard not found"}))
 
 (defn- get-field-values [id->values id]
@@ -63,6 +50,14 @@
       (map #(m/assoc-some % :field-values (some->> % :id (get-field-values id->values))) cols))
     cols))
 
+(defn- add-table-reference
+  [query col]
+  (cond-> col
+    (and (:fk-field-id col)
+         (:table-id col))
+    (assoc :table-reference (-> (lib/display-name query (lib.metadata/field query (:fk-field-id col)))
+                                lib.util/strip-id))))
+
 (defn metric-details
   "Get metric details as returned by tools."
   ([id] (metric-details id nil))
@@ -83,7 +78,7 @@
                       (lib/remove-all-breakouts metric-query))
          visible-cols (when query-needed?
                         (->> (lib/visible-columns base-query)
-                             (map #(metabot-v3.tools.u/add-table-reference base-query %))))
+                             (map #(add-table-reference base-query %))))
          col->index (when query-needed?
                       (into {} (map-indexed (fn [i col] [col i])) visible-cols))
          col-index (when query-needed?
@@ -103,11 +98,10 @@
                                                       default-temporal-breakout
                                                       (col-index default-temporal-breakout)
                                                       field-id-prefix)
-                                                     :field-id))
-              :verified (verified-review? id "card")}
+                                                     :field-id))}
        with-queryable-dimensions?
        (assoc :queryable-dimensions (into []
-                                          (comp (map #(metabot-v3.tools.u/add-table-reference base-query %))
+                                          (comp (map #(add-table-reference base-query %))
                                                 (map #(metabot-v3.tools.u/->result-column
                                                        metric-query % (col-index %) field-id-prefix)))
                                           (->> (lib/filterable-columns base-query)
@@ -135,29 +129,25 @@
         :as   options}]
    (when-let [base (if metadata-provider
                      (lib.metadata/table metadata-provider id)
-                     (metabot-v3.tools.u/get-table id :db_id :description :name :schema))]
+                     (metabot-v3.tools.u/get-table id :db_id :description :name))]
      (let [query-needed? (or with-fields? with-metrics?)
-           db-id (if metadata-provider (:db-id base) (:db_id base))
            mp (when query-needed?
                 (or metadata-provider
-                    (lib.metadata.jvm/application-database-metadata-provider db-id)))
+                    (lib.metadata.jvm/application-database-metadata-provider (:db_id base))))
            table-query (when query-needed?
                          (lib/query mp (lib.metadata/table mp id)))
            cols (when with-fields?
                   (->> (lib/visible-columns table-query)
                        field-values-fn
-                       (map #(metabot-v3.tools.u/add-table-reference table-query %))))
+                       (map #(add-table-reference table-query %))))
            field-id-prefix (when with-fields?
                              (metabot-v3.tools.u/table-field-id-prefix id))]
        (-> {:id id
             :type :table
             :fields (into [] (map-indexed #(metabot-v3.tools.u/->result-column table-query %2 %1 field-id-prefix)) cols)
-            :name (:name base)
-            ;; :display_name should be (lib/display-name table-query), but we want to avoid creating the query if possible
-            :display_name (some->> (:name base)
-                                   (u.humanization/name->human-readable-name :simple))
-            :database_id db-id
-            :database_schema (:schema base)}
+            ;; :name should be (lib/display-name table-query), but we want to avoid creating the query if possible
+            :name (some->> (:name base)
+                           (u.humanization/name->human-readable-name :simple))}
            (m/assoc-some :description (:description base)
                          :metrics (when with-metrics?
                                     (not-empty (mapv #(convert-metric % mp options)
@@ -191,18 +181,13 @@
          cols (when with-fields?
                 (->> (lib/visible-columns card-query)
                      field-values-fn
-                     (map #(metabot-v3.tools.u/add-table-reference card-query %))))
+                     (map #(add-table-reference card-query %))))
          field-id-prefix (metabot-v3.tools.u/card-field-id-prefix id)]
      (-> {:id id
           :type card-type
           :fields (into [] (map-indexed #(metabot-v3.tools.u/->result-column card-query %2 %1 field-id-prefix)) cols)
           :name (:name base)
-          :display_name (some->> (:name base)
-                                 (u.humanization/name->human-readable-name :simple))
-          :database_id (:database_id base)
-          :queryable-foreign-key-tables []
-          :verified (verified-review? id "card")}
-
+          :queryable-foreign-key-tables []}
          (m/assoc-some :description (:description base)
                        :metrics (when with-metrics?
                                   (not-empty (mapv #(convert-metric % metadata-provider options)
@@ -306,7 +291,7 @@
           details (if (int? report-id)
                     (let [details (card-details report-id options)]
                       (some-> details
-                              (select-keys [:id :type :description :name :verified])
+                              (select-keys [:id :type :description :name])
                               (assoc :result-columns (:fields details))))
                     "invalid report_id")]
       (if (map? details)

enterprise/backend/src/metabase_enterprise/metabot_v3/models/metabot.clj

@@ -18,16 +18,15 @@
 
 ;;; --------------------------------------------------- Hydration ----------------------------------------------------
 
-(methodical/defmethod t2/batched-hydrate [:model/Metabot :prompts]
-  "Hydrate the list of prompts for a collection of metabots."
+(methodical/defmethod t2/batched-hydrate [:model/Metabot :entities]
+  "Hydrate the list of entities for a collection of metabots."
   [_model k metabots]
   (mi/instances-with-hydrated-data
    metabots k
    #(group-by :metabot_id
-              (t2/select :model/MetabotPrompt {:where [:in :metabot_id (map :id metabots)]}))
+              (t2/select :model/MetabotEntity {:where [:in :metabot_id (map :id metabots)]}))
    :id
    {:default []}))
-
 ;;; ------------------------------------------------ Serdes Hashing -------------------------------------------------
 
 (defmethod serdes/hash-fields :model/Metabot
@@ -37,15 +36,14 @@
 ;;; ------------------------------------------------- Serialization -------------------------------------------------
 
 (defmethod serdes/dependencies "Metabot"
-  [{:keys [prompts]}]
-  (set (mapcat serdes/dependencies prompts)))
+  [{:keys [entities]}]
+  (into #{} (mapcat serdes/dependencies) entities))
 
 (defmethod serdes/generate-path "Metabot" [_ metabot]
   [(serdes/infer-self-path "Metabot" metabot)])
 
 (defmethod serdes/make-spec "Metabot" [_model-name opts]
-  {:copy      [:name :description :entity_id :use_verified_content]
-   :transform {:created_at    (serdes/date)
-               :updated_at    (serdes/date)
-               :collection_id (serdes/fk :model/Collection)
-               :prompts       (serdes/nested :model/MetabotPrompt :metabot_id opts)}})
+  {:copy      [:name :description :entity_id]
+   :transform {:created_at (serdes/date)
+               :updated_at (serdes/date)
+               :entities   (serdes/nested :model/MetabotEntity :metabot_id opts)}})

enterprise/backend/src/metabase_enterprise/metabot_v3/models/metabot_entity.clj

@@ -0,0 +1,56 @@
+(ns metabase-enterprise.metabot-v3.models.metabot-entity
+  (:require
+   [metabase.models.interface :as mi]
+   [metabase.models.serialization :as serdes]
+   [methodical.core :as methodical]
+   [toucan2.core :as t2]))
+
+;;; --------------------------------------------------- Lifecycle ----------------------------------------------------
+
+(methodical/defmethod t2/table-name :model/MetabotEntity [_model] :metabot_entity)
+
+(doto :model/MetabotEntity
+  (derive :metabase/model)
+  (derive ::mi/read-policy.superuser)
+  (derive ::mi/write-policy.superuser)
+  (derive :hook/entity-id)
+  (derive :hook/created-at-timestamped?))
+
+(t2/deftransforms :model/MetabotEntity
+  {:model mi/transform-keyword})
+
+;;; ------------------------------------------------ Serdes Hashing -------------------------------------------------
+
+(defmethod serdes/hash-fields :model/MetabotEntity
+  [_table]
+  [:metabot_id :model :model_id])
+
+;;; ------------------------------------------------- Serialization -------------------------------------------------
+
+(defmethod serdes/dependencies "MetabotEntity"
+  [{:keys [model model_id prompts]}]
+  (into #{[{:model (case model
+                     "collection" "Collection"
+                     ("dataset" "metric") "Card")
+            :id model_id}]}
+        (mapcat serdes/dependencies prompts)))
+
+(defmethod serdes/generate-path "MetabotEntity" [_ entity]
+  [(serdes/infer-self-path "Metabot" (t2/select-one :model/Metabot :id (:metabot_id entity)))
+   (serdes/infer-self-path "MetabotEntity" entity)])
+
+(defmethod serdes/make-spec "MetabotEntity" [_model-name opts]
+  {:copy      [:entity_id]
+   :transform {:created_at (serdes/date)
+               :model      (serdes/kw)
+               :model_id   {::fk true
+                            :export-with-context (fn [{:keys [model model_id]} _ _]
+                                                   (serdes/*export-fk* model_id (case model
+                                                                                  :collection        :model/Collection
+                                                                                  (:dataset :metric) :model/Card)))
+                            :import-with-context (fn [{:keys [model model_id]} _ _]
+                                                   (serdes/*import-fk* model_id (case model
+                                                                                  "collection"         :model/Collection
+                                                                                  ("dataset" "metric") :model/Card)))}
+               :metabot_id (serdes/parent-ref)
+               :prompts    (serdes/nested :model/MetabotPrompt :metabot_entity_id opts)}})

enterprise/backend/src/metabase_enterprise/metabot_v3/models/metabot_prompt.clj

@@ -22,12 +22,12 @@
 
 (defmethod serdes/hash-fields :model/MetabotPrompt
   [_table]
-  [:metabot_id :model :card_id :prompt])
+  [:metabot_entity_id :model :card_id :prompt])
 
 ;;; ------------------------------------------------- Serialization -------------------------------------------------
 
 (defmethod serdes/generate-path "MetabotPrompt" [_ entity]
-  (conj (serdes/generate-path "Metabot" (t2/select-one :model/Metabot (:metabot_id entity)))
+  (conj (serdes/generate-path "MetabotEntity" (t2/select-one :model/MetabotEntity (:metabot_entity_id entity)))
         (serdes/infer-self-path "MetabotPrompt" entity)))
 
 (defmethod serdes/dependencies "MetabotPrompt" [prompt]
@@ -35,8 +35,8 @@
 
 (defmethod serdes/make-spec "MetabotPrompt" [_model-name _opts]
   {:copy      [:entity_id :prompt]
-   :transform {:created_at (serdes/date)
-               :updated_at (serdes/date)
-               :model      (serdes/kw)
-               :card_id    (serdes/fk :model/Card)
-               :metabot_id (serdes/parent-ref)}})
+   :transform {:created_at        (serdes/date)
+               :updated_at        (serdes/date)
+               :model             (serdes/kw)
+               :card_id           (serdes/fk :model/Card)
+               :metabot_entity_id (serdes/parent-ref)}})

enterprise/backend/src/metabase_enterprise/metabot_v3/table_utils.clj

@@ -3,14 +3,9 @@
   (:require
    [clojure.set :as set]
    [metabase-enterprise.metabot-v3.query-analyzer :as query-analyzer]
-   [metabase-enterprise.metabot-v3.tools.util :as metabot-v3.tools.u]
    [metabase.api.common :as api]
-   [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]
-   [metabase.lib.core :as lib]
-   [metabase.lib.metadata :as lib.metadata]
    [metabase.models.interface :as mi]
    [metabase.util :as u]
-   [metabase.util.humanization :as u.humanization]
    [toucan2.core :as t2]
    [toucan2.realize :as t2.realize])
   (:import
@@ -64,62 +59,13 @@
          all-tables (concat priority-tables fill-tables)
          all-tables (take all-tables-limit all-tables)]
      (mapv (fn [{:keys [fields] :as table}]
-             (merge (select-keys table [:id :name :schema :description])
+             (merge (select-keys table [:name :schema :description])
                     {:columns (mapv (fn [{:keys [database_type] :as field}]
-                                      (merge (select-keys field [:id :name :description])
+                                      (merge (select-keys field [:name :description])
                                              {:data_type database_type}))
                                     fields)}))
            all-tables))))
 
-(defn enhanced-database-tables
-  "Get database tables formatted with the new metabot tools schema format.
-
-  Returns tables with :type, :display_name, :database_id, :database_schema, :fields (with field-id), :metrics.
-  This format is used by metabot context and other modern tools."
-  ([database-id]
-   (enhanced-database-tables database-id nil))
-  ([database-id {:keys [all-tables-limit priority-tables exclude-table-ids]
-                 :or {all-tables-limit max-database-tables
-                      priority-tables []
-                      exclude-table-ids #{}}}]
-   (let [priority-table-ids (set (map :id priority-tables))
-         ;; Fetch most viewed tables, excluding priority tables and excluded tables
-         fill-tables (t2/select [:model/Table :id :db_id :name :schema :description]
-                                :db_id database-id
-                                :active true
-                                :visibility_type nil
-                                {:where    (mi/visible-filter-clause :model/Table
-                                                                     :id
-                                                                     {:user-id       api/*current-user-id*
-                                                                      :is-superuser? api/*is-superuser?*}
-                                                                     {:perms/view-data      :unrestricted
-                                                                      :perms/create-queries :query-builder-and-native})
-                                 :order-by [[:view_count :desc]]
-                                 :limit    all-tables-limit})
-         fill-tables (remove #(or (priority-table-ids (:id %))
-                                  (exclude-table-ids (:id %))) fill-tables)
-         all-tables (concat priority-tables fill-tables)
-         all-tables (take all-tables-limit all-tables)]
-     (lib.metadata.jvm/with-metadata-provider-cache
-       (let [mp (lib.metadata.jvm/application-database-metadata-provider database-id)
-             table-ids (map :id all-tables)
-             _ (lib.metadata/bulk-metadata mp :metadata/table table-ids)]
-         (mapv (fn [{:keys [id name schema description]}]
-                 (let [table-query (lib/query mp (lib.metadata/table mp id))
-                       cols (->> (lib/visible-columns table-query)
-                                 (map #(metabot-v3.tools.u/add-table-reference table-query %)))
-                       field-id-prefix (metabot-v3.tools.u/table-field-id-prefix id)]
-                   {:id id
-                    :type :table
-                    :name name
-                    :display_name (u.humanization/name->human-readable-name :simple name)
-                    :database_id database-id
-                    :database_schema schema
-                    :description description
-                    :fields (into [] (map-indexed #(metabot-v3.tools.u/->result-column table-query %2 %1 field-id-prefix) cols))
-                    :metrics []}))
-               all-tables))))))
-
 (defn similar?
   "Check if two strings are similar using Levenshtein distance with a max distance of 4."
   [left right]

enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.clj

@@ -18,12 +18,9 @@
     :as metabot-v3.tools.create-dashboard-subscription]
    [metabase-enterprise.metabot-v3.tools.field-stats :as metabot-v3.tools.field-stats]
    [metabase-enterprise.metabot-v3.tools.filters :as metabot-v3.tools.filters]
+   [metabase-enterprise.metabot-v3.tools.find-metric :as metabot-v3.tools.find-metric]
    [metabase-enterprise.metabot-v3.tools.find-outliers :as metabot-v3.tools.find-outliers]
    [metabase-enterprise.metabot-v3.tools.generate-insights :as metabot-v3.tools.generate-insights]
-   [metabase-enterprise.metabot-v3.tools.search :as metabot-v3.tools.search]
-   [metabase-enterprise.metabot-v3.tools.search-data-sources :as metabot-v3.tools.search-data-sources]
-   [metabase-enterprise.metabot-v3.tools.snippets :as metabot-v3.tools.snippets]
-   [metabase-enterprise.metabot-v3.tools.transforms :as metabot-v3.tools.transforms]
    [metabase-enterprise.metabot-v3.util :as metabot-v3.u]
    [metabase.api.common :as api]
    [metabase.api.macros :as api.macros]
@@ -244,23 +241,6 @@
     [:limit {:optional true} [:maybe :int]]]
    [:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])
 
-(mr/def ::query-datasource-arguments
-  [:and
-   [:map
-    [:table_id {:optional true} :int]
-    [:model_id {:optional true} :int]
-    [:fields {:optional true} [:maybe [:sequential ::field]]]
-    [:filters {:optional true} [:maybe [:sequential ::filter]]]
-    [:aggregations {:optional true} [:maybe [:sequential ::aggregation]]]
-    [:group_by {:optional true} [:maybe [:sequential ::group-by]]]
-    [:order_by {:optional true} [:maybe [:sequential [:map
-                                                      [:field ::field]
-                                                      [:direction [:enum {:encode/tool-api-request keyword} "asc" "desc"]]]]]]
-    [:limit {:optional true} [:maybe :int]]]
-   [:fn {:error/message "Exactly one of table_id and model_id required"}
-    #(= (count (select-keys % [:table_id :model_id])) 1)]
-   [:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])
-
 (mr/def ::count
   [:and
    :int
@@ -422,8 +402,7 @@
   [:merge
    ::basic-metric
    [:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
-    [:queryable_dimensions {:optional true} ::columns]
-    [:verified {:optional true} :boolean]]])
+    [:queryable_dimensions {:optional true} ::columns]]])
 
 (mr/def ::find-metric-result
   [:or
@@ -487,6 +466,7 @@
                          [:name :string]
                          [:email_address :string]]]]
    [:map [:output :string]]])
+
 (mr/def ::get-dashboard-details-result
   [:or
    [:map
@@ -495,8 +475,7 @@
                          [:id :int]
                          [:type [:= :dashboard]]
                          [:name :string]
-                         [:description {:optional true} :string]
-                         [:verified {:optional true} :boolean]]]]
+                         [:description {:optional true} :string]]]]
    [:map [:output :string]]])
 
 (mr/def ::get-metric-details-arguments
@@ -551,8 +530,7 @@
                          [:type [:= :question]]
                          [:name :string]
                          [:description {:optional true} [:maybe :string]]
-                         [:result_columns ::columns]
-                         [:verified {:optional true} :boolean]]]]
+                         [:result_columns ::columns]]]]
    [:map [:output :string]]])
 
 (mr/def ::get-document-details-arguments
@@ -626,9 +604,6 @@
    [:id :int]
    [:type [:enum :model :table]]
    [:name :string]
-   [:display_name :string]
-   [:database_id :int]
-   [:database_schema {:optional true} [:maybe :string]] ; Schema name, if applicable
    [:fields ::columns]
    [:description {:optional true} [:maybe :string]]
    [:metrics {:optional true} [:sequential ::basic-metric]]])
@@ -645,43 +620,6 @@
     [:structured_output ::full-table]]
    [:map [:output :string]]])
 
-(mr/def ::basic-transform
-  [:map
-   {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
-   [:id :int]
-   [:name :string]
-   [:description {:optional true} [:maybe :string]]
-   [:entity_id {:optional true} [:maybe :string]]
-   [:source ::metabot-v3.tools.transforms/transform-source]])
-
-(mr/def ::full-transform
-  [:merge
-   ::basic-transform
-   [:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
-    [:created_at ms/TemporalString]
-    [:updated_at ms/TemporalString]
-    [:target ::metabot-v3.tools.transforms/transform-target]]])
-
-(mr/def ::get-transforms-result
-  [:or
-   [:map
-    {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
-    [:structured_output [:sequential ::basic-transform]]]
-   [:map [:output :string]]])
-
-(mr/def ::get-transform-details-arguments
-  [:and
-   [:map
-    [:transform_id :int]]
-   [:map {:encode/tool-api-request
-          #(set/rename-keys % {:transform_id :transform-id})}]])
-
-(mr/def ::get-transform-details-result
-  [:or
-   [:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
-    [:structured_output [:sequential ::full-transform]]]
-   [:map [:output :string]]])
-
 (mr/def ::answer-sources-result
   [:or
    [:map
@@ -691,109 +629,6 @@
                          [:models  [:sequential ::full-table]]]]]
    [:map [:output :string]]])
 
-(mr/def ::search-data-sources-arguments
-  [:and
-   [:map
-    [:keywords {:optional true} [:maybe [:sequential :string]]]
-    [:description {:optional true} [:maybe :string]]
-    [:database_id {:optional true} [:maybe :int]]
-    [:entity_types {:optional true} [:maybe [:sequential [:enum "table" "model" "question" "dashboard" "metric"]]]]
-    [:limit {:optional true, :default 50} [:and :int [:fn #(<= 1 % 100)]]]]
-   [:map {:encode/tool-api-request
-          #(set/rename-keys % {:database_id :database-id
-                               :entity_types :entity-types})}]])
-
-(mr/def ::search-data-sources-table-result
-  "Schema for table/model search results"
-  [:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
-   [:id :int]
-   [:type [:enum :table :model]]
-   [:name :string]                                   ; Table name (technical for tables, display for models)
-   [:display_name {:optional true} [:maybe :string]] ; Display name
-   [:description {:optional true} [:maybe :string]]
-   [:database_id {:optional true} [:maybe :int]]
-   [:database_schema {:optional true} [:maybe :string]]])
-
-(mr/def ::search-data-sources-dashboard-result
-  "Schema for dashboard search results"
-  [:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
-   [:id :int]
-   [:type [:= :dashboard]]
-   [:name :string]                                    ; Display name
-   [:description {:optional true} [:maybe :string]]
-   [:verified {:optional true} :boolean]])
-
-(mr/def ::search-data-sources-question-result
-  "Schema for question search results"
-  [:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
-   [:id :int]
-   [:type [:= :question]]
-   [:name :string]                                    ; Display name
-   [:description {:optional true} [:maybe :string]]
-   [:verified {:optional true} :boolean]])
-
-(mr/def ::search-data-sources-metric-result
-  "Schema for metric search results"
-  [:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
-   [:id :int]
-   [:type [:= :metric]]
-   [:name :string]                                    ; Display name
-   [:description {:optional true} [:maybe :string]]
-   [:verified {:optional true} :boolean]])
-
-(mr/def ::search-data-sources-result-item
-  "Union of all search result types.
-
-   We use dedicated schemas per entity type because:
-   - Tables need technical names and database context
-   - Models need display names and database context
-   - Other entities use display names and don't need database fields
-   - Type-specific validation ensures correct field usage"
-  [:or
-   ::search-data-sources-table-result
-   ::search-data-sources-dashboard-result
-   ::search-data-sources-question-result
-   ::search-data-sources-metric-result])
-
-(mr/def ::search-data-sources-result
-  [:or
-   [:map
-    {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
-    [:structured_output [:map
-                         [:data [:sequential ::search-data-sources-result-item]]
-                         [:total_count :int]]]]
-   [:map [:output :string]]])
-
-(mr/def ::basic-snippet
-  [:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
-   [:id :int]
-   [:name :string]
-   [:description {:optional true} [:maybe :string]]])
-
-(mr/def ::full-snippet
-  [:merge
-   ::basic-snippet
-   [:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
-    [:content :string]]])
-
-(mr/def ::get-snippets-result
-  "Schema for SQL snippet list results"
-  [:map
-   {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
-   [:structured_output [:sequential ::basic-snippet]]])
-
-(mr/def ::get-snippet-details-arguments
-  [:and
-   [:map
-    [:snippet_id :int]]
-   [:map {:encode/tool-api-request
-          #(set/rename-keys % {:snippet_id :snippet-id})}]])
-
-(mr/def ::get-snippet-details-result
-  "Schema for SQL snippet detail results"
-  [:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
-   [:structured_output ::full-snippet]])
-
 (api.macros/defendpoint :post "/answer-sources" :- [:merge ::answer-sources-result ::tool-request]
   "Return top level meta information about available information sources."
   [_route-params
@@ -862,6 +697,20 @@
               (assoc :conversation_id conversation_id))
       (metabot-v3.context/log :llm.log/be->llm))))
 
+(api.macros/defendpoint :post "/find-metric" :- [:merge ::find-metric-result ::tool-request]
+  "Find a metric matching a description."
+  [_route-params
+   _query-params
+   {:keys [arguments conversation_id] :as body} :- [:merge
+                                                    [:map [:arguments [:map [:message :string]]]]
+                                                    ::tool-request]]
+  (metabot-v3.context/log (assoc body :api :find-metric) :llm.log/llm->be)
+  (doto (-> (mc/decode ::find-metric-result
+                       (metabot-v3.tools.find-metric/find-metric arguments)
+                       (mtx/transformer {:name :tool-api-response}))
+            (assoc :conversation_id conversation_id))
+    (metabot-v3.context/log :llm.log/be->llm)))
+
 (api.macros/defendpoint :post "/find-outliers" :- [:merge ::find-outliers-result ::tool-request]
   "Find outliers in the values provided by a data source for a given column."
   [_route-params
@@ -1015,34 +864,6 @@
               (assoc :conversation_id conversation_id))
       (metabot-v3.context/log :llm.log/be->llm))))
 
-(api.macros/defendpoint :post "/get-transforms" :- [:merge ::get-transforms-result ::tool-request]
-  "Get a list of all known transforms."
-  [_route-params
-   _query-params
-   {:keys [conversation_id] :as body} :- ::tool-request]
-  (metabot-v3.context/log (assoc body :api :get-transforms) :llm.log/llm->be)
-  (doto (-> (mc/decode ::get-transforms-result
-                       (metabot-v3.tools.transforms/get-transforms)
-                       (mtx/transformer {:name :tool-api-response}))
-            (assoc :conversation_id conversation_id))
-    (metabot-v3.context/log :llm.log/be->llm)))
-
-(api.macros/defendpoint :post "/get-transform-details" :- [:merge ::get-transform-details-result ::tool-request]
-  "Get information about a transform."
-  [_route-params
-   _query-params
-   {:keys [arguments conversation_id] :as body} :- [:merge
-                                                    [:map [:arguments ::get-transform-details-arguments]]
-                                                    ::tool-request]]
-  (metabot-v3.context/log (assoc body :api :get-transform-details) :llm.log/llm->be)
-  (let [arguments (mc/encode ::get-transform-details-arguments arguments (mtx/transformer {:name :tool-api-request}))
-        transform-id (:transform-id arguments)]
-    (doto (-> (mc/decode ::get-transform-details-result
-                         (metabot-v3.tools.transforms/get-transform-details transform-id)
-                         (mtx/transformer {:name :tool-api-response}))
-              (assoc :conversation_id conversation_id))
-      (metabot-v3.context/log :llm.log/be->llm))))
-
 (api.macros/defendpoint :post "/query-metric" :- [:merge ::filtering-result ::tool-request]
   "Construct a query from a metric."
   [_route-params
@@ -1075,147 +896,6 @@
               (assoc :conversation_id conversation_id))
       (metabot-v3.context/log :llm.log/be->llm))))
 
-;; TODO tsplude - drop the `/query-model` endpoint and filter logic in favor of this
-(api.macros/defendpoint :post "/query-datasource" :- [:merge ::filtering-result ::tool-request]
-  "Construct a query from a model or table data source."
-  [_route-params
-   _query-params
-   {:keys [arguments conversation_id] :as body} :- [:merge
-                                                    [:map [:arguments ::query-datasource-arguments]]
-                                                    ::tool-request]]
-  (metabot-v3.context/log (assoc body :api :query-datasource) :llm.log/llm->be)
-  (let [arguments (mc/encode ::query-datasource-arguments
-                             arguments (mtx/transformer {:name :tool-api-request}))]
-    (doto (-> (mc/decode ::filtering-result
-                         (metabot-v3.tools.filters/query-datasource arguments)
-                         (mtx/transformer {:name :tool-api-response}))
-              (assoc :conversation_id conversation_id))
-      (metabot-v3.context/log :llm.log/be->llm))))
-
-(api.macros/defendpoint :post "/search-data-sources" :- [:merge ::search-data-sources-result ::tool-request]
-  "Search for data sources (tables, models, cards, dashboards, metrics) in Metabase."
-  [_route-params
-   _query-params
-   {:keys [arguments conversation_id] :as body} :- [:merge
-                                                    [:map [:arguments {:optional true} ::search-data-sources-arguments]]
-                                                    ::tool-request]
-   request]
-  (metabot-v3.context/log (assoc body :api :search-data-sources) :llm.log/llm->be)
-  (try
-    (let [options (mc/encode ::search-data-sources-arguments
-                             arguments (mtx/transformer {:name :tool-api-request}))
-          metabot-id (:metabot-v3/metabot-id request)
-          results (metabot-v3.tools.search-data-sources/search-data-sources
-                   (assoc options :metabot-id metabot-id))
-          response-data {:data results
-                         :total_count (count results)}]
-      (doto (-> (mc/decode ::search-data-sources-result
-                           {:structured_output response-data}
-                           (mtx/transformer {:name :tool-api-response}))
-                (assoc :conversation_id conversation_id))
-        (metabot-v3.context/log :llm.log/be->llm)))
-    (catch Exception e
-      (log/error e "Error in search-data-sources")
-      (doto (-> {:output (str "Search failed: " (or (ex-message e) "Unknown error"))}
-                (assoc :conversation_id conversation_id))
-        (metabot-v3.context/log :llm.log/be->llm)))))
-
-(mr/def ::search-arguments
-  [:and
-   [:map
-    [:term_queries     {:optional true} [:maybe [:sequential :string]]]
-    [:semantic_queries {:optional true} [:maybe [:sequential :string]]]
-    [:entity_types     {:optional true} [:maybe [:sequential [:enum "table" "model" "question" "dashboard" "metric" "database"]]]]
-    [:database_id      {:optional true} [:maybe :int]]
-    [:created_at       {:optional true} [:maybe ms/NonBlankString]]
-    [:last_edited_at   {:optional true} [:maybe ms/NonBlankString]]
-    [:limit            {:optional true, :default 50} [:and :int [:fn #(<= 1 % 100)]]]]
-   [:map {:encode/tool-api-request
-          #(update-keys % (comp keyword u/->kebab-case-en))}]])
-
-(mr/def ::search-result-item
-  "Unified schema for search result items."
-  [:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
-   [:id :int]
-   [:type [:enum :table :model :dashboard :question :metric :database]]
-   [:name :string]
-   [:display_name {:optional true} [:maybe :string]]
-   [:description {:optional true} [:maybe :string]]
-   [:database_id {:optional true} [:maybe :int]]
-   [:database_schema {:optional true} [:maybe :string]]
-   [:verified {:optional true} [:maybe :boolean]]
-   [:updated_at {:optional true} [:maybe :string]]
-   [:created_at {:optional true} [:maybe :string]]
-   [:collection {:optional true} [:maybe [:map
-                                          [:name {:optional true} [:maybe :string]]
-                                          [:authority_level {:optional true} [:maybe :string]]]]]])
-
-(mr/def ::search-result
-  [:or
-   [:map
-    {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
-    [:structured_output [:map
-                         [:data [:sequential ::search-result-item]]
-                         [:total_count :int]]]]
-   [:map [:output :string]]])
-
-(api.macros/defendpoint :post "/search" :- [:merge ::search-result ::tool-request]
-  "Enhanced search with term and semantic queries using Reciprocal Rank Fusion."
-  [_route-params
-   _query-params
-   {:keys [arguments conversation_id] :as body} :- [:merge
-                                                    [:map [:arguments {:optional true} ::search-arguments]]
-                                                    ::tool-request]
-   request]
-  (metabot-v3.context/log (assoc body :api :search) :llm.log/llm->be)
-  (try
-    (let [options (mc/encode ::search-arguments
-                             arguments (mtx/transformer {:name :tool-api-request}))
-          metabot-id (:metabot-v3/metabot-id request)
-          results (metabot-v3.tools.search/search
-                   (assoc options :metabot-id metabot-id))
-          response-data {:data results
-                         :total_count (count results)}]
-      (doto (-> (mc/decode ::search-result
-                           {:structured_output response-data}
-                           (mtx/transformer {:name :tool-api-response}))
-                (assoc :conversation_id conversation_id))
-        (metabot-v3.context/log :llm.log/be->llm)))
-    (catch Exception e
-      (log/error e "Error in search")
-      (doto (-> {:output (str "Search failed: " (or (ex-message e) "Unknown error"))}
-                (assoc :conversation_id conversation_id))
-        (metabot-v3.context/log :llm.log/be->llm)))))
-
-(api.macros/defendpoint :post "/get-snippets" :- [:merge ::get-snippets-result ::tool-request]
-  "Get a list of all known SQL snippets."
-  [_route-params
-   _query-params
-   {:keys [conversation_id] :as body} :- ::tool-request]
-  (metabot-v3.context/log (assoc body :api :get-snippets) :llm.log/llm->be)
-  (doto (-> (mc/decode ::get-snippets-result
-                       (metabot-v3.tools.snippets/get-snippets)
-                       (mtx/transformer {:name :tool-api-response}))
-            (assoc :conversation_id conversation_id))
-    (metabot-v3.context/log :llm.log/be->llm)))
-
-(api.macros/defendpoint :post "/get-snippet-details" :- [:merge ::get-snippet-details-result ::tool-request]
-  "Get the content of a single SQL snippet."
-  [_route-params
-   _query-params
-   {:keys [arguments conversation_id] :as body} :- [:merge
-                                                    [:map [:arguments ::get-snippet-details-arguments]]
-                                                    ::tool-request]]
-
-  (metabot-v3.context/log (assoc body :api :get-snippet-details) :llm.log/llm->be)
-  (let [arguments (mc/encode ::get-snippet-details-arguments arguments (mtx/transformer {:name :tool-api-request}))
-        snippet-id (:snippet-id arguments)]
-    (doto (-> (mc/decode ::get-snippet-details-result
-                         (metabot-v3.tools.snippets/get-snippet-details snippet-id)
-                         (mtx/transformer {:name :tool-api-response}))
-              (assoc :conversation_id conversation_id))
-      (metabot-v3.context/log :llm.log/be->llm))))
-
 (defn- enforce-authentication
   "Middleware that returns a 401 response if no `ai-session` can be found for  `request`."
   [handler]

enterprise/backend/src/metabase_enterprise/metabot_v3/tools/filters.clj

@@ -278,73 +278,6 @@
         {:output (str "No model found with model_id " model-id)}
         (metabot-v3.tools.u/handle-agent-error e)))))
 
-(defn- resolve-datasource
-  "Resolve datasource parameters to [field-id-prefix base-query] tuple.
-   Accepts either {:table-id id} or {:model-id id}."
-  [{:keys [table-id model-id]}]
-  (cond
-    model-id
-    [(metabot-v3.tools.u/card-field-id-prefix model-id) (metabot-v3.tools.u/card-query model-id)]
-
-    table-id
-    [(metabot-v3.tools.u/table-field-id-prefix table-id) (metabot-v3.tools.u/table-query table-id)]
-
-    :else
-    (throw (ex-info "Either table-id or model-id must be provided" {:agent-error? true}))))
-
-(defn- query-datasource*
-  [{:keys [fields filters aggregations group-by order-by limit] :as arguments}]
-  (let [[filter-field-id-prefix base-query] (resolve-datasource arguments)
-        visible-cols (lib/visible-columns base-query)
-        resolve-visible-column  #(metabot-v3.tools.u/resolve-column % filter-field-id-prefix visible-cols)
-        resolve-order-by-column (fn [{:keys [field direction]}] {:field (resolve-visible-column field) :direction direction})
-        projection (map (comp (juxt filter-bucketed-column (fn [{:keys [column bucket]}]
-                                                             (let [column (cond-> column
-                                                                            bucket (assoc :unit bucket))]
-                                                               (lib/display-name base-query -1 column :long))))
-                              resolve-visible-column)
-                        fields)
-        reduce-query (fn [query f coll] (reduce f query coll))
-        query (-> base-query
-                  (reduce-query (fn [query [expr-or-column expr-name]]
-                                  (lib/expression query expr-name expr-or-column))
-                                (filter (comp expression? first) projection))
-                  (add-fields projection)
-                  (reduce-query add-filter (map resolve-visible-column filters))
-                  (reduce-query add-aggregation (map resolve-visible-column aggregations))
-                  (reduce-query add-breakout (map resolve-visible-column group-by))
-                  (reduce-query add-order-by (map resolve-order-by-column order-by))
-                  (add-limit limit))
-        query-id (u/generate-nano-id)
-        query-field-id-prefix (metabot-v3.tools.u/query-field-id-prefix query-id)
-        returned-cols (lib/returned-columns query)]
-    {:type :query
-     :query-id query-id
-     ;; existing usage, don't do this going forward -- use Lib instead and persist MBQL 5 to the app DB
-     :query #_{:clj-kondo/ignore [:discouraged-var]} (lib/->legacy-MBQL query)
-     :result-columns (into []
-                           (map-indexed #(metabot-v3.tools.u/->result-column query %2 %1 query-field-id-prefix))
-                           returned-cols)}))
-
-(defn query-datasource
-  "Create a query based on a datasource (table or model)."
-  [{:keys [table-id model-id] :as arguments}]
-  (try
-    (cond
-      (and table-id model-id) (throw (ex-info "Cannot provide both table_id and model_id" {:agent-error? true}))
-      (int? model-id) {:structured-output (query-datasource* arguments)}
-      (int? table-id) {:structured-output (query-datasource* arguments)}
-      model-id        {:output (str "Invalid model_id " model-id)}
-      table-id        {:output (str "Invalid table_id " table-id)}
-      :else           {:output "Either table_id or model_id must be provided"})
-    (catch Exception e
-      (if (= (:status-code (ex-data e)) 404)
-        {:output (cond
-                   table-id (str "No table found with table_id " table-id)
-                   model-id (str "No model found with model_id " model-id)
-                   :else "Resource not found")}
-        (metabot-v3.tools.u/handle-agent-error e)))))
-
 (defn- base-query
   [data-source]
   (let [{:keys [table-id query query-id report-id]} data-source

enterprise/backend/src/metabase_enterprise/metabot_v3/tools/find_metric.clj

@@ -0,0 +1,24 @@
+(ns metabase-enterprise.metabot-v3.tools.find-metric
+  (:require
+   [metabase-enterprise.metabot-v3.client :as client]
+   [metabase-enterprise.metabot-v3.dummy-tools :as dummy-tools]
+   [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]
+   [metabase.models.interface :as mi]
+   [toucan2.core :as t2]))
+
+(defn find-metric
+  "Find a metric described by `message`.
+
+  See [[metabase-enterprise.metabot-v3.dummy-tools/metric-details]] for the output if a metric can be found."
+  [{:keys [message]}]
+  (lib.metadata.jvm/with-metadata-provider-cache
+    (let [{:keys [id]} (client/select-metric-request
+                        (->> (t2/select [:model/Card :id :name :description]
+                                        :type [:= "metric"]
+                                        :archived [:= false])
+                             (filter mi/can-read?))
+                        message)]
+      (if-let [result (when id
+                        (dummy-tools/metric-details id))]
+        {:structured-output result}
+        {:output "Metric not found."}))))

enterprise/backend/src/metabase_enterprise/metabot_v3/tools/search.clj

@@ -1,135 +0,0 @@
-(ns metabase-enterprise.metabot-v3.tools.search
-  (:require
-   [clojure.set :as set]
-   [metabase-enterprise.metabot-v3.config :as metabot-v3.config]
-   [metabase-enterprise.metabot-v3.reactions]
-   [metabase.api.common :as api]
-   [metabase.permissions.core :as perms]
-   [metabase.search.core :as search]
-   [toucan2.core :as t2]))
-
-(def ^:private metabot-search-models
-  #{"table" "dataset" "card" "dashboard" "metric" "database"})
-
-(def ^:private search-model-mappings
-  "Maps metabot entity types to search engine model types"
-  {"model"    "dataset"
-   "question" "card"})
-
-(defn- entity-type->search-model
-  [entity-type]
-  (get search-model-mappings entity-type entity-type))
-
-(defn- search-model->result-type
-  [search-model]
-  (get (set/map-invert search-model-mappings) search-model search-model))
-
-(defn- transform-search-result
-  "Transform a single search result to match the appropriate entity-specific schema."
-  [{:keys [verified moderated_status collection] :as result}]
-  (let [model (:model result)
-        verified? (or (boolean verified) (= moderated_status "verified"))
-        collection-info (select-keys collection [:name :authority_level])
-        common-fields {:id           (:id result)
-                       :type         (search-model->result-type model)
-                       :name         (:name result)
-                       :description  (:description result)
-                       :updated_at   (:updated_at result)
-                       :created_at   (:created_at result)}]
-    (case model
-      "database"
-      common-fields
-
-      "table"
-      (merge common-fields
-             {:name            (:table_name result)
-              :display_name    (:name result)
-              :database_id     (:database_id result)
-              :database_schema (:table_schema result)})
-
-      "dashboard"
-      (merge common-fields
-             {:verified        verified?
-              :collection      collection-info})
-
-      ;; Questions, metrics, and datasets
-      (merge common-fields
-             {:database_id     (:database_id result)
-              :verified        verified?
-              :collection      collection-info}))))
-
-(defn- search-result-id
-  "Generate a unique identifier for a search result based on its id and model."
-  [search-result]
-  (juxt :id :model) search-result)
-
-(defn- reciprocal-rank-fusion
-  "Combine multiple ranked search result lists using Reciprocal Rank Fusion (RRF).
-
-  Takes a list of search result lists and combines them by:
-  1. Calculating RRF scores for each item based on its rank in each list
-  2. Summing scores for items that appear in multiple lists
-  3. Returning items sorted by total RRF score (descending)
-
-  The RRF score is calculated as: 1 / (k + r) where k defaults to 60 (typical RRF constant)"
-  ([result-lists]
-   (reciprocal-rank-fusion result-lists 60))
-  ([result-lists k]
-   (let [rrf-results (reduce
-                      (fn [acc-map result-list]
-                        (reduce-kv
-                         (fn [acc rank search-result]
-                           (let [id (search-result-id search-result)
-                                 rrf-score (/ 1.0 (+ k (inc rank)))]
-                             (if (contains? acc id)
-                               (update-in acc [id :rrf] + rrf-score)
-                               (assoc acc id {:search-result search-result
-                                              :rrf rrf-score}))))
-                         acc-map
-                         (vec result-list)))
-                      {}
-                      result-lists)]
-     (->> rrf-results
-          vals
-          (sort-by :rrf >)
-          (map :search-result)))))
-
-(defn search
-  "Search for data sources (tables, models, cards, dashboards, metrics) in Metabase.
-  Abstracted from the API endpoint logic."
-  [{:keys [term-queries semantic-queries database-id created-at last-edited-at
-           entity-types limit metabot-id]}]
-  (let [search-models (if (seq entity-types)
-                        (set (distinct (keep entity-type->search-model entity-types)))
-                        metabot-search-models)
-        all-queries   (distinct (concat (or term-queries []) (or semantic-queries [])))
-        metabot (t2/select-one :model/Metabot :entity_id (get-in metabot-v3.config/metabot-config [metabot-id :entity-id] metabot-id))
-        use-verified-content? (if metabot-id
-                                (:use_verified_content metabot)
-                                false)
-        search-fn (fn [query]
-                    (let [search-context (search/search-context
-                                          (merge
-                                           {:search-string query
-                                            :models search-models
-                                            :table-db-id database-id
-                                            :created-at created-at
-                                            :last-edited-at last-edited-at
-                                            :current-user-id api/*current-user-id*
-                                            :is-impersonated-user? (perms/impersonated-user?)
-                                            :is-sandboxed-user? (perms/sandboxed-user?)
-                                            :is-superuser? api/*is-superuser?*
-                                            :current-user-perms @api/*current-user-permissions-set*
-                                            :context :metabot
-                                            :archived false
-                                            :limit (or limit 50)
-                                            :offset 0}
-                                           (when use-verified-content?
-                                             {:verified true})))
-                          search-results (search/search search-context)]
-                      (:data search-results)))
-        ;; Create futures for parallel execution
-        futures (mapv #(future (search-fn %)) all-queries)
-        result-lists (mapv deref futures)
-        fused-results (reciprocal-rank-fusion result-lists)]
-    (map transform-search-result fused-results)))

enterprise/backend/src/metabase_enterprise/metabot_v3/tools/search_data_sources.clj

@@ -1,130 +0,0 @@
-(ns metabase-enterprise.metabot-v3.tools.search-data-sources
-  "Tools for searching data sources in the Metabot v3 context."
-  (:require
-   [clojure.string :as str]
-   [medley.core :as m]
-   [metabase-enterprise.metabot-v3.config :as metabot-v3.config]
-   [metabase-enterprise.metabot-v3.reactions]
-   [metabase.api.common :as api]
-   [metabase.permissions.core :as perms]
-   [metabase.search.core :as search]
-   [toucan2.core :as t2]))
-
-(def ^:private entity-type->search-model
-  "Maps API entity types to search engine model types"
-  {"table" "table"
-   "model" "dataset"
-   "question" "card"
-   "dashboard" "dashboard"
-   "metric" "metric"})
-
-(def ^:private search-model->result-type
-  "Maps search engine model types to result types"
-  {"dashboard" :dashboard
-   "table" :table
-   "dataset" :model
-   "card" :question
-   "metric" :metric})
-
-(defn transform-search-result
-  "Transform a single search result to match the appropriate entity-specific schema.
-
-   Field mapping:
-   - Tables: name = technical table name (table_name), include database_id and database_schema
-   - Models: name = display name, include database_id and database_schema
-   - Other entities: name = display name, no database fields"
-  [result]
-  (let [model (:model result)
-        result-type (search-model->result-type model)]
-    (case model
-      "table"
-      {:id (:id result)
-       :type result-type
-       :name (:table_name result)           ; Technical table name for tables
-       :display_name (:name result)
-       :description (:description result)
-       :database_id (:database_id result)
-       :database_schema (:table_schema result)}
-
-      "dataset"  ; Models
-      {:id (:id result)
-       :type result-type
-       :name (:name result)                 ; Use the display name here as well since there is no technical name
-       :display_name (:name result)
-       :description (:description result)
-       :database_id (:database_id result)
-       :verified (boolean (:verified result))}  ;; Fallback to False if not provided
-
-      ;; For dashboards, questions, and metrics:
-      {:id (:id result)
-       :type result-type
-       :name (:name result)                 ; Display name
-       :description (:description result)
-       :verified (boolean (or (:verified result)  ; Fallback to False if not provided
-                              (= "verified" (:moderated_status result))))})))
-
-(defn- build-search-context
-  "Build the search context for a given search term"
-  [term database-id search-models search-limit verified?]
-  (search/search-context
-   (cond-> {:search-string term
-            :models (set search-models)
-            :table-db-id database-id
-            :current-user-id api/*current-user-id*
-            :is-impersonated-user? (perms/impersonated-user?)
-            :is-sandboxed-user? (perms/sandboxed-user?)
-            :is-superuser? api/*is-superuser?*
-            :current-user-perms @api/*current-user-permissions-set*
-            :context :metabot
-            :archived false
-            :limit search-limit
-            :offset 0}
-     verified? (assoc :verified true))))
-
-(defn- filter-by-collection
-  "Filter search results by collection, always including tables"
-  [metabot-coll-id results]
-  (if metabot-coll-id
-    (filter (fn [result]
-              (or (= (:model result) "table")
-                  (= (get-in result [:collection :id]) metabot-coll-id)))
-            results)
-    results))
-
-(defn- dedupe-by-id-and-model
-  "Remove duplicate results based on id and model"
-  [results]
-  (vals (m/index-by (juxt :id :model) results)))
-
-(defn- search-for-term
-  "Execute a search for a single term"
-  [{:keys [term database-id search-models search-limit verified?]}]
-  (-> (build-search-context term database-id search-models search-limit verified?)
-      search/search
-      :data))
-
-(defn search-data-sources
-  "Search for data sources (tables, models, cards, dashboards, metrics) in Metabase.
-  Abstracted from the API endpoint logic."
-  [{:keys [keywords description database-id entity-types limit metabot-id]}]
-  (let [metabot-entity-id (get-in metabot-v3.config/metabot-config [metabot-id :entity-id] metabot-id)
-        metabot (t2/select-one :model/Metabot :entity_id metabot-entity-id)
-        metabot-coll-id (:collection_id metabot)
-
-        normalized-description (when-not (str/blank? description) description)
-        entity-types (if (seq entity-types)
-                       entity-types
-                       ["table" "model" "question" "dashboard" "metric"])
-
-        search-terms (distinct (concat (or keywords []) (when normalized-description [normalized-description])))
-        search-models (keep entity-type->search-model entity-types)
-        search-limit (or limit (if metabot-coll-id 75 50))]
-    (->> search-terms
-         (mapcat #(search-for-term {:term %
-                                    :database-id database-id
-                                    :search-models search-models
-                                    :search-limit search-limit
-                                    :verified? (:use_verified_content metabot)}))
-         (filter-by-collection metabot-coll-id)
-         dedupe-by-id-and-model
-         (map transform-search-result))))

enterprise/backend/src/metabase_enterprise/metabot_v3/tools/snippets.clj

@@ -1,25 +0,0 @@
-(ns metabase-enterprise.metabot-v3.tools.snippets
-  (:require
-   [metabase-enterprise.metabot-v3.tools.util :as metabot-v3.tools.u]
-   [metabase.native-query-snippets.core :as snippets]))
-
-(defn get-snippets
-  "Lists SQL snippets available to the current user."
-  []
-  (try
-    {:structured_output
-     (->> (snippets/list-native-query-snippets)
-          (map #(select-keys % [:id :name :description])))}
-    (catch Exception e
-      (metabot-v3.tools.u/handle-agent-error e))))
-
-(defn get-snippet-details
-  "Retrieve a specific SQL snippet by ID, including its content."
-  [snippet-id]
-  (try
-    {:structured_output
-     (when snippet-id
-       (when-let [snippet (snippets/get-native-query-snippet snippet-id)]
-         (select-keys snippet [:id :name :description :content])))}
-    (catch Exception e
-      (metabot-v3.tools.u/handle-agent-error e))))

enterprise/backend/src/metabase_enterprise/metabot_v3/tools/transforms.clj

@@ -1,46 +0,0 @@
-(ns metabase-enterprise.metabot-v3.tools.transforms
-  (:require
-   [metabase-enterprise.metabot-v3.tools.util :as metabot-v3.tools.u]
-   [metabase-enterprise.transforms.api :as api.transforms]
-   [metabase.util.malli.registry :as mr]))
-
-(mr/def ::transform-source ::api.transforms/transform-source)
-(mr/def ::transform-target ::api.transforms/transform-target)
-
-(defn- transform-source-type
-  [transform]
-  (-> transform :source :type))
-
-(defn- transform-source-query-type
-  [transform]
-  (-> transform :source :query :type))
-
-(defn- python-transform?
-  [transform]
-  (= "python" (transform-source-type transform)))
-
-(defn- native-query-transform?
-  [transform]
-  (and (= "query" (transform-source-type transform))
-       (= "native" (transform-source-query-type transform))))
-
-(defn get-transforms
-  "Get a list of all known transforms."
-  []
-  (try
-    {:structured_output
-     (->> (api.transforms/get-transforms)
-          (into [] (comp (map #(select-keys % [:id :entity_id :name :description :source]))
-                         (filter #(or (python-transform? %)
-                                      (native-query-transform? %))))))}
-    (catch Exception e
-      (metabot-v3.tools.u/handle-agent-error e))))
-
-(defn get-transform-details
-  "Get information about a transform."
-  [transform-id]
-  (try
-    {:structured_output
-     (api.transforms/get-transform transform-id)}
-    (catch Exception e
-      (metabot-v3.tools.u/handle-agent-error e))))

enterprise/backend/src/metabase_enterprise/metabot_v3/tools/util.clj

@@ -8,8 +8,6 @@
    [metabase.lib.core :as lib]
    [metabase.lib.metadata :as lib.metadata]
    [metabase.lib.types.isa :as lib.types.isa]
-   [metabase.lib.util :as lib.util]
-   [metabase.premium-features.core :as premium-features]
    [metabase.util :as u]
    [toucan2.core :as t2]))
 
@@ -32,15 +30,6 @@
       (isa? (:effective-type column) :type/Time)     :time
       (lib.types.isa/temporal? column)               :date)))
 
-(defn add-table-reference
-  "Add table-reference to columns that have FK relationships."
-  [query col]
-  (cond-> col
-    (and (:fk-field-id col)
-         (:table-id col))
-    (assoc :table-reference (-> (lib/display-name query (lib.metadata/field query (:fk-field-id col)))
-                                lib.util/strip-id))))
-
 (defn table-field-id-prefix
   "Return the field ID prefix for `table-id`."
   [table-id]
@@ -66,8 +55,7 @@
   [query column index field-id-prefix]
   (let [semantic-type (some-> (:semantic-type column) name u/->snake_case_en)]
     (-> {:field-id (str field-id-prefix index)
-         :name (:name column)
-         :display_name (lib/display-name query column)
+         :name (lib/display-name query column)
          :type (convert-field-type column)}
         (m/assoc-some :description (:description column)
                       :semantic-type semantic-type
@@ -133,49 +121,74 @@
     (let [mp (lib.metadata.jvm/application-database-metadata-provider (:db_id table))]
       (lib/query mp (lib.metadata/table mp table-id)))))
 
-(defn metabot-metrics-and-models-query
+(defn metabot-scope-query
   "Return the metric and model cards in metabot scope visible to the current user.
 
-  Takes a metabot-id and returns all metric and model cards in that metabot's collection
-  and its subcollections. If the metabot has use_verified_content enabled, only verified
-  content is returned."
-  [metabot-id]
-  (let [metabot (t2/select-one :model/Metabot :id metabot-id)
-        metabot-collection-id (:collection_id metabot)
-        use-verified-content? (:use_verified_content metabot)
-        collection-filter (if metabot-collection-id
-                            (let [collection (t2/select-one :model/Collection :id metabot-collection-id)
-                                  collection-ids (conj (collection/descendant-ids collection) metabot-collection-id)]
-                              [:in :collection_id collection-ids])
-                            [:and true])
-        base-query {:select [:report_card.*]
-                    :from   [[:report_card]]
-                    :where  [:and
-                             collection-filter
-                             [:in :type [:inline ["metric" "model"]]]
-                             [:= :archived false]
-                             (collection/visible-collection-filter-clause :collection_id)]}]
-    (if (and use-verified-content? (premium-features/has-feature? :content-verification))
-      (-> base-query
-          (assoc :left-join [[:moderation_review :mr] [:and
-                                                       [:= :mr.moderated_item_id :report_card.id]
-                                                       [:= :mr.moderated_item_type [:inline "card"]]
-                                                       [:= :mr.most_recent true]]])
-          (update :where conj [:= :mr.status [:inline "verified"]]))
-      base-query)))
+  If provided, the filter clause `metabot-entity-condition` is used to filter the metabot_entity table. The clause can
+  refer to that table using the :mbe alias. The cards are returned with an additional field called :metabot_entity_id
+  containing the ID of the metabot entity that brought the card in scope. (Situations in which several metabot
+  entities can bring a card into scope is not supported. It is expected that at some point this query will be extended
+  to deal with tables as well.)
+
+  For example,
+    [:in [:mbe.id [1 7 42]]] can be used to consider only specific entities,
+    [:= :mbe.metabot_id 1]   can be used to consider only entities of metabot 1."
+  ([]
+   (metabot-scope-query nil))
+  ([metabot-entity-condition]
+   {:select [:*]
+    :from   [[{:union-all
+               [{:select [:card.* [:mbe.id :metabot_entity_id]]
+                 :from   [[:report_card :card]]
+                 :join   [[:metabot_entity :mbe] (cond-> [:and [:= :mbe.model [:inline "collection"]]]
+                                                   metabot-entity-condition (conj metabot-entity-condition))
+                          [:collection :ecoll]   [:= :ecoll.id :mbe.model_id]
+                          [:collection :coll]    [:or
+                                                  [:= :coll.id :ecoll.id]
+                                                  [:like :coll.location [:concat :ecoll.location :ecoll.id [:inline "/%"]]]]]
+                 :where  [:= :card.collection_id :coll.id]}
+                {:select [:card.* [:mbe.id :metabot_entity_id]]
+                 :from   [[:report_card :card]]
+                 :join   [[:metabot_entity :mbe] (cond-> [:and
+                                                          [:in :mbe.model [:inline ["dataset"  "metric"]]]]
+                                                   metabot-entity-condition (conj metabot-entity-condition))]
+                 :where  [:= :card.id :mbe.model_id]}]}
+              :card]]
+    :where  [:and
+             [:in :card.type [:inline ["metric" "model"]]]
+             [:= :card.archived false]
+             ;; check that the current user can see the card
+             (collection/visible-collection-filter-clause :card.collection_id)]}))
 
 (comment
   (binding [api/*current-user-id* 2
             api/*is-superuser?* true]
-    (t2/select-fn-vec #(select-keys % [:id :name :type])
+    (t2/select-fn-vec #(select-keys % [:id :name :type :metabot_entity_id])
                       :model/Card
-                      (metabot-metrics-and-models-query 1)))
+                      (metabot-scope-query [:= :mbe.metabot_id 1])))
   -)
 
 (defn get-metrics-and-models
   "Retrieve the metric and model cards for the Metabot instance with ID `metabot-id` from the app DB.
 
-  Only cards visible to the current user are returned."
+  Only card visible to the current user are returned."
   [metabot-id]
-  (t2/select :model/Card (-> (metabot-metrics-and-models-query metabot-id)
-                             (assoc :order-by [:id]))))
+  (let [scope-query (metabot-scope-query [:= :mbe.metabot_id metabot-id])]
+    (t2/select :model/Card {:select   [:card.*]
+                            :from     [[scope-query :card]]
+                            :order-by [:card.id]})))
+
+(defn metabot-scope
+  "Return a map from cards (models or metrics) to the ID of metabot entity they belong to.
+
+  Warning: this function assumes that each card is bought by one entity into scope. This assumption holds
+  in the known special cases when there is but one collection entity, or when there are only dataset (model)
+  and metric type entities."
+  [entity-ids]
+  (when (seq entity-ids)
+    (let [cards (t2/select :model/Card
+                           (metabot-scope-query [:in :mbe.id entity-ids]))]
+      (reduce (fn [m c]
+                (assoc m (dissoc c :metabot_entity_id) (:metabot_entity_id c)))
+              {}
+              cards))))

enterprise/backend/src/metabase_enterprise/semantic_search/core.clj

@@ -12,8 +12,7 @@
    [metabase.analytics.core :as analytics]
    [metabase.premium-features.core :refer [defenterprise]]
    [metabase.search.engine :as search.engine]
-   [metabase.util.log :as log]
-   [toucan2.realize :as t2.realize]))
+   [metabase.util.log :as log]))
 
 (defn- fallback-engine
   "Find the highest priority search engine available for fallback."
@@ -59,12 +58,7 @@
           (analytics/inc! :metabase-search/semantic-fallback-triggered {:fallback-engine fallback})
           (analytics/observe! :metabase-search/semantic-results-before-fallback final-count)
           (let [fallback-results (try
-                                   (cond->> (search.engine/results (assoc search-ctx :search-engine fallback))
-                                     ;; The in-place engine returns a reducible (but not seqable) result that needs to
-                                     ;; be realized before we concat and dedup with the semantic engine results.
-                                     (= :search.engine/in-place fallback)
-                                     (into [] (comp (map t2.realize/realize)
-                                                    (take (- (semantic.settings/semantic-search-results-limit) final-count)))))
+                                   (search.engine/results (assoc search-ctx :search-engine fallback))
                                    (catch Throwable t
                                      (log/warn t "Semantic search fallback errored, ignoring")
                                      []))

enterprise/backend/src/metabase_enterprise/semantic_search/index.clj

@@ -127,7 +127,7 @@
    :creator_id          creator_id
    :database_id         database_id
    :last_editor_id      last_editor_id
-   :name                (or (:name doc) "")
+   :name                (:name doc)
    :content             searchable_text
    :display_type        display_type
    :archived            (some-> archived to-boolean)
@@ -145,18 +145,18 @@
    :text_search_vector  (if (:name doc)
                           [:||
                            (search/weighted-tsvector "A" (:name doc))
-                           (search/weighted-tsvector "B" (or (:searchable_text doc) ""))]
-                          (search/weighted-tsvector "A" (or (:searchable_text doc) "")))
+                           (search/weighted-tsvector "B" (:searchable_text doc ""))]
+                          (search/weighted-tsvector "A" (:searchable_text doc "")))
    :text_search_with_native_query_vector
    (if (:name doc)
      [:||
       (search/weighted-tsvector "A" (:name doc))
       (search/weighted-tsvector "B"
-                                (str/join " " (remove str/blank? [(or (:searchable_text doc) "")
-                                                                  (or (:native_query doc) "")])))]
+                                (str/join " " (remove str/blank? [(:searchable_text doc "")
+                                                                  (:native_query doc "")])))]
      (search/weighted-tsvector "A"
-                               (str/join " " (remove str/blank? [(or (:searchable_text doc) "")
-                                                                 (or (:native_query doc) "")]))))})
+                               (str/join " " (remove str/blank? [(:searchable_text doc "")
+                                                                 (:native_query doc "")]))))})
 
 (defn index-size
   "Fetches the number of documents in the index table."

enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj

@@ -37,6 +37,7 @@
    "Dimension"
    "ParameterCard"
    "DashboardCardSeries"
+   "MetabotEntity"
    "MetabotPrompt"
    "TimelineEvent"])
 
@@ -52,8 +53,6 @@
    "CloudMigration"
    "CollectionBookmark"
    "CollectionPermissionGraphRevision"
-   "Comment"
-   "CommentReaction"
    "ConnectionImpersonation"
    "ContentTranslation"
    "DatabaseRouter"

enterprise/backend/src/metabase_enterprise/transforms/api.clj

@@ -1,7 +1,10 @@
 (ns metabase-enterprise.transforms.api
   (:require
+   [clojure.core.async :as a]
+   [clojure.string :as str]
    [metabase-enterprise.transforms.api.transform-job]
    [metabase-enterprise.transforms.api.transform-tag]
+   [metabase-enterprise.transforms.canceling :as trasnforms.canceling]
    [metabase-enterprise.transforms.execute :as transforms.execute]
    [metabase-enterprise.transforms.models.transform :as transform.model]
    [metabase-enterprise.transforms.models.transform-run :as transform-run]
@@ -29,12 +32,21 @@
 (set! *warn-on-reflection* true)
 
 (mr/def ::transform-source
-  [:map
-   [:type [:= "query"]]
-   [:query [:map [:database :int]]]])
+  [:multi {:dispatch (comp keyword :type)}
+   [:query
+    [:map
+     [:type [:= "query"]]
+     [:query [:map [:database :int]]]]]
+   [:python
+    [:map {:closed true}
+     [:source-database {:optional true} :int]
+     [:source-tables   [:map-of :string :int]]
+     [:type [:= "python"]]
+     [:body :string]]]])
 
 (mr/def ::transform-target
   [:map
+   [:database {:optional true} :int]
    [:type [:enum "table"]]
    [:schema {:optional true} [:or ms/NonBlankString :nil]]
    [:name :string]])
@@ -57,33 +69,31 @@
 
 (defn- source-database-id
   [transform]
-  (-> transform :source :query :database))
+  (if (transforms.util/python-transform? transform)
+    (-> transform :source :source-database)
+    (-> transform :source :query :database)))
 
 (defn- check-database-feature
   [transform]
-  (let [database (api/check-400 (t2/select-one :model/Database (source-database-id transform))
-                                (deferred-tru "The source database cannot be found."))
-        feature (transforms.util/required-database-feature transform)]
-    (api/check-400 (not (:is_sample database))
-                   (deferred-tru "Cannot run transforms on the sample database."))
-    (api/check-400 (not (:is_audit database))
-                   (deferred-tru "Cannot run transforms on audit databases."))
-    (api/check-400 (driver.u/supports? (:engine database) feature database)
-                   (deferred-tru "The database does not support the requested transform target type."))))
-
-(defn get-transforms
-  "Get a list of transforms."
-  []
-  (api/check-superuser)
-  (-> (t2/select :model/Transform)
-      (t2/hydrate :last_run :transform_tag_ids)
-      (->> (map #(update % :last_run transforms.util/localize-run-timestamps)))))
+  (when (transforms.util/query-transform? transform)
+    (let [database (api/check-400 (t2/select-one :model/Database (source-database-id transform))
+                                  (deferred-tru "The source database cannot be found."))
+          feature (transforms.util/required-database-feature transform)]
+      (api/check-400 (not (:is_sample database))
+                     (deferred-tru "Cannot run transforms on the sample database."))
+      (api/check-400 (not (:is_audit database))
+                     (deferred-tru "Cannot run transforms on audit databases."))
+      (api/check-400 (driver.u/supports? (:engine database) feature database)
+                     (deferred-tru "The database does not support the requested transform target type.")))))
 
 (api.macros/defendpoint :get "/"
   "Get a list of transforms."
   [_route-params
    _query-params]
-  (get-transforms))
+  (api/check-superuser)
+  (-> (t2/select :model/Transform)
+      (t2/hydrate :last_run :transform_tag_ids)
+      (->> (map #(update % :last_run transforms.util/localize-run-timestamps)))))
 
 (api.macros/defendpoint :post "/"
   "Create a new transform."
@@ -111,9 +121,11 @@
       ;; Return with hydrated tag_ids
       (t2/hydrate transform :transform_tag_ids))))
 
-(defn get-transform
+(api.macros/defendpoint :get "/:id"
   "Get a specific transform."
-  [id]
+  [{:keys [id]} :- [:map
+                    [:id ms/PositiveInt]]]
+  (log/info "get transform" id)
   (api/check-superuser)
   (let [{:keys [target] :as transform} (api/check-404 (t2/select-one :model/Transform id))
         database-id (source-database-id transform)
@@ -123,13 +135,6 @@
         (u/update-some :last_run transforms.util/localize-run-timestamps)
         (assoc :table target-table))))
 
-(api.macros/defendpoint :get "/:id"
-  "Get a specific transform."
-  [{:keys [id]} :- [:map
-                    [:id ms/PositiveInt]]]
-  (log/info "get transform" id)
-  (get-transform id))
-
 (api.macros/defendpoint :get "/:id/dependencies"
   "Get the dependencies of a specific transform."
   [{:keys [id]} :- [:map
@@ -182,9 +187,10 @@
           new (merge old body)
           target-fields #(-> % :target (select-keys [:schema :name]))]
       (check-database-feature new)
-      (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]
-        (throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)
-                        {:status-code 400})))
+      (when (transforms.util/query-transform? old)
+        (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]
+          (throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)
+                          {:status-code 400}))))
       (api/check (not (and (not= (target-fields old) (target-fields new))
                            (transforms.util/target-table-exists? new)))
                  403
@@ -217,11 +223,14 @@
 (api.macros/defendpoint :post "/:id/cancel"
   "Cancel the current run for a given transform."
   [{:keys [id]} :- [:map
-                    [:id :string]]]
+                    [:id ms/PositiveInt]]]
   (log/info "canceling transform " id)
   (api/check-superuser)
-  (let [run (api/check-404 (transform-run/running-run-for-run-id id))]
-    (transform-run-cancelation/mark-cancel-started-run! (:id run)))
+  (let [transform (api/check-404 (t2/select-one :model/Transform id))
+        run (api/check-404 (transform-run/running-run-for-transform-id id))]
+    (transform-run-cancelation/mark-cancel-started-run! (:id run))
+    (when (transforms.util/python-transform? transform)
+      (trasnforms.canceling/cancel-run! (:id run))))
   nil)
 
 (api.macros/defendpoint :post "/:id/run"
@@ -232,9 +241,16 @@
   (api/check-superuser)
   (let [transform (api/check-404 (t2/select-one :model/Transform id))
         start-promise (promise)]
-    (u.jvm/in-virtual-thread*
-     (transforms.execute/run-mbql-transform! transform {:start-promise start-promise
-                                                        :run-method :manual}))
+    (if (transforms.util/python-transform? transform)
+      (u.jvm/in-virtual-thread*
+       (transforms.execute/execute-python-transform! transform {:start-promise start-promise
+                                                                :message-log   (atom {:pre-python  []
+                                                                                      :python      nil
+                                                                                      :post-python []})
+                                                                :run-method :manual}))
+      (u.jvm/in-virtual-thread*
+       (transforms.execute/run-mbql-transform! transform {:start-promise start-promise
+                                                          :run-method :manual})))
     (when (instance? Throwable @start-promise)
       (throw @start-promise))
     (let [result @start-promise
@@ -244,6 +260,53 @@
                               :run_id run-id})
           (assoc :status 202)))))
 
+;; hack
+(def ^:private python-test-run-id Integer/MAX_VALUE)
+
+(api.macros/defendpoint :post "/test-python"
+  "Test Python code execution without creating a transform."
+  [_route-params
+   _query-params
+   body :- [:map
+            [:code :string]
+            [:tables [:map-of :string :int]]]]
+  (log/info "test python code execution")
+  (api/check-superuser)
+  (let [run-id python-test-run-id
+        cancel-chan (a/promise-chan)]
+    (trasnforms.canceling/chan-start-run! run-id cancel-chan)
+    (try
+      (let [{:keys [response output events]} (transforms.execute/test-python-transform! (:code body) (:tables body) run-id cancel-chan)
+            {:keys [body status]} response]
+        (if (= status 200)
+          (do
+            (log/info "Python test execution succeeded")
+            (-> (response/response {:message (deferred-tru "Python code executed successfully")
+                                    :result  {:body (assoc body :output output)}})
+                (assoc :status 200)))
+          (do
+            (log/error "Error executing Python test code")
+            (-> (response/response {:message   (deferred-tru "Python code execution failed")
+                                    :error     body
+                                    :stdout    (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
+                                    :stderr    (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))
+                                    :exit_code (:exit_code (:exit_code body))})
+                (assoc :status status)))))
+      (finally
+        (trasnforms.canceling/chan-end-run! run-id)))))
+
+(api.macros/defendpoint :post "/test-python/cancel"
+  "Cancel the current test-python execution."
+  [_route-params
+   _query-params]
+  (log/info "canceling test python execution")
+  (api/check-superuser)
+  (if (trasnforms.canceling/chan-signal-cancel! python-test-run-id)
+    (-> (response/response {:message (deferred-tru "Python test canceled")})
+        (assoc :status 200))
+    (-> (response/response {:message (deferred-tru "No running Python test to cancel")})
+        (assoc :status 404))))
+
 (def ^{:arglists '([request respond raise])} routes
   "`/api/ee/transform` routes."
   (handlers/routes

enterprise/backend/src/metabase_enterprise/transforms/canceling.clj

@@ -49,7 +49,9 @@
    (chan-signal-cancel! run-id)
    (transform-run/timeout-run! run-id)))
 
-(defn- cancel-run! [run-id]
+(defn cancel-run!
+  "Cancel run"
+  [run-id]
   (when (chan-signal-cancel! run-id)
     (transform-run/cancel-run! run-id)))
 

enterprise/backend/src/metabase_enterprise/transforms/execute.clj

@@ -1,17 +1,27 @@
 (ns metabase-enterprise.transforms.execute
   (:require
    [clojure.core.async :as a]
+   [clojure.java.io :as io]
+   [clojure.string :as str]
    [metabase-enterprise.transforms.canceling :as canceling]
+   [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
    [metabase-enterprise.transforms.models.transform-run :as transform-run]
+   [metabase-enterprise.transforms.python-runner :as python-runner]
    [metabase-enterprise.transforms.settings :as transforms.settings]
    [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.driver :as driver]
    [metabase.driver.util :as driver.u]
    [metabase.lib.schema.common :as schema.common]
    [metabase.query-processor.pipeline :as qp.pipeline]
+   [metabase.util :as u]
+   [metabase.util.jvm :as u.jvm]
    [metabase.util.log :as log]
    [metabase.util.malli.registry :as mr]
-   [toucan2.core :as t2]))
+   [toucan2.core :as t2])
+  (:import
+   (clojure.lang IDeref)
+   (java.io Closeable File)
+   (java.time Duration)))
 
 (set! *warn-on-reflection* true)
 
@@ -37,24 +47,38 @@
    (log/info "Syncing target" (pr-str target) "for transform")
    (transforms.util/activate-table-and-mark-computed! database target)))
 
-(defn run-transform!
+(defn run-cancelable-transform!
   "Run a compiled transform"
-  [run-id driver {:keys [db-id conn-spec output-schema] :as transform-details} opts]
-  ;; local run is responsible for status
+  [run-id driver {:keys [db conn-spec output-schema]} run-transform!]
+  ;; local run is responsible for status, using canceling lifecycle
   (try
-    (when-not (driver/schema-exists? driver db-id output-schema)
-      (driver/create-schema-if-needed! driver conn-spec output-schema))
+    (when (driver.u/supports? driver :schemas db)
+      (when-not (driver/schema-exists? driver (:id db) output-schema)
+        (driver/create-schema-if-needed! driver conn-spec output-schema)))
     (canceling/chan-start-timeout-vthread! run-id (transforms.settings/transform-timeout))
-    (binding [qp.pipeline/*canceled-chan* (a/promise-chan)]
-      (canceling/chan-start-run! run-id qp.pipeline/*canceled-chan*)
-      (driver/run-transform! driver transform-details opts))
-    (transform-run/succeed-started-run! run-id)
+    (let [cancel-chan (a/promise-chan)
+          ret (binding [qp.pipeline/*canceled-chan* cancel-chan]
+                (canceling/chan-start-run! run-id cancel-chan)
+                (run-transform! cancel-chan))]
+      (transform-run/succeed-started-run! run-id)
+      ret)
     (catch Throwable t
       (transform-run/fail-started-run! run-id {:message (.getMessage t)})
       (throw t))
     (finally
       (canceling/chan-end-run! run-id))))
 
+(defn- try-start-unless-already-running [id run-method]
+  (try
+    (transform-run/start-run! id {:run_method run-method})
+    (catch java.sql.SQLException e
+      (if (= (.getSQLState e) "23505")
+        (throw (ex-info "Transform is already running"
+                        {:error :already-running
+                         :transform-id id}
+                        e))
+        (throw e)))))
+
 (defn run-mbql-transform!
   "Run `transform` and sync its target table.
 
@@ -67,7 +91,7 @@
      (let [db (get-in source [:query :database])
            {driver :engine :as database} (t2/select-one :model/Database db)
            feature (transforms.util/required-database-feature transform)
-           transform-details {:db-id db
+           transform-details {:db db
                               :transform-type (keyword (:type target))
                               :conn-spec (driver/connection-spec driver database)
                               :query (transforms.util/compile-source source)
@@ -78,20 +102,13 @@
          (throw (ex-info "The database does not support the requested transform target type."
                          {:driver driver, :database database, :feature feature})))
        ;; mark the execution as started and notify any observers
-       (let [{run-id :id} (try
-                            (transform-run/start-run! id {:run_method run-method})
-                            (catch java.sql.SQLException e
-                              (if (= (.getSQLState e) "23505")
-                                (throw (ex-info "Transform is already running"
-                                                {:error :already-running
-                                                 :transform-id id}
-                                                e))
-                                (throw e))))]
+       (let [{run-id :id} (try-start-unless-already-running id run-method)]
          (when start-promise
            (deliver start-promise [:started run-id]))
          (log/info "Executing transform" id "with target" (pr-str target))
-         (run-transform! run-id driver transform-details opts)
-         (sync-target! target database run-id)))
+         (run-cancelable-transform! run-id driver transform-details (fn [_cancel-chan] (driver/run-transform! driver transform-details opts)))
+         (transforms.instrumentation/with-stage-timing [run-id :sync :table-sync]
+           (sync-target! target database run-id))))
      (catch Throwable t
        (log/error t "Error executing transform")
        (when start-promise
@@ -99,3 +116,236 @@
          ;; but we assume nobody would catch the exception anyway
          (deliver start-promise t))
        (throw t)))))
+
+(defn- log! [message-log s]
+  (when message-log
+    (swap! message-log (fn [m] (update m (if (:python m) :post-python :pre-python) conj s))))
+  nil)
+
+(defn- replace-python-logs! [message-log events]
+  (when message-log (swap! message-log assoc :python events))
+  nil)
+
+(defn- save-log-as-message! [run-id message-log]
+  (when message-log
+    (let [{:keys [pre-python python post-python]} @message-log]
+      (t2/update! :model/TransformRun
+                  :id run-id
+                  {:message (str/join "\n" (concat pre-python
+                                                   (for [{:keys [stream message]} python]
+                                                     (if (= "stdout" stream)
+                                                       (str "\033[32m" message "\033[0m")
+                                                       (str "\033[31m" message "\033[0m")))
+                                                   post-python))}))))
+
+(defn- log-loop! [run-id message-log]
+  (let [poll #(python-runner/get-logs run-id)]
+    (try
+      (loop []
+        (if (.isInterrupted (Thread/currentThread))
+          (log/debug "Message update loop interrupted")
+          (do (Thread/sleep 1000)
+              (let [{:keys [status body]} (poll)]
+                (cond
+                  (<= 200 status 299)
+                  (let [{:keys [execution_id events]} body]
+                    (if-not (= run-id execution_id)
+                      (do (log/debugf "Run id did not match expected: %s actual: %s" run-id execution_id)
+                          (recur))
+                      (do
+                        (replace-python-logs! message-log events)
+                        (save-log-as-message! run-id message-log)
+                        (recur))))
+                  (= 404 status)
+                  (do
+                    (log/debugf "No logs yet (or run finished), run-id: %s" run-id)
+                    (recur))
+                  :else
+                  (do
+                    (log/warnf "Unexpected status polling for logs %s %s, run-id: %s" status body run-id)
+                    (log/debug "Exiting due to poll error")))))))
+      (catch InterruptedException _)
+      (catch Throwable e
+        (log/errorf e "An exception was caught during msg update loop, run-id: %s" run-id)))))
+
+(defn- open-log-thread! ^Closeable [run-id message-log]
+  (let [log-thread-promise (promise)
+        cleanup (fn []
+                  (let [^Thread log-thread (deref log-thread-promise 1000 nil)]
+                    (when-not log-thread (log/fatalf "Log thread reference not bound, run-id: %s" run-id))
+                    (when (.isAlive log-thread)
+                      (.interrupt log-thread)
+                      (when-not (.join log-thread (Duration/ofSeconds 10))
+                        (log/fatalf "Log thread could not be interrupted, run-id: %s" run-id)))))]
+    (u.jvm/in-virtual-thread*
+     (deliver log-thread-promise (Thread/currentThread))
+     (log-loop! run-id message-log))
+    (when-not (deref log-thread-promise 1000 nil) (log/fatalf "Log thread reference not bound, run-id: %s" run-id))
+    (reify
+      IDeref
+      (deref [_] @log-thread-promise)
+      Closeable
+      (close [_] (cleanup)))))
+
+(defn- debug-info-str [{:keys [exit-code events]}]
+  (str/join "\n"
+            ;; todo this was temporary before the log stuff, needs a rethink at some point
+            ["stdout"
+             "======"
+             (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
+             "stderr"
+             "======"
+             (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))
+             "======"
+             (format "exit code %d" exit-code)]))
+
+(defn- create-table-and-insert-data!
+  "Create a table from metadata and insert data from source."
+  [driver db-id table-name metadata data-source]
+  (let [table-schema {:name (if (keyword? table-name) table-name (keyword table-name))
+                      :columns (mapv (fn [{:keys [name dtype base_type database_type]}]
+                                       {:name name
+                                        :type (or (some->> base_type (keyword "type"))
+                                                  (transforms.util/dtype->base-type dtype))
+                                        :database-type database_type
+                                        :nullable? true})
+                                     (:fields metadata))}
+        data-source (assoc data-source :table-schema table-schema)]
+    (transforms.util/create-table-from-schema! driver db-id table-schema)
+    (driver/insert-from-source! driver db-id table-name (mapv :name (:columns table-schema)) data-source)))
+
+(defn- transfer-file-to-db [driver db {:keys [target] :as transform} metadata temp-file]
+  (let [table-name (transforms.util/qualified-table-name driver target)
+        table-exists? (transforms.util/target-table-exists? transform)
+        data-source {:type :csv-file
+                     :file temp-file}]
+    (if table-exists?
+      ;; Table exists - use temp table + atomic swap pattern
+      (let [temp-table-name (keyword (str (u/qualified-name table-name) "_temp_" (System/currentTimeMillis)))]
+        (log/info "Existing table deletected, Create then swap")
+        (try
+          (create-table-and-insert-data! driver (:id db) temp-table-name metadata data-source)
+          ;; TODO: These operations should be within a transaction for atomicity
+          (transforms.util/drop-table! driver (:id db) table-name)
+          (transforms.util/rename-table! driver (:id db) temp-table-name table-name)
+          (catch Exception e
+            (log/error e "Failed to transfer data to table")
+            (try
+              (transforms.util/drop-table! driver (:id db) temp-table-name)
+              (catch Exception _))
+            (throw e))))
+      ;; Table doesn't exist - create directly with target name
+      (do
+        (log/info "New table")
+        (create-table-and-insert-data! driver (:id db) table-name metadata data-source)))))
+
+(defn test-python-transform!
+  "Execute a transform in test mode (does not write result into a table)."
+  [code tables->id run-id cancel-chan]
+  (with-open [shared-storage-ref (python-runner/open-s3-shared-storage! tables->id)]
+    (let [server-url              (transforms.settings/python-execution-server-url)
+          _                       (python-runner/copy-tables-to-s3! {:run-id         run-id
+                                                                     :shared-storage @shared-storage-ref
+                                                                     :table-name->id tables->id
+                                                                     :cancel-chan    cancel-chan})
+          _                       (python-runner/open-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan
+          response
+          (python-runner/execute-python-code-http-call!
+           {:server-url     server-url
+            :code           code
+            :run-id         run-id
+            :table-name->id tables->id
+            :shared-storage @shared-storage-ref})
+          {:keys [output events]} (python-runner/read-output-objects @shared-storage-ref)]
+      {:response response
+       :events   events
+       :output   output})))
+
+(defn- run-python-transform! [{:keys [source] :as transform} db run-id cancel-chan message-log]
+  (with-open [log-thread-ref     (open-log-thread! run-id message-log)
+              shared-storage-ref (python-runner/open-s3-shared-storage! (:source-tables source))]
+    (let [driver     (:engine db)
+          server-url (transforms.settings/python-execution-server-url)
+          _          (python-runner/copy-tables-to-s3! {:run-id                  run-id
+                                                        :shared-storage @shared-storage-ref
+                                                        :table-name->id (:source-tables source)
+                                                        :cancel-chan    cancel-chan})
+          _          (python-runner/open-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan
+          {:keys [status body] :as response}
+          (python-runner/execute-python-code-http-call!
+           {:server-url     server-url
+            :code           (:body source)
+            :run-id         run-id
+            :table-name->id (:source-tables source)
+            :shared-storage @shared-storage-ref})
+          {:keys [exit_code]} body
+          ;; TODO temporary to keep more code stable while refactoring
+          ;; no need to materialize these early (i.e output we can stream directly into a tmp file or db if small)
+          {:keys [output output-manifest events]} (python-runner/read-output-objects @shared-storage-ref)]
+      (.close log-thread-ref)           ; early close to force any writes to flush
+      (when (seq events)
+        (replace-python-logs! message-log events))
+      (if (not= 200 status)
+        (throw (ex-info (debug-info-str {:exit-code exit_code :events events}) ;; todo do better here
+                        {:status-code     400
+                         :api-status-code status
+                         :body            body
+                         :events          events
+                         :error           (:error body)}))
+        (try
+          (let [temp-file (File/createTempFile "transform-output-" ".csv")]
+            (when-not (seq (:fields output-manifest))
+              (throw (ex-info "No fields in metadata"
+                              {:metadata output-manifest
+                               :raw-body body
+                               :events   events})))
+            (try
+              (with-open [writer (io/writer temp-file)]
+                (.write writer ^String output))
+              (let [file-size (.length temp-file)]
+                (transforms.instrumentation/with-stage-timing [run-id :data-transfer :file-to-dwh]
+                  (transfer-file-to-db driver db transform output-manifest temp-file))
+                (transforms.instrumentation/record-data-transfer! run-id :file-to-dwh file-size nil))
+              (finally
+                (.delete temp-file))))
+          response
+          (catch Exception e
+            (log/error e "Failed to to create resulting table")
+            (throw (ex-info "Failed to create the resulting table" {:error (.getMessage e)}))))))))
+
+(defn execute-python-transform!
+  "Execute a Python transform by calling the python runner.
+
+  This is executing synchronously, but supports being kicked off in the background
+  by delivering the `start-promise` just before the start when the beginning of the execution has been booked
+  in the database."
+  [transform {:keys [run-method start-promise message-log]}]
+  (when (transforms.util/python-transform? transform)
+    (try
+      (let [message-log (or message-log (atom {:pre-python  []
+                                               :python      nil
+                                               :post-python []}))
+            {:keys [target] transform-id :id} transform
+            {driver :engine :as db} (t2/select-one :model/Database (:database target))
+            {run-id :id} (try-start-unless-already-running transform-id run-method)]
+        (some-> start-promise (deliver [:started run-id]))
+        (log! message-log "Executing Python transform")
+        (log/info "Executing Python transform" transform-id "with target" (pr-str target))
+        (let [start-ms (u/start-timer)
+              transform-details
+              {:db db
+               :transform-type (keyword (:type target))
+               :conn-spec (driver/connection-spec driver db)
+               :output-schema (:schema target)
+               :output-table (transforms.util/qualified-table-name driver target)}
+              result   (run-cancelable-transform! run-id driver transform-details (fn [cancel-chan] (run-python-transform! transform db run-id cancel-chan message-log)))]
+          (transforms.instrumentation/with-stage-timing [run-id :sync :table-sync]
+            (sync-target! target db run-id))
+          (log! message-log (format "Python execution finished in %s" (Duration/ofMillis (u/since-ms start-ms))))
+          (save-log-as-message! run-id message-log)
+          {:run_id run-id
+           :result result}))
+      (catch Throwable t
+        (log/error t "Error executing Python transform")
+        (log! message-log "Error executing python transform")
+        (throw t)))))

enterprise/backend/src/metabase_enterprise/transforms/instrumentation.clj

@@ -0,0 +1,181 @@
+(ns metabase-enterprise.transforms.instrumentation
+  (:require
+   [metabase.analytics.prometheus :as prometheus]
+   [metabase.util :as u]
+   [metabase.util.log :as log]
+   [metabase.util.malli :as mu]
+   [metabase.util.malli.registry :as mr]))
+
+(set! *warn-on-reflection* true)
+
+(mr/def ::stage-type
+  [:enum :data-transfer :computation :sync])
+
+(mr/def ::stage-label
+  [:enum
+   :dwh-to-file
+   :file-to-s3
+   :file-to-dwh
+
+   :python-execution
+   :mbql-query
+   :table-sync])
+
+(mu/defn record-stage-start!
+  "Record the start of a transform stage."
+  [job-run-id :- pos-int?
+   stage-type :- ::stage-type
+   stage-label :- ::stage-label]
+  (log/infof "Transform stage started: run-id=%d type=%s label=%s" job-run-id (name stage-type) (name stage-label))
+  (prometheus/inc! :metabase-transforms/stage-started
+                   {:job-run-id (str job-run-id)
+                    :stage-type (name stage-type)
+                    :stage-label (name stage-label)}))
+
+(mu/defn record-stage-completion!
+  "Record the successful completion of a transform stage."
+  [job-run-id :- pos-int?
+   stage-type :- ::stage-type
+   stage-label :- ::stage-label
+   duration-ms :- int?]
+  (log/infof "Transform stage completed: run-id=%d type=%s label=%s duration=%dms" job-run-id (name stage-type) (name stage-label) duration-ms)
+  (let [labels {:job-run-id (str job-run-id)
+                :stage-type (name stage-type)
+                :stage-label (name stage-label)}]
+    (prometheus/inc! :metabase-transforms/stage-completed labels)
+    (prometheus/observe! :metabase-transforms/stage-duration-ms labels duration-ms)))
+
+(mu/defn record-stage-failure!
+  "Record the failure of a transform stage."
+  [job-run-id :- pos-int?
+   stage-type :- ::stage-type
+   stage-label :- ::stage-label
+   duration-ms :- [:maybe int?]]
+  (log/warnf "Transform stage failed: run-id=%d type=%s label=%s duration=%s" job-run-id (name stage-type) (name stage-label) (if duration-ms (str duration-ms "ms") "unknown"))
+  (let [labels {:job-run-id (str job-run-id)
+                :stage-type (name stage-type)
+                :stage-label (name stage-label)}]
+    (prometheus/inc! :metabase-transforms/stage-failed labels)
+    (when duration-ms
+      (prometheus/observe! :metabase-transforms/stage-duration-ms labels duration-ms))))
+
+(defn with-timing
+  "Generic timing function that executes body while recording metrics.
+
+   Takes a map of optional functions:
+   - :start-fn - function to call when starting
+   - :success-fn - function to call on success (receives duration-ms as last arg)
+   - :error-fn - function to call on error (receives duration-ms as last arg)
+
+   Usage:
+   (with-timing {:start-fn #(record-stage-start! %1 %2 %3)
+                 :success-fn #(record-stage-completion! %1 %2 %3 %4)
+                 :error-fn #(record-stage-failure! %1 %2 %3 %4)}
+                [job-run-id stage-type stage-label]
+                #(do-work)) "
+  [callbacks args body-fn]
+  (let [start-time (u/start-timer)]
+    (when-let [start-fn (:start-fn callbacks)]
+      (apply start-fn args))
+    (try
+      (let [result (body-fn)]
+        (when-let [success-fn (:success-fn callbacks)]
+          (apply success-fn (conj (vec args) (long (u/since-ms start-time)))))
+        result)
+      (catch Throwable t
+        (when-let [error-fn (:error-fn callbacks)]
+          (apply error-fn (conj (vec args) (long (u/since-ms start-time)))))
+        (throw t)))))
+
+(defmacro with-stage-timing
+  "Execute body while timing a transform stage. Automatically records start, completion or failure.
+
+   Usage:
+   (with-stage-timing [job-run-id :data-transfer :dwh-to-file]
+     ;; stage implementation
+     result)"
+  [args & body]
+  `(with-timing {:start-fn record-stage-start!
+                 :success-fn record-stage-completion!
+                 :error-fn record-stage-failure!}
+     ~args
+     (^:once fn* [] ~@body)))
+
+(mu/defn record-data-transfer!
+  "Record metrics about data transfer (size and rows)."
+  [job-run-id :- pos-int?
+   stage-label :- ::stage-label
+   bytes :- [:maybe int?]
+   rows :- [:maybe int?]]
+  (log/infof "Data transfer recorded: run-id=%d stage=%s%s%s"
+             job-run-id (name stage-label)
+             (if bytes (str " bytes=" bytes) "")
+             (if rows (str " rows=" rows) ""))
+  (let [labels {:job-run-id (str job-run-id)
+                :stage-label (name stage-label)}]
+    (when bytes
+      (prometheus/observe! :metabase-transforms/data-transfer-bytes labels bytes))
+    (when rows
+      (prometheus/observe! :metabase-transforms/data-transfer-rows labels rows))))
+
+(defn record-job-start!
+  "Record the start of a transform job run."
+  [job-id run-method]
+  (log/infof "Transform job started: job-id=%d run-method=%s" job-id (name run-method))
+  (prometheus/inc! :metabase-transforms/job-runs-total
+                   {:job-id (str job-id)
+                    :run-method (name run-method)}))
+
+(defn record-job-completion!
+  "Record the successful completion of a transform job run."
+  [job-id run-method duration-ms]
+  (log/infof "Transform job completed: job-id=%d run-method=%s duration=%dms" job-id (name run-method) duration-ms)
+  (let [labels {:job-id (str job-id)
+                :run-method (name run-method)}]
+    (prometheus/inc! :metabase-transforms/job-runs-completed labels)
+    (prometheus/observe! :metabase-transforms/job-run-duration-ms labels duration-ms)))
+
+(defn record-job-failure!
+  "Record the failure of a transform job run."
+  [job-id run-method duration-ms]
+  (log/warnf "Transform job failed: job-id=%d run-method=%s duration=%s" job-id (name run-method) (if duration-ms (str duration-ms "ms") "unknown"))
+  (let [labels {:job-id (str job-id)
+                :run-method (name run-method)}]
+    (prometheus/inc! :metabase-transforms/job-runs-failed labels)
+    (when duration-ms
+      (prometheus/observe! :metabase-transforms/job-run-duration-ms labels duration-ms))))
+
+(defmacro with-job-timing
+  "Execute body while timing a transform job run. Automatically records start, completion or failure.
+
+   Usage:
+   (with-job-timing [job-id run-method]
+     ;; job implementation
+     result)"
+  [[job-id run-method] & body]
+  `(with-timing {:start-fn record-job-start!
+                 :success-fn record-job-completion!
+                 :error-fn record-job-failure!}
+     [~job-id ~run-method]
+     (^:once fn* [] ~@body)))
+
+(mu/defn record-python-api-call!
+  "Record metrics about Python API calls."
+  [job-run-id :- pos-int?
+   duration-ms :- int?
+   status :- [:enum :success :error :timeout]]
+  (log/infof "Python API call %s: run-id=%d duration=%dms" (name status) job-run-id duration-ms)
+  (let [labels {:job-run-id (str job-run-id)}]
+    (prometheus/inc! :metabase-transforms/python-api-calls-total
+                     (assoc labels :status (name status)))
+    (prometheus/observe! :metabase-transforms/python-api-call-duration-ms labels duration-ms)))
+
+(defmacro with-python-api-timing
+  "Execute body while timing a Python API call."
+  [[job-run-id] & body]
+  `(with-timing {:success-fn (fn [job-run-id# duration-ms#]
+                               (record-python-api-call! job-run-id# duration-ms# :success))
+                 :error-fn (fn [job-run-id# duration-ms#]
+                             (record-python-api-call! job-run-id# duration-ms# :error))}
+     [~job-run-id]
+     (^:once fn* [] ~@body)))

enterprise/backend/src/metabase_enterprise/transforms/jobs.clj

@@ -6,10 +6,12 @@
    [clojurewerkz.quartzite.triggers :as triggers]
    [flatland.ordered.set :as ordered-set]
    [metabase-enterprise.transforms.execute :as transforms.execute]
+   [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
    [metabase-enterprise.transforms.models.job-run :as transforms.job-run]
    [metabase-enterprise.transforms.models.transform-run :as transform-run]
    [metabase-enterprise.transforms.ordering :as transforms.ordering]
    [metabase-enterprise.transforms.settings :as transforms.settings]
+   [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.task.core :as task]
    [metabase.util.log :as log]
    [toucan2.core :as t2]))
@@ -64,14 +66,21 @@
         (map transforms-by-id complete)))))
 
 (defn- run-transform! [run-id run-method {transform-id :id :as transform}]
-  (when (transform-run/running-run-for-run-id transform-id)
+  (when
+   (transform-run/running-run-for-transform-id transform-id)
     (log/warn "Transform" (pr-str transform-id) "already running, waiting")
     (loop []
       (Thread/sleep 2000)
-      (when (transform-run/running-run-for-run-id transform-id)
+      (when (transform-run/running-run-for-transform-id transform-id)
         (recur))))
+
   (log/info "Executing job transform" (pr-str transform-id))
-  (transforms.execute/run-mbql-transform! transform {:run-method run-method})
+  (if (transforms.util/python-transform? transform)
+    (transforms.instrumentation/with-stage-timing [run-id :computation :python-execution]
+      (transforms.execute/execute-python-transform! transform {:run-method run-method}))
+    (transforms.instrumentation/with-stage-timing [run-id :computation :mbql-query]
+      (transforms.execute/run-mbql-transform! transform {:run-method run-method})))
+
   (transforms.job-run/add-run-activity! run-id))
 
 (defn run-transforms!
@@ -110,12 +119,13 @@
     (let [transforms (job-transform-ids job-id)]
       (log/info "Executing transform job" (pr-str job-id) "with transforms" (pr-str transforms))
       (let [{run-id :id} (transforms.job-run/start-run! job-id run-method)]
-        (try
-          (run-transforms! run-id transforms opts)
-          (transforms.job-run/succeed-started-run! run-id)
-          (catch Throwable t
-            (transforms.job-run/fail-started-run! run-id {:message (.getMessage t)})
-            (throw t)))))))
+        (transforms.instrumentation/with-job-timing [job-id run-method]
+          (try
+            (run-transforms! run-id transforms opts)
+            (transforms.job-run/succeed-started-run! run-id)
+            (catch Throwable t
+              (transforms.job-run/fail-started-run! run-id {:message (.getMessage t)})
+              (throw t))))))))
 
 (def ^:private job-key "metabase-enterprise.transforms.jobs.timeout-job")
 

enterprise/backend/src/metabase_enterprise/transforms/models/transform_run.clj

@@ -147,11 +147,11 @@
                         :message   "Canceled by user but could not guarantee run stopped."})
     (cancel/delete-old-canceling-runs!)))
 
-(defn running-run-for-run-id
+(defn running-run-for-transform-id
   "Return a single active transform run or nil."
-  [id]
+  [transform-id]
   (t2/select-one :model/TransformRun
-                 :id id
+                 :transform_id transform-id
                  :is_active true))
 
 (defn- timestamp-constraint

enterprise/backend/src/metabase_enterprise/transforms/models/transform_run_cancelation.clj

@@ -14,14 +14,10 @@
   ([run-id]
    (mark-cancel-started-run! run-id {}))
   ([run-id properties]
-   (t2/insert! :model/TransformRunCancelation
-               (assoc properties
-                      :run_id run-id)
-               {:where [:exists {:select [1]
-                                 :from   [:transform_run]
-                                 :where  [:and
-                                          [:= :transform_run.id run-id]
-                                          :transform_run.is_active]}]})))
+   (when (t2/exists? :transform_run :id run-id :is_active true)
+     (t2/insert! :model/TransformRunCancelation
+                 (assoc properties
+                        :run_id run-id)))))
 
 (defn reducible-canceled-local-runs
   "Return a reducible sequence of local canceled runs."

enterprise/backend/src/metabase_enterprise/transforms/ordering.clj

@@ -16,23 +16,29 @@
 (set! *warn-on-reflection* true)
 
 (defn- transform-deps [transform]
-  (let [query (-> (get-in transform [:source :query])
-                  transforms.util/massage-sql-query
-                  qp.preprocess/preprocess
-                  ;; legacy usage -- don't do things like this going forward
-                  #_{:clj-kondo/ignore [:discouraged-var]}
-                  lib/->legacy-MBQL)]
-    (case (:type query)
-      :native (driver/native-query-deps (-> (qp.store/metadata-provider)
-                                            lib.metadata/database
-                                            :engine)
-                                        (get-in query [:native :query]))
-      :query (into #{}
-                   (keep #(clojure.core.match/match %
-                            [:source-table source] (when (int? source)
-                                                     {:table source})
-                            _ nil))
-                   (tree-seq coll? seq query)))))
+  (if (transforms.util/python-transform? transform)
+    ;; Python transforms have dependencies via source-tables mapping
+    (into #{}
+          (map #(hash-map :table %))
+          (vals (get-in transform [:source :source-tables])))
+    ;; Query transforms have dependencies via SQL query analysis
+    (let [query (-> (get-in transform [:source :query])
+                    transforms.util/massage-sql-query
+                    qp.preprocess/preprocess
+                    ;; legacy usage -- don't do things like this going forward
+                    #_{:clj-kondo/ignore [:discouraged-var]}
+                    lib/->legacy-MBQL)]
+      (case (:type query)
+        :native (driver/native-query-deps (-> (qp.store/metadata-provider)
+                                              lib.metadata/database
+                                              :engine)
+                                          (get-in query [:native :query]))
+        :query (into #{}
+                     (keep #(clojure.core.match/match %
+                              [:source-table source] (when (int? source)
+                                                       {:table source})
+                              _ nil))
+                     (tree-seq coll? seq query))))))
 
 (defn- dependency-map [transforms]
   (into {}
@@ -56,24 +62,32 @@
 
   The result is a map of transform id -> #{transform ids the transform depends on}. Dependencies are limited to just
   the transforms in the original list -- if a transform depends on some transform not in the list, the 'extra'
-  dependency is ignored."
+  dependency is ignored. Both query and Python transforms can have dependencies on tables produced by other transforms."
   [transforms]
-  (let [transforms-by-db (->> transforms
+
+  (let [;; Group all transforms by their database
+        transforms-by-db (->> transforms
                               (map (fn [transform]
-                                     {(get-in transform [:source :query :database]) [transform]}))
+                                     (let [db-id (if (transforms.util/python-transform? transform)
+                                                   (get-in transform [:target :database])
+                                                   (get-in transform [:source :query :database]))]
+                                       {db-id [transform]})))
                               (apply merge-with into))
+
         transform-ids (into #{} (map :id) transforms)
         {:keys [output-tables dependencies]} (->> transforms-by-db
                                                   (map (fn [[db-id db-transforms]]
                                                          (qp.store/with-metadata-provider db-id
                                                            {:output-tables (output-table-map db-transforms)
                                                             :dependencies (dependency-map db-transforms)})))
-                                                  (apply merge-with merge))]
-    (update-vals dependencies #(into #{}
-                                     (keep (fn [{:keys [table transform]}]
-                                             (or (output-tables table)
-                                                 (transform-ids transform))))
-                                     %))))
+                                                  (apply merge-with merge))
+
+        all-deps (update-vals dependencies #(into #{}
+                                                  (keep (fn [{:keys [table transform]}]
+                                                          (or (output-tables table)
+                                                              (transform-ids transform))))
+                                                  %))]
+    all-deps))
 
 (defn find-cycle
   "Finds a path containing a cycle in the directed graph `node->children`.

enterprise/backend/src/metabase_enterprise/transforms/python_runner.clj

@@ -0,0 +1,442 @@
+(ns metabase-enterprise.transforms.python-runner
+  (:require
+   [clj-http.client :as http]
+   [clojure.core.async :as a]
+   [clojure.java.io :as io]
+   [clojure.string :as str]
+   [medley.core :as m]
+   [metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
+   [metabase-enterprise.transforms.settings :as transforms.settings]
+   [metabase.driver :as driver]
+   [metabase.query-processor.compile :as qp.compile]
+   [metabase.query-processor.pipeline :as qp.pipeline]
+   ^{:clj-kondo/ignore [:discouraged-namespace]}
+   [metabase.query-processor.store :as qp.store]
+   [metabase.util.json :as json]
+   [metabase.util.log :as log]
+   [toucan2.core :as t2])
+  (:import
+   (clojure.lang IDeref)
+   (java.io BufferedWriter Closeable File OutputStream OutputStreamWriter)
+   (java.net URI)
+   (java.nio.charset StandardCharsets)
+   (java.time Duration)
+   (software.amazon.awssdk.auth.credentials AwsBasicCredentials StaticCredentialsProvider)
+   (software.amazon.awssdk.auth.credentials DefaultCredentialsProvider)
+   (software.amazon.awssdk.core.sync RequestBody)
+   (software.amazon.awssdk.regions Region)
+   (software.amazon.awssdk.services.s3 S3Client S3ClientBuilder S3Configuration)
+   (software.amazon.awssdk.services.s3.model DeleteObjectRequest GetObjectRequest NoSuchKeyException PutObjectRequest)
+   (software.amazon.awssdk.services.s3.presigner S3Presigner S3Presigner$Builder)
+   (software.amazon.awssdk.services.s3.presigner.model GetObjectPresignRequest PresignedGetObjectRequest PresignedPutObjectRequest PutObjectPresignRequest)))
+
+(set! *warn-on-reflection* true)
+
+(defn- safe-delete
+  "Safely delete a file."
+  [^File file]
+  (try (.delete file) (catch Exception _)))
+
+(defn- write-to-stream! [^OutputStream os col-names reducible-rows]
+  (let [writer (-> os
+                   (OutputStreamWriter. StandardCharsets/UTF_8)
+                   (BufferedWriter.))]
+
+    (run! (fn [row]
+            (let [row-map (zipmap col-names row)]
+              (json/encode-to row-map writer {})
+              (.newLine writer)))
+          reducible-rows)
+
+    (doto writer
+      (.flush)
+      (.close))))
+
+(defn- execute-mbql-query
+  [driver db-id query respond cancel-chan]
+  (driver/with-driver driver
+    (let [native (qp.compile/compile {:type :query, :database db-id :query query})
+
+          query  {:database db-id
+                  :type     :native
+                  :native   native}]
+      (qp.store/with-metadata-provider db-id
+        (binding [qp.pipeline/*canceled-chan* cancel-chan]
+          (driver/execute-reducible-query driver query {:canceled-chan cancel-chan} respond))))))
+
+(defn- root-type
+  [base-type]
+  (when base-type
+    (some #(when (isa? base-type %) %)
+          [:type/Float
+           :type/Integer
+           :type/Boolean
+           :type/DateTimeWithTZ
+           :type/DateTime
+           :type/Dictionary
+           :type/JSON
+           :type/Date
+           :type/Text])))
+
+(defn- closest-ancestor [t pred]
+  (loop [remaining (conj clojure.lang.PersistentQueue/EMPTY [t])]
+    (when-let [t (first remaining)]
+      (if (pred t)
+        t
+        (recur (into (pop t) (parents t)))))))
+
+(defn- effective-semantic-type-i-think
+  "Kinda sketchy but maybe reasonable way to infer the effective-semantic-type"
+  [{:keys [base_type effective_type semantic_type]}]
+  (or semantic_type (closest-ancestor (or effective_type base_type) #(isa? % :Semantic/*))))
+
+(defn- generate-manifest
+  "Generate a metadata manifest for the table columns."
+  [table-id cols-meta]
+  {:version        "0.1.0"
+   :fields         (mapv (fn [col-meta]
+                           {:name           (:name col-meta)
+                            :base_type      (some-> (:base_type col-meta) name)
+                            :database_type  (some-> (:database_type col-meta) name)
+                            :root_type      (some-> (root-type (:base_type col-meta)) name)
+                            ;; replace nil values with values indicating how they behave in practice.
+                            ;; there may be better ways of doing this already, but i worry it's just implicit in QP
+                            :semantic_type  (some-> (effective-semantic-type-i-think col-meta) name)
+                            :effective_type (some-> (or (:effective_type col-meta) (:database_type col-meta)) name)
+                            ;; TODO get this passed through
+                            :field_id       (:id col-meta)})
+                         cols-meta)
+   :table_metadata {:table_id table-id}})
+
+(defn- maybe-fixup-v [col v]
+  ;; clickhouse returns bigdecimals for int64 values
+  (if (and (isa? (:base_type col) :type/Integer)
+           (or (instance? BigDecimal v)
+               (float? v)))
+    (bigint v)
+    v))
+
+(defn- write-table-data-to-file! [id temp-file cancel-chan]
+  (let [db-id           (t2/select-one-fn :db_id (t2/table-name :model/Table) :id id)
+        driver          (t2/select-one-fn :engine :model/Database db-id)
+        all-fields-meta (t2/select [:model/Field :id :name :base_type :effective_type :semantic_type :database_type :database_position :nfc_path :parent_id]
+                                   :table_id id
+                                   :active true
+                                   {:order-by [[:database_position :asc]]})
+        fields-meta (filter #(and (nil? (:parent_id %)) (nil? (:nfc_path %))) all-fields-meta)
+        query {:source-table id}
+        manifest (generate-manifest id fields-meta)]
+    (execute-mbql-query driver db-id query
+                        (fn [{cols-meta :cols} reducible-rows]
+                          (with-open [os (io/output-stream temp-file)]
+                            (let [filtered-col-meta (m/index-by :name fields-meta)
+                                  filtered-cols (filter #(contains? filtered-col-meta (:name %)) cols-meta)
+                                  col-name->index (into {} (map-indexed (fn [i col] [(:name col) i]) cols-meta))
+                                  filtered-indices (mapv (fn [col] [(col-name->index (:name col)) col]) filtered-cols)
+                                  filtered-rows (eduction (map (fn [row] (mapv (fn [[idx col]]
+                                                                                 (let [v (nth row idx)]
+                                                                                   (maybe-fixup-v col v))) filtered-indices)))
+                                                          reducible-rows)]
+                              (write-to-stream! os (mapv :name filtered-cols) filtered-rows))))
+                        cancel-chan)
+    manifest))
+
+(defn- maybe-with-endpoint-s3-client [^S3ClientBuilder builder endpoint]
+  (let [region (transforms.settings/python-storage-s-3-region)]
+    (case [(some? endpoint) (some? region)]
+      [true true]   (doto builder
+                      (.endpointOverride (URI/create endpoint))
+                      (.region (Region/of region)))
+      [true false]  (log/warn "Ignoring endpoint because region is not defined")
+      [false true]  (.region builder (Region/of region))
+      [false false] builder)))
+
+(defn- maybe-with-endpoint-s3-presigner [^S3Presigner$Builder builder endpoint]
+  (let [region (transforms.settings/python-storage-s-3-region)]
+    (case [(some? endpoint) (some? region)]
+      [true true]   (doto builder
+                      (.endpointOverride (URI/create endpoint))
+                      (.region (Region/of region)))
+      [true false]  (log/warn "Ignoring endpoint because region is not defined")
+      [false true]  (.region builder (Region/of region))
+      [false false] (.region builder (Region/US_EAST_1)))))
+
+(defn- create-s3-configuration
+  "Create S3Configuration with path-style access setting"
+  ^S3Configuration []
+  (-> (S3Configuration/builder)
+      (.pathStyleAccessEnabled (transforms.settings/python-storage-s-3-path-style-access))
+      (.build)))
+
+(defn- build-put-object-request
+  "Build a PutObjectRequest with bucket and key"
+  [^String bucket-name ^String key]
+  (-> (PutObjectRequest/builder)
+      (.bucket bucket-name)
+      (.key key)
+      (.build)))
+
+(defn- build-get-object-request
+  "Build a GetObjectRequest with bucket and key"
+  [^String bucket-name ^String key]
+  (-> (GetObjectRequest/builder)
+      (.bucket bucket-name)
+      (.key key)
+      (.build)))
+
+(defn- build-delete-object-request
+  "Build a DeleteObjectRequest with bucket and key"
+  [^String bucket-name ^String key]
+  (-> (DeleteObjectRequest/builder)
+      (.bucket bucket-name)
+      (.key key)
+      (.build)))
+
+(def ^:private ^Duration presigned-url-duration
+  "Default duration for presigned URLs"
+  (Duration/ofHours 1))
+
+(defn- maybe-with-credentials-s3-client [^S3ClientBuilder builder]
+  (let [access-key (transforms.settings/python-storage-s-3-access-key)
+        secret-key (transforms.settings/python-storage-s-3-secret-key)]
+    (if (or access-key secret-key)
+      (if-not (and access-key secret-key)
+        (do (log/warnf "Ignoring %s because %s is not defined"
+                       (if access-key "access-key" "secret-key")
+                       (if (not access-key) "access-key" "secret-key"))
+            (.credentialsProvider builder (DefaultCredentialsProvider/create)))
+        (.credentialsProvider builder
+                              (StaticCredentialsProvider/create
+                               (AwsBasicCredentials/create access-key secret-key))))
+      (.credentialsProvider builder (DefaultCredentialsProvider/create)))))
+
+(defn- maybe-with-credentials-s3-presigner [^S3Presigner$Builder builder]
+  (let [access-key (transforms.settings/python-storage-s-3-access-key)
+        secret-key (transforms.settings/python-storage-s-3-secret-key)]
+    (if (or access-key secret-key)
+      (if-not (and access-key secret-key)
+        (do (log/warnf "Ignoring %s because %s is not defined"
+                       (if access-key "access-key" "secret-key")
+                       (if (not access-key) "access-key" "secret-key"))
+            builder)
+        (.credentialsProvider builder
+                              (StaticCredentialsProvider/create
+                               (AwsBasicCredentials/create access-key secret-key))))
+      builder)))
+
+;; We just recreate the client every time, to keep things simple if config is changed.
+(defn- create-s3-client
+  "Create S3 client for host operations (uploads, reads)"
+  ^S3Client []
+  (let [^S3ClientBuilder builder (S3Client/builder)]
+    (doto builder
+      (maybe-with-endpoint-s3-client (transforms.settings/python-storage-s-3-endpoint))
+      maybe-with-credentials-s3-client
+      (.serviceConfiguration (create-s3-configuration)))
+    (.build builder)))
+
+(defn- create-s3-presigner-for-container
+  "Create S3 presigner for container operations (presigned URLs).
+  Uses container-endpoint if different from host endpoint."
+  ^S3Presigner []
+  (let [container-endpoint (transforms.settings/python-storage-s-3-container-endpoint)
+        builder (S3Presigner/builder)]
+    (doto builder
+      (maybe-with-endpoint-s3-presigner (or container-endpoint (transforms.settings/python-storage-s-3-endpoint)))
+      maybe-with-credentials-s3-presigner
+      (.serviceConfiguration (create-s3-configuration)))
+    (.build builder)))
+
+(defn- upload-file-to-s3
+  "Upload file using host client"
+  [^S3Client s3-client ^String bucket-name ^String key ^File file]
+  (let [^PutObjectRequest request (build-put-object-request bucket-name key)]
+    (.putObject s3-client request (RequestBody/fromFile file))))
+
+(defn- generate-presigned-get-url
+  "Generate GET URL using container presigner"
+  [^S3Presigner presigner ^String bucket-name ^String key]
+  (let [^GetObjectRequest get-request (build-get-object-request bucket-name key)
+        ^GetObjectPresignRequest presign-request (-> (GetObjectPresignRequest/builder)
+                                                     (.signatureDuration presigned-url-duration)
+                                                     (.getObjectRequest get-request)
+                                                     (.build))
+        presigned ^PresignedGetObjectRequest (.presignGetObject presigner presign-request)]
+    (.toString (.url presigned))))
+
+(defn- generate-presigned-put-url
+  "Generate PUT URL using container presigner"
+  [^S3Presigner presigner ^String bucket-name ^String key]
+  (let [^PutObjectRequest put-request (build-put-object-request bucket-name key)
+        ^PutObjectPresignRequest presign-request (-> (PutObjectPresignRequest/builder)
+                                                     (.signatureDuration presigned-url-duration)
+                                                     (.putObjectRequest put-request)
+                                                     (.build))
+        presigned ^PresignedPutObjectRequest (.presignPutObject presigner presign-request)]
+    (.toString (.url presigned))))
+
+(defn- delete-s3-object [^S3Client s3-client ^String bucket-name ^String key]
+  (try
+    (let [^DeleteObjectRequest request (build-delete-object-request bucket-name key)]
+      (.deleteObject s3-client request))
+    (catch Exception e
+      (log/debugf e "Error deleting s3 object %s" key)
+      ;; Ignore deletion errors - object might not exist, or we might not have permissions
+      ;; NOTE: we plan to put general retention on the bucket so that objects will eventually be deleted
+      nil)))
+
+(defn- cleanup-s3-objects [^S3Client s3-client bucket-name s3-keys]
+  (run! (partial delete-s3-object s3-client bucket-name) s3-keys))
+
+(defn- read-from-s3
+  ([s3-client bucket-name key] (read-from-s3 s3-client bucket-name key ::throw))
+  ([^S3Client s3-client ^String bucket-name ^String key not-found]
+   (try
+     (let [^GetObjectRequest request (build-get-object-request bucket-name key)
+           response                  (.getObject s3-client request)]
+       (slurp response))
+     (catch NoSuchKeyException e
+       (if (identical? ::throw not-found)
+         (throw e)
+         not-found)))))
+
+(defn get-logs
+  "Return the logs of the current running python process"
+  [run-id]
+  (let [server-url (transforms.settings/python-execution-server-url)]
+    (http/get (str server-url "/logs")
+              {:content-type     :json
+               :accept           :json
+               :throw-exceptions false
+               :as               :json
+               :query-params     {:request_id run-id}})))
+
+(defn- s3-shared-storage [table-name->id]
+  (let [prefix              (some-> (transforms.settings/python-storage-s-3-prefix) (str "/"))
+        work-dir-name       (str prefix "run-" (System/nanoTime) "-" (rand-int 10000))
+        container-presigner (create-s3-presigner-for-container)
+        bucket-name         (transforms.settings/python-storage-s-3-bucket)
+        loc
+        (fn [method relative-path]
+          ;; save a tracking root to a well known location
+          ;; later can ls these to collect those objects whose ttl has elapsed
+          ;; (OR setup a retention policy in s3?)
+          (let [path (str work-dir-name "/" relative-path)]
+            {:path   path
+             :method method
+             :url    (case method :put (generate-presigned-put-url container-presigner bucket-name path)
+                           :get (generate-presigned-get-url container-presigner bucket-name path))}))]
+    {:s3-client   (create-s3-client)                        ;; do not like mixing interactive things with descriptions, but its damn convenient to have it here for now
+     :bucket-name bucket-name
+     :objects
+     (into
+      {:output          (loc :put "output.csv")
+       :output-manifest (loc :put "output-manifest.json")
+       :events          (loc :put "events.jsonl")}
+      (for [[table-name id] table-name->id]
+        {[:table id :manifest] (loc :get (str "-table-" (name table-name) "-" id ".manifest.json"))
+         [:table id :data]     (loc :get (str "-table-" (name table-name) "-" id ".jsonl"))}))}))
+
+(defn open-s3-shared-storage!
+  "Returns a deref'able shared storage value, (.close) will optimistically delete any s3 objects named in storage (data files for tables, metadata files etc).
+  The intention is the bucket specifies a generic object retention policy to ensure objects are eventually deleted (e.g. because the process dies during writing and .close never gets called)"
+  ^Closeable [table-name->id]
+  (let [shared-storage (s3-shared-storage table-name->id)]
+    (reify IDeref
+      (deref [_] shared-storage)
+      Closeable
+      (close [_] (cleanup-s3-objects (:s3-client shared-storage) (:bucket-name shared-storage) (map :path (vals (:objects shared-storage))))))))
+
+(defn copy-tables-to-s3!
+  "Writes table content to their corresponding objects named in shared-storage, see (open-shared-storage!).
+  Blocks until all tables are fully written and committed to shared storage."
+  [{:keys [run-id
+           shared-storage
+           table-name->id
+           cancel-chan]}]
+  (doseq [id (vals table-name->id)
+          :let [{:keys [s3-client bucket-name objects]} shared-storage
+                {data-path :path}     (get objects [:table id :data])
+                {manifest-path :path} (get objects [:table id :manifest])]]
+    (let [temp-file       (File/createTempFile data-path "")
+          manifest-file   (File/createTempFile manifest-path "")]
+      (try
+        ;; Write table data to temporary file and get manifest
+        (let [manifest (transforms.instrumentation/with-stage-timing [run-id :data-transfer :dwh-to-file]
+                         (write-table-data-to-file! id temp-file cancel-chan))]
+          ;; Write manifest to file
+          (with-open [writer (io/writer manifest-file)]
+            (json/encode-to manifest writer {}))
+          (let [file-size (.length temp-file)
+                manifest-size (.length manifest-file)]
+            (transforms.instrumentation/record-data-transfer! run-id :dwh-to-file file-size nil)
+
+            ;; Upload both files to S3
+            (transforms.instrumentation/with-stage-timing [run-id :data-transfer :file-to-s3]
+              (upload-file-to-s3 s3-client bucket-name data-path temp-file)
+              (upload-file-to-s3 s3-client bucket-name manifest-path manifest-file))
+
+            (transforms.instrumentation/record-data-transfer! run-id :file-to-s3 (+ file-size manifest-size) nil)))
+        (finally
+          ;; Clean up temporary files
+          (safe-delete temp-file)
+          (safe-delete manifest-file))))))
+
+(defn execute-python-code-http-call!
+  "Calls the /execute endpoint of the python runner. Blocks until the run either succeeds or fails and returns
+  the response from the server."
+  [{:keys [server-url code run-id table-name->id shared-storage]}]
+  (let [{:keys [objects]} shared-storage
+        {:keys [output output-manifest events]} objects
+
+        table-name->url          (update-vals table-name->id (comp :url #(get objects [:table % :data])))
+        table-name->manifest-url (update-vals table-name->id (comp :url #(get objects [:table % :manifest])))
+
+        payload                  {:code                code
+                                  :timeout             30
+                                  :request_id          run-id
+                                  :output_url          (:url output)
+                                  :output_manifest_url (:url output-manifest)
+                                  :events_url          (:url events)
+                                  :table_mapping       table-name->url
+                                  :manifest_mapping    table-name->manifest-url}
+
+        response                 (transforms.instrumentation/with-python-api-timing [run-id]
+                                   (http/post (str server-url "/execute")
+                                              {:content-type     :json
+                                               :accept           :json
+                                               :body             (json/encode payload)
+                                               :throw-exceptions false
+                                               :as               :json}))]
+    ;; when a 500 is returned we observe a string in the body (despite the python returning json)
+    ;; always try to parse the returned string as json before yielding (could tighten this up at some point)
+    (update response :body (fn [string-if-error]
+                             (if (string? string-if-error)
+                               (try
+                                 (json/decode+kw string-if-error)
+                                 (catch Exception _
+                                   {:error string-if-error}))
+                               string-if-error)))))
+
+(defn open-cancellation-process!
+  "Starts a core.async process that optimistically sends a cancellation request to the python executor if cancel-chan receives a value.
+  Returns a channel that will receive either the async http call j.u.c.FutureTask in the case of cancellation, or nil when the cancel-chan is closed."
+  [server-url run-id cancel-chan]
+  (a/go (when (a/<! cancel-chan)
+          (http/post (str server-url "/cancel")
+                     {:content-type :json
+                      :body         (json/encode {:request_id run-id})
+                      :async?       true}
+                     identity identity))))
+
+;; temporary, we should not need to realize data/events files into memory longer term
+(defn read-output-objects
+  "Temporary function that strings/jsons stuff in S3 and returns it for compatibility."
+  [{:keys [s3-client bucket-name objects]}]
+  (let [{:keys [output output-manifest events]} objects
+        output-content          (read-from-s3 s3-client bucket-name (:path output) nil)
+        output-manifest-content (read-from-s3 s3-client bucket-name (:path output-manifest) "{}")
+        events-content          (read-from-s3 s3-client bucket-name (:path events))]
+    {:output output-content
+     :output-manifest (json/decode+kw output-manifest-content)
+     :events (mapv json/decode+kw (str/split-lines events-content))}))

enterprise/backend/src/metabase_enterprise/transforms/schedule.clj

@@ -101,6 +101,7 @@
         trigger (build-trigger job-id schedule)]
     (task/schedule-task! job trigger)))
 
+;; TODO: should this be part of transformjob hooks?
 (defn update-job!
   "Update the trigger for a transform job."
   [job-id schedule]

enterprise/backend/src/metabase_enterprise/transforms/settings.clj

@@ -1,17 +1,119 @@
 (ns metabase-enterprise.transforms.settings
   (:require
+   [metabase.config.core :as config]
    [metabase.settings.core :as setting]
    [metabase.util.i18n :refer [deferred-tru]]))
 
 (set! *warn-on-reflection* true)
 
+(def ^:private is-not-prod? (or config/is-dev? config/is-test?))
+
 (setting/defsetting transform-timeout
   (deferred-tru "The timeout for a transform job, in minutes.")
-  :type       :integer
+  :type :integer
   :visibility :internal
-  :default    (* 4 60)
-  :feature    :transforms
-  :doc        false
-  :export?    false
+  :default (* 4 60)
+  :feature :transforms
+  :doc false
+  :export? false
   :encryption :no
-  :audit      :getter)
+  :audit :getter)
+
+(setting/defsetting python-execution-server-url
+  (deferred-tru "URL for the Python execution server that runs transform functions.")
+  :type :string
+  :visibility :admin
+  :default (when is-not-prod? "http://localhost:5001")
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-endpoint
+  (deferred-tru "S3 endpoint URL for storing Python execution artifacts.")
+  :type :string
+  :visibility :admin
+  :default (when is-not-prod? "http://localhost:4566")
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-region
+  (deferred-tru "AWS region for S3 storage.")
+  :type :string
+  :visibility :admin
+  :default (when is-not-prod? "us-east-1")
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-bucket
+  (deferred-tru "S3 bucket name for storing Python execution artifacts.")
+  :type :string
+  :visibility :admin
+  :default (if is-not-prod? "metabase-python-runner" "metabase-cloud-python-runner-staging")
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-prefix
+  (deferred-tru "Prefix to use for S3 objects. Necessary in prod where we only have access to a particular prefix.")
+  :type :string
+  :visibility :admin
+  :feature :transforms
+  :default (if is-not-prod? "test-prefix" "hosting-4b9351bb22b84791")
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-access-key
+  (deferred-tru "AWS access key ID for S3 authentication.")
+  :type :string
+  :visibility :admin
+  :feature :transforms
+  :default (when is-not-prod? "test")
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-secret-key
+  (deferred-tru "AWS secret access key for S3 authentication.")
+  :type :string
+  :visibility :admin
+  :feature :transforms
+  :default (when is-not-prod? "test")
+  :doc false
+  :export? false
+  :encryption :when-encryption-key-set
+  :audit :never)
+
+(setting/defsetting python-storage-s-3-container-endpoint
+  (deferred-tru "Alternative S3 endpoint accessible from containers. Leave empty if same as main endpoint.")
+  :type :string
+  :visibility :admin
+  :default (when is-not-prod? "http://localstack:4566")
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)
+
+(setting/defsetting python-storage-s-3-path-style-access
+  (deferred-tru "Use path-style access for S3 requests (required for LocalStack and some S3-compatible services).")
+  :type :boolean
+  :visibility :admin
+  :default (when is-not-prod? true)
+  :feature :transforms
+  :doc false
+  :export? false
+  :encryption :no
+  :audit :getter)

enterprise/backend/src/metabase_enterprise/transforms/util.clj

@@ -1,12 +1,16 @@
 (ns metabase-enterprise.transforms.util
   (:require
+   [clojure.string :as str]
    [java-time.api :as t]
    [metabase.driver :as driver]
+   [metabase.lib.schema.common :as lib.schema.common]
    [metabase.query-processor.compile :as qp.compile]
    [metabase.sync.core :as sync]
    [metabase.util :as u]
    [metabase.util.date-2 :as u.date]
    [metabase.util.log :as log]
+   [metabase.util.malli :as mu]
+   [metabase.util.malli.registry :as mr]
    [toucan2.core :as t2])
   (:import
    (java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)
@@ -21,10 +25,22 @@
     (keyword schema name)
     (keyword name)))
 
+(defn query-transform?
+  "Check if this is a query transform: native query / mbql query."
+  [transform]
+  (= :query (-> transform :source :type keyword)))
+
+(defn python-transform?
+  "Check if this is a Python transform."
+  [transform]
+  (= :python (-> transform :source :type keyword)))
+
 (defn target-table-exists?
   "Test if the target table of a transform already exists."
   [{:keys [source target] :as _transform}]
-  (let [db-id (-> source :query :database)
+  (let [db-id (or (-> source :query :database)
+                  ;; python transform target
+                  (-> target :database))
         {driver :engine :as database} (t2/select-one :model/Database db-id)]
     (driver/table-exists? driver database target)))
 
@@ -66,7 +82,9 @@
   [{:keys [id target source], :as _transform}]
   (when target
     (let [target (update target :type keyword)
-          database-id (-> source :query :database)
+          database-id (or (-> source :query :database)
+                          ;; python transform target
+                          (-> target :database))
           {driver :engine :as database} (t2/select-one :model/Database database-id)]
       (driver/drop-transform-target! driver database target)
       (log/info "Deactivating  target " (pr-str target) "for transform" id)
@@ -122,3 +140,60 @@
   (-> run
       (u/update-some :start_time utc-timestamp-string)
       (u/update-some :end_time   utc-timestamp-string)))
+
+(mr/def ::column-definition
+  [:map
+   [:name :string]
+   [:type ::lib.schema.common/base-type]
+   [:nullable? {:optional true} :boolean]])
+
+(mr/def ::table-definition
+  [:map
+   [:name :keyword]
+   [:columns [:sequential ::column-definition]]
+   [:primary-key {:optional true} [:sequential :string]]])
+
+(defn dtype->base-type
+  "Maps pandas dtype strings directly to Metabase base types in the type hierarchy."
+  [dtype-str]
+  (cond
+    (str/starts-with? dtype-str "int") :type/Integer
+    (str/starts-with? dtype-str "float") :type/Float
+    (str/starts-with? dtype-str "bool") :type/Boolean
+    ;; datetime64[ns, timezone] indicates timezone-aware datetime
+    (str/starts-with? dtype-str "datetime64[ns, ") :type/DateTimeWithTZ
+    (str/starts-with? dtype-str "datetime") :type/DateTime
+    ;; this is not a real dtype, pandas uses 'object', but we override it if there's source or custom field metadata
+    (str/starts-with? dtype-str "date") :type/Date
+    :else :type/Text))
+
+(mu/defn create-table-from-schema!
+  "Create a table from a table-schema"
+  [driver :- :keyword
+   database-id :- pos-int?
+   table-schema :- ::table-definition]
+  (let [{:keys [columns] table-name :name} table-schema
+        column-definitions (into {} (map (fn [{:keys [name type database-type]}]
+                                           (let [database-type (when-not (and (= driver :bigquery-cloud-sdk)
+                                                                              (#{"ARRAY" "RECORD"} database-type))
+                                                                 database-type)
+                                                 db-type (if database-type
+                                                           [[:raw database-type]]
+                                                           (driver/type->database-type driver type))]
+                                             [name db-type])))
+                                 columns)
+        primary-key-opts (select-keys table-schema [:primary-key])]
+    (log/infof "Creating table %s with %d columns" table-name (count columns))
+    (driver/create-table! driver database-id table-name column-definitions primary-key-opts)))
+
+(defn rename-table!
+  "kename a table in the database."
+  [driver database-id old-table-name new-table-name]
+  (log/infof "Renaming table %s to %s" old-table-name new-table-name)
+  (driver/rename-table! driver database-id old-table-name new-table-name))
+
+(defn drop-table!
+  "Drop a table in the database."
+  [driver database-id table-name]
+  (log/infof "Dropping table %s" table-name)
+  (driver/drop-table! driver database-id table-name))

enterprise/backend/test/metabase_enterprise/comments/api_test.clj

@@ -1,301 +0,0 @@
-(ns metabase-enterprise.comments.api-test
-  "Tests for /api/ee/comment/ endpoints."
-  (:require
-   [clojure.set :as set]
-   [clojure.string :as str]
-   [clojure.test :refer :all]
-   [metabase.notification.seed :as notification.seed]
-   [metabase.test :as mt]
-   [metabase.util :as u]
-   [toucan2.core :as t2]))
-
-(use-fixtures :each (fn [f] (mt/with-premium-features #{:documents} (f))))
-
-(defn- relaxed-re [& s]
-  (re-pattern (str "(?s).*" (str/join ".*" s) ".*")))
-
-;;; tiptap helpers
-
-(defn- render-tiptap [node]
-  (if (string? node)
-    {:type "text" :text node}
-    (let [tag     (case (first node)
-                    :p "paragraph"
-                    (name (first node)))
-          block?  #{"paragraph"}
-          attrs   (when (map? (second node)) (second node))
-          content (if attrs (drop 2 node) (drop 1 node))]
-      (u/remove-nils
-       {:type    tag
-        :attrs   (cond-> attrs
-                   (block? tag) (assoc :_id (str (random-uuid)))
-                   true         not-empty)
-        :content (when (seq content)
-                   (mapv render-tiptap content))}))))
-
-(defn tiptap
-  "Little helpers to generate tiptap docs"
-  [& content]
-  {:type    "doc"
-   :content (mapv render-tiptap content)})
-
-(deftest tiptap-helpers-test
-  (testing "can generate some content but less verbose"
-    (is (=? {:type "doc"
-             :content
-             [{:type  "paragraph"
-               :attrs {:_id string?},
-               :content
-               [{:type "text" :text "omg is that you? "}
-                {:type  "smartLink"
-                 :attrs {:entityId 6 :model "user"}}]}]}
-            (tiptap [:p
-                     "omg is that you? "
-                     [:smartLink {:entityId 6 :model "user"}]])))))
-
-;;;
-
-(deftest comments-require-documents-feature
-  (testing "GET /api/ee/comment/"
-    (mt/with-premium-features #{}
-      (mt/with-temp [:model/Document {doc-id :id} {}]
-        (testing "should not return the comments"
-          (mt/user-http-request :crowberto :get 402 "ee/comment/"
-                                :target_type "document"
-                                :target_id doc-id))))))
-
-(deftest basic-comments-test
-  (testing "GET /api/ee/comment/"
-    (mt/with-temp [:model/Document {doc-id :id doc :document} {:name       "New Document"
-                                                               :creator_id (mt/user->id :lucky)}]
-      (mt/with-model-cleanup [:model/CommentReaction
-                              :model/Comment
-                              :model/Notification]
-        (mt/with-fake-inbox
-          (notification.seed/seed-notification!)
-          (is (set/subset? #{:event/comment-created}
-                           (t2/select-fn-set :event_name :model/NotificationSubscription)))
-          (testing "returns empty comments list for entity with no comments"
-            (is (= {:comments []}
-                   (mt/user-http-request :rasta :get 200 "ee/comment/"
-                                         :target_type "document"
-                                         :target_id doc-id))))
-          (testing "creates and returns comments for entity"
-            (let [created   (mt/user-http-request :rasta :post 200 "ee/comment/"
-                                                  {:target_type "document"
-                                                   :target_id   doc-id
-                                                   :content     {:text "New comment"}})
-                  expected1 {:id          int?
-                             :content     {:text "New comment"}
-                             :target_type "document"
-                             :target_id   doc-id
-                             :creator     {:id (mt/user->id :rasta)}
-                             :reactions   []}]
-              (is (=? expected1 created))
-              (is (=? {:comments [expected1]}
-                      (mt/user-http-request :rasta :get 200 "ee/comment/"
-                                            :target_type "document"
-                                            :target_id doc-id)))
-              (testing "document creator receives notifications for top-level comments"
-                (is (=? {(:email (mt/fetch-user :lucky))
-                         [{:subject "Comment on New Document"
-                           :body    [{:content (relaxed-re
-                                                (str (:common_name (mt/fetch-user :rasta)) " left a comment")
-                                                (format "http://localhost:\\d+/document/%s#comment-%s"
-                                                        doc-id
-                                                        (:id created)))}]}]}
-                        (first (swap-vals! mt/inbox empty)))))
-
-              (testing "creates a reply to an existing comment"
-                (let [child     (mt/user-http-request :crowberto :post 200 "ee/comment/"
-                                                      {:target_type       "document"
-                                                       :target_id         doc-id
-                                                       :parent_comment_id (:id created)
-                                                       :content           {:text "Other comment"}})
-                      expected2 {:id                int?
-                                 :content           {:text "Other comment"}
-                                 :target_type       "document"
-                                 :target_id         doc-id
-                                 :creator           {:id (mt/user->id :crowberto)}
-                                 :parent_comment_id (:id created)
-                                 :reactions         []}]
-                  (is (=? expected2 child))
-                  (is (=? {:comments [expected1 expected2]}
-                          (mt/user-http-request :rasta :get 200 "ee/comment/"
-                                                :target_type "document"
-                                                :target_id doc-id)))
-                  (testing "participants in the thread receive notifications for new replies"
-                    (is (=? {(:email (mt/fetch-user :rasta))
-                             [{:subject "Comment on New Document"
-                               :body    [{:content (relaxed-re
-                                                    (str (:common_name (mt/fetch-user :crowberto)) " replied to a thread"))}]}]}
-                            (first (swap-vals! mt/inbox empty)))))))
-
-              (testing "comment in a thread should send emails to all participants of the thread"
-                (let [_another (mt/user-http-request :lucky :post 200 "ee/comment/"
-                                                     {:target_type       "document"
-                                                      :target_id         doc-id
-                                                      :parent_comment_id (:id created)
-                                                      :content           {:text "Third comment in a thread"}})]
-                  (is (=? {(:email (mt/fetch-user :rasta))
-                           [{:subject "Comment on New Document"
-                             :body    [{:content (relaxed-re
-                                                  (str (:common_name (mt/fetch-user :lucky)) " replied to a thread"))}]}]
-                           (:email (mt/fetch-user :crowberto))
-                           [{:subject "Comment on New Document"
-                             :body    [{:content (relaxed-re
-                                                  (str (:common_name (mt/fetch-user :lucky)) " replied to a thread"))}]}]}
-                          (first (swap-vals! mt/inbox empty))))))))
-
-          (testing "creates a comment for part of an entity"
-            (let [part-id (-> doc :content first :attrs :_id)
-                  created (mt/user-http-request :rasta :post 200 "ee/comment/"
-                                                {:target_type     "document"
-                                                 :target_id       doc-id
-                                                 :child_target_id part-id
-                                                 :content         {:text "Part comment"}})]
-              (is (=? {:id              int?
-                       :content         {:text "Part comment"}
-                       :target_type     "document"
-                       :target_id       doc-id
-                       :child_target_id part-id
-                       :creator         {:id (mt/user->id :rasta)}
-                       :reactions       []}
-                      created))
-              (testing "Comments on concrete paragraphs are also sent as notifications"
-                (is (=? {(:email (mt/fetch-user :lucky))
-                         [{:subject "Comment on New Document"
-                           :body    [{:content (relaxed-re (format "http://localhost:\\d+/document/%s/comments/%s#comment-%s"
-                                                                   doc-id
-                                                                   part-id
-                                                                   (:id created)))}]}]}
-                        (first (swap-vals! mt/inbox empty)))))))
-
-          (testing "Comments with mentions send notification emails"
-            (let [_created (mt/user-http-request :rasta :post 200 "ee/comment/"
-                                                 {:target_type "document"
-                                                  :target_id   doc-id
-                                                  :content     (tiptap
-                                                                [:smartLink {:model    "user"
-                                                                             :entityId (mt/user->id :crowberto)}])})]
-              (is (=? {(:email (mt/fetch-user :lucky))     [{:subject "Comment on New Document"}]
-                       (:email (mt/fetch-user :crowberto)) [{:subject "Comment on New Document"}]}
-                      (first (swap-vals! mt/inbox empty)))))))))))
-
-(deftest update-comment-test
-  (testing "PUT /api/ee/comment/:comment-id"
-    (mt/with-temp [:model/Document {doc-id :id}     {}
-                   :model/Comment  {comment-id :id} {:target_id doc-id}]
-      (testing "updates comment content"
-        (is (=? {:content {:text "Updated content"}}
-                (mt/user-http-request :rasta :put 200 (str "ee/comment/" comment-id)
-                                      {:content {"text" "Updated content"}}))))
-
-      (testing "updates comment resolution status"
-        (is (=? {:is_resolved true}
-                (mt/user-http-request :rasta :put 200 (str "ee/comment/" comment-id)
-                                      {:is_resolved true})))))))
-
-(deftest delete-comment-test
-  (testing "DELETE /api/ee/comment/:comment-id"
-    (mt/with-temp [:model/Document {doc-id :id} {}
-                   :model/Comment  {c1 :id}     {:target_id doc-id}
-                   :model/Comment  {c2 :id}     {:target_id doc-id}
-                   :model/Comment  {c3 :id}     {:target_id         doc-id
-                                                 :parent_comment_id c2}]
-      (testing "soft deletes a comment"
-        (is (= nil
-               (mt/user-http-request :rasta :delete 204 (str "ee/comment/" c1))))
-        (testing "comment is marked as deleted in database"
-          (let [comment (t2/select-one :model/Comment :id c1)]
-            (is (some? (:deleted_at comment)))))
-        (testing "deleting a comment twice leaves sour taste in the mouth"
-          ;; NOTE: maybe it's fine and we should just noop here rather than return an error?
-          (is (= "Comment is already deleted"
-                 (mt/user-http-request :rasta :delete 400 (str "ee/comment/" c1))))))
-
-      (testing "deleting parent comment still leaves it in a response"
-        (is (= nil
-               (mt/user-http-request :rasta :delete 204 (str "ee/comment/" c2))))
-        (is (=? {:comments [{:id         c2
-                             :deleted_at some?}
-                            {:id         c3
-                             :deleted_at nil}]}
-                (mt/user-http-request :rasta :get 200 "ee/comment/"
-                                      :target_type "document"
-                                      :target_id doc-id)))))))
-
-(deftest toggle-reaction-test
-  (testing "POST /api/ee/comment/:comment-id/reaction"
-    (mt/with-temp [:model/Document {doc-id :id}     {}
-                   :model/Comment  {comment-id :id} {:target_id doc-id}]
-      (testing "adds a reaction to a comment"
-        (is (=? {:reacted true}
-                (mt/user-http-request :rasta :post 200 (str "ee/comment/" comment-id "/reaction")
-                                      {:emoji "👍"})))
-
-        (is (=? {:comments [{:id        comment-id
-                             :reactions [{:emoji "👍"
-                                          :count 1
-                                          :users [{:id (mt/user->id :rasta)}]}]}]}
-                (mt/user-http-request :rasta :get 200 "ee/comment/"
-                                      :target_type "document"
-                                      :target_id doc-id))))
-
-      (testing "removes an existing reaction when toggled again"
-        (is (=? {:reacted false}
-                (mt/user-http-request :rasta :post 200 (str "ee/comment/" comment-id "/reaction")
-                                      {:emoji "👍"})))
-
-        (is (=? {:comments [{:id        comment-id
-                             :reactions []}]}
-                (mt/user-http-request :rasta :get 200 "ee/comment/"
-                                      :target_type "document"
-                                      :target_id doc-id)))))))
-
-(deftest comments-permissions-test
-  (testing "Comment permissions - users without document access cannot read or write comments"
-    (mt/with-non-admin-groups-no-root-collection-perms
-      (mt/with-temp [:model/Collection {restricted-col :id}        {:name "Restricted Collection"}
-                     :model/Document   {restricted-doc-id :id}     {:collection_id restricted-col
-                                                                    :name "Restricted Document"}
-                     :model/Comment    {restricted-comment-id :id} {:target_id restricted-doc-id}]
-
-        (mt/with-model-cleanup [:model/Comment]
-          (testing "GET /api/ee/comment/ - users without document read permissions cannot see comments"
-            (is (= "You don't have permissions to do that."
-                   (mt/user-http-request :lucky :get 403 "ee/comment/"
-                                         :target_type "document"
-                                         :target_id restricted-doc-id))))
-
-          (testing "POST /api/ee/comment/ - users without document read permissions cannot create comments"
-            (is (= "You don't have permissions to do that."
-                   (mt/user-http-request :lucky :post 403 "ee/comment/"
-                                         {:target_type "document"
-                                          :target_id   restricted-doc-id
-                                          :content     {:text "Comment by lucky"}}))))
-
-          (testing "PUT /api/ee/comment/:id - users without document access cannot update comments"
-            (is (= "You don't have permissions to do that."
-                   (mt/user-http-request :lucky :put 403 (str "ee/comment/" restricted-comment-id)
-                                         {:content {:text "Updated by lucky"}}))))
-
-          (testing "DELETE /api/ee/comment/:id - users without document access cannot delete comments"
-            (is (= "You don't have permissions to do that."
-                   (mt/user-http-request :lucky :delete 403 (str "ee/comment/" restricted-comment-id)))))
-
-          (testing "POST /api/ee/comment/:id/reaction - users without document access cannot react to comments"
-            (is (= "You don't have permissions to do that."
-                   (mt/user-http-request :lucky :post 403 (str "ee/comment/" restricted-comment-id "/reaction")
-                                         {:emoji "👍"})))))))))
-
-;; (deftest mention-entities-test
-;;   (testing "We can get users to mention"
-;;     (is (=? {:data   [{:id int? :common_name "Crowberto Corv" :model "user"}
-;;                       {:id int? :common_name "Lucky Pigeon" :model "user"}
-;;                       {:id int? :common_name "Rasta Toucan" :model "user"}]
-;;              :total  int?
-;;              :limit  50
-;;              :offset 0}
-;;             (-> (mt/user-http-request :rasta :get 200 "ee/comment/mentions" :limit 50)
-;;                 (update :data #(filter mt/test-user? %)))))))

enterprise/backend/test/metabase_enterprise/comments/models/comment_test.clj

@@ -1,18 +0,0 @@
-(ns metabase-enterprise.comments.models.comment-test
-  (:require
-   [clojure.test :refer :all]
-   [metabase-enterprise.comments.api-test :as at]
-   [metabase-enterprise.comments.models.comment :as comment]))
-
-(deftest comment-test
-  (testing "mentions are parsed correctly"
-    (is (= [6]
-           (comment/mentions
-            (at/tiptap
-             [:p
-              "omg is that you? "
-              [:smartLink {:entityId 6 :model "user"}]]))))
-    (is (= [6]
-           (comment/mentions
-            (at/tiptap
-             [:smartLink {:entityId 6 :model "user"}]))))))

enterprise/backend/test/metabase_enterprise/metabot_v3/api/metabot_test.clj

@@ -4,7 +4,6 @@
    [clojure.test :refer :all]
    [metabase-enterprise.metabot-v3.api.metabot :as metabot-v3.api.metabot]
    [metabase-enterprise.metabot-v3.client :as metabot-v3.client]
-   [metabase-enterprise.metabot-v3.config :as metabot-v3.config]
    [metabase.collections.models.collection :as collection]
    [metabase.lib.core :as lib]
    [metabase.lib.metadata :as lib.metadata]
@@ -25,137 +24,323 @@
          (when (seq original-entities#)
            (t2/insert! :model/Metabot original-entities#))))))
 
+(deftest metabot-entities-get-cards-test
+  (testing "GET /api/ee/metabot-v3/metabot/:id/entities"
+    (mt/with-premium-features #{:metabot-v3}
+      (mt/with-temp [:model/Metabot {metabot-id :id} {:name "Test Metabot"}
+                     :model/Collection {collection-id :id} {:name "Test Collection"}
+                     :model/Card {card-name-1 :name
+                                  card-id-1 :id} {:name "Test Card 1"
+                                                  :collection_id collection-id}
+                     :model/Card {card-name-2 :name
+                                  card-id-2 :id} {:name "Test Card 2"
+                                                  :collection_id nil}
+                     :model/MetabotEntity _ {:metabot_id metabot-id
+                                             :model "dataset"
+                                             :model_id card-id-1}
+                     :model/MetabotEntity _ {:metabot_id metabot-id
+                                             :model "dataset"
+                                             :model_id card-id-2}]
+
+        (testing "should return entities for a metabot"
+          (let [response (mt/user-http-request :crowberto :get 200
+                                               (format "ee/metabot-v3/metabot/%d/entities" metabot-id))]
+            (is (= 2 (:total response)))
+            (is (= 2 (count (:items response))))
+            (is (= #{card-name-1 card-name-2}
+                   (set (map :name (:items response)))))
+            (is (= #{card-id-1 card-id-2}
+                   (set (map :model_id (:items response)))))
+            (is (= #{collection-id "root"} (into #{} (map :collection_id (:items response)))))
+            (is (= #{"Test Collection" "Our analytics"} (into #{} (map :collection_name (:items response)))))))
+
+        (testing "should support pagination"
+          (let [response (mt/user-http-request :crowberto :get 200
+                                               (format "ee/metabot-v3/metabot/%d/entities?limit=1" metabot-id))]
+            (is (= 2 (:total response)))
+            (is (= 1 (count (:items response))))
+            (is (= 1 (:limit response)))))
+
+        (testing "should require superuser permissions"
+          (is (= "You don't have permissions to do that."
+                 (mt/user-http-request :rasta :get 403
+                                       (format "ee/metabot-v3/metabot/%d/entities" metabot-id)))))
+
+        (testing "should return 404 for non-existent metabot"
+          (is (= "Not found."
+                 (mt/user-http-request :crowberto :get 404
+                                       (format "ee/metabot-v3/metabot/%d/entities" Integer/MAX_VALUE)))))))))
+
+(deftest metabot-entities-get-collections-test
+  (testing "GET /api/ee/metabot-v3/metabot/:id/entities"
+    (mt/with-premium-features #{:metabot-v3}
+      (mt/with-temp [:model/Metabot {metabot-id :id} {:name "Test Metabot"}
+                     :model/Collection {collection-id :id} {:name "Test Collection"}
+                     :model/MetabotEntity _ {:metabot_id metabot-id
+                                             :model "collection"
+                                             :model_id collection-id}]
+
+        (testing "should return entities for a metabot"
+          (let [response (mt/user-http-request :crowberto :get 200
+                                               (format "ee/metabot-v3/metabot/%d/entities" metabot-id))]
+            (is (= #{collection-id}
+                   (set (map :model_id (:items response)))))
+            (is (= #{"Test Collection"} (into #{} (map :name (:items response)))))
+            (is (= #{nil} (into #{} (map :collection_id (:items response)))))
+            (is (= #{nil} (into #{} (map :collection_name (:items response)))))))
+
+        (testing "should support pagination"
+          (let [response (mt/user-http-request :crowberto :get 200
+                                               (format "ee/metabot-v3/metabot/%d/entities?limit=1" metabot-id))]
+            (is (= 1 (:total response)))
+            (is (= 1 (count (:items response))))
+            (is (= 1 (:limit response)))))
+
+        (testing "should require superuser permissions"
+          (is (= "You don't have permissions to do that."
+                 (mt/user-http-request :rasta :get 403
+                                       (format "ee/metabot-v3/metabot/%d/entities" metabot-id)))))
+
+        (testing "should return 404 for non-existent metabot"
+          (is (= "Not found."
+                 (mt/user-http-request :crowberto :get 404
+                                       (format "ee/metabot-v3/metabot/%d/entities" Integer/MAX_VALUE)))))))))
+
+(deftest metabot-entities-put-test-add-entities
+  (testing "PUT /api/ee/metabot-v3/metabot/:id/entities"
+    (mt/with-premium-features #{:metabot-v3}
+      (mt/with-temp [:model/Metabot {metabot-id :id} {:name "Test Metabot"}
+                     :model/Collection {collection-id :id} {:name "Test Collection"}
+                     :model/Card {card-id-1 :id} {:name "Test Card 1"
+                                                  :collection_id collection-id
+                                                  :type :model}
+                     :model/Card {card-id-2 :id} {:name "Test Card 2"
+                                                  :collection_id collection-id
+                                                  :type :metric}]
+
+        (testing "should add entities to metabot access list"
+          (let [entities {:items [{:id card-id-1 :model "dataset"}
+                                  {:id card-id-2 :model "metric"}]}]
+
+            ;; Make the API call to add entities
+            (with-redefs [metabot-v3.api.metabot/generate-sample-prompts identity]
+              (mt/user-http-request :crowberto :put 204
+                                    (format "ee/metabot-v3/metabot/%d/entities" metabot-id)
+                                    entities))
+
+            ;; Verify entities were added to the database
+            (let [added-entities (t2/select :model/MetabotEntity
+                                            :metabot_id metabot-id)]
+              (is (= #{card-id-1 card-id-2}
+                     (set (map :model_id added-entities))))
+              (is (= [:dataset :metric] (mapv :model added-entities))))))))))
+
+(deftest metabot-entities-put-test-add-entities-collections
+  (testing "PUT /api/ee/metabot-v3/metabot/:id/entities"
+    (mt/with-premium-features #{:metabot-v3}
+      (mt/with-temp [:model/Metabot {metabot-id :id} {:name "Test Metabot"}
+                     :model/Collection {collection-id :id} {:name "Test Collection"}]
+
+        (testing "should add entities to metabot access list"
+          (let [entities {:items [{:id collection-id :model "collection"}]}]
+
+            ;; Make the API call to add entities
+            (with-redefs [metabot-v3.api.metabot/generate-sample-prompts identity]
+              (mt/user-http-request :crowberto :put 204
+                                    (format "ee/metabot-v3/metabot/%d/entities" metabot-id)
+                                    entities))
+
+            ;; Verify entities were added to the database
+            (let [added-entities (t2/select :model/MetabotEntity
+                                            :metabot_id metabot-id)]
+              (is (= #{collection-id}
+                     (set (map :model_id added-entities))))
+              (is (= [:collection] (mapv :model added-entities))))))))))
+
+(deftest metabot-entities-put-duplicates-test
+  (testing "PUT /api/ee/metabot-v3/metabot/:id/entities"
+    (mt/with-premium-features #{:metabot-v3}
+      (mt/with-temp [:model/Metabot {metabot-id :id} {:name "Test Metabot"}
+                     :model/Collection {collection-id :id} {:name "Test Collection"}
+                     :model/Card {card-id-1 :id} {:name "Test Card 1"
+                                                  :collection_id collection-id}
+                     :model/Card {card-id-2 :id} {:name "Test Card 2"
+                                                  :collection_id collection-id}
+                     :model/MetabotEntity _ {:metabot_id metabot-id :model_id card-id-1 :model "metric"}
+                     :model/MetabotEntity _ {:metabot_id metabot-id :model_id card-id-2 :model "dataset"}]
+        (testing "should not add duplicate entities"
+          (let [entities {:items [{:id card-id-1 :model "metric"}]}] ;; This entity already exists
+
+            ;; Make the API call again with an existing entity
+            (with-redefs [metabot-v3.api.metabot/generate-sample-prompts identity]
+              (mt/user-http-request :crowberto :put 204
+                                    (format "ee/metabot-v3/metabot/%d/entities" metabot-id)
+                                    entities))
+
+            ;; Verify no new entities were added
+            (is (= 2 (t2/count :model/MetabotEntity :metabot_id metabot-id)))))))))
+
+(deftest metabot-entities-put-super-user-only-test
+  (testing "PUT /api/ee/metabot-v3/metabot/:id/entities"
+    (mt/with-premium-features #{:metabot-v3}
+      (mt/with-temp [:model/Metabot {metabot-id :id} {:name "Test Metabot"}
+                     :model/Collection {collection-id :id} {:name "Test Collection"}
+                     :model/Card {card-id-1 :id} {:name "Test Card 1"
+                                                  :collection_id collection-id}]
+        (testing "should require superuser permissions"
+          (is (= "You don't have permissions to do that."
+                 (mt/user-http-request :rasta :put 403
+                                       (format "ee/metabot-v3/metabot/%d/entities" metabot-id)
+                                       {:items [{:id (str card-id-1) :model "metric"}]}))))))))
+
+(deftest metabot-entities-put-404-non-existent-test
+  (testing "PUT /api/ee/metabot-v3/metabot/:id/entities"
+    (mt/with-premium-features #{:metabot-v3}
+      (mt/with-temp [:model/Collection {collection-id :id} {:name "Test Collection"}
+                     :model/Card {card-id-1 :id} {:name "Test Card 1"
+                                                  :collection_id collection-id}]
+        (testing "should return 404 for non-existent metabot"
+          (is (= "Not found."
+                 (mt/user-http-request :crowberto :put 404
+                                       (format "ee/metabot-v3/metabot/%d/entities" Integer/MAX_VALUE)
+                                       {:items [{:id (str card-id-1) :model "dataset"}]}))))))))
+
 (deftest sample-prompts-e2e-test
-  (mt/dataset test-data
-    (let [mp (mt/metadata-provider)
-          model-source-query (lib/query mp (lib.metadata/table mp (mt/id :products)))
-          metric-source-query (-> model-source-query
-                                  (lib/aggregate (lib/avg (lib.metadata/field mp (mt/id :products :rating))))
-                                  (lib/breakout (lib/with-temporal-bucket
-                                                  (lib.metadata/field mp (mt/id :products :created_at)) :week)))
-          metric-data  {:description "Metric description"
-                        :dataset_query (lib/->legacy-MBQL metric-source-query)
-                        :type :metric}
-          model-data   {:description "Model desc"
-                        :dataset_query (lib/->legacy-MBQL model-source-query)
-                        :type :model}]
-      (mt/with-premium-features #{:metabot-v3}
-        (mt/with-temp [:model/Collection {coll-id :id}   {}
-                       :model/Collection {child-id1 :id} {:location (collection/location-path coll-id)}
-                       :model/Collection {child-id2 :id} {:location (collection/location-path coll-id)}
-                       :model/Collection {child-id3 :id} {:location (collection/location-path coll-id child-id2)}
-                       :model/Card _ (assoc model-data  :name "Model1"  :collection_id coll-id)
-                       :model/Card _ (assoc metric-data :name "Metric1" :collection_id child-id1)
-                       :model/Card _ (assoc model-data  :name "Model2"  :collection_id child-id2)
-                       :model/Card _ (assoc metric-data :name "Metric2" :collection_id child-id3)
-                       :model/Metabot {metabot-id :id} {:name "metabot" :collection_id coll-id}]
-          (let [prompts {"Metric1" ["metric question1" "metric question2" "metric question3"]
-                         "Metric2" ["metric question4" "metric question5" "metric question6"]
-                         "Model1"  ["model question1"  "model question2"  "model question3"]
-                         "Model2"  ["model question4"  "model question5"  "model question6"]}
-                generate-prompt (fn [promptables]
-                                  (map (fn [promptable]
-                                         {:questions (-> promptable :name prompts)})
-                                       promptables))
-                prompt-generator (fn [request]
-                                   (-> request
-                                       (update :metrics generate-prompt)
-                                       (update :tables  generate-prompt)
-                                       (set/rename-keys {:metrics :metric_questions
-                                                         :tables :table_questions})))]
-                      ;; --------------------------- Generating sample prompts ---------------------------
-            (testing "should generate prompt suggestions for metabot"
-              (with-redefs [metabot-v3.client/generate-example-questions prompt-generator]
-                          ;; Trigger prompt generation by calling the regenerate endpoint
-                (mt/user-http-request :crowberto :post 204
-                                      (format "ee/metabot-v3/metabot/%d/prompt-suggestions/regenerate" metabot-id)))
-              (let [added-prompts (t2/select [:model/MetabotPrompt [:card.name :model_name] :prompt]
-                                             :metabot_id metabot-id
-                                             {:join     [[:report_card :card] [:= :card.id :card_id]]
-                                              :order-by [:metabot_prompt.id]})]
-                          ;; Verify prompts were added to the database
-                (is (= prompts
-                       (-> (group-by :model_name added-prompts)
-                           (update-vals #(map :prompt %)))))))
-                      ;; --------------------------- Querying sample prompts ---------------------------
-            (let [expected-prompts (into #{} (mapcat val) prompts)
-                  all-prompts (mt/user-http-request :rasta :get 200
-                                                    (format "ee/metabot-v3/metabot/%d/prompt-suggestions" metabot-id))]
-              (testing "can get all prompts"
-                (is (= (count expected-prompts) (:total all-prompts)))
-                (is (= expected-prompts  (into #{} (map :prompt) (:prompts all-prompts)))))
-              (testing "can page through the prompts"
-                (let [offset 3
-                      limit 5
-                      url (format "ee/metabot-v3/metabot/%d/prompt-suggestions?offset=%d&limit=%d"
-                                  metabot-id offset limit)
-                      page-prompts (mt/user-http-request :rasta :get 200 url)]
-                  (is (= {:prompts (->> all-prompts :prompts (drop offset) (take limit))
-                          :limit limit, :offset offset, :total (:total all-prompts)}
-                         page-prompts))))
-              (testing "can filter by model type"
-                (doseq [model-type ["metric" "model"]]
-                  (let [url (format "ee/metabot-v3/metabot/%d/prompt-suggestions?model=%s" metabot-id model-type)
-                        expected-prompts (->> all-prompts :prompts (filter (comp #{model-type} :model)))
-                        filtered-prompts (mt/user-http-request :rasta :get 200 url)]
-                    (is (= {:prompts expected-prompts
-                            :limit nil, :offset nil, :total (count expected-prompts)}
-                           filtered-prompts)))))
-              (testing "can filter by model type and model ID"
-                (let [selected-prompt (rand-nth (:prompts all-prompts))
-                      url (format "ee/metabot-v3/metabot/%d/prompt-suggestions?model=%s&model_id=%d"
-                                  metabot-id (:model selected-prompt) (:model_id selected-prompt))
-                      expected-prompts (->> all-prompts :prompts (filter (comp #{(:model_id selected-prompt)} :model_id)))
+  (let [mp (mt/metadata-provider)
+        model-source-query (lib/query mp (lib.metadata/table mp (mt/id :products)))
+        metric-source-query (-> model-source-query
+                                (lib/aggregate (lib/avg (lib.metadata/field mp (mt/id :products :rating))))
+                                (lib/breakout (lib/with-temporal-bucket
+                                                (lib.metadata/field mp (mt/id :products :created_at)) :week)))
+        metric-data  {:description "Metric description"
+                      :dataset_query (lib/->legacy-MBQL metric-source-query)
+                      :type :metric}
+        model-data   {:description "Model desc"
+                      :dataset_query (lib/->legacy-MBQL model-source-query)
+                      :type :model}]
+    (mt/with-premium-features #{:metabot-v3}
+      (mt/with-temp [:model/Collection {coll-id :id}   {}
+                     :model/Collection {child-id1 :id} {:location (collection/location-path coll-id)}
+                     :model/Collection {child-id2 :id} {:location (collection/location-path coll-id)}
+                     :model/Collection {child-id3 :id} {:location (collection/location-path coll-id child-id2)}
+                     :model/Card _ (assoc model-data  :name "Model1"  :collection_id coll-id)
+                     :model/Card _ (assoc metric-data :name "Metric1" :collection_id child-id1)
+                     :model/Card _ (assoc model-data  :name "Model2"  :collection_id child-id2)
+                     :model/Card _ (assoc metric-data :name "Metric2" :collection_id child-id3)
+                     :model/Metabot {metabot-id :id} {:name "metabot"}]
+        (let [entities {:items [{:id coll-id :model "collection"}]}
+              prompts {"Metric1" ["metric question1" "metric question2" "metric question3"]
+                       "Metric2" ["metric question4" "metric question5" "metric question6"]
+                       "Model1"  ["model question1"  "model question2"  "model question3"]
+                       "Model2"  ["model question4"  "model question5"  "model question6"]}
+              generate-prompt (fn [promptables]
+                                (map (fn [promptable]
+                                       {:questions (-> promptable :name prompts)})
+                                     promptables))
+              prompt-generator (fn [request]
+                                 (-> request
+                                     (update :metrics generate-prompt)
+                                     (update :tables  generate-prompt)
+                                     (set/rename-keys {:metrics :metric_questions
+                                                       :tables :table_questions})))]
+          ;; --------------------------- Generating sample prompts ---------------------------
+          (testing "should add entities to metabot access list and generate prompt suggestions"
+            (with-redefs [metabot-v3.client/generate-example-questions prompt-generator]
+              ;; Make the API call to add entities
+              (mt/user-http-request :crowberto :put 204
+                                    (format "ee/metabot-v3/metabot/%d/entities" metabot-id)
+                                    entities))
+            (let [entity-id     (t2/select-one-fn :id :model/MetabotEntity :metabot_id metabot-id)
+                  added-prompts (t2/select [:model/MetabotPrompt [:card.name :model_name] :prompt]
+                                           :metabot_entity_id entity-id
+                                           {:join     [[:report_card :card] [:= :card.id :card_id]]
+                                            :order-by [:metabot_prompt.id]})]
+              ;; Verify prompts were added to the database
+              (is (= prompts
+                     (-> (group-by :model_name added-prompts)
+                         (update-vals #(map :prompt %)))))))
+          ;; --------------------------- Querying sample prompts ---------------------------
+          (let [expected-prompts (into #{} (mapcat val) prompts)
+                all-prompts (mt/user-http-request :rasta :get 200
+                                                  (format "ee/metabot-v3/metabot/%d/prompt-suggestions" metabot-id))]
+            (testing "can get all prompts"
+              (is (= (count expected-prompts) (:total all-prompts)))
+              (is (= expected-prompts  (into #{} (map :prompt) (:prompts all-prompts)))))
+            (testing "can page through the prompts"
+              (let [offset 3
+                    limit 5
+                    url (format "ee/metabot-v3/metabot/%d/prompt-suggestions?offset=%d&limit=%d"
+                                metabot-id offset limit)
+                    page-prompts (mt/user-http-request :rasta :get 200 url)]
+                (is (= {:prompts (->> all-prompts :prompts (drop offset) (take limit))
+                        :limit limit, :offset offset, :total (:total all-prompts)}
+                       page-prompts))))
+            (testing "can filter by model type"
+              (doseq [model-type ["metric" "model"]]
+                (let [url (format "ee/metabot-v3/metabot/%d/prompt-suggestions?model=%s" metabot-id model-type)
+                      expected-prompts (->> all-prompts :prompts (filter (comp #{model-type} :model)))
                       filtered-prompts (mt/user-http-request :rasta :get 200 url)]
                   (is (= {:prompts expected-prompts
                           :limit nil, :offset nil, :total (count expected-prompts)}
-                         filtered-prompts))))
-              (testing "can return a sample"
-                (let [limit 5
-                      url (format "ee/metabot-v3/metabot/%d/prompt-suggestions?sample=true&limit=%d"
-                                  metabot-id limit)
-                      expected-prompts? (fn [prompts]
-                                          (and (= (count prompts) limit)
-                                               (set/subset? (set prompts) (set (:prompts all-prompts)))))
-                      sample-prompts (mt/user-http-request :rasta :get 200 url)]
-                  (is (=? {:prompts expected-prompts?
-                           :limit limit, :offset nil, :total (:total all-prompts)}
-                          sample-prompts))))
-                        ;; --------------------------- Deleting & regenerating sample prompts ---------------------------
-              (let [all-prompt-ids (into #{} (map :id) (:prompts all-prompts))
-                    current-prompt-ids #(t2/select-pks-set :model/MetabotPrompt :metabot_id metabot-id)
-                    selected-prompt (rand-nth (:prompts all-prompts))
-                    url (format "ee/metabot-v3/metabot/%d/prompt-suggestions/%d" metabot-id (:id selected-prompt))
-                    remaining-prompt-ids (disj all-prompt-ids (:id selected-prompt))]
+                         filtered-prompts)))))
+            (testing "can filter by model type and model ID"
+              (let [selected-prompt (rand-nth (:prompts all-prompts))
+                    url (format "ee/metabot-v3/metabot/%d/prompt-suggestions?model=%s&model_id=%d"
+                                metabot-id (:model selected-prompt) (:model_id selected-prompt))
+                    expected-prompts (->> all-prompts :prompts (filter (comp #{(:model_id selected-prompt)} :model_id)))
+                    filtered-prompts (mt/user-http-request :rasta :get 200 url)]
+                (is (= {:prompts expected-prompts
+                        :limit nil, :offset nil, :total (count expected-prompts)}
+                       filtered-prompts))))
+            (testing "can return a sample"
+              (let [limit 5
+                    url (format "ee/metabot-v3/metabot/%d/prompt-suggestions?sample=true&limit=%d"
+                                metabot-id limit)
+                    expected-prompts? (fn [prompts]
+                                        (and (= (count prompts) limit)
+                                             (set/subset? (set prompts) (set (:prompts all-prompts)))))
+                    sample-prompts (mt/user-http-request :rasta :get 200 url)]
+                (is (=? {:prompts expected-prompts?
+                         :limit limit, :offset nil, :total (:total all-prompts)}
+                        sample-prompts))))
+            ;; --------------------------- Deleting & regenerating sample prompts ---------------------------
+            (let [all-prompt-ids (into #{} (map :id) (:prompts all-prompts))
+                  current-prompt-ids #(t2/select-pks-set :model/MetabotPrompt
+                                                         {:join [[:metabot_entity :mbe]
+                                                                 [:= :mbe.id :metabot_prompt.metabot_entity_id]]
+                                                          :where [:= :mbe.metabot_id metabot-id]})
+                  selected-prompt (rand-nth (:prompts all-prompts))
+                  url (format "ee/metabot-v3/metabot/%d/prompt-suggestions/%d" metabot-id (:id selected-prompt))
+                  remaining-prompt-ids (disj all-prompt-ids (:id selected-prompt))]
 
-                (testing "deleting a specific prompt"
-                  (testing "normal users cannot delete"
-                    (mt/user-http-request :rasta :delete 403 url)
-                    (is (= all-prompt-ids (current-prompt-ids))))
-                  (testing "admins can delete"
-                    (mt/user-http-request :crowberto :delete 204 url)
-                    (is (= remaining-prompt-ids (current-prompt-ids)))))
+              (testing "deleting a specific prompt"
+                (testing "normal users cannot delete"
+                  (mt/user-http-request :rasta :delete 403 url)
+                  (is (= all-prompt-ids (current-prompt-ids))))
+                (testing "admins can delete"
+                  (mt/user-http-request :crowberto :delete 204 url)
+                  (is (= remaining-prompt-ids (current-prompt-ids)))))
 
-                (testing "generating new prompts"
-                  (let [url (format "ee/metabot-v3/metabot/%d/prompt-suggestions/regenerate" metabot-id)]
-                    (testing "normal users are not allowed"
-                      (mt/user-http-request :rasta :post 403 url)
-                      (is (= remaining-prompt-ids (current-prompt-ids))))
-                    (testing "admin users are allowed"
-                      (with-redefs [metabot-v3.client/generate-example-questions prompt-generator]
-                        (mt/user-http-request :crowberto :post 204 url)))))
+              (testing "generating new prompts"
+                (let [url (format "ee/metabot-v3/metabot/%d/prompt-suggestions/regenerate" metabot-id)]
+                  (testing "normal users are not allowed"
+                    (mt/user-http-request :rasta :post 403 url)
+                    (is (= remaining-prompt-ids (current-prompt-ids))))
+                  (testing "admin users are allowed"
+                    (with-redefs [metabot-v3.client/generate-example-questions prompt-generator]
+                      (mt/user-http-request :crowberto :post 204 url)))))
 
-                (let [new-prompt-ids (current-prompt-ids)]
-                  (is (= (count all-prompt-ids) (count new-prompt-ids)))
-                  (is (empty? (set/intersection all-prompt-ids new-prompt-ids)))
-                  (testing "can delete all prompts"
-                    (let [url (format "ee/metabot-v3/metabot/%d/prompt-suggestions" metabot-id)]
-                      (testing "normal users cannot delete"
-                        (mt/user-http-request :rasta :delete 403 url)
-                        (is (= new-prompt-ids (current-prompt-ids))))
-                      (testing "admins can delete"
-                        (mt/user-http-request :crowberto :delete 204 url)
-                        (is (nil? (current-prompt-ids)))))))))))))))
+              (let [new-prompt-ids (current-prompt-ids)]
+                (is (= (count all-prompt-ids) (count new-prompt-ids)))
+                (is (empty? (set/intersection all-prompt-ids new-prompt-ids)))
+                (testing "can delete all prompts"
+                  (let [url (format "ee/metabot-v3/metabot/%d/prompt-suggestions" metabot-id)]
+                    (testing "normal users cannot delete"
+                      (mt/user-http-request :rasta :delete 403 url)
+                      (is (= new-prompt-ids (current-prompt-ids))))
+                    (testing "admins can delete"
+                      (mt/user-http-request :crowberto :delete 204 url)
+                      (is (nil? (current-prompt-ids))))))))))))))
 
 (deftest metabot-list-test
   (testing "GET /api/ee/metabot-v3/metabot"
@@ -177,214 +362,72 @@
             (is (= "You don't have permissions to do that."
                    (mt/user-http-request :rasta :get 403 "ee/metabot-v3/metabot")))))))))
 
-(deftest metabot-get-single-test
-  (testing "GET /api/ee/metabot-v3/metabot/:id"
+(deftest metabot-entities-delete-collection-test
+  (testing "DELETE /api/ee/metabot-v3/metabot/:id/entities/:model-type/:model-id"
     (mt/with-premium-features #{:metabot-v3}
-      (mt/with-temp [:model/Collection {collection-id :id} {:name "Test Collection"}
-                     :model/Metabot {metabot-id :id} {:name "Test Metabot"
-                                                      :description "Test Description"
-                                                      :use_verified_content true
-                                                      :collection_id collection-id}]
+      (mt/with-temp [:model/Metabot {metabot-id :id} {:name "Test Metabot"}
+                     :model/Collection {collection-id :id} {:name "Test Collection"}
+                     :model/MetabotEntity _ {:metabot_id metabot-id
+                                             :model_id collection-id
+                                             :model "collection"}]
 
-        (testing "should return metabot with all fields"
-          (let [response (mt/user-http-request :crowberto :get 200
-                                               (format "ee/metabot-v3/metabot/%d" metabot-id))]
-            (is (= metabot-id (:id response)))
-            (is (= "Test Metabot" (:name response)))
-            (is (= "Test Description" (:description response)))
-            (is (true? (:use_verified_content response)))
-            (is (= collection-id (:collection_id response)))))
+        (testing "should delete the specified entity"
+          (is (= 1 (t2/count :model/MetabotEntity :metabot_id metabot-id)))
+
+          ;; Delete one entity
+          (mt/user-http-request :crowberto :delete 204
+                                (format "ee/metabot-v3/metabot/%d/entities/collection/%d"
+                                        metabot-id collection-id))
+
+          (let [remaining-entities (t2/select :model/MetabotEntity :metabot_id metabot-id)]
+            (is (= 0 (count remaining-entities)))))))))
+
+(deftest metabot-entities-delete-test
+  (testing "DELETE /api/ee/metabot-v3/metabot/:id/entities/:model-type/:model-id"
+    (mt/with-premium-features #{:metabot-v3}
+      (mt/with-temp [:model/Metabot {metabot-id :id} {:name "Test Metabot"}
+                     :model/Collection {collection-id :id} {:name "Test Collection"}
+                     :model/Card {card-id-1 :id} {:name "Test Card 1"
+                                                  :collection_id collection-id}
+                     :model/Card {card-id-2 :id} {:name "Test Card 2"
+                                                  :collection_id collection-id}
+                     :model/MetabotEntity _ {:metabot_id metabot-id
+                                             :model_id card-id-1
+                                             :model "dataset"}
+                     :model/MetabotEntity {entity-id-2 :id} {:metabot_id metabot-id
+                                                             :model_id card-id-2
+                                                             :model "metric"}]
+
+        (testing "should delete the specified entity"
+          ;; Verify both entities exist before deletion
+          (is (= 2 (t2/count :model/MetabotEntity :metabot_id metabot-id)))
+
+          ;; Delete one entity
+          (mt/user-http-request :crowberto :delete 204
+                                (format "ee/metabot-v3/metabot/%d/entities/dataset/%d"
+                                        metabot-id card-id-1))
+
+          ;; Verify only one entity remains and it's the correct one
+          (let [remaining-entities (t2/select :model/MetabotEntity :metabot_id metabot-id)]
+            (is (= 1 (count remaining-entities)))
+            (is (= entity-id-2 (:id (first remaining-entities))))
+            (is (= card-id-2 (:model_id (first remaining-entities))))
+            (is (= :metric (:model (first remaining-entities))))))
 
         (testing "should require superuser permissions"
           (is (= "You don't have permissions to do that."
-                 (mt/user-http-request :rasta :get 403
-                                       (format "ee/metabot-v3/metabot/%d" metabot-id)))))
+                 (mt/user-http-request :rasta :delete 403
+                                       (format "ee/metabot-v3/metabot/%d/entities/metric/%d"
+                                               metabot-id card-id-2)))))
 
         (testing "should return 404 for non-existent metabot"
           (is (= "Not found."
-                 (mt/user-http-request :crowberto :get 404
-                                       (format "ee/metabot-v3/metabot/%d" Integer/MAX_VALUE)))))))))
+                 (mt/user-http-request :crowberto :delete 404
+                                       (format "ee/metabot-v3/metabot/%d/entities/metric/%d"
+                                               Integer/MAX_VALUE card-id-2)))))
 
-(deftest metabot-put-test
-  (testing "PUT /api/ee/metabot-v3/metabot/:id"
-    (mt/with-premium-features #{:metabot-v3 :content-verification}
-      (mt/with-temp [:model/Collection {collection-id-1 :id} {:name "Collection 1"}
-                     :model/Collection {collection-id-2 :id} {:name "Collection 2"}
-                     :model/Metabot {metabot-id :id} {:name "Test Metabot"
-                                                      :use_verified_content false
-                                                      :collection_id collection-id-1}]
-
-        (testing "should update use_verified_content field"
-          (with-redefs [metabot-v3.api.metabot/generate-sample-prompts (constantly nil)]
-            (let [response (mt/user-http-request :crowberto :put 200
-                                                 (format "ee/metabot-v3/metabot/%d" metabot-id)
-                                                 {:use_verified_content true})]
-              (is (true? (:use_verified_content response)))
-              (is (= collection-id-1 (:collection_id response))) ; Should remain unchanged
-              ;; Verify in database
-              (let [updated-metabot (t2/select-one :model/Metabot :id metabot-id)]
-                (is (true? (:use_verified_content updated-metabot)))))))
-
-        (testing "should update collection_id field"
-          (with-redefs [metabot-v3.api.metabot/generate-sample-prompts (constantly nil)]
-            (let [response (mt/user-http-request :crowberto :put 200
-                                                 (format "ee/metabot-v3/metabot/%d" metabot-id)
-                                                 {:collection_id collection-id-2})]
-              (is (= collection-id-2 (:collection_id response)))
-              (is (true? (:use_verified_content response))) ; Should remain from previous test
-              ;; Verify in database
-              (let [updated-metabot (t2/select-one :model/Metabot :id metabot-id)]
-                (is (= collection-id-2 (:collection_id updated-metabot)))))))
-
-        (testing "should update collection_id to null"
-          (with-redefs [metabot-v3.api.metabot/generate-sample-prompts (constantly nil)]
-            (let [response (mt/user-http-request :crowberto :put 200
-                                                 (format "ee/metabot-v3/metabot/%d" metabot-id)
-                                                 {:collection_id nil})]
-              (is (= nil (:collection_id response)))
-              ;; Verify in database
-              (let [updated-metabot (t2/select-one :model/Metabot :id metabot-id)]
-                (is (= nil (:collection_id updated-metabot)))))))
-
-        (testing "should update all fields simultaneously"
-          (with-redefs [metabot-v3.api.metabot/generate-sample-prompts (constantly nil)]
-            (let [response (mt/user-http-request :crowberto :put 200
-                                                 (format "ee/metabot-v3/metabot/%d" metabot-id)
-                                                 {:use_verified_content false
-                                                  :collection_id collection-id-1})]
-              (is (= false (:use_verified_content response)))
-              (is (= collection-id-1 (:collection_id response))))))
-
-        (testing "should require superuser permissions"
-          (is (= "You don't have permissions to do that."
-                 (mt/user-http-request :rasta :put 403
-                                       (format "ee/metabot-v3/metabot/%d" metabot-id)
-                                       {:use_verified_content true}))))
-
-        (testing "should return 404 for non-existent metabot"
-          (is (= "Not found."
-                 (mt/user-http-request :crowberto :put 404
-                                       (format "ee/metabot-v3/metabot/%d" Integer/MAX_VALUE)
-                                       {:use_verified_content true}))))
-
-        (testing "should prevent enabling verified content without premium feature"
-          (mt/with-premium-features #{:metabot-v3}  ; Only metabot-v3, no content-verification
-            (is (= "Content verification is a paid feature not currently available to your instance. Please upgrade to use it. Learn more at metabase.com/upgrade/"
-                   (:message (mt/user-http-request :crowberto :put 402
-                                                   (format "ee/metabot-v3/metabot/%d" metabot-id)
-                                                   {:use_verified_content true}))))))
-
-        (testing "should prevent updating collection_id on primary metabot instance"
-          (let [metabot-id (metabot-v3.config/normalize-metabot-id metabot-v3.config/internal-metabot-id)]
-            (is (= "Cannot update collection_id for the primary metabot instance."
-                   (mt/user-http-request :crowberto :put 400
-
-                                         (format "ee/metabot-v3/metabot/%d" metabot-id)
-                                         {:collection_id collection-id-1})))
-            ;; Verify the collection_id was not updated
-            (let [unchanged-metabot (t2/select-one :model/Metabot :id metabot-id)]
-              (is (= nil (:collection_id unchanged-metabot))))
-
-            ;; Verify that updating other fields still works
-            (with-redefs [metabot-v3.api.metabot/generate-sample-prompts (constantly nil)]
-              (let [response (mt/user-http-request :crowberto :put 200
-                                                   (format "ee/metabot-v3/metabot/%d" metabot-id)
-                                                   {:use_verified_content true})]
-                (is (true? (:use_verified_content response)))
-                (is (= nil (:collection_id response)))))))))))
-
-(deftest metabot-prompt-regeneration-on-config-change-test
-  (mt/dataset test-data
-    (testing "PUT /api/ee/metabot-v3/metabot/:id should regenerate prompts when config changes"
-      (mt/with-premium-features #{:metabot-v3 :content-verification}
-        (let [metric-query {:type :query, :database (mt/id), :query {:source-table (mt/id :products)}}
-              model-query {:type :query, :database (mt/id), :query {:source-table (mt/id :products)}}]
-          (mt/with-temp [:model/Collection {collection-id-1 :id} {:name "Collection 1"}
-                         :model/Collection {collection-id-2 :id} {:name "Collection 2"}
-                         :model/Card {card-id-1 :id} {:name "Test Metric Card"
-                                                      :type :metric
-                                                      :dataset_query metric-query}
-                         :model/Card {card-id-2 :id} {:name "Test Model Card"
-                                                      :type :model
-                                                      :dataset_query model-query}
-                         :model/Card {card-id-3 :id} {:name "Test Card for New Prompt"
-                                                      :type :metric
-                                                      :dataset_query metric-query}
-                         :model/Card {card-id-4 :id} {:name "Test Card for Collection Change"
-                                                      :type :model
-                                                      :dataset_query model-query}
-                         :model/Card {card-id-5 :id} {:name "Test Card for Baseline"
-                                                      :type :metric
-                                                      :dataset_query metric-query}
-                         :model/Metabot {metabot-id :id} {:name "Test Metabot"
-                                                          :use_verified_content false
-                                                          :collection_id collection-id-1}
-                         :model/MetabotPrompt {prompt-id-1 :id} {:metabot_id metabot-id
-                                                                 :prompt "old prompt 1"
-                                                                 :model :metric
-                                                                 :card_id card-id-1}
-                         :model/MetabotPrompt {prompt-id-2 :id} {:metabot_id metabot-id
-                                                                 :prompt "old prompt 2"
-                                                                 :model :model
-                                                                 :card_id card-id-2}]
-
-            (let [original-prompt-ids #{prompt-id-1 prompt-id-2}]
-
-              (testing "should regenerate prompts when use_verified_content changes"
-                (with-redefs [metabot-v3.api.metabot/generate-sample-prompts
-                              (fn [metabot-id]
-                                (t2/insert! :model/MetabotPrompt {:metabot_id metabot-id
-                                                                  :prompt "new prompt after verified change"
-                                                                  :model :metric
-                                                                  :card_id card-id-3}))]
-                  (mt/user-http-request :crowberto :put 200
-                                        (format "ee/metabot-v3/metabot/%d" metabot-id)
-                                        {:use_verified_content true})
-
-                            ;; Verify old prompts were deleted and new ones created
-                  (let [current-prompts (t2/select :model/MetabotPrompt :metabot_id metabot-id)
-                        current-prompt-ids (set (map :id current-prompts))]
-                    (is (= 1 (count current-prompts)))
-                    (is (empty? (set/intersection original-prompt-ids current-prompt-ids)))
-                    (is (= "new prompt after verified change" (:prompt (first current-prompts)))))))
-
-              (testing "should regenerate prompts when collection_id changes"
-                (with-redefs [metabot-v3.api.metabot/generate-sample-prompts
-                              (fn [metabot-id]
-                                (t2/insert! :model/MetabotPrompt {:metabot_id metabot-id
-                                                                  :prompt "new prompt after collection change"
-                                                                  :model :model
-                                                                  :card_id card-id-4}))]
-                  (mt/user-http-request :crowberto :put 200
-                                        (format "ee/metabot-v3/metabot/%d" metabot-id)
-                                        {:collection_id collection-id-2})
-
-                            ;; Verify prompts were regenerated again
-                  (let [current-prompts (t2/select :model/MetabotPrompt :metabot_id metabot-id)]
-                    (is (= 1 (count current-prompts)))
-                    (is (= "new prompt after collection change" (:prompt (first current-prompts)))))))
-
-              (testing "should NOT regenerate prompts when no relevant fields change"
-                          ;; First, establish a baseline
-                (t2/delete! :model/MetabotPrompt :metabot_id metabot-id)
-                (t2/insert! :model/MetabotPrompt {:metabot_id metabot-id
-                                                  :prompt "baseline prompt"
-                                                  :model :metric
-                                                  :card_id card-id-5})
-                (let [baseline-prompts (t2/select :model/MetabotPrompt :metabot_id metabot-id)
-                      baseline-ids (set (map :id baseline-prompts))]
-
-                            ;; Make a PUT request that doesn't change verified content or collection_id
-                            ;; (This would be if we add other fields to update in the future)
-                  (with-redefs [metabot-v3.api.metabot/generate-sample-prompts
-                                (fn [_] (throw (Exception. "Should not be called")))]
-                    (mt/user-http-request :crowberto :put 200
-                                          (format "ee/metabot-v3/metabot/%d" metabot-id)
-                                          {:use_verified_content true  ; Same as current value
-                                           :collection_id collection-id-2})) ; Same as current value
-
-                            ;; Verify prompts were NOT changed
-                  (let [current-prompts (t2/select :model/MetabotPrompt :metabot_id metabot-id)
-                        current-ids (set (map :id current-prompts))]
-                    (is (= baseline-ids current-ids))
-                    (is (= "baseline prompt" (:prompt (first current-prompts))))))))))))))
+        (testing "should return 204 even if entity doesn't exist"
+          ;; This tests idempotency - deleting a non-existent entity should still return success
+          (mt/user-http-request :crowberto :delete 204
+                                (format "ee/metabot-v3/metabot/%d/entities/dataset/%d"
+                                        metabot-id 99999)))))))

enterprise/backend/test/metabase_enterprise/metabot_v3/config_test.clj

@@ -36,9 +36,9 @@
                    (metabot-v3.config/resolve-dynamic-profile-id nil "any-metabot-id")))))
 
         (testing "metabot-id mapping takes third precedence"
-          (is (= "internal"
+          (is (= "experimental"
                  (metabot-v3.config/resolve-dynamic-profile-id nil metabot-v3.config/internal-metabot-id)))
-          (is (= "embedding"
+          (is (= "default"
                  (metabot-v3.config/resolve-dynamic-profile-id nil metabot-v3.config/embedded-metabot-id))))
 
         (testing "falls back to default when no matches"
@@ -47,7 +47,7 @@
 
         (testing "single arity version uses dynamic metabot resolution"
           (mt/with-temporary-setting-values [metabot-v3.settings/metabot-id metabot-v3.config/embedded-metabot-id]
-            (is (= "embedding"
+            (is (= "default"
                    (metabot-v3.config/resolve-dynamic-profile-id nil)))))))))
 
 (deftest integrated-resolution-test
@@ -75,4 +75,4 @@
           (let [metabot-id (metabot-v3.config/resolve-dynamic-metabot-id nil)
                 profile-id (metabot-v3.config/resolve-dynamic-profile-id nil metabot-id)]
             (is (= metabot-v3.config/internal-metabot-id metabot-id))
-            (is (= "internal" profile-id))))))))
+            (is (= "experimental" profile-id))))))))

enterprise/backend/test/metabase_enterprise/metabot_v3/context_test.clj

@@ -10,16 +10,13 @@
         extra [{:id 3 :name "extra1"} {:id 4 :name "extra2"}]
         all-tables (concat used extra)]
     (with-redefs [table-utils/used-tables (fn [_] used)
-                  table-utils/enhanced-database-tables (fn [_ {:keys [priority-tables all-tables-limit] :or {all-tables-limit 100}}]
-                                                         (let [priority-ids (set (map :id priority-tables))
-                                                               non-priority (remove #(priority-ids (:id %)) all-tables)
-                                                               result (concat priority-tables (take (- all-tables-limit (count priority-tables)) non-priority))]
-                                                           (take all-tables-limit result)))]
-      (let [result (#'context/database-tables-for-context {:query {:database 123 :native {:query "SELECT *"}}, :all-tables-limit 3})]
-        (is (= used result) "Should only return used tables, in order")
-        (is (= (count used) (count result)) "Should return all used tables")
-        (is (not-any? #(= 3 (:id %)) result) "Should not include extra tables")
-        (is (not-any? #(= 4 (:id %)) result) "Should not include extra tables")
+                  table-utils/database-tables (fn [_ opts]
+                                                (take (:all-tables-limit opts 100) all-tables))]
+      (let [result (#'context/database-tables-for-context 123 {:query {:native {:query "SELECT *"}} :all-tables-limit 3})]
+        (is (= (take 2 result) used) "Used tables should be first")
+        (is (= 3 (count result)) "Should respect limit")
+        (is (some #(= 3 (:id %)) result) "Should include extra table if room")
+        (is (not (some #(= 4 (:id %)) result)) "Should not exceed limit")
         (is (= (count (distinct (map :id result))) (count result))) ; no duplicates
         ))))
 
@@ -28,101 +25,74 @@
         extra (mapv #(hash-map :id % :name (str "extra" %)) (range 6 10))
         all-tables (concat used extra)]
     (with-redefs [table-utils/used-tables (fn [_] used)
-                  table-utils/enhanced-database-tables (fn [_ {:keys [priority-tables all-tables-limit] :or {all-tables-limit 100}}]
-                                                         (let [priority-ids (set (map :id priority-tables))
-                                                               non-priority (remove #(priority-ids (:id %)) all-tables)
-                                                               result (concat priority-tables (take (- all-tables-limit (count priority-tables)) non-priority))]
-                                                           (take all-tables-limit result)))]
-      (let [result (#'context/database-tables-for-context {:query {:database 123 :native {:query "SELECT *"}}, :all-tables-limit 3})]
-        (is (= used result) "Should return all used tables")
-        (is (= (count used) (count result)) "Should return all used tables")
+                  table-utils/database-tables (fn [_ opts]
+                                                (take (:all-tables-limit opts 100) all-tables))]
+      (let [result (#'context/database-tables-for-context 123 {:query {:native {:query "SELECT *"}} :all-tables-limit 3})]
+        (is (= 3 (count result)) "Should respect limit")
+        (is (every? (set used) result) "Only used tables should be present if they exceed limit")
         (is (= (count (distinct (map :id result))) (count result))) ; no duplicates
         ))))
 
 (deftest database-tables-for-context-no-used-tables
   (let [extra (mapv #(hash-map :id % :name (str "extra" %)) (range 1 5))]
     (with-redefs [table-utils/used-tables (fn [_] [])
-                  table-utils/enhanced-database-tables (fn [_ opts]
-                                                         (take (:all-tables-limit opts 100) extra))]
-      (let [result (#'context/database-tables-for-context {:query {:database 123 :native {:query "SELECT *"}}, :all-tables-limit 3})]
-        (is (empty? result) "Should return empty when no used tables")))))
+                  table-utils/database-tables (fn [_ opts]
+                                                (take (:all-tables-limit opts 100) extra))]
+      (let [result (#'context/database-tables-for-context 123 {:query {:native {:query "SELECT *"}} :all-tables-limit 3})]
+        (is (= 3 (count result)))
+        (is (every? (set extra) result))
+        (is (= (count (distinct (map :id result))) (count result))) ; no duplicates
+        ))))
 
 (deftest database-tables-for-context-used-tables-exact-limit
   (let [used (mapv #(hash-map :id % :name (str "used" %)) (range 1 4)) ; 3 used tables
         extra (mapv #(hash-map :id % :name (str "extra" %)) (range 4 7))
         all-tables (concat used extra)]
     (with-redefs [table-utils/used-tables (fn [_] used)
-                  table-utils/enhanced-database-tables (fn [_ {:keys [priority-tables all-tables-limit] :or {all-tables-limit 100}}]
-                                                         (let [priority-ids (set (map :id priority-tables))
-                                                               non-priority (remove #(priority-ids (:id %)) all-tables)
-                                                               result (concat priority-tables (take (- all-tables-limit (count priority-tables)) non-priority))]
-                                                           (take all-tables-limit result)))]
-      (let [result (#'context/database-tables-for-context {:query {:database 123 :native {:query "SELECT *"}}, :all-tables-limit 3})]
+                  table-utils/database-tables (fn [_ opts]
+                                                (take (:all-tables-limit opts 100) all-tables))]
+      (let [result (#'context/database-tables-for-context 123 {:query {:native {:query "SELECT *"}} :all-tables-limit 3})]
+        (is (= 3 (count result)))
         (is (= used result) "Should be exactly the used tables, in order")
-        (is (= (count used) (count result)))
         (is (= (count (distinct (map :id result))) (count result))) ; no duplicates
         ))))
 
-(deftest database-tables-for-context-exception-handling
-  (testing "Returns empty when table-utils/enhanced-database-tables throws"
-    (with-redefs [table-utils/used-tables (fn [_] [{:id 1}])
-                  table-utils/enhanced-database-tables (fn [_ _] (throw (Exception. "boom")))]
-      (let [result (#'context/database-tables-for-context {:query {:database 123 :native {:query "SELECT *"}}})]
-        (is (empty? result))))))
-
-(deftest database-tables-for-context-nil-used-tables
-  (testing "Handles nil used-tables gracefully"
-    (with-redefs [table-utils/used-tables (fn [_] nil)
-                  table-utils/enhanced-database-tables (fn [_ opts]
-                                                         (take (:all-tables-limit opts 100) [{:id 1} {:id 2}]))]
-      (let [result (#'context/database-tables-for-context {:query {:database 123 :native {:query "SELECT *"}}})]
-        (is (empty? result) "Should return empty when used-tables is nil")))))
-
-(deftest database-tables-for-context-duplicate-ids
-  (testing "Deduplicates tables returned by database-tables"
-    (let [used [{:id 1}]
-          dup [{:id 1} {:id 1} {:id 2}]]
-      (with-redefs [table-utils/used-tables (fn [_] used)
-                    table-utils/enhanced-database-tables (fn [_ _] dup)]
-        (let [result (#'context/database-tables-for-context {:query {:database 123 :native {:query "SELECT *"}}})]
-          (is (= [1 2] (map :id result)) "Should preserve order and remove duplicates"))))))
-
 ;; --- enhance-context-with-schema tests ---
 
 (deftest enhance-context-with-schema-native
   (let [mock-tables [{:id 1 :name "table1"} {:id 2 :name "table2"}]]
-    (with-redefs [context/database-tables-for-context (fn [_] mock-tables)]
+    (with-redefs [context/database-tables-for-context (fn [_ _] mock-tables)]
       (testing "Enhances context with schema for native queries"
         (let [input {:user_is_viewing [{:query {:type :native :database 123}}]}
               result (#'context/enhance-context-with-schema input)]
-          (is (= (get-in result [:user_is_viewing 0 :used_tables]) mock-tables)))))))
+          (is (= (get-in result [:user_is_viewing 0 :database_schema]) mock-tables)))))))
 
 (deftest enhance-context-with-schema-non-native
   (testing "Does not enhance context for non-native queries"
     (let [input {:user_is_viewing [{:query {:type :query :database 123}}]}
           result (#'context/enhance-context-with-schema input)]
-      (is (nil? (get-in result [:user_is_viewing 0 :used_tables]))))))
+      (is (nil? (get-in result [:user_is_viewing 0 :database_schema]))))))
 
 (deftest enhance-context-with-schema-no-database
   (testing "Does not enhance context if no database present"
     (let [input {:user_is_viewing [{:query {:type :native}}]}
           result (#'context/enhance-context-with-schema input)]
-      (is (nil? (get-in result [:user_is_viewing 0 :used_tables]))))))
+      (is (nil? (get-in result [:user_is_viewing 0 :database_schema]))))))
 
 (deftest enhance-context-with-schema-complete-native-query
   (testing "Enhances context with schema for complete native query structure"
     (let [mock-tables [{:id 1 :name "users"} {:id 2 :name "orders"}]]
-      (with-redefs [context/database-tables-for-context (fn [_] mock-tables)]
+      (with-redefs [context/database-tables-for-context (fn [_ _] mock-tables)]
         (let [input {:user_is_viewing [{:query {:type :native
                                                 :database 123
                                                 :native {:query "SELECT * FROM users WHERE active = true"}}}]}
               result (#'context/enhance-context-with-schema input)]
-          (is (= (get-in result [:user_is_viewing 0 :used_tables]) mock-tables)))))))
+          (is (= (get-in result [:user_is_viewing 0 :database_schema]) mock-tables)))))))
 
 (deftest enhance-context-with-schema-complete-mbql-query
   (testing "Does not enhance context for complete MBQL query structure and doesn't call database-tables-for-context"
     (let [call-count (atom 0)]
-      (with-redefs [context/database-tables-for-context (fn [_]
+      (with-redefs [context/database-tables-for-context (fn [_ _]
                                                           (swap! call-count inc)
                                                           [{:id 1 :name "should-not-be-used"}])]
         (let [input {:user_is_viewing [{:query {:type :query
@@ -131,7 +101,7 @@
                                                         :aggregation [[:count]]
                                                         :breakout [[:field 789 nil]]}}}]}
               result (#'context/enhance-context-with-schema input)]
-          (is (nil? (get-in result [:user_is_viewing 0 :used_tables]))
+          (is (nil? (get-in result [:user_is_viewing 0 :database_schema]))
               "Should not add database_schema for MBQL queries")
           (is (= 0 @call-count)
               "Should not call database-tables-for-context for MBQL queries"))))))

enterprise/backend/test/metabase_enterprise/metabot_v3/models/metabot_test.clj

@@ -2,41 +2,30 @@
   (:require
    [clojure.test :refer :all]
    [metabase-enterprise.metabot-v3.models.metabot]
-   [metabase-enterprise.metabot-v3.models.metabot-prompt]
    [metabase.test :as mt]
    [metabase.test.fixtures :as fixtures]
    [toucan2.core :as t2]))
 
 (use-fixtures :once (fixtures/initialize :db))
 
-(deftest metabot-prompts-hydration-test
-  (mt/dataset test-data
-    (testing "batched hydration of metabot prompts"
-      (mt/with-temp
-        [:model/Card {card1-id :id} {:name "Test Card 1"
-                                     :type :metric
-                                     :dataset_query {:database (mt/id)
-                                                     :type :query
-                                                     :query {:source-table (mt/id :products)}}}
-         :model/Card {card2-id :id} {:name "Test Card 2"
-                                     :type :model
-                                     :dataset_query {:database (mt/id)
-                                                     :type :query
-                                                     :query {:source-table (mt/id :products)}}}
-         :model/Metabot {metabot1-id :id} {:name "Test Metabot 1"}
-         :model/Metabot {metabot2-id :id} {:name "Test Metabot 2"}
-         :model/MetabotPrompt _ {:metabot_id metabot1-id :prompt "Prompt 1" :model :metric :card_id card1-id}
-         :model/MetabotPrompt _ {:metabot_id metabot1-id :prompt "Prompt 2" :model :model :card_id card2-id}]
+(deftest metabot-entities-hydration-test
+  (testing "batched hydration of metabot entities"
+    (mt/with-temp
+      [:model/Database {db-id :id} {}
+       :model/Card card1 {:name "Test Card 1" :type :model :database_id db-id}
+       :model/Card card2 {:name "Test Card 2" :type :model :database_id db-id}
+       :model/Metabot {metabot1-id :id} {:name "Test Metabot 1"}
+       :model/Metabot {metabot2-id :id} {:name "Test Metabot 2"}
+       :model/MetabotEntity _ {:metabot_id metabot1-id :model "dataset" :model_id (:id card1)}
+       :model/MetabotEntity _ {:metabot_id metabot1-id :model "dataset" :model_id (:id card2)}]
 
-        (let [hydrated-metabots (t2/hydrate (t2/select :model/Metabot :id [:in [metabot1-id metabot2-id]]) :prompts)]
-          (testing "should hydrate prompts for metabots with prompts"
-            (let [metabot1 (first (filter #(= (:id %) metabot1-id) hydrated-metabots))]
-              (is (= 2 (count (:prompts metabot1))))
-              (is (= #{"Prompt 1" "Prompt 2"}
-                     (set (map :prompt (:prompts metabot1)))))
-              (is (= #{:metric :model}
-                     (set (map :model (:prompts metabot1)))))))
+      (let [hydrated-metabots (t2/hydrate (t2/select :model/Metabot :id [:in [metabot1-id metabot2-id]]) :entities)]
+        (testing "should hydrate entities for metabots with entities"
+          (let [metabot1 (first (filter #(= (:id %) metabot1-id) hydrated-metabots))]
+            (is (= 2 (count (:entities metabot1))))
+            (is (= #{(:id card1) (:id card2)}
+                   (set (map :model_id (:entities metabot1)))))))
 
-          (testing "should return empty list for metabots without prompts"
-            (let [metabot2 (first (filter #(= (:id %) metabot2-id) hydrated-metabots))]
-              (is (= [] (:prompts metabot2))))))))))
+        (testing "should return empty list for metabots without entities"
+          (let [metabot2 (first (filter #(= (:id %) metabot2-id) hydrated-metabots))]
+            (is (= [] (:entities metabot2)))))))))

enterprise/backend/test/metabase_enterprise/metabot_v3/table_utils_test.clj

@@ -263,67 +263,6 @@
           (is (every? #(not= "hidden_table" (:name %)) tables))
           (is (some #(= "visible_table" (:name %)) tables)))))))
 
-(deftest enhanced-database-tables-test
-  (testing "enhanced-database-tables function with new format"
-    (mt/with-temp [:model/Database {db-id :id} {}
-                   :model/Table    {table1-id :id} {:db_id db-id, :name "users", :schema "public", :active true, :visibility_type nil}
-                   :model/Table    {table2-id :id} {:db_id db-id, :name "orders", :schema "public", :active true, :visibility_type nil}
-                   :model/Field    {user-id-field :id} {:table_id table1-id, :name "id", :database_type "INTEGER", :base_type :type/Integer, :semantic_type :type/PK}
-                   :model/Field    {} {:table_id table1-id, :name "name", :database_type "VARCHAR", :base_type :type/Text}
-                   :model/Field    {} {:table_id table2-id, :name "id", :database_type "INTEGER", :base_type :type/Integer, :semantic_type :type/PK}
-                   :model/Field    {} {:table_id table2-id, :name "user_id", :database_type "INTEGER", :base_type :type/Integer, :semantic_type :type/FK, :fk_target_field_id user-id-field}
-                   :model/Field    {} {:table_id table2-id, :name "total", :database_type "DECIMAL", :base_type :type/Decimal}]
-
-      (testing "returns tables with new enhanced formatting"
-        (mt/with-current-user (mt/user->id :crowberto)
-          (let [tables (table-utils/enhanced-database-tables db-id)]
-            (is (vector? tables))
-            (is (every? #(contains? % :name) tables))
-            (is (every? #(contains? % :database_schema) tables))
-            (is (every? #(contains? % :fields) tables))
-            (is (every? #(contains? % :type) tables))
-            (is (every? #(contains? % :display_name) tables))
-            (is (every? #(= :table (:type %)) tables))
-            (is (every? #(every? (fn [field] (contains? field :field-id)) (:fields %)) tables))
-            (is (every? #(every? (fn [field] (contains? field :name)) (:fields %)) tables))
-            (is (every? #(every? (fn [field] (contains? field :type)) (:fields %)) tables))
-            (is (every? #(contains? % :metrics) tables)))))
-
-      (testing "includes table-reference for implicitly joined fields"
-        (mt/dataset test-data
-          (mt/with-current-user (mt/user->id :crowberto)
-            (let [test-db-id (mt/id)
-                  tables (table-utils/enhanced-database-tables test-db-id)
-                  orders-table (first (filter #(= "ORDERS" (:name %)) tables))
-                  all-fields (:fields orders-table)
-                  user-fields (filter #(= "User" (:table-reference %)) all-fields)
-                  product-fields (filter #(= "Product" (:table-reference %)) all-fields)]
-              (is (some? orders-table) "Expected to find ORDERS table")
-              (is (seq user-fields) "Expected to find fields with table-reference 'User' from implicit join")
-              (is (some #(= "NAME" (:name %)) user-fields) "Expected to find User NAME field from implicit join")
-              (is (seq product-fields) "Expected to find fields with table-reference 'Product' from implicit join")
-              (is (some #(= "TITLE" (:name %)) product-fields) "Expected to find Product TITLE field from implicit join")))))
-
-      (testing "enhanced format respects all-tables-limit option"
-        (mt/with-current-user (mt/user->id :crowberto)
-          (let [tables (table-utils/enhanced-database-tables db-id {:all-tables-limit 1})]
-            (is (<= (count tables) 1)))))
-
-      (testing "enhanced format excludes specified table IDs"
-        (mt/with-current-user (mt/user->id :crowberto)
-          (let [all-tables (table-utils/enhanced-database-tables db-id)
-                filtered-tables (table-utils/enhanced-database-tables db-id {:exclude-table-ids #{table1-id}})]
-            (is (< (count filtered-tables) (count all-tables)))
-            (is (not-any? #(= table1-id (:id %)) filtered-tables)))))
-
-      (testing "enhanced format prioritizes specified tables"
-        (mt/with-current-user (mt/user->id :crowberto)
-          (let [priority-table {:id table2-id :name "orders" :schema "public"}
-                tables (table-utils/enhanced-database-tables db-id {:priority-tables [priority-table]})]
-            (is (seq tables))
-            ;; Priority table should appear first (if it appears at all)
-            (is (= "orders" (-> tables first :name)))))))))
-
 (deftest ^:parallel format-escaped-test
   (are [in out] (= out
                    (with-out-str (#'table-utils/format-escaped in *out*)))

enterprise/backend/test/metabase_enterprise/metabot_v3/tools/filters_test.clj

@@ -13,9 +13,9 @@
 (defn- by-name
   [dimensions dimension-name]
   (let [pred (if (string? dimension-name)
-               (comp #{dimension-name} :display_name)
+               (comp #{dimension-name} :name)
                (every-pred (comp #{(first dimension-name)} :table-reference)
-                           (comp #{(second dimension-name)} :display_name)))]
+                           (comp #{(second dimension-name)} :name)))]
     (m/find-first pred dimensions)))
 
 (deftest ^:parallel query-metric-test
@@ -489,227 +489,3 @@
           (is (=? expected
                   (metabot-v3.tools.filters/filter-records
                    (assoc input :data-source (select-keys query-details [:query]))))))))))
-
-(deftest ^:parallel query-datasource-table-test
-  (mt/with-current-user (mt/user->id :crowberto)
-    (let [mp (mt/metadata-provider)
-          table-id (mt/id :orders)
-          table-details (#'metabot-v3.dummy-tools/table-details table-id {:metadata-provider mp})
-          ->field-id #(u/prog1 (-> table-details :fields (by-name %) :field-id)
-                        (when-not <>
-                          (throw (ex-info (str "Column " % " not found") {:column %}))))]
-      (testing "Basic table query works"
-        (is (=? {:structured-output {:type :query
-                                     :query-id string?
-                                     :query {:database (mt/id)
-                                             :type :query
-                                             :query {:source-table table-id}}}}
-                (metabot-v3.tools.filters/query-datasource
-                 {:table-id table-id}))))
-
-      (testing "Table query with fields selection"
-        (is (=? {:structured-output {:type :query
-                                     :query-id string?
-                                     :query {:database (mt/id)
-                                             :type :query
-                                             :query {:source-table table-id
-                                                     :fields [[:field (mt/id :orders :created_at) {:base-type :type/DateTimeWithLocalTZ}]
-                                                              [:field (mt/id :orders :total) {:base-type :type/Float}]]}}}}
-                (metabot-v3.tools.filters/query-datasource
-                 {:table-id table-id
-                  :fields [{:field-id (->field-id "Created At")}
-                           {:field-id (->field-id "Total")}]}))))
-
-      (testing "Table query with filters"
-        (is (=? {:structured-output {:type :query
-                                     :query-id string?
-                                     :query {:database (mt/id)
-                                             :type :query
-                                             :query {:source-table table-id
-                                                     :filter [:and
-                                                              [:> [:field (mt/id :orders :discount) {:base-type :type/Float}] 3]
-                                                              [:= [:field (mt/id :orders :user_id) {:base-type :type/Integer}] 10]]}}}}
-                (metabot-v3.tools.filters/query-datasource
-                 {:table-id table-id
-                  :filters [{:field-id (->field-id "Discount")
-                             :operation :number-greater-than
-                             :value 3}
-                            {:field-id (->field-id "User ID")
-                             :operation :equals
-                             :value 10}]}))))
-
-      (testing "Table query with aggregations and grouping"
-        (is (=? {:structured-output {:type :query
-                                     :query-id string?
-                                     :query {:database (mt/id)
-                                             :type :query
-                                             :query {:source-table table-id
-                                                     :aggregation [[:sum [:field (mt/id :orders :subtotal) {:base-type :type/Float}]]
-                                                                   [:avg [:field (mt/id :orders :discount) {:base-type :type/Float}]]]
-                                                     :breakout [[:field (mt/id :orders :product_id) {:base-type :type/Integer}]]}}}}
-                (metabot-v3.tools.filters/query-datasource
-                 {:table-id table-id
-                  :aggregations [{:field-id (->field-id "Subtotal")
-                                  :function :sum}
-                                 {:field-id (->field-id "Discount")
-                                  :function :avg}]
-                  :group-by [{:field-id (->field-id "Product ID")}]}))))
-
-      (testing "Table query with order by and limit"
-        (is (=? {:structured-output {:type :query
-                                     :query-id string?
-                                     :query {:database (mt/id)
-                                             :type :query
-                                             :query {:source-table table-id
-                                                     :order-by [[:asc [:field (mt/id :orders :created_at) {:base-type :type/DateTimeWithLocalTZ}]]
-                                                                [:desc [:field (mt/id :orders :total) {:base-type :type/Float}]]]
-                                                     :limit 100}}}}
-                (metabot-v3.tools.filters/query-datasource
-                 {:table-id table-id
-                  :order-by [{:field {:field-id (->field-id "Created At")}
-                              :direction :asc}
-                             {:field {:field-id (->field-id "Total")}
-                              :direction :desc}]
-                  :limit 100}))))
-
-      (testing "Table query with temporal bucketing"
-        (is (=? {:structured-output {:type :query
-                                     :query-id string?
-                                     :query {:database (mt/id)
-                                             :type :query
-                                             :query {:source-table table-id
-                                                     :aggregation [[:count]]
-                                                     :breakout [[:field (mt/id :orders :created_at)
-                                                                 {:base-type :type/DateTimeWithLocalTZ
-                                                                  :temporal-unit :month}]]}}}}
-                (metabot-v3.tools.filters/query-datasource
-                 {:table-id table-id
-                  :aggregations [{:field-id (->field-id "Product ID")
-                                  :function :count}]
-                  :group-by [{:field-id (->field-id "Created At")
-                              :field-granularity :month}]})))))))
-
-(deftest ^:parallel query-datasource-model-test
-  (mt/with-temp [:model/Card {model-id :id} {:dataset_query (mt/mbql-query orders {})
-                                             :database_id (mt/id)
-                                             :name "Orders Model"
-                                             :description "Test model for orders"
-                                             :type :model}]
-    (mt/with-current-user (mt/user->id :crowberto)
-      (let [model-details (-> (metabot-v3.dummy-tools/get-table-details {:model-id model-id})
-                              :structured-output)
-            model-card-id (str "card__" model-id)
-            ->field-id #(u/prog1 (-> model-details :fields (by-name %) :field-id)
-                          (when-not <>
-                            (throw (ex-info (str "Column " % " not found") {:column %}))))]
-        (testing "Basic model query works"
-          (is (=? {:structured-output {:type :query
-                                       :query-id string?
-                                       :query (mt/mbql-query orders {:source-table model-card-id})}}
-                  (metabot-v3.tools.filters/query-datasource
-                   {:model-id model-id}))))
-
-        (testing "Model query with fields selection"
-          (is (=? {:structured-output {:type :query
-                                       :query-id string?
-                                       :query (mt/mbql-query orders
-                                                {:source-table model-card-id
-                                                 :fields [[:field "CREATED_AT" {:base-type :type/DateTimeWithLocalTZ}]
-                                                          [:field "TOTAL" {:base-type :type/Float}]]})}}
-                  (metabot-v3.tools.filters/query-datasource
-                   {:model-id model-id
-                    :fields [{:field-id (->field-id "Created At")}
-                             {:field-id (->field-id "Total")}]}))))
-
-        (testing "Model query with filters"
-          (is (=? {:structured-output {:type :query
-                                       :query-id string?
-                                       :query (mt/mbql-query orders
-                                                {:source-table model-card-id
-                                                 :filter [:and
-                                                          [:> [:field "DISCOUNT" {:base-type :type/Float}] 5]
-                                                          [:< [:field "SUBTOTAL" {:base-type :type/Float}] 100]]})}}
-                  (metabot-v3.tools.filters/query-datasource
-                   {:model-id model-id
-                    :filters [{:field-id (->field-id "Discount")
-                               :operation :number-greater-than
-                               :value 5}
-                              {:field-id (->field-id "Subtotal")
-                               :operation :number-less-than
-                               :value 100}]}))))
-
-        (testing "Model query with aggregations and grouping"
-          (is (=? {:structured-output {:type :query
-                                       :query-id string?
-                                       :query (mt/mbql-query orders
-                                                {:source-table model-card-id
-                                                 :aggregation [[:count]
-                                                               [:max [:field "TOTAL" {:base-type :type/Float}]]]
-                                                 :breakout [[:field "USER_ID" {:base-type :type/Integer}]]})}}
-                  (metabot-v3.tools.filters/query-datasource
-                   {:model-id model-id
-                    :aggregations [{:field-id (->field-id "Product ID")
-                                    :function :count}
-                                   {:field-id (->field-id "Total")
-                                    :function :max}]
-                    :group-by [{:field-id (->field-id "User ID")}]}))))
-
-        (testing "Model query with order by and limit"
-          (is (=? {:structured-output {:type :query
-                                       :query-id string?
-                                       :query (mt/mbql-query orders
-                                                {:source-table model-card-id
-                                                 :order-by [[:desc [:field "CREATED_AT" {:base-type :type/DateTimeWithLocalTZ}]]]
-                                                 :limit 50})}}
-                  (metabot-v3.tools.filters/query-datasource
-                   {:model-id model-id
-                    :order-by [{:field {:field-id (->field-id "Created At")}
-                                :direction :desc}]
-                    :limit 50}))))))))
-
-(deftest ^:parallel query-datasource-validation-test
-  (mt/with-current-user (mt/user->id :crowberto)
-    (testing "Error when neither table-id nor model-id provided"
-      (is (= {:output "Either table_id or model_id must be provided"}
-             (metabot-v3.tools.filters/query-datasource {}))))
-
-    (testing "Error when both table-id and model-id provided"
-      (is (= {:output "Cannot provide both table_id and model_id"}
-             (metabot-v3.tools.filters/query-datasource
-              {:table-id (mt/id :orders)
-               :model-id 123}))))
-
-    (testing "Error with invalid table-id"
-      (is (= {:output "Invalid table_id not-a-number"}
-             (metabot-v3.tools.filters/query-datasource
-              {:table-id "not-a-number"}))))
-
-    (testing "Error with invalid model-id"
-      (is (= {:output "Invalid model_id not-a-number"}
-             (metabot-v3.tools.filters/query-datasource
-              {:model-id "not-a-number"}))))
-
-    (testing "Error with non-existent table"
-      (is (= {:output (str "No table found with table_id " Integer/MAX_VALUE)}
-             (metabot-v3.tools.filters/query-datasource
-              {:table-id Integer/MAX_VALUE}))))
-
-    (testing "Error with non-existent model"
-      (is (= {:output (str "No model found with model_id " Integer/MAX_VALUE)}
-             (metabot-v3.tools.filters/query-datasource
-              {:model-id Integer/MAX_VALUE}))))))
-
-(deftest ^:parallel query-datasource-permissions-test
-  (mt/with-temp [:model/Card {model-id :id} {:dataset_query (mt/mbql-query orders {})
-                                             :database_id (mt/id)
-                                             :name "Orders Model"
-                                             :type :model}]
-    (testing "User without permissions gets error for table"
-      (is (thrown-with-msg? clojure.lang.ExceptionInfo #"You don't have permissions to do that."
-                            (metabot-v3.tools.filters/query-datasource
-                             {:table-id (mt/id :orders)}))))
-
-    (testing "User without permissions gets error for model"
-      (is (thrown-with-msg? clojure.lang.ExceptionInfo #"You don't have permissions to do that."
-                            (metabot-v3.tools.filters/query-datasource
-                             {:model-id model-id}))))))

enterprise/backend/test/metabase_enterprise/metabot_v3/tools/find_outliers_test.clj

@@ -13,7 +13,7 @@
 
 (defn- by-name
   [dimensions dimension-name]
-  (m/find-first (comp #{dimension-name} :display_name) dimensions))
+  (m/find-first (comp #{dimension-name} :name) dimensions))
 
 (defn- test-card
   []

enterprise/backend/test/metabase_enterprise/metabot_v3/tools/search_data_sources_test.clj

@@ -1,152 +0,0 @@
-(ns metabase-enterprise.metabot-v3.tools.search-data-sources-test
-  (:require
-   [clojure.test :refer :all]
-   [metabase-enterprise.metabot-v3.tools.search-data-sources :as sds]
-   [metabase.api.common :as api]
-   [metabase.permissions.core :as perms]
-   [metabase.search.core :as search]
-   [metabase.test :as mt]
-   [toucan2.core :as t2]))
-
-(deftest transform-search-result-test
-  (testing "table result transformation"
-    (let [result {:model "table"
-                  :id 1
-                  :table_name "orders"
-                  :name "Orders"
-                  :description "Order table"
-                  :database_id 42
-                  :table_schema "public"}
-          expected {:id 1
-                    :type :table
-                    :name "orders"
-                    :display_name "Orders"
-                    :description "Order table"
-                    :database_id 42
-                    :database_schema "public"}]
-      (is (= expected (sds/transform-search-result result)))))
-
-  (testing "model result transformation"
-    (let [result {:model "dataset"
-                  :id 2
-                  :name "Sales Model"
-                  :description "Model for sales"
-                  :database_id 43
-                  :verified true}
-          expected {:id 2
-                    :type :model
-                    :name "Sales Model"
-                    :display_name "Sales Model"
-                    :description "Model for sales"
-                    :database_id 43
-                    :verified true}]
-      (is (= expected (sds/transform-search-result result)))))
-
-  (testing "dashboard result transformation"
-    (let [result {:model "dashboard"
-                  :id 3
-                  :name "Main Dashboard"
-                  :description "Dashboard desc"
-                  :verified false}
-          expected {:id 3
-                    :type :dashboard
-                    :name "Main Dashboard"
-                    :description "Dashboard desc"
-                    :verified false}]
-      (is (= expected (sds/transform-search-result result)))))
-
-  (testing "question result transformation with moderated_status"
-    (let [result {:model "card"
-                  :id 4
-                  :name "Q1"
-                  :description "Question desc"
-                  :moderated_status "verified"}
-          expected {:id 4
-                    :type :question
-                    :name "Q1"
-                    :description "Question desc"
-                    :verified true}]
-      (is (= expected (sds/transform-search-result result)))))
-
-  (testing "metric result transformation"
-    (let [result {:model "metric"
-                  :id 5
-                  :name "Revenue"
-                  :description "Metric desc"
-                  :verified nil}
-          expected {:id 5
-                    :type :metric
-                    :name "Revenue"
-                    :description "Metric desc"
-                    :verified false}]
-      (is (= expected (sds/transform-search-result result))))))
-
-(deftest search-data-sources-test
-  (mt/with-premium-features #{:content-verification}
-    (mt/with-test-user :rasta
-      (let [order-table {:id 1
-                         :model "table"
-                         :table_name "orders"
-                         :name "Orders"
-                         :description "Order table"
-                         :database_id 42
-                         :table_schema "public"}
-            dashboard {:id 2
-                       :model "dashboard"
-                       :name "Sales Dashboard"
-                       :description "Dashboard for sales"
-                       :verified true}]
-
-        (with-redefs [perms/impersonated-user? (fn [] false)
-                      perms/sandboxed-user? (fn [] false)
-                      api/*current-user-id* 1]
-          (testing "search-data-sources returns transformed and deduplicated results for multiple keywords"
-            (with-redefs [search/search (fn [_] {:data [order-table]})]
-              (let [args {:keywords ["orders" "sales"]
-                          :entity-types ["table"]}
-                    results (sds/search-data-sources args)
-                    expected [(sds/transform-search-result order-table)]]
-                (is (= expected results)))))
-
-          (testing "search-data-sources handles multiple results and deduplication"
-            (with-redefs [search/search (fn [_] {:data [order-table
-                                                        order-table
-                                                        dashboard]})]
-              (let [args {:keywords ["orders"]
-                          :entity-types ["table" "dashboard"]}
-                    results (sds/search-data-sources args)
-                    expected [(sds/transform-search-result order-table)
-                              (sds/transform-search-result dashboard)]]
-                (is (= expected results)))))
-
-          (testing "search-data-sources handles empty results"
-            (with-redefs [search/search (fn [_] {:data []})]
-              (let [args {:keywords ["none"]
-                          :entity-types ["table"]}
-                    results (sds/search-data-sources args)]
-                (is (empty? results)))))
-
-          (testing "search-data-sources with metabot collection and verified content"
-            (let [metabot {:entity_id "test-bot"
-                           :collection_id 100
-                           :use_verified_content true}
-                  table-result {:id 1 :model "table" :name "orders"}
-                  card-in-collection {:id 2 :model "card" :name "Q1" :collection {:id 100} :verified true}
-                  card-other-collection {:id 3 :model "card" :name "Q2" :collection {:id 200} :verified true}]
-
-              (with-redefs [t2/select-one (fn [model & _]
-                                            (is (= :model/Metabot model) "Should query for Metabot model")
-                                            metabot)
-                            search/search (fn [context]
-                                            ;; Verify that verified flag is set when metabot has use_verified_content
-                                            (is (true? (:verified context)))
-                                            {:data [table-result card-in-collection card-other-collection]})]
-
-                (let [results (sds/search-data-sources {:keywords ["test"]
-                                                        :metabot-id "test-bot"
-                                                        :entity-types ["table" "question"]})]
-                  ;; Should only return table and card in collection 100
-                  (is (= 2 (count results)))
-                  (is (some #(= (:id %) 1) results) "Table should be included")
-                  (is (some #(= (:id %) 2) results) "Card in metabot collection should be included")
-                  (is (not (some #(= (:id %) 3) results)) "Card in different collection should be excluded"))))))))))

enterprise/backend/test/metabase_enterprise/metabot_v3/tools/search_test.clj

@@ -1,305 +0,0 @@
-(ns metabase-enterprise.metabot-v3.tools.search-test
-  (:require
-   [clojure.test :refer :all]
-   [metabase-enterprise.metabot-v3.tools.search :as search]
-   [metabase.api.common :as api]
-   [metabase.permissions.core :as perms]
-   [metabase.search.core :as search-core]
-   [metabase.test :as mt]
-   [toucan2.core :as t2]))
-
-(deftest reciprocal-rank-fusion-test
-  (testing "Basic RRF with single list"
-    (let [single-list [[{:id 1 :model "card" :name "Card 1"}
-                        {:id 2 :model "dashboard" :name "Dashboard 1"}
-                        {:id 3 :model "table" :name "Table 1"}]]
-          result (#'search/reciprocal-rank-fusion single-list)]
-      (is (= 3 (count result)))
-      (is (= 1 (-> result first :id)))
-      (is (= 2 (-> result second :id)))
-      (is (= 3 (-> result last :id)))))
-
-  (testing "RRF with multiple lists - no overlap"
-    (let [list1 [{:id 1 :model "card" :name "Card 1"}
-                 {:id 2 :model "dashboard" :name "Dashboard 1"}]
-          list2 [{:id 3 :model "table" :name "Table 1"}
-                 {:id 4 :model "metric" :name "Metric 1"}]
-          result (#'search/reciprocal-rank-fusion [list1 list2])]
-      (is (= 4 (count result)))
-      (is (every? #(contains? #{1 2 3 4} (:id %)) result))))
-
-  (testing "RRF with overlapping results - should boost common items"
-    (let [list1 [{:id 1 :model "card" :name "Revenue Report"}
-                 {:id 2 :model "dashboard" :name "Sales Dashboard"}
-                 {:id 3 :model "table" :name "Orders"}]
-          list2 [{:id 2 :model "dashboard" :name "Sales Dashboard"}
-                 {:id 1 :model "card" :name "Revenue Report"}
-                 {:id 4 :model "metric" :name "Total Revenue"}]
-          result (#'search/reciprocal-rank-fusion [list1 list2])]
-      (is (= 4 (count result)))
-      ;; Items appearing in both lists should rank higher
-      (let [top-two-ids (set (map :id (take 2 result)))]
-        (is (contains? top-two-ids 1))
-        (is (contains? top-two-ids 2)))))
-
-  (testing "RRF with identical items at different positions"
-    (let [list1 [{:id 1 :model "card" :name "First"}
-                 {:id 2 :model "dashboard" :name "Second"}
-                 {:id 3 :model "table" :name "Third"}]
-          list2 [{:id 3 :model "table" :name "Third"}
-                 {:id 2 :model "dashboard" :name "Second"}
-                 {:id 1 :model "card" :name "First"}]
-          list3 [{:id 2 :model "dashboard" :name "Second"}
-                 {:id 3 :model "table" :name "Third"}
-                 {:id 1 :model "card" :name "First"}]
-          result (#'search/reciprocal-rank-fusion [list1 list2 list3])]
-      (is (= 3 (count result)))
-      ;; Item 2 appears first in list3, second in list1 and list2, so should rank highest
-      (is (= 2 (-> result first :id)))))
-
-  (testing "RRF with empty lists"
-    (let [list1 []
-          list2 [{:id 1 :model "card" :name "Card 1"}]
-          result (#'search/reciprocal-rank-fusion [list1 list2])]
-      (is (= 1 (count result)))
-      (is (= 1 (-> result first :id)))))
-
-  (testing "RRF with all empty lists"
-    (let [result (#'search/reciprocal-rank-fusion [[] [] []])]
-      (is (empty? result))))
-
-  (testing "RRF score calculation correctness"
-    ;; Test that the RRF formula 1/(k+r) where k=60 is correctly applied
-    (let [list1 [{:id 1 :model "card" :name "Rank 1"}]  ; rank=1, score=1/61
-          list2 [{:id 2 :model "dashboard" :name "Other"}
-                 {:id 1 :model "card" :name "Rank 1"}]  ; rank=2, score=1/62
-          result (#'search/reciprocal-rank-fusion [list1 list2])
-          first-item (first result)
-          second-item (second result)]
-      ;; Item 1 appears at rank 1 in list1 (score=1/61) and rank 2 in list2 (score=1/62)
-      ;; Total score = 1/61 + 1/62 ≈ 0.0164 + 0.0161 = 0.0325
-      ;; Item 2 appears only at rank 1 in list2 (score=1/61 ≈ 0.0164)
-      ;; So item 1 should rank higher than item 2
-      (is (= 1 (:id first-item)))
-      (is (= 2 (:id second-item)))))
-
-  (testing "RRF preserves item data"
-    (let [complex-item {:id 42
-                        :model "dataset"
-                        :name "Complex Dataset"
-                        :description "A detailed description"
-                        :database_id 1
-                        :created_at "2024-01-01"
-                        :extra_field "preserved"}
-          result (#'search/reciprocal-rank-fusion [[complex-item]])]
-      (is (= 1 (count result)))
-      (is (= complex-item (first result)))))
-
-  (testing "RRF with many lists"
-    (let [lists (for [i (range 5)]
-                  [{:id (inc i) :model "card" :name (str "Card " (inc i))}
-                   {:id 99 :model "dashboard" :name "Common Dashboard"}
-                   {:id (+ i 10) :model "table" :name (str "Table " (+ i 10))}])
-          result (#'search/reciprocal-rank-fusion lists)]
-      ;; Item 99 appears in all 5 lists at position 2, so should rank very high
-      (is (= 99 (:id (first result)))))))
-
-(deftest transform-search-result-test
-  (testing "table result transformation"
-    (let [result {:model "table"
-                  :id 1
-                  :table_name "orders"
-                  :name "Orders"
-                  :description "Order table"
-                  :database_id 42
-                  :table_schema "public"
-                  :updated_at "2024-01-01"
-                  :created_at "2024-01-01"}
-          expected {:id 1
-                    :type "table"
-                    :name "orders"
-                    :display_name "Orders"
-                    :description "Order table"
-                    :database_id 42
-                    :database_schema "public"
-                    :updated_at "2024-01-01"
-                    :created_at "2024-01-01"}]
-      (is (= expected (#'search/transform-search-result result)))))
-
-  (testing "model (dataset) result transformation"
-    (let [result {:model "dataset"
-                  :id 2
-                  :name "Sales Model"
-                  :description "Model for sales"
-                  :database_id 43
-                  :verified true
-                  :collection nil
-                  :updated_at "2024-01-02"
-                  :created_at "2024-01-02"}
-          expected {:id 2
-                    :type "model"
-                    :name "Sales Model"
-                    :description "Model for sales"
-                    :database_id 43
-                    :verified true
-                    :collection {}
-                    :updated_at "2024-01-02"
-                    :created_at "2024-01-02"}]
-      (is (= expected (#'search/transform-search-result result)))))
-
-  (testing "dashboard result transformation"
-    (let [result {:model "dashboard"
-                  :id 3
-                  :name "Main Dashboard"
-                  :description "Dashboard desc"
-                  :verified false
-                  :collection {:name "Finance" :authority_level "official"}
-                  :updated_at "2024-01-03"
-                  :created_at "2024-01-03"}
-          expected {:id 3
-                    :type "dashboard"
-                    :name "Main Dashboard"
-                    :description "Dashboard desc"
-                    :verified false
-                    :collection {:name "Finance" :authority_level "official"}
-                    :updated_at "2024-01-03"
-                    :created_at "2024-01-03"}]
-      (is (= expected (#'search/transform-search-result result)))))
-
-  (testing "question (card) result transformation with moderated_status"
-    (let [result {:model "card"
-                  :id 4
-                  :name "Q1"
-                  :description "Question desc"
-                  :moderated_status "verified"
-                  :collection {:name "Analytics" :authority_level nil}
-                  :updated_at "2024-01-04"
-                  :created_at "2024-01-04"}
-          expected {:id 4
-                    :type "question"
-                    :name "Q1"
-                    :description "Question desc"
-                    :database_id nil
-                    :verified true
-                    :collection {:name "Analytics" :authority_level nil}
-                    :updated_at "2024-01-04"
-                    :created_at "2024-01-04"}]
-      (is (= expected (#'search/transform-search-result result)))))
-
-  (testing "metric result transformation"
-    (let [result {:model "metric"
-                  :id 5
-                  :name "Revenue"
-                  :description "Metric desc"
-                  :verified nil
-                  :updated_at "2024-01-05"
-                  :created_at "2024-01-05"}
-          expected {:id 5
-                    :type "metric"
-                    :name "Revenue"
-                    :description "Metric desc"
-                    :database_id nil
-                    :verified false
-                    :collection {}
-                    :updated_at "2024-01-05"
-                    :created_at "2024-01-05"}]
-      (is (= expected (#'search/transform-search-result result)))))
-
-  (testing "database result transformation"
-    (let [result {:model "database"
-                  :id 6
-                  :name "Production DB"
-                  :description "Main database"
-                  :updated_at "2024-01-06"
-                  :created_at "2024-01-06"}
-          expected {:id 6
-                    :type "database"
-                    :name "Production DB"
-                    :description "Main database"
-                    :updated_at "2024-01-06"
-                    :created_at "2024-01-06"}]
-      (is (= expected (#'search/transform-search-result result))))))
-
-(deftest search-test
-  (mt/with-premium-features #{:content-verification}
-    (mt/with-test-user :rasta
-      (let [order-table {:id 1
-                         :model "table"
-                         :table_name "orders"
-                         :name "Orders"
-                         :description "Order table"
-                         :database_id 42
-                         :table_schema "public"}
-            dashboard {:id 2
-                       :model "dashboard"
-                       :name "Sales Dashboard"
-                       :description "Dashboard for sales"
-                       :verified true}]
-
-        (with-redefs [perms/impersonated-user? (fn [] false)
-                      perms/sandboxed-user? (fn [] false)
-                      api/*current-user-id* 1]
-
-          (testing "search returns transformed results for term queries"
-            (with-redefs [search-core/search (fn [_] {:data [order-table]})]
-              (let [args {:term-queries ["orders"]
-                          :entity-types ["table"]}
-                    results (search/search args)
-                    expected [(#'search/transform-search-result order-table)]]
-                (is (= expected results)))))
-
-          (testing "search returns transformed results for semantic queries"
-            (with-redefs [search-core/search (fn [_] {:data [dashboard]})]
-              (let [args {:semantic-queries ["sales metrics"]
-                          :entity-types ["dashboard"]}
-                    results (search/search args)
-                    expected [(#'search/transform-search-result dashboard)]]
-                (is (= expected results)))))
-
-          (testing "search combines term and semantic queries using RRF"
-            (with-redefs [search-core/search (fn [context]
-                                               (if (= (:search-string context) "orders")
-                                                 {:data [order-table]}
-                                                 {:data [dashboard]}))]
-              (let [args {:term-queries ["orders"]
-                          :semantic-queries ["sales"]
-                          :entity-types ["table" "dashboard"]}
-                    results (search/search args)]
-                ;; Should return both results combined via RRF
-                (is (= 2 (count results)))
-                (is (some #(= (:id %) 1) results))
-                (is (some #(= (:id %) 2) results)))))
-
-          (testing "search applies RRF to overlapping results"
-            (with-redefs [search-core/search (fn [_]
-                                               {:data [order-table dashboard]})]
-              (let [args {:term-queries ["orders" "sales"]
-                          :entity-types ["table" "dashboard"]}
-                    results (search/search args)]
-                ;; Both queries return same results, RRF should boost them
-                (is (= 2 (count results)))
-                (is (some #(= (:id %) 1) results))
-                (is (some #(= (:id %) 2) results)))))
-
-          (testing "search handles empty results"
-            (with-redefs [search-core/search (fn [_] {:data []})]
-              (let [args {:term-queries ["nonexistent"]
-                          :entity-types ["table"]}
-                    results (search/search args)]
-                (is (empty? results)))))
-
-          (testing "search with metabot verified content flag"
-            (let [metabot {:entity_id "test-bot"
-                           :use_verified_content true}]
-              (with-redefs [t2/select-one (fn [model & _]
-                                            (is (= :model/Metabot model) "Should query for Metabot model")
-                                            metabot)
-                            search-core/search (fn [context]
-                                                ;; Verify that verified flag is set when metabot has use_verified_content
-                                                 (is (true? (:verified context)))
-                                                 {:data [dashboard]})]
-                (let [results (search/search {:term-queries ["test"]
-                                              :metabot-id "test-bot"
-                                              :entity-types ["dashboard"]})]
-                  (is (= 1 (count results)))
-                  (is (= 2 (:id (first results)))))))))))))

enterprise/backend/test/metabase_enterprise/metabot_v3/tools/util_test.clj

@@ -3,165 +3,64 @@
    [clojure.test :refer :all]
    [metabase-enterprise.metabot-v3.tools.util :as metabot-v3.tools.util]
    [metabase.collections.models.collection :as collection]
-   [metabase.content-verification.core :as moderation]
-   [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]
-   [metabase.lib.core :as lib]
-   [metabase.lib.metadata :as lib.metadata]
    [metabase.permissions.models.permissions :as perms]
    [metabase.test :as mt]))
 
-(deftest metabot-scope-query-test
-  (testing "metabot-scope-query with collection hierarchy"
-    (mt/dataset test-data
-      (mt/with-non-admin-groups-no-root-collection-perms
-        (mt/with-temp [:model/PermissionsGroup {group-id :id} {}
-                       :model/PermissionsGroupMembership _ {:user_id (mt/user->id :rasta)
-                                                            :group_id group-id}
-                       :model/Collection container-coll {:name "container coll"}
-                       :model/Collection metabot-coll   {:name "mb coll"
-                                                         :location (collection/location-path container-coll)}
-                       :model/Collection mb-child-coll1 {:name "mbc1"
-                                                         :location (collection/location-path container-coll metabot-coll)}
-                       :model/Collection mb-child-coll2 {:name "mbc2"
-                                                         :location (collection/location-path container-coll metabot-coll)}
-                       :model/Collection non-mb-coll    {:name "non-mbc"
-                                                         :location (collection/location-path container-coll)}
-                       :model/Card mb-model1  {:type :model,  :collection_id (:id metabot-coll)}
-                       :model/Card mb-model2  {:type :model,  :collection_id (:id mb-child-coll1)}
-                       :model/Card mb-metric1 {:type :metric, :collection_id (:id metabot-coll)}
-                       :model/Card mb-metric2 {:type :metric, :collection_id (:id mb-child-coll2)}
-                       :model/Card outside-model {:type :model, :collection_id (:id non-mb-coll)}
-                       :model/Card outside-metric {:type :metric, :collection_id (:id non-mb-coll)}
-                       :model/Metabot metabot {:name "metabot"
-                                               :collection_id (:id metabot-coll)
-                                               :use_verified_content false}]
-          (perms/grant-collection-read-permissions! group-id metabot-coll)
-          (perms/grant-collection-read-permissions! group-id mb-child-coll1)
-          (perms/grant-collection-read-permissions! group-id mb-child-coll2)
-
-          (testing "admin can see all cards in metabot collection and subcollections"
-            (let [admin-result (mt/with-test-user :crowberto
-                                 (metabot-v3.tools.util/get-metrics-and-models (:id metabot)))
-                  card-ids (set (map :id admin-result))]
-              (is (contains? card-ids (:id mb-model1)))
-              (is (contains? card-ids (:id mb-model2)))
-              (is (contains? card-ids (:id mb-metric1)))
-              (is (contains? card-ids (:id mb-metric2)))
-              (is (not (contains? card-ids (:id outside-model))))
-              (is (not (contains? card-ids (:id outside-metric))))))
-
-          (testing "normal user sees only permitted cards"
-            (let [user-result (mt/with-test-user :rasta
-                                (metabot-v3.tools.util/get-metrics-and-models (:id metabot)))
-                  card-ids (set (map :id user-result))]
-              (is (contains? card-ids (:id mb-model1)))
-              (is (contains? card-ids (:id mb-model2)))
-              (is (contains? card-ids (:id mb-metric1)))
-              (is (contains? card-ids (:id mb-metric2))))))))))
-
-(deftest metabot-scope-query-root-collection-test
-  (testing "metabot-scope-query with root collection (null collection_id)"
-    (mt/dataset test-data
-      (mt/with-temp [:model/Card root-model  {:type :model, :collection_id nil}
-                     :model/Card root-metric {:type :metric, :collection_id nil}
-                     :model/Collection some-coll {:name "some collection"}
-                     :model/Card coll-model {:type :model, :collection_id (:id some-coll)}
-                     :model/Metabot metabot {:name "root metabot"
-                                             :collection_id nil
-                                             :use_verified_content false}]
-        (testing "metabot with root collection sees all content"
-          (let [result (mt/with-test-user :crowberto
-                         (metabot-v3.tools.util/get-metrics-and-models (:id metabot)))
-                card-ids (set (map :id result))]
-            (is (contains? card-ids (:id root-model)))
-            (is (contains? card-ids (:id root-metric)))
-            (is (contains? card-ids (:id coll-model)))))))))
-
-(deftest add-table-reference-test
-  (testing "add-table-reference function adds table-reference for FK fields"
-    (mt/dataset test-data
-      (mt/with-current-user (mt/user->id :crowberto)
-        (let [test-db-id (mt/id)
-              mp (lib.metadata.jvm/application-database-metadata-provider test-db-id)
-              orders-query (lib/query mp (lib.metadata/table mp (mt/id :orders)))
-              columns (lib/visible-columns orders-query)]
-
-          (testing "adds table-reference for implicitly joined columns"
-            (let [processed-columns (map #(metabot-v3.tools.util/add-table-reference orders-query %) columns)
-                  user-name-column (first (filter #(and (= "NAME" (:name %))
-                                                        (:fk-field-id %)) processed-columns))]
-              (is (some? user-name-column) "Expected to find implicitly joined User NAME column")
-              (is (contains? user-name-column :table-reference))
-              (is (string? (:table-reference user-name-column)))
-              (is (seq (:table-reference user-name-column)))
-              (is (= "User" (:table-reference user-name-column)))))
-
-          (testing "does not add table-reference for direct table columns"
-            (let [processed-columns (map #(metabot-v3.tools.util/add-table-reference orders-query %) columns)
-                  id-column (first (filter #(and (= "ID" (:name %))
-                                                 (not (:fk-field-id %))) processed-columns))]
-              (is (some? id-column) "Expected to find direct ORDERS ID column")
-              (is (not (contains? id-column :table-reference)))))
-
-          (testing "handles columns without fk-field-id or table-id gracefully"
-            (let [mock-column {:name "test-column" :type :string}
-                  result (metabot-v3.tools.util/add-table-reference orders-query mock-column)]
-              (is (= mock-column result))
-              (is (not (contains? result :table-reference)))))
-
-          (testing "handles columns with fk-field-id but no table-id"
-            (let [mock-column {:name "test-fk" :fk-field-id 123}
-                  result (metabot-v3.tools.util/add-table-reference orders-query mock-column)]
-              (is (= mock-column result))
-              (is (not (contains? result :table-reference)))))
-
-          (testing "handles columns with table-id but no fk-field-id"
-            (let [mock-column {:name "test-field" :table-id (mt/id :orders)}
-                  result (metabot-v3.tools.util/add-table-reference orders-query mock-column)]
-              (is (= mock-column result))
-              (is (not (contains? result :table-reference))))))))))
-
-(deftest metabot-verified-content-test
-  (testing "metabot-scope-query with verified content filtering"
-    (mt/dataset test-data
-      (mt/with-premium-features #{:content-verification}
-        (mt/with-temp [:model/Collection metabot-coll {:name "mb coll"}
-                       :model/Card verified-model {:type :model, :collection_id (:id metabot-coll)}
-                       :model/Card unverified-model {:type :model, :collection_id (:id metabot-coll)}
-                       :model/Card verified-metric {:type :metric, :collection_id (:id metabot-coll)}
-                       :model/Card unverified-metric {:type :metric, :collection_id (:id metabot-coll)}
-                       :model/Metabot verified-metabot {:name "verified metabot"
-                                                        :collection_id (:id metabot-coll)
-                                                        :use_verified_content true}
-                       :model/Metabot unverified-metabot {:name "unverified metabot"
-                                                          :collection_id (:id metabot-coll)
-                                                          :use_verified_content false}]
-        ;; Mark some content as verified
-          (moderation/create-review! {:moderated_item_id (:id verified-model)
-                                      :moderated_item_type "card"
-                                      :moderator_id (mt/user->id :crowberto)
-                                      :status "verified"
-                                      :text "This is verified"})
-          (moderation/create-review! {:moderated_item_id (:id verified-metric)
-                                      :moderated_item_type "card"
-                                      :moderator_id (mt/user->id :crowberto)
-                                      :status "verified"
-                                      :text "This is verified"})
-
-          (testing "metabot with use_verified_content=true sees only verified content"
-            (let [result (mt/with-test-user :crowberto
-                           (metabot-v3.tools.util/get-metrics-and-models (:id verified-metabot)))
-                  card-ids (set (map :id result))]
-              (is (contains? card-ids (:id verified-model)))
-              (is (contains? card-ids (:id verified-metric)))
-              (is (not (contains? card-ids (:id unverified-model))))
-              (is (not (contains? card-ids (:id unverified-metric))))))
-
-          (testing "metabot with use_verified_content=false sees all content"
-            (let [result (mt/with-test-user :crowberto
-                           (metabot-v3.tools.util/get-metrics-and-models (:id unverified-metabot)))
-                  card-ids (set (map :id result))]
-              (is (contains? card-ids (:id verified-model)))
-              (is (contains? card-ids (:id verified-metric)))
-              (is (contains? card-ids (:id unverified-model)))
-              (is (contains? card-ids (:id unverified-metric))))))))))
+(deftest metabot-scope-test
+  (mt/dataset test-data
+    (mt/with-non-admin-groups-no-root-collection-perms
+      (mt/with-temp [:model/PermissionsGroup {group-id :id} {}
+                     :model/PermissionsGroupMembership _ {:user_id (mt/user->id :rasta)
+                                                          :group_id group-id}
+                     :model/Collection container-coll {:name "container coll"}
+                     :model/Collection metabot-coll   {:name "mb coll"
+                                                       :location (collection/location-path container-coll)}
+                     :model/Collection mb-child-coll1 {:name "mbc1"
+                                                       :location (collection/location-path container-coll metabot-coll)}
+                     :model/Collection mb-child-coll2 {:name "mbc2"
+                                                       :location (collection/location-path container-coll metabot-coll)}
+                     :model/Collection mb-child-coll3 {:name "mbc3"
+                                                       :location (collection/location-path
+                                                                  container-coll metabot-coll mb-child-coll2)}
+                     :model/Collection non-mb-coll    {:name "non-mbc"
+                                                       :location (collection/location-path container-coll)}
+                     :model/Card mb-model1  {:type :model,  :collection_id (:id metabot-coll)}
+                     :model/Card mb-model2  {:type :model,  :collection_id (:id mb-child-coll1)}
+                     :model/Card mb-model3  {:type :model,  :collection_id (:id mb-child-coll1)}
+                     :model/Card mb-model4  {:type :model,  :collection_id (:id non-mb-coll)}
+                     :model/Card _          {:type :model,  :collection_id (:id non-mb-coll)}
+                     :model/Card mb-metric1 {:type :metric, :collection_id (:id metabot-coll)}
+                     :model/Card mb-metric2 {:type :metric, :collection_id (:id mb-child-coll2)}
+                     :model/Card mb-metric3 {:type :metric, :collection_id (:id mb-child-coll3)}
+                     :model/Card mb-metric4 {:type :metric, :collection_id (:id non-mb-coll)}
+                     :model/Card _          {:type :metric, :collection_id (:id non-mb-coll)}
+                     :model/Metabot metabot {:name "metabot"}
+                     :model/MetabotEntity {mb-coll-entity-id :id}   {:metabot_id (:id metabot)
+                                                                     :model :collection
+                                                                     :model_id (:id metabot-coll)}
+                     :model/MetabotEntity {mb-model-entity-id :id}  {:metabot_id (:id metabot)
+                                                                     :model :dataset
+                                                                     :model_id (:id mb-model4)}
+                     :model/MetabotEntity {mb-metric-entity-id :id} {:metabot_id (:id metabot)
+                                                                     :model :metric
+                                                                     :model_id (:id mb-metric4)}]
+        (perms/grant-collection-read-permissions! group-id mb-child-coll2)
+        (testing "admins can see all cards"
+          (let [admin-result (mt/with-test-user :crowberto
+                               (metabot-v3.tools.util/metabot-scope
+                                [mb-coll-entity-id mb-model-entity-id mb-metric-entity-id]))]
+            (is (= {mb-model1 mb-coll-entity-id
+                    mb-model2 mb-coll-entity-id
+                    mb-model3 mb-coll-entity-id
+                    mb-model4 mb-model-entity-id
+                    mb-metric1 mb-coll-entity-id
+                    mb-metric2 mb-coll-entity-id
+                    mb-metric3 mb-coll-entity-id
+                    mb-metric4 mb-metric-entity-id}
+                   admin-result))))
+        (testing "normal users can see permitted cards"
+          (let [user-result (mt/with-test-user :rasta
+                              (metabot-v3.tools.util/metabot-scope
+                               [mb-coll-entity-id mb-model-entity-id mb-metric-entity-id]))]
+            (is (= {mb-metric2 mb-coll-entity-id}
+                   user-result))))))))

enterprise/backend/test/metabase_enterprise/models/entity_id_test.clj

@@ -96,9 +96,7 @@
     :model/ViewLog
     :model/GroupTableAccessPolicy
     :model/ConnectionImpersonation
-    :model/CloudMigration
-    :model/Comment
-    :model/CommentReaction})
+    :model/CloudMigration})
 
 (deftest ^:parallel comprehensive-entity-id-test
   (let [entity-id-models (->> (v2.entity-ids/toucan-models)
@@ -107,7 +105,7 @@
     (testing "All exported models should get entity id except those with other unique property (like name)"
       (is (= (set (concat serdes.models/exported-models
                           ;; those are inline models which still have entity_id
-                          ["DashboardCard" "DashboardTab" "Dimension" "MetabotPrompt"]))
+                          ["DashboardCard" "DashboardTab" "Dimension" "MetabotEntity" "MetabotPrompt"]))
              (set (->> (concat entity-id-models
                                entities-external-name)
                        (map name))))))

enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/row_level_restrictions_test.clj

@@ -186,12 +186,13 @@
                    :query {:source-query {:source-table $$checkins
                                           :fields [$id $date $user_id $venue_id]
                                           :filter [:and
-                                                   ;; This still gets :default bucketing! auto-bucket-datetimes
-                                                   ;; puts :day bucketing on both parts of this filter, since it's
-                                                   ;; matching a YYYY-mm-dd string. Then optimize-temporal-filters
-                                                   ;; sees that the
-                                                   ;; :type/Date column already has :day
-                                                   ;; granularity, and switches both to :default
+                                                                          ;; This still gets :default bucketing!
+                                                                          ;; auto-bucket-datetimes puts :day bucketing
+                                                                          ;; on both parts of this filter, since it's
+                                                                          ;; matching a YYYY-mm-dd string. Then
+                                                                          ;; optimize-temporal-filters sees that the
+                                                                          ;; :type/Date column already has :day
+                                                                          ;; granularity, and switches both to :default
                                                    [:> !default.date [:absolute-datetime
                                                                       #t "2014-01-01"
                                                                       :default]]