create model using mbql

right now it's populating the static datasets with transforms tables

.clj-kondo/config.edn

@@ -1,7 +1,7 @@
 ;; -*- comment-column: 100; -*-
 {:config-paths ["macros" "src" "config"]
  :linters
- {;;
+ { ;;
   ;; enabled optional linters (these are disabled by default)
   ;;
 
@@ -49,7 +49,7 @@
   ;; disabled linters
   ;;
 
-  :redundant-ignore {:level :off} ; lots of false positives, see https://github.com/clj-kondo/clj-kondo/issues/2414
+  :redundant-ignore {:level :off}    ; lots of false positives, see https://github.com/clj-kondo/clj-kondo/issues/2414
   :unexpected-recur {:level :off} ; TODO (cam): I think we just need to tell it how to handle MBQL match and we can enable this?
 
   ;;
@@ -187,7 +187,7 @@
   :metabase/disallow-hardcoded-driver-names-in-tests
   ;; don't include `:h2` since that will probably lead to a mountain of tests where we are specifically testing
   ;; middleware or compilation behavior with just H2 since it's the default.
-  {:drivers #{;; core drivers
+  {:drivers #{ ;; core drivers
               #_:h2
               :postgres
               :mysql
@@ -215,7 +215,7 @@
   ;; misc config for built-in linters
   ;;
 
-  :invalid-arity       {:skip-args [metabase.lib.util.match/match]} ; TODO (cam): can we fix these?
+  :invalid-arity       {:skip-args [metabase.lib.util.match/match]}                    ; TODO (cam): can we fix these?
   :refer-all           {:level :warning, :exclude [clojure.test]}
   :unused-referred-var {:exclude {compojure.core [GET DELETE POST PUT]}}
 
@@ -410,9 +410,12 @@
   clojurewerkz.quartzite.jobs/defjob                                                    clojure.core/defn
   instaparse.core/defparser                                                             clojure.core/def
   metabase-enterprise.action-v2.test-util/with-test-tables!                             clojure.core/let
+  metabase-enterprise.content-translation.routes-test/with-new-secret-key!              clojure.core/fn
   metabase-enterprise.serialization.test-util/with-dbs                                  clojure.core/fn
   metabase-enterprise.serialization.test-util/with-random-dump-dir                      clojure.core/let
-  metabase-enterprise.content-translation.routes-test/with-new-secret-key!              clojure.core/fn
+  metabase-enterprise.transforms.api-test/with-transform-tags clojure.core/let
+  metabase-enterprise.transforms.api-test/with-worker-runs clojure.core/let
+  metabase-enterprise.transforms.test-util/with-transform-cleanup!                      clojure.core/let
   metabase.actions.test-util/with-actions                                               clojure.core/let
   metabase.api.common/let-404                                                           clojure.core/let
   metabase.app-db.custom-migrations.util/with-temp-scheduler!                           clojure.core/fn
@@ -643,6 +646,7 @@
 
   :macroexpand
   {clojurewerkz.quartzite.jobs/build                                                       macros.quartz/build-job
+   clojurewerkz.quartzite.schedule.calendar-interval/build                                 macros.quartz/build-job
    clojurewerkz.quartzite.schedule.cron/schedule                                           macros.quartz/schedule
    clojurewerkz.quartzite.schedule.simple/schedule                                         macros.quartz/simple-schedule
    clojurewerkz.quartzite.triggers/build                                                   macros.quartz/build-trigger
@@ -722,7 +726,10 @@
   ;; `metabase-enterprise.sandbox.query-processor*-test`
   ;;
   {:pattern "^metabase(?:((?:-enterprise\\.sandbox)?\\.query-processor)|(?:\\.driver\\.sql(?!server)(?!ite))).*-test$"
-   :name    general-qp-and-driver-test-namespaces}]
+   :name    general-qp-and-driver-test-namespaces}
+
+  {:pattern "^metabase-enterprise\\.workspaces.*$"
+   :name    workspaces-module}]
 
  :config-in-ns
  {test-namespaces
@@ -818,4 +825,11 @@
      metabase.test.util/with-column-remappings {:message "Use metabase.lib.test-util/remap-metadata-provider instead of with-column-remappings in QP tests"}}
 
     :metabase/disallow-hardcoded-driver-names-in-tests
-    {:level :warning}}}}}
+    {:level :warning}}}
+
+  workspaces-module
+  {:linters
+   {:docstring-leading-trailing-whitespace {:level :warning}
+    :used-underscored-binding              {:level :warning}
+    :keyword-binding                       {:level :warning}
+    :shadowed-var                          {:level :warning}}}}}

.clj-kondo/config/modules/config.edn

@@ -72,13 +72,16 @@
 
   activity-feed
   {:team "UX West"
-   :api  #{metabase.activity-feed.api metabase.activity-feed.init}
+   :api #{metabase.activity-feed.api
+          metabase.activity-feed.core
+          metabase.activity-feed.init}
    :uses #{api
            app-db
            collections
            config
            events
            models
+           premium-features
            util}}
 
   analytics
@@ -1640,6 +1643,28 @@
            warehouse-schema
            warehouses}}
 
+  enterprise/documents
+  {:team "UX West"
+   :api #{metabase-enterprise.documents.api
+          metabase-enterprise.documents.core
+          metabase-enterprise.documents.init}
+   :uses #{activity-feed
+           api
+           app-db
+           batch-processing
+           collections
+           events
+           models
+           permissions
+           premium-features
+           queries
+           query-permissions
+           revisions
+           search
+           users
+           util
+           view-log}}
+
   enterprise/email
   {:team "UX West"
    :api  #{metabase-enterprise.email.api}
@@ -1726,7 +1751,8 @@
            system
            users
            util
-           warehouse-schema}}
+           warehouse-schema
+           enterprise/documents}}
 
   enterprise/permission-debug
   {:team "UX West"
@@ -1857,13 +1883,57 @@
            settings
            util}}
 
+  enterprise/transforms
+  {:team "Querying"
+   :api  #{metabase-enterprise.transforms.api
+           metabase-enterprise.transforms.core
+           metabase-enterprise.transforms.init
+           ;; added during workspace poc:
+           metabase-enterprise.transforms.execute}
+   :uses #{api
+           config
+           driver
+           events
+           lib
+           models
+           permissions
+           query-processor
+           settings
+           sync
+           task
+           task-history
+           util
+           enterprise/worker}}
+
   enterprise/upload-management
   {:team "UX West"
-   :api  #{metabase-enterprise.upload-management.api}
+   :api #{metabase-enterprise.upload-management.api}
    :uses #{api
            models
            premium-features
            upload
+           util}}
+
+  enterprise/workspaces
+  {:team "DevEx"
+   :api #{metabase-enterprise.workspaces.api
+          metabase-enterprise.workspaces.common}
+   :uses #{api
+           api-keys
+           collections
+           dashboards
+           util
+           xrays
+           enterprise/transforms}}
+
+  enterprise/worker
+  {:team "Querying"
+   :api  #{metabase-enterprise.worker.api
+           metabase-enterprise.worker.core
+           metabase-enterprise.worker.init}
+   :uses #{config
+           system
+           task
            util}}}
 
  ;; namespaces matching these patterns (with `re-find`) are excluded from the module linter. Since regex literals

.github/actions/test-driver/action.yml

@@ -27,7 +27,7 @@ runs:
       env:
         MB_EDITION: ee
     - name: Test database driver
-      run: clojure -X:dev:ci:ee:ee-dev:drivers:drivers-dev:test ${{ inputs.test-args }}
+      run: clojure -X:dev:ci:ee:ee-dev:drivers:drivers-dev:test :only metabase-enterprise.transforms.api-test ${{ inputs.test-args }}
       shell: bash
     - name: Publish Test Report (JUnit)
       uses: dorny/test-reporter@v1

.storybook/main.ts

@@ -8,6 +8,7 @@ const { CSS_CONFIG } = require("../frontend/build/shared/rspack/css-config");
 const mainAppStories = [
   "../frontend/**/*.mdx",
   "../frontend/**/*.stories.@(js|jsx|ts|tsx)",
+  "../enterprise/frontend/**/*.stories.@(js|jsx|ts|tsx)",
 ];
 
 const config: StorybookConfig = {

deps.edn

@@ -630,7 +630,8 @@
                                    "src/metabase/util/currency\\.cljc$"
                                    "test/metabase/channel/render/png_test.clj"
                                    "#.+#$"
-                                   "\\.transit\\.json$"]}}
+                                   "\\.transit\\.json$"
+                                   "enterprise/frontend/src/metabase-enterprise/documents/components/Editor/data/data\\.json$"]}}
 
   ;; clojure -X:dev:ee:ee-dev:test:cloverage
   :cloverage
@@ -876,6 +877,12 @@
   :test/kondo
   {:exec-args {:only [".clj-kondo/test"]}}
 
+  ;; test workspaces.
+  ;;
+  ;;    clj -X:dev:ee:ee-dev:test:test/workspaces
+  :test/workspaces
+  {:exec-args {:only "enterprise/backend/test/metabase_enterprise/workspaces"}}
+
   ;; DB configs -- start DBs with:
   ;; ./bin/mage start-db postgres oldest
   :db/postgres-oldest

dev/src/dev.clj

@@ -51,6 +51,7 @@
   "Put everything needed for REPL development within easy reach"
   (:require
    [clojure.core.async :as a]
+   [clojure.main]
    [clojure.string :as str]
    [clojure.test]
    [dev.debug-qp :as debug-qp]
@@ -100,6 +101,8 @@
   model-tracking/keep-me
   dev.h2/keep-me)
 
+(apply require clojure.main/repl-requires)
+
 #_:clj-kondo/ignore
 (defn tap>-spy [x]
   (doto x tap>))
@@ -194,6 +197,16 @@
   (stop!)
   (start!))
 
+(defn start-worker!
+  "Start metabase worker"
+  []
+  ((requiring-resolve 'metabase-enterprise.worker.core/start!) {:dev true}))
+
+(defn stop-worker!
+  "Stop metabase worker"
+  []
+  ((requiring-resolve 'metabase-enterprise.worker.core/stop!)))
+
 (defn ns-unmap-all
   "Unmap all interned vars in a namespace. Reset the namespace to a blank slate! Perfect for when you rename everything
   and want to make sure you didn't miss a reference or when you redefine a multimethod.

dev/src/dev/model_tracking.clj

@@ -22,7 +22,6 @@
    [toucan2.tools.before-update :as t2.before-update]
    [toucan2.util :as t2.util]))
 
-
 (def changes*
   "An atom to store all the changes of models that we currently track."
   (atom {}))
@@ -79,7 +78,7 @@
                            :update
                            (into {} (t2/changes row-or-instance))
                            (into {} row-or-instance))
-                        clean-change)
+                         clean-change)
         path       [(t2/table-name model) action]]
     ;; ideally this should be debug, but for some reasons this doesn't get logged
     (on-change path change-info)
@@ -117,6 +116,9 @@
     (track-one! model)
     (swap! tracked-models conj model)))
 
+(comment
+  (track! :model/Collection :model/Workspace))
+
 (defn- untrack-one!
   [model]
   (doseq [[hook aux-method _action] hook+aux-method+action+deriveable]

docs/developers-guide/driver-changelog.md

@@ -37,11 +37,13 @@ title: Driver interface changelog
 
 - Added the driver multi-method `driver/set-database-used!` for drivers to set a database on the connection with statements like `USE DATABASE`.
 
+- Added the driver features `:transforms/table`, `:transforms/view`, and `:transforms/matarialized-view` for drivers
+  that support transforms with table, view, and materialized view as target, respectively.
+
 ## Metabase 0.55.9
 
 - Add multi-method `driver/do-with-resilient-connection` for executing functions in a context where closed connections may be automatically reopened
 
-
 ## Metabase 0.55.0
 
 - Add the multi-method `->date` that allows the driver to control how to cast strings and temporal types to dates.

e2e/support/helpers/api/createDocument.ts

@@ -0,0 +1,31 @@
+import type {
+  Document,
+  DocumentContent,
+  RegularCollectionId,
+} from "metabase-types/api";
+
+export const createDocument = ({
+  name,
+  collection_id,
+  document,
+  alias,
+}: {
+  name: string;
+  collection_id?: RegularCollectionId | null;
+  document: DocumentContent;
+  alias?: string;
+}): Cypress.Chainable<Cypress.Response<Document>> => {
+  cy.log(`Create a collection: ${name}`);
+
+  return cy
+    .request("POST", "/api/ee/document", {
+      name,
+      collection_id,
+      document,
+    })
+    .then((response) => {
+      if (alias) {
+        cy.wrap(response.body.id).as(alias);
+      }
+    });
+};

e2e/support/helpers/api/createTransform.ts

@@ -0,0 +1,19 @@
+import type { CreateTransformRequest, Transform } from "metabase-types/api";
+
+export type Options = {
+  wrapId?: boolean;
+  idAlias?: string;
+};
+
+export function createTransform(
+  request: CreateTransformRequest,
+  { wrapId = false, idAlias = "transformId" }: Options = {},
+): Cypress.Chainable<Cypress.Response<Transform>> {
+  return cy
+    .request<Transform>("POST", "/api/ee/transform", request)
+    .then(({ body }) => {
+      if (wrapId) {
+        cy.wrap(body.id).as(idAlias);
+      }
+    });
+}

e2e/support/helpers/api/index.ts

@@ -10,6 +10,7 @@ export { createDashboard } from "./createDashboard";
 export type { DashboardDetails } from "./createDashboard";
 export { createDashboardWithQuestions } from "./createDashboardWithQuestions";
 export { createDashboardWithTabs } from "./createDashboardWithTabs";
+export { createDocument } from "./createDocument";
 export { createModerationReview } from "./createModerationReview";
 export { createNativeQuestion } from "./createNativeQuestion";
 export type { NativeQuestionDetails } from "./createNativeQuestion";
@@ -26,6 +27,7 @@ export { createSnippet } from "./createSnippet";
 export { createTimeline } from "./createTimeline";
 export { createTimelineEvent } from "./createTimelineEvent";
 export { createTimelineWithEvents } from "./createTimelineWithEvents";
+export { createTransform } from "./createTransform";
 export { editDashboardCard } from "./editDashboardCard";
 export { getCurrentUser } from "./getCurrentUser";
 export { remapDisplayValueToFK } from "./remapDisplayValueToFK";

e2e/support/helpers/e2e-document-helpers.ts

@@ -0,0 +1,20 @@
+export const documentContent = () => cy.findByTestId("document-content");
+
+export const addToDocument = (text: string, newLine: boolean = true) => {
+  cy.realType(`${text}`);
+  if (newLine) {
+    cy.realType("{enter}");
+  }
+};
+
+export const documentSuggestionDialog = () =>
+  cy.findByRole("dialog", { name: "Mention Dialog" });
+
+export const documentSuggestionItem = (name: string) =>
+  documentSuggestionDialog().findByRole("listitem", { name });
+
+export const commandSuggestionDialog = () =>
+  cy.findByRole("dialog", { name: "Command Dialog" });
+
+export const commandSuggestionItem = (name: string) =>
+  commandSuggestionDialog().findByRole("listitem", { name });

e2e/support/helpers/index.ts

@@ -14,6 +14,7 @@ export * from "./e2e-dashboard-visualizer-helpers";
 export * from "./e2e-database-metadata-helpers";
 export * from "./e2e-datamodel-helpers";
 export * from "./e2e-dimension-list-helpers";
+export * from "./e2e-document-helpers";
 export * from "./e2e-downloads-helpers";
 export * from "./e2e-dragndrop-helpers";
 export * from "./e2e-element-visibility-helpers";

e2e/test/scenarios/admin/transforms.cy.spec.ts

@@ -0,0 +1,202 @@
+import { WRITABLE_DB_ID } from "e2e/support/cypress_data";
+import type { TransformId } from "metabase-types/api";
+
+const { H } = cy;
+
+const DB_NAME = "Writable Postgres12";
+
+describe("scenarios > admin > transforms", () => {
+  beforeEach(() => {
+    H.restore("postgres-writable");
+    H.resetTestTable({ type: "postgres", table: "many_schemas" });
+    cy.signInAsAdmin();
+    H.activateToken("bleeding-edge");
+    H.resyncDatabase({ dbId: WRITABLE_DB_ID });
+  });
+
+  it("should be able to create and run an mbql transform", () => {
+    visitTransformListPage();
+    getTransformListPage().button("Create a transform").click();
+    H.popover().findByText("Query builder").click();
+
+    H.entityPickerModal().within(() => {
+      cy.findByText(DB_NAME).click();
+      cy.findByText("Schema a").click();
+      cy.findByText("Animals").click();
+    });
+    getQueryEditor().button("Save").click();
+    H.modal().within(() => {
+      cy.findByLabelText("Name").type("MBQL transform");
+      cy.findByLabelText("Table name").type("transform_table");
+      cy.button("Save").click();
+    });
+
+    cy.log("run the transform and make sure its table can be queried");
+    runAndWaitForSuccess();
+    getTableLink().click();
+    H.queryBuilderHeader().findByText("Transform Table").should("be.visible");
+    H.assertQueryBuilderRowCount(3);
+  });
+
+  it("should be able to change the target before running a transform", () => {
+    cy.log("create but do not run the transform");
+    createMbqlTransform();
+    cy.get<TransformId>("@transformId").then(visitTransformPage);
+
+    cy.log("modify the transform before running");
+    getTransformPage().button("Change target").click();
+    H.modal().within(() => {
+      cy.findByLabelText("Table name").should("have.value", "transform_table");
+      cy.findByLabelText("Schema").should("have.value", "Schema A");
+      cy.findByLabelText("Table name").clear().type("transform_table_2");
+      cy.findByLabelText("Schema").click();
+    });
+    H.popover().findByText("Schema B").click();
+    H.modal().button("Save").click();
+    getTableLink().should("have.text", "transform_table_2");
+    getSchemaLink().should("have.text", "Schema B");
+
+    cy.log("run the transform and verify the table");
+    runAndWaitForSuccess();
+    getTableLink().click();
+    H.queryBuilderHeader().findByText("Transform Table 2").should("be.visible");
+    H.assertQueryBuilderRowCount(3);
+  });
+
+  it("should be able to change the target after running a transform and keep the old target", () => {
+    cy.log("create and run a transform");
+    createMbqlTransform();
+    cy.get<TransformId>("@transformId").then(visitTransformPage);
+    runAndWaitForSuccess();
+
+    cy.log("modify the transform after running");
+    getTransformPage().button("Change target").click();
+    H.modal().within(() => {
+      cy.findByLabelText("Table name").should("have.value", "transform_table");
+      cy.findByLabelText("Schema").should("have.value", "Schema A");
+      cy.findByLabelText("Keep transform_table").should("be.checked");
+      cy.findByLabelText("Table name").clear().type("transform_table_2");
+      cy.button("Change target").click();
+    });
+    getTableLink().should("have.text", "transform_table_2");
+
+    cy.log("run the transform and verify the new table");
+    runAndWaitForSuccess();
+    getTableLink().click();
+    H.queryBuilderHeader().findByText("Transform Table 2").should("be.visible");
+    H.assertQueryBuilderRowCount(3);
+
+    cy.log("verify that the original question still works");
+    H.createNativeQuestion(
+      {
+        database: WRITABLE_DB_ID,
+        native: { query: 'SELECT * FROM "Schema A".transform_table' },
+      },
+      { visitQuestion: true },
+    );
+    H.assertQueryBuilderRowCount(3);
+  });
+
+  it("should be able to change the target after running a transform and delete the old target", () => {
+    cy.log("create and run a transform");
+    createMbqlTransform();
+    cy.get<TransformId>("@transformId").then(visitTransformPage);
+    runAndWaitForSuccess();
+
+    cy.log("modify the transform after running");
+    getTransformPage().button("Change target").click();
+    H.modal().within(() => {
+      cy.findByLabelText("Table name").should("have.value", "transform_table");
+      cy.findByLabelText("Schema").should("have.value", "Schema A");
+      cy.findByLabelText("Table name").clear().type("transform_table_2");
+      cy.findByLabelText("Delete transform_table").click();
+      cy.button("Change target and delete the old one").click();
+    });
+    getTableLink().should("have.text", "transform_table_2");
+
+    cy.log("run the transform and verify the new table");
+    runAndWaitForSuccess();
+    getTableLink().click();
+    H.queryBuilderHeader().findByText("Transform Table 2").should("be.visible");
+    H.assertQueryBuilderRowCount(3);
+
+    cy.log("verify that the original question still works");
+    H.createNativeQuestion(
+      {
+        database: WRITABLE_DB_ID,
+        native: { query: 'SELECT * FROM "Schema A".transform_table' },
+      },
+      { visitQuestion: true },
+    );
+    getQueryVisualization()
+      .findByText(/ERROR: relation "Schema A.transform_table" does not exist/)
+      .should("be.visible");
+  });
+});
+
+function getTransformListPage() {
+  return cy.findByTestId("transform-list-page");
+}
+
+function getTransformPage() {
+  return cy.findByTestId("transform-page");
+}
+
+function getQueryEditor() {
+  return cy.findByTestId("transform-query-editor");
+}
+
+function getRunButton() {
+  return cy.findByTestId("run-button");
+}
+
+function getTableLink() {
+  return cy.findByTestId("table-link");
+}
+
+function getSchemaLink() {
+  return cy.findByTestId("table-link");
+}
+
+function getQueryVisualization() {
+  return cy.findByTestId("query-visualization-root");
+}
+
+function visitTransformListPage() {
+  return cy.visit("/admin/transforms");
+}
+
+function visitTransformPage(id: TransformId) {
+  return cy.visit(`/admin/transforms/${id}`);
+}
+
+function runAndWaitForSuccess() {
+  getRunButton().click();
+  getRunButton().should("have.text", "Ran successfully");
+}
+
+function createMbqlTransform() {
+  H.getTableId({ name: "Animals" }).then((tableId) => {
+    H.createTransform(
+      {
+        name: "MBQL transform",
+        source: {
+          type: "query",
+          query: {
+            database: WRITABLE_DB_ID,
+            type: "query",
+            query: {
+              "source-table": tableId,
+            },
+          },
+        },
+        target: {
+          type: "table",
+          name: "transform_table",
+          schema: "Schema A",
+        },
+      },
+      { wrapId: true },
+    );
+  });
+}

e2e/test/scenarios/documents/documents.cy.spec.ts

@@ -0,0 +1,178 @@
+import { ORDERS_BY_YEAR_QUESTION_ID } from "e2e/support/cypress_sample_instance_data";
+
+const { H } = cy;
+
+describe("documents", () => {
+  beforeEach(() => {
+    H.restore();
+    cy.signInAsAdmin();
+    H.activateToken("bleeding-edge");
+  });
+
+  it("should allow you to create a new document from the new button and save", () => {
+    cy.visit("/");
+
+    H.newButton("Document").click();
+
+    cy.findByRole("textbox", { name: "Document Title" })
+      .should("be.focused")
+      .type("Test Document");
+
+    H.documentContent().type("This is a paragraph\nAnd this is another");
+
+    cy.findByRole("button", { name: "Save" }).click();
+
+    H.entityPickerModalTab("Collections").click();
+    H.entityPickerModalItem(0, "Our analytics").should(
+      "have.attr",
+      "data-active",
+      "true",
+    );
+    H.entityPickerModalItem(1, "First collection").click();
+    H.entityPickerModal().findByRole("button", { name: "Select" }).click();
+
+    H.appBar()
+      .findByRole("link", { name: /First collection/ })
+      .click();
+
+    H.collectionTable()
+      .findByRole("link", { name: "Test Document" })
+      .should("exist");
+
+    H.openCollectionItemMenu("Test Document");
+
+    H.popover().findByText("Move").click();
+
+    H.entityPickerModalTab("Collections").click();
+    H.entityPickerModalItem(0, "Our analytics")
+      .should("have.attr", "data-active", "true")
+      .click();
+    H.entityPickerModal().findByRole("button", { name: "Move" }).click();
+  });
+
+  describe("document editing", () => {
+    beforeEach(() => {
+      H.createDocument({
+        name: "Foo Document",
+        document: {
+          content: [],
+          type: "doc",
+        },
+        collection_id: null,
+      }).then((data) => {
+        cy.visit(`/document/${data.body.id}`);
+      });
+    });
+
+    it("should support typing with a markdown syntax", () => {
+      H.documentContent().click();
+
+      H.addToDocument("# This is a heading level 1");
+      H.addToDocument("## This is a heading level 2");
+      H.addToDocument("### This is a heading level 3");
+      H.addToDocument("#### This is a heading level 4");
+
+      H.addToDocument("**Some Bold Text**");
+      H.addToDocument("*Some Italic Text*");
+
+      H.addToDocument("Lets start an unordered list");
+      H.addToDocument("- First Item");
+      H.addToDocument("Second Item");
+      // New Line to break out of the list;
+      H.addToDocument("");
+
+      H.addToDocument("Lets start an ordered list");
+      H.addToDocument("1. First Ordered Item");
+      H.addToDocument("Second Ordered Item");
+      // New Line to break out of the list;
+      H.addToDocument("");
+
+      H.addToDocument("http://metabase.com");
+
+      H.documentContent().within(() => {
+        cy.findByRole("heading", { name: "This is a heading level 1" }).should(
+          "exist",
+        );
+        cy.findByRole("heading", { name: "This is a heading level 2" }).should(
+          "exist",
+        );
+        cy.findByRole("heading", { name: "This is a heading level 3" }).should(
+          "exist",
+        );
+        cy.findByRole("heading", { name: "This is a heading level 4" }).should(
+          "exist",
+        );
+        cy.findByRole("strong").should("contain.text", "Some Bold Text");
+        cy.findByRole("emphasis").should("contain.text", "Some Italic Text");
+
+        cy.findAllByRole("list").should("have.length", 2);
+        cy.findAllByRole("listitem")
+          .should("contain.text", "First Item")
+          .should("contain.text", "Second Item")
+          .should("contain.text", "First Ordered Item")
+          .should("contain.text", "Second Ordered Item");
+
+        cy.findByRole("link", { name: "http://metabase.com" }).should("exist");
+      });
+    });
+
+    it("should support adding cards and smart links to documents", () => {
+      cy.intercept({
+        method: "PUT",
+        path: "/api/ee/document/*",
+      }).as("documentUpdate");
+      cy.intercept({
+        method: "GET",
+        path: "/api/ee/document/*",
+      }).as("documentGet");
+
+      //initial load
+      cy.wait("@documentGet");
+      H.documentContent().click();
+
+      H.addToDocument("@ord", false);
+
+      H.documentSuggestionDialog()
+        .should("contain.text", "Orders, Count, Grouped by Created At (year)")
+        .should("contain.text", "Orders, Count")
+        .should("contain.text", "Orders Model");
+
+      H.documentSuggestionItem(
+        "Orders, Count, Grouped by Created At (year)",
+      ).click();
+
+      //Adding a new line
+      H.addToDocument("");
+      H.addToDocument("Adding a static link: /", false);
+      H.commandSuggestionItem("Link to...").click();
+      H.addToDocument("Ord", false);
+      H.commandSuggestionItem("Orders, Count").click();
+      H.addToDocument(" And continue typing", false);
+
+      H.documentContent().within(() => {
+        cy.findAllByTestId("document-card-embed")
+          .should("have.length", 1)
+          .should(
+            "contain.text",
+            "Orders, Count, Grouped by Created At (year)",
+          );
+
+        cy.findByRole("link", { name: /Orders, Count$/ }).should("exist");
+      });
+
+      cy.findByRole("button", { name: "Save" }).click();
+
+      cy.wait("@documentUpdate");
+      cy.wait("@documentGet");
+
+      cy.findByTestId("document-card-embed")
+        .findByText("Orders, Count, Grouped by Created At (year)")
+        .click();
+
+      cy.location("pathname").should(
+        "not.include",
+        ORDERS_BY_YEAR_QUESTION_ID.toString(),
+      );
+    });
+  });
+});

enterprise/backend/src/metabase_enterprise/api/routes.clj

@@ -18,6 +18,7 @@
    [metabase-enterprise.content-verification.api.routes]
    [metabase-enterprise.database-replication.api :as database-replication.api]
    [metabase-enterprise.database-routing.api]
+   [metabase-enterprise.documents.api]
    [metabase-enterprise.email.api]
    [metabase-enterprise.gsheets.api :as gsheets.api]
    [metabase-enterprise.llm.api]
@@ -28,7 +29,9 @@
    [metabase-enterprise.scim.routes]
    [metabase-enterprise.serialization.api]
    [metabase-enterprise.stale.api]
+   [metabase-enterprise.transforms.api]
    [metabase-enterprise.upload-management.api]
+   [metabase-enterprise.workspaces.api]
    [metabase.api.macros :as api.macros]
    [metabase.api.util.handlers :as handlers]
    [metabase.util.i18n :refer [deferred-tru]]))
@@ -49,12 +52,15 @@
    :etl-connections-pg         (deferred-tru "ETL Connections PG replication")
    :llm-autodescription        (deferred-tru "LLM Auto-description")
    :metabot-v3                 (deferred-tru "MetaBot")
+   :documents                  (deferred-tru "Documents")
    :scim                       (deferred-tru "SCIM configuration")
    :serialization              (deferred-tru "Serialization")
    :table-data-editing         (deferred-tru "Table Data Editing")
+   :transforms                 (deferred-tru "Transforms")
    :upload-management          (deferred-tru "Upload Management")
    :database-routing           (deferred-tru "Database Routing")
-   :cloud-custom-smtp          (deferred-tru "Custom SMTP")})
+   :cloud-custom-smtp          (deferred-tru "Custom SMTP")
+   :workspaces                 (deferred-tru "Workspaces")})
 
 (defn- premium-handler [handler required-feature]
   (let [handler (cond-> handler
@@ -98,10 +104,16 @@
    "/metabot-tools"                metabase-enterprise.metabot-v3.tools.api/routes
    "/metabot-v3"                   (premium-handler metabase-enterprise.metabot-v3.api/routes :metabot-v3)
    "/permission_debug"             (premium-handler metabase-enterprise.permission-debug.api/routes :advanced-permissions)
+   "/document"                     (premium-handler metabase-enterprise.documents.api/routes :documents)
    "/scim"                         (premium-handler metabase-enterprise.scim.routes/routes :scim)
    "/serialization"                (premium-handler metabase-enterprise.serialization.api/routes :serialization)
    "/stale"                        (premium-handler metabase-enterprise.stale.api/routes :collection-cleanup)
-   "/upload-management"            (premium-handler metabase-enterprise.upload-management.api/routes :upload-management)})
+   "/transform"                    (premium-handler metabase-enterprise.transforms.api/routes :transforms)
+   "/transform-job"                (premium-handler metabase-enterprise.transforms.api/transform-job-routes :transforms)
+   "/transform-tag"                (premium-handler metabase-enterprise.transforms.api/transform-tag-routes :transforms)
+   "/upload-management"            (premium-handler metabase-enterprise.upload-management.api/routes :upload-management)
+   "/workspace"                    (premium-handler metabase-enterprise.workspaces.api/routes :workspaces)})
+
 ;;; ↑↑↑ KEEP THIS SORTED OR ELSE ↑↑↑
 
 (def ^:private routes-map

enterprise/backend/src/metabase_enterprise/bookmarks/models/bookmark.clj

@@ -0,0 +1,10 @@
+(ns metabase-enterprise.bookmarks.models.bookmark
+  "Enterprise DocumentBookmark model. Provides bookmarking functionality for documents."
+  (:require
+   [methodical.core :as methodical]
+   [toucan2.core :as t2]))
+
+(methodical/defmethod t2/table-name :model/DocumentBookmark [_model] :document_bookmark)
+
+(doto :model/DocumentBookmark
+  (derive :metabase/model))

enterprise/backend/src/metabase_enterprise/core/init.clj

@@ -11,9 +11,12 @@
    [metabase-enterprise.audit-app.init]
    [metabase-enterprise.cache.init]
    [metabase-enterprise.database-replication.init]
+   [metabase-enterprise.documents.init]
    [metabase-enterprise.gsheets.init]
    [metabase-enterprise.llm.init]
    [metabase-enterprise.metabot-v3.init]
    [metabase-enterprise.scim.init]
    [metabase-enterprise.sso.init]
-   [metabase-enterprise.stale.init]))
+   [metabase-enterprise.stale.init]
+   [metabase-enterprise.transforms.init]
+   [metabase-enterprise.worker.init]))

enterprise/backend/src/metabase_enterprise/documents/api.clj

@@ -0,0 +1,10 @@
+(ns metabase-enterprise.documents.api
+  "`/api/ee/document/` routes"
+  (:require
+   [metabase-enterprise.documents.api.document]
+   [metabase.api.util.handlers :as handlers]))
+
+(def ^{:arglists '([request respond raise])} routes
+  "`/api/ee/document/` routes."
+  (handlers/routes
+   metabase-enterprise.documents.api.document/routes))

enterprise/backend/src/metabase_enterprise/documents/api/document.clj

@@ -0,0 +1,239 @@
+(ns metabase-enterprise.documents.api.document
+  "`/api/ee/document/` routes"
+  (:require
+   [metabase-enterprise.documents.models.document :as m.document]
+   [metabase-enterprise.documents.prose-mirror :as prose-mirror]
+   [metabase.api.common :as api]
+   [metabase.api.macros :as api.macros]
+   [metabase.api.routes.common :refer [+auth]]
+   [metabase.collections.models.collection :as collection]
+   [metabase.events.core :as events]
+   [metabase.models.interface :as mi]
+   [metabase.queries.core :as card]
+   [metabase.query-permissions.core :as query-perms]
+   [metabase.util :as u]
+   [metabase.util.i18n :refer [tru]]
+   [metabase.util.malli :as mu]
+   [metabase.util.malli.schema :as ms]
+   [toucan2.core :as t2]))
+
+(def ^:private CardCreateSchema
+  "Schema for creating a new card - simplified version to avoid circular dependencies"
+  [:map
+   [:name ms/NonBlankString]
+   [:dataset_query ms/Map]
+   [:entity_id {:optional true} [:maybe ms/NonBlankString]]
+   [:parameters {:optional true} [:maybe [:sequential ms/Map]]]
+   [:parameter_mappings {:optional true} [:maybe [:sequential ms/Map]]]
+   [:description {:optional true} [:maybe ms/NonBlankString]]
+   [:display ms/NonBlankString]
+   [:visualization_settings ms/Map]
+   [:result_metadata {:optional true} [:maybe [:sequential ms/Map]]]
+   [:cache_ttl {:optional true} [:maybe ms/PositiveInt]]])
+
+(defn- create-card!
+  "Checks that the query is runnable by the current user then saves"
+  [{query :dataset_query :as card} creator]
+  (query-perms/check-run-permissions-for-query query)
+  (card/create-card! (assoc card :type :question :dashboard_id nil) creator))
+
+(mu/defn- update-cards-in-ast :- [:map [:document :any]
+                                  [:content_type :string]]
+
+  [document :- [:map
+                [:document :any]
+                [:content_type :string]]
+   card-id-map :- [:maybe [:map-of :int ms/PositiveInt]]]
+  (cond-> document
+    (map? document)
+    (prose-mirror/update-ast (fn match-card-to-update [{:keys [type attrs]}]
+                               (and (= type prose-mirror/card-embed-type)
+                                    (contains? card-id-map (:id attrs))))
+                             (fn update-card-id [embed]
+                               (update-in embed [:attrs :id] card-id-map)))))
+
+(mu/defn- create-cards-for-document! :- [:map-of ms/NegativeInt ms/PositiveInt]
+  "Creates cards for a document from the cards map.
+   Returns a mapping from the original negative integer keys to the newly created card IDs.
+
+   Args:
+   - cards-to-create: Map of negative-int -> CardCreateSchema data
+   - document-id: ID of the document these cards belong to
+   - document-collection-id: Collection ID of the document (for inheritance)
+   - creator: User creating the cards
+
+   Returns:
+   - Map of negative-int -> actual-card-id"
+  [cards-to-create :- [:map-of [:int {:max -1}] CardCreateSchema]
+   document-id :- ms/PositiveInt
+   document-collection-id :- [:or :nil ms/PositiveInt]
+   creator :- [:map [:id ms/PositiveInt]]]
+  (when (seq cards-to-create)
+    (reduce-kv
+     (fn [result-map original-key card-data]
+       (let [;; Merge document info into card data
+             ;; Cards inherit document's collection_id if not explicitly specified
+             merged-card-data (-> card-data
+                                  (assoc :document_id document-id)
+                                  (cond-> (nil? (:collection_id card-data))
+                                    (assoc :collection_id document-collection-id)))
+             ;; Create the card using the queries core function
+             new-card (create-card! merged-card-data creator)]
+         (assoc result-map original-key (:id new-card))))
+     {}
+     cards-to-create)))
+
+(mu/defn- clone-cards-in-document! :- [:map-of ms/PositiveInt ms/PositiveInt]
+  "Finds all cards in the document that are not associated with the document and clones the cards.
+
+  Args:
+  - document: the document model to clone cards within
+
+  Returns:
+  - map of old-card-id -> cloned-card-id"
+  [{:keys [id collection_id] :as document}]
+  (let [card-ids (prose-mirror/collect-ast document #(when (and (= prose-mirror/card-embed-type (:type %))
+                                                                (pos? (-> % :attrs :id)))
+                                                       (-> % :attrs :id)))
+        to-clone (when (seq card-ids)
+                   (t2/select :model/Card {:where [:and [:in :id card-ids]
+                                                   [:or [:<> :document_id id]
+                                                    [:= :document_id nil]]]}))]
+    (reduce (fn [accum card]
+              (api/read-check card)
+              (assoc accum
+                     (:id card)
+                     (:id (create-card! (assoc card :document_id id :collection_id collection_id)
+                                        @api/*current-user*))))
+            {}
+            to-clone)))
+
+(defn get-document
+  "Get document by id checking if the current user has permission to access and if the document exists."
+  [id]
+  (u/prog1 (api/check-404
+            (api/read-check
+             (t2/hydrate (t2/select-one :model/Document :id id) :creator :can_write)))
+    (events/publish-event! :event/document-read
+                           {:object-id id
+                            :user-id api/*current-user-id*})))
+
+(api.macros/defendpoint :get "/"
+  "Gets existing `Documents`."
+  [_route-params
+   _query-params]
+  {:items (t2/hydrate (t2/select :model/Document {:where [:and
+                                                          (collection/visible-collection-filter-clause)
+                                                          [:= :archived false]]})
+                      :creator :can_write)})
+
+(api.macros/defendpoint :post "/"
+  "Create a new `Document`."
+  [_route-params
+   _query-params
+   {:keys [name document collection_id collection_position cards]}
+   :- [:map
+       [:name ms/NonBlankString]
+       [:document :any]
+       [:collection_id {:optional true} [:maybe ms/PositiveInt]]
+       [:collection_position {:optional true} [:maybe ms/PositiveInt]]
+       [:cards {:optional true} [:maybe [:map-of [:int {:max -1}] CardCreateSchema]]]]]
+  (collection/check-write-perms-for-collection collection_id)
+  (let [document-id (t2/with-transaction [_conn]
+                      (when collection_position
+                        (api/maybe-reconcile-collection-position! {:collection_id collection_id
+                                                                   :collection_position collection_position}))
+                      (let [document-id (t2/insert-returning-pk! :model/Document {:name name
+                                                                                  :collection_id collection_id
+                                                                                  :collection_position collection_position
+                                                                                  :document document
+                                                                                  :content_type prose-mirror/prose-mirror-content-type
+                                                                                  :creator_id api/*current-user-id*})
+                            cards-to-update-in-ast (merge (clone-cards-in-document! {:id document-id
+                                                                                     :collection_id collection_id
+                                                                                     :document document
+                                                                                     :content_type prose-mirror/prose-mirror-content-type})
+                                                          (when-not (empty? cards)
+                                                            (create-cards-for-document! cards document-id collection_id @api/*current-user*)))]
+                        (when (seq cards-to-update-in-ast)
+                          (t2/update! :model/Document :id document-id
+                                      (update-cards-in-ast
+                                       {:document document
+                                        :content_type prose-mirror/prose-mirror-content-type}
+                                       cards-to-update-in-ast)))
+                        document-id))
+        created-document (get-document document-id)]
+    ;; Publish event after successful creation
+    (events/publish-event! :event/document-create
+                           {:object created-document
+                            :user-id api/*current-user-id*})
+    created-document))
+
+(api.macros/defendpoint :get "/:document-id"
+  "Returns an existing Document by ID."
+  [{:keys [document-id]} :- [:map [:document-id ms/PositiveInt]]]
+  (api/read-check (get-document document-id)))
+
+(api.macros/defendpoint :put "/:document-id"
+  "Updates an existing `Document`."
+  [{:keys [document-id]} :- [:map
+                             [:document-id ms/PositiveInt]]
+   _query-params
+   {:keys [name document collection_id collection_position cards] :as body} :- [:map
+                                                                                [:name {:optional true} ms/NonBlankString]
+                                                                                [:document {:optional true} :any]
+                                                                                [:collection_id {:optional true} [:maybe ms/PositiveInt]]
+                                                                                [:collection_position {:optional true} [:maybe ms/PositiveInt]]
+                                                                                [:cards {:optional true} [:maybe [:map-of :int CardCreateSchema]]]
+                                                                                [:archived {:optional true} [:maybe :boolean]]]]
+  (let [existing-document (api/check-404 (get-document document-id))]
+    (api/check-403 (mi/can-write? existing-document))
+    (when (api/column-will-change? :collection_id existing-document body)
+      (m.document/validate-collection-move-permissions (:collection_id existing-document) collection_id))
+
+    ;; Handle archiving logic
+    (let [document-updates (dissoc (api/updates-with-archived-directly existing-document body) :cards)]
+      (t2/with-transaction [_conn]
+        (when collection_position
+          (api/maybe-reconcile-collection-position! existing-document {:collection_id (if (contains? body :collection_id)
+                                                                                        collection_id
+                                                                                        (:collection_id existing-document))
+                                                                       :collection_position collection_position}))
+        (t2/update! :model/Document document-id
+                    (cond-> document-updates
+                      document (merge (update-cards-in-ast
+                                       {:document document
+                                        :content_type (:content_type existing-document)}
+                                       (merge
+                                        (clone-cards-in-document! (assoc existing-document :document document))
+                                        (when-not (empty? cards) (create-cards-for-document! cards document-id collection_id @api/*current-user*)))))
+                      name (assoc :name name)
+                      (contains? body :collection_id) (assoc :collection_id collection_id))))
+      (let [updated-document (get-document document-id)]
+        ;; Publish appropriate events
+        (if (:archived document-updates)
+          (events/publish-event! :event/document-delete
+                                 {:object updated-document
+                                  :user-id api/*current-user-id*})
+          (events/publish-event! :event/document-update
+                                 {:object updated-document
+                                  :user-id api/*current-user-id*}))
+        updated-document))))
+
+(api.macros/defendpoint :delete "/:document-id"
+  "Permanently deletes an archived Document."
+  [{:keys [document-id]} :- [:map [:document-id ms/PositiveInt]]]
+  (let [document (api/check-404 (t2/select-one :model/Document :id document-id))]
+    (api/check-403 (mi/can-write? document))
+    (when-not (:archived document)
+      (let [msg (tru "Document must be archived before it can be deleted.")]
+        (throw (ex-info msg {:status-code 400, :errors {:archived msg}}))))
+    (t2/delete! :model/Document :id document-id)
+    (events/publish-event! :event/document-delete
+                           {:object document
+                            :user-id api/*current-user-id*})
+    api/generic-204-no-content))
+
+(def ^{:arglists '([request respond raise])} routes
+  "`/api/ee/document/` routes."
+  (api.macros/ns-handler *ns* +auth))

enterprise/backend/src/metabase_enterprise/documents/core.clj

@@ -0,0 +1,15 @@
+(ns metabase-enterprise.documents.core
+  (:require
+   [metabase-enterprise.documents.api.document]
+   [metabase-enterprise.documents.recent-views]
+   [potemkin :as p]))
+
+(comment
+  metabase-enterprise.documents.api.document/keep-me
+  metabase-enterprise.documents.recent-views/keep-me)
+
+(p/import-vars
+ [metabase-enterprise.documents.api.document
+  get-document]
+ [metabase-enterprise.documents.recent-views
+  select-documents-for-recents])

enterprise/backend/src/metabase_enterprise/documents/init.clj

@@ -0,0 +1,5 @@
+(ns metabase-enterprise.documents.init
+  (:require
+   [metabase-enterprise.documents.recent-views]
+   [metabase-enterprise.documents.revisions.impl]
+   [metabase-enterprise.documents.view-log]))

enterprise/backend/src/metabase_enterprise/documents/models/document.clj

@@ -0,0 +1,205 @@
+(ns metabase-enterprise.documents.models.document
+  (:require
+   [metabase-enterprise.documents.prose-mirror :as prose-mirror]
+   [metabase.api.common :as api]
+   [metabase.collections.models.collection :as collection]
+   [metabase.models.interface :as mi]
+   [metabase.models.serialization :as serdes]
+   [metabase.search.spec :as search.spec]
+   [metabase.util :as u]
+   [metabase.util.log :as log]
+   [methodical.core :as methodical]
+   [toucan2.core :as t2]))
+
+(methodical/defmethod t2/table-name :model/Document [_model] :document)
+
+(t2/deftransforms :model/Document
+  {:document mi/transform-json})
+
+(doto :model/Document
+  (derive :metabase/model)
+  (derive :perms/use-parent-collection-perms)
+  (derive :hook/timestamped?)
+  (derive :hook/entity-id))
+
+(defn validate-collection-move-permissions
+  "Validates that the current user has write permissions for both old and new collections
+   when moving a document. Uses the same permission pattern as check-allowed-to-change-collection.
+   Throws 403 exception if permissions are insufficient."
+  [old-collection-id new-collection-id]
+  (when old-collection-id
+    (collection/check-write-perms-for-collection old-collection-id))
+  (when new-collection-id
+    (collection/check-write-perms-for-collection new-collection-id))
+  (when new-collection-id
+    (api/check-400 (t2/exists? :model/Collection :id new-collection-id :archived false))))
+
+(methodical/defmethod t2/batched-hydrate [:model/Document :creator]
+  "Hydrate the creator (user) of a document based on the creator_id."
+  [_model k documents]
+  (mi/instances-with-hydrated-data
+   documents k
+   #(-> (t2/select [:model/User :id :email :first_name :last_name] :id (keep :creator_id documents))
+        (map (juxt :id identity))
+        (into {}))
+   :creator_id {:default {}}))
+
+(defn sync-document-cards-collection!
+  "Updates all cards associated with a document to match the document's collection.
+  If the document is archived, also archives all associated cards. "
+  [document-id collection-id & {:keys [archived archived-directly]}]
+  (let [update-map {:collection_id collection-id
+                    :archived (boolean archived)
+                    :archived_directly (boolean archived-directly)}]
+    (t2/update! :model/Card
+                :document_id document-id
+                update-map)))
+
+(t2/define-after-update :model/Document
+  [{:keys [id collection_id archived archived_directly]}]
+  (sync-document-cards-collection! id collection_id
+                                   :archived archived
+                                   :archived-directly archived_directly))
+
+;;; ------------------------------------------------ Serdes Hashing -------------------------------------------------
+
+(defmethod serdes/hash-fields :model/Document
+  [_table]
+  [:name (serdes/hydrated-hash :collection) :created-at])
+
+(defmethod serdes/make-spec "Document"
+  [_model-name _opts]
+  {:copy [:archived :archived_directly :collection_position :description :entity_id :name :view_count]
+   :skip [;; instance-specific stats
+          :last_viewed_at
+               ;; skip until we implement serdes for documents
+          :document_id]
+   :transform {:created_at (serdes/date)
+               :updated_at (serdes/date)
+               :collection_id (serdes/fk :model/Collection)
+               :creator_id (serdes/fk :model/User)
+               :document :skip}})
+
+;;; ----------------------------------------------- Search ----------------------------------------------------------
+
+(search.spec/define-spec "document"
+  {:model :model/Document
+   :attrs {:archived true
+           :collection-id :collection_id
+           :creator-id :creator_id
+           :view-count :view_count
+           :created-at :created_at
+           :updated-at :updated_at
+           :last-viewed-at :last_viewed_at
+           :pinned [:> [:coalesce :collection_position [:inline 0]] [:inline 0]]}
+   :search-terms [:name]
+   :render-terms {:document-name :name
+                  :document-id :id
+                  :collection-position true}})
+
+;;; ----------------------------------------------- Serdes Hashing -------------------------------------------------
+
+(defmethod serdes/hash-fields :model/Document
+  [_table]
+  [:name (serdes/hydrated-hash :collection) :created-at])
+
+;;; ---------------------------------------------- Serialization --------------------------------------------------
+
+(def ^:private ast-model->db-model
+  {"card"      :model/Card
+   "dataset"   :model/Card
+   "table"     :model/Table
+   "dashboard" :model/Dashboard})
+
+(def ^:private model->serdes-model
+  {"card"      "Card"
+   "dataset"   "Card"
+   "dashboard" "Dashboard"
+   "table"     "Table"})
+
+(defn- id->entity-id
+  [{{:keys [model] :or {model "card"} :as attrs} :attrs type :type :as node}]
+  (let [id-key (if (= prose-mirror/smart-link-type type) :entityId :id)
+        id (id-key attrs)]
+    (if-let [db-model (t2/select-one (ast-model->db-model model) :id id)]
+      (assoc-in node [:attrs id-key] (mapv #(dissoc % :label) (serdes/generate-path (model->serdes-model model) db-model)))
+      (u/prog1 node
+        (log/warnf "entity_id not found for %s at id: %s" model id)))))
+
+(defn- entity-id->id
+  [{:keys [attrs type] :as node}]
+  (let [id-key (if (= prose-mirror/smart-link-type type) :entityId :id)
+        id (:id (serdes/load-find-local (id-key attrs)))]
+    (if id
+      (assoc-in node [:attrs id-key] id)
+      (u/prog1 node
+        (log/warn "Model not found at path" (id-key attrs))))))
+
+(defn- export-document-content
+  "Transform cardEmbed/smartLink nodes to use entity IDs instead of database IDs"
+  [document serdes-key _]
+  (serdes-key
+   (if (= (:content_type document) prose-mirror/prose-mirror-content-type)
+     (prose-mirror/update-ast
+      document
+      #(contains? #{prose-mirror/smart-link-type prose-mirror/card-embed-type} (:type %))
+      id->entity-id)
+     document)))
+
+(defn- import-document-content
+  "Transform cardEmbed/smartLink nodes to use database IDs instead of entity IDs"
+  [document serdes-key _]
+  (serdes-key
+   (if (= (:content_type document) prose-mirror/prose-mirror-content-type)
+     (prose-mirror/update-ast
+      document
+      #(contains? #{prose-mirror/smart-link-type prose-mirror/card-embed-type} (:type %))
+      entity-id->id)
+     document)))
+
+(defmethod serdes/make-spec "Document"
+  [_model-name _opts]
+  {:copy [:archived :archived_directly :content_type :entity_id :name :collection_position]
+   :skip [:view_count :last_viewed_at]
+   :transform {:created_at (serdes/date)
+               :updated_at (serdes/date)
+               :document {:export-with-context export-document-content
+                          :import-with-context import-document-content}
+               :collection_id (serdes/fk :model/Collection)
+               :creator_id (serdes/fk :model/User)}})
+
+(defn- document-deps
+  [{:keys [content_type] :as document}]
+  (when (= content_type prose-mirror/prose-mirror-content-type)
+    (set (prose-mirror/collect-ast document (fn document-deps [{:keys [type attrs]}]
+                                              (cond
+                                                (and (= prose-mirror/smart-link-type type)
+                                                     (contains? model->serdes-model (:model attrs)))
+                                                (:entityId attrs)
+
+                                                (= prose-mirror/card-embed-type type)
+                                                (:id attrs)
+
+                                                :else
+                                                nil))))))
+
+(defmethod serdes/dependencies "Document"
+  [{:keys [collection_id] :as document}]
+  (set (concat
+        (document-deps document)
+        (when collection_id #{[{:model "Collection" :id collection_id}]}))))
+
+(defmethod serdes/descendants "Document"
+  [_model-name id]
+  (when-let [document (t2/select-one :model/Document :id id)]
+    (when (= prose-mirror/prose-mirror-content-type (:content_type document))
+      (merge
+       (into {}
+             (for [embedded-card-id (prose-mirror/card-ids document)]
+               {["Card" embedded-card-id] {"Document" id}}))
+       (into {}
+             (for [{model :model link-id :id} (prose-mirror/collect-ast document
+                                                                        #(when (= prose-mirror/smart-link-type (:type %))
+                                                                           (:attrs %)))
+                   :when (contains? model->serdes-model model)]
+               {[(model->serdes-model model) link-id] {"Document" id}}))))))

enterprise/backend/src/metabase_enterprise/documents/prose_mirror.clj

@@ -0,0 +1,64 @@
+(ns metabase-enterprise.documents.prose-mirror
+  "Manipulate the prose mirror ast for documents"
+  (:require
+   [clojure.walk :as walk]))
+
+(def card-embed-type
+  "Type of a card-embed node"
+  "cardEmbed")
+
+(def smart-link-type
+  "Type of a smart-link node"
+  "smartLink")
+
+(def prose-mirror-content-type
+  "The vendored 'mime-type' for documents saved using the prose-mirror ast."
+  "application/json+vnd.prose-mirror")
+
+(defn- assert-prose-mirror
+  "Asserts the content-type is correct for the document or throw"
+  [{:keys [content_type]}]
+  (when-not (= content_type prose-mirror-content-type)
+    (throw (ex-info "Document does not have the prose mirror content-type"
+                    {:content-type content_type
+                     :status-code 400}))))
+
+(defn update-ast
+  "Update a node that matches a predicate using a post-walk.
+
+  Args:
+  - document - a :model/Document, this will check that content type is valid for prose mirror
+  - predicate - a one-arg function returning true a given node should be updated
+  - updater - a one-arg function taking the node and returning the new node
+
+  Returns:
+  - the updated prose-mirror ast"
+  [{:keys [document] :as doc} predicate updater]
+  (assert-prose-mirror doc)
+  (assoc doc :document
+         (walk/postwalk (fn ast-walker
+                          [node]
+                          (cond-> node
+                            (predicate node) updater))
+                        document)))
+
+(defn collect-ast
+  "Collect values from the ast lazily removes nils
+
+  Args:
+  - document - a :model/Document, this will check that the content-type is valid for prose mirror
+  - collector - a function that extracts values from a given node
+
+  Returns:
+  - a lazy seq of results from collector"
+  [{:keys [document] :as doc} collector]
+  (assert-prose-mirror doc)
+  (->> (tree-seq :content :content document)
+       (keep collector)))
+
+(defn card-ids
+  "Get all card-ids"
+  [document]
+  (collect-ast document #(when (and (= card-embed-type (:type %))
+                                    (pos? (-> % :attrs :id)))
+                           (-> % :attrs :id))))

enterprise/backend/src/metabase_enterprise/documents/recent_views.clj

@@ -0,0 +1,70 @@
+(ns metabase-enterprise.documents.recent-views
+  "Enterprise multimethod implementations for documents in recent views."
+  (:require
+   [metabase.activity-feed.core :as activity-feed]
+   [metabase.collections.models.collection.root :as root]
+   [metabase.models.interface :as mi]
+   [metabase.premium-features.core :refer [defenterprise]]
+   [toucan2.core :as t2]))
+
+(defenterprise select-documents-for-recents
+  "Query to select recent document data. Returns documents with their collection information
+  for use in recent views display."
+  :feature :documents
+  [document-ids]
+  (if-not (seq document-ids)
+    []
+    (let [documents (t2/select :model/Document
+                               {:select [:d.id
+                                         :d.name
+                                         :d.archived
+                                         [:d.collection_id :entity-coll-id]
+                                         [:c.id :collection_id]
+                                         [:c.name :collection_name]
+                                         [:c.authority_level :collection_authority_level]]
+                                :from [[:document :d]]
+                                :where [:in :d.id document-ids]
+                                :left-join [[:collection :c]
+                                            [:and
+                                             [:= :c.id :d.collection_id]
+                                             [:= :c.archived false]]]})]
+      documents)))
+
+(defn- fill-parent-coll
+  "Fill in parent collection information for a document, following the same pattern
+  as other recent view implementations."
+  [model-object]
+  (if (:collection_id model-object)
+    {:id (:collection_id model-object)
+     :name (:collection_name model-object)
+     :authority_level (some-> (:collection_authority_level model-object) name)}
+    (select-keys
+     (root/root-collection-with-ui-details {})
+     [:id :name :authority_level])))
+
+(defn- parent-collection-valid?
+  "Returns true when a parent collection actually exists for this document.
+  Follows the same pattern as cards - returns false when the collection has a
+  collection_id but no collection_name (indicating the collection no longer exists)."
+  [{:keys [collection_id entity-coll-id]}]
+  (not (and entity-coll-id (nil? collection_id))))
+
+(defn- ellide-archived
+  "Returns the model when it's not archived.
+  We use this to ensure that archived models are not returned in the recent views."
+  [model]
+  (when (false? (:archived model)) model))
+
+(defmethod activity-feed/fill-recent-view-info :document
+  [{:keys [_model model_id timestamp model_object]}]
+  (when-let [document (and
+                       (mi/can-read? model_object)
+                       (parent-collection-valid? model_object)
+                       (ellide-archived model_object))]
+    {:id model_id
+     :name (:name document)
+     :description ""
+     :model :document
+     :can_write (mi/can-write? document)
+     :timestamp (str timestamp)
+     :parent_collection (fill-parent-coll document)}))

enterprise/backend/src/metabase_enterprise/documents/revisions/impl.clj

@@ -0,0 +1,27 @@
+(ns metabase-enterprise.documents.revisions.impl
+  "Document revision serialization implementation.
+
+  Implements Document revision serialization following the Card exclusion pattern,
+  excluding metadata columns while preserving content, name, and other Document-specific fields."
+  (:require
+   [metabase.revisions.core :as revisions]))
+
+(def ^:private excluded-columns-for-document-revision
+  "Columns to exclude from Document revision serialization.
+
+  Excludes metadata columns (timestamps, IDs, collection_id, creator_id, etc.)
+  following the same pattern established for Card revisions.
+
+  The :document field contains the actual content and is preserved."
+  #{:id
+    :creator_id
+    :created_at
+    :updated_at
+    :view_count
+    :last_viewed_at
+    :collection_id
+    :collection_position})
+
+(defmethod revisions/serialize-instance :model/Document
+  [_model _id instance]
+  (apply dissoc instance excluded-columns-for-document-revision))

enterprise/backend/src/metabase_enterprise/documents/view_log.clj

@@ -0,0 +1,65 @@
+(ns metabase-enterprise.documents.view-log
+  "Integrates view logging events for documents"
+  (:require
+   [java-time.api :as t]
+   [metabase.activity-feed.core :as activity-feed]
+   [metabase.app-db.cluster-lock :as cluster-lock]
+   [metabase.batch-processing.core :as grouper]
+   [metabase.events.core :as events]
+   [metabase.util.log :as log]
+   [metabase.view-log.core :as view-log]
+   [methodical.core :as m]
+   [steffan-westcott.clj-otel.api.trace.span :as span]
+   [toucan2.core :as t2]))
+
+(def ^:private document-statistics-lock
+  "keyword to use for locking document updates that can deadlock"
+  ::document-statistics-lock)
+
+(derive ::document-read :metabase/event)
+(derive :event/document-read ::document-read)
+
+(def ^:private update-document-last-viewed-at-interval-seconds 20)
+
+(defn- update-document-last-viewed-at!* [document-id-timestamps]
+  (let [document-id->timestamp (update-vals (group-by :id document-id-timestamps)
+                                            (fn [xs] (apply t/max (map :timestamp xs))))]
+    (try
+      (cluster-lock/with-cluster-lock document-statistics-lock
+        (t2/update! :model/Document :id [:in (keys document-id->timestamp)]
+                    {:last_viewed_at (into [:case]
+                                           (mapcat (fn [[id timestamp]]
+                                                     [[:= :id id] [:greatest [:coalesce :last_viewed_at (t/offset-date-time 0)] timestamp]])
+                                                   document-id->timestamp))
+                     :updated_at :updated_at})) ;; setting last_viewed_at should not update the updated_at column
+      (catch Exception e
+        (log/error e "Failed to update document last_viewed_at")))))
+
+(def ^:private update-document-last-viewed-at-queue
+  (delay (grouper/start!
+          #'update-document-last-viewed-at!*
+          :capacity 500
+          :interval (* update-document-last-viewed-at-interval-seconds 1000))))
+
+(defn- update-document-last-viewed-at!
+  "Update the `last_viewed_at` of a document asynchronously"
+  [document-id]
+  (let [now (t/offset-date-time)]
+    (grouper/submit! @update-document-last-viewed-at-queue {:id document-id
+                                                            :timestamp now})))
+
+(m/defmethod events/publish-event! ::document-read
+  "Handle processing for the document read event. Logs the document view and increments view count."
+  [topic {:keys [object-id user-id] :as event}]
+  (span/with-span!
+    {:name "view-log-document-read"
+     :topic topic
+     :user-id user-id}
+    (try
+      (view-log/increment-view-counts! :model/Document object-id)
+      (update-document-last-viewed-at! object-id)
+      (view-log/record-views! (view-log/generate-view :model :model/Document event))
+      ;; Update recent views alongside existing view log functionality
+      (activity-feed/update-users-recent-views! user-id :model/Document object-id :view)
+      (catch Throwable e
+        (log/warnf e "Failed to process document view event. %s" topic)))))

enterprise/backend/src/metabase_enterprise/metabot_v3/api.clj

@@ -3,6 +3,7 @@
   (:require
    [malli.core :as mc]
    [malli.transform :as mtx]
+   [metabase-enterprise.metabot-v3.api.document]
    [metabase-enterprise.metabot-v3.api.metabot]
    [metabase-enterprise.metabot-v3.client.schema :as metabot-v3.client.schema]
    [metabase-enterprise.metabot-v3.config :as metabot-v3.config]
@@ -93,4 +94,5 @@
   (handlers/routes
    (api.macros/ns-handler *ns* +auth)
    (handlers/route-map-handler
-    {"/metabot" metabase-enterprise.metabot-v3.api.metabot/routes})))
+    {"/metabot" metabase-enterprise.metabot-v3.api.metabot/routes
+     "/document" metabase-enterprise.metabot-v3.api.document/routes})))

enterprise/backend/src/metabase_enterprise/metabot_v3/api/document.clj

@@ -0,0 +1,23 @@
+(ns metabase-enterprise.metabot-v3.api.document
+  "`/api/ee/metabot-v3/document` routes"
+  (:require
+   [metabase-enterprise.metabot-v3.client :as metabot-v3.client]
+   [metabase.api.common :as api]
+   [metabase.api.macros :as api.macros]
+   [metabase.api.routes.common :refer [+auth]]
+   [metabase.util.malli.schema :as ms]))
+
+(api.macros/defendpoint :post "/generate-content"
+  "Create a new piece of content to insert into the document."
+  [_route-params
+   _query-params
+
+   body :- [:map
+            [:instructions ms/NonBlankString]]]
+  (metabot-v3.client/document-generate-content {:instructions (:instructions body)
+                                                :user_id         api/*current-user-id*
+                                                :conversation_id (str (random-uuid))}))
+
+(def ^{:arglists '([request respond raise])} routes
+  "`/api/ee/metabot-v3/document` routes."
+  (api.macros/ns-handler *ns* +auth))

enterprise/backend/src/metabase_enterprise/metabot_v3/client.clj

@@ -1,12 +1,16 @@
 (ns metabase-enterprise.metabot-v3.client
   (:require
+   [buddy.core.hash :as buddy-hash]
+   [buddy.sign.jwt :as jwt]
    [clj-http.client :as http]
+   [clj-time.core :as time]
    [clojure.java.io :as io]
    [clojure.set :as set]
    [clojure.string :as str]
    [malli.core :as mc]
    [malli.transform :as mtx]
    [metabase-enterprise.metabot-v3.client.schema :as metabot-v3.client.schema]
+   [metabase-enterprise.metabot-v3.config :as metabot-v3.config]
    [metabase-enterprise.metabot-v3.context :as metabot-v3.context]
    [metabase-enterprise.metabot-v3.settings :as metabot-v3.settings]
    [metabase-enterprise.metabot-v3.util :as metabot-v3.u]
@@ -26,6 +30,18 @@
 
 (def ^:dynamic ^:private *debug* false)
 
+(defn get-ai-service-token
+  "Get the token for the AI service."
+  ([]
+   (get-ai-service-token api/*current-user-id* metabot-v3.config/internal-metabot-id))
+
+  ([user-id metabot-id]
+   (let [secret (buddy-hash/sha256 (metabot-v3.settings/site-uuid-for-metabot-tools))
+         claims {:user       user-id
+                 :exp        (time/plus (time/now) (time/seconds (metabot-v3.settings/metabot-ai-service-token-ttl)))
+                 :metabot-id metabot-id}]
+     (jwt/encrypt claims secret {:alg :dir, :enc :a128cbc-hs256}))))
+
 (defn- ->json-bytes ^bytes [x]
   (with-open [os (java.io.ByteArrayOutputStream.)
               w  (java.io.OutputStreamWriter. os)]
@@ -91,6 +107,9 @@
 (defn- example-question-generation-endpoint []
   (str (metabot-v3.settings/ai-service-base-url) "/v1/example-question-generation/batch"))
 
+(defn- document-generate-content-endpoint []
+  (str (metabot-v3.settings/ai-service-base-url) "/v1/document/generate-content"))
+
 (mu/defn request :- ::metabot-v3.client.schema/ai-service.response
   "Make a V2 request to the AI Service."
   [{:keys [context messages profile-id conversation-id session-id state]}
@@ -280,6 +299,23 @@
                       {:request (assoc options :body body)
                        :response response})))))
 
+(mu/defn document-generate-content
+  "Ask the AI service to generate a new node for a document."
+  [body :- [:map
+            [:instructions :string]]]
+  (let [url (document-generate-content-endpoint)
+        options (build-request-options body)
+        headers (merge (:headers options) {"x-metabase-session-token"  (get-ai-service-token)
+                                           "x-metabase-url"            (system/site-url)})
+        options (assoc options :headers headers)
+        response (post! url options)]
+    (if (= (:status response) 200)
+      (:body response)
+      (throw (ex-info (format "Error in request to AI service: unexpected status code: %d %s"
+                              (:status response) (:reason-phrase response))
+                      {:request (assoc options :body body)
+                       :response response})))))
+
 (def chart-analysis-schema
   "Schema for chart analysis data input."
   [:map

enterprise/backend/src/metabase_enterprise/metabot_v3/dummy_tools.clj

@@ -1,6 +1,7 @@
 (ns metabase-enterprise.metabot-v3.dummy-tools
   (:require
    [medley.core :as m]
+   [metabase-enterprise.documents.core :as documents]
    [metabase-enterprise.metabot-v3.tools.util :as metabot-v3.tools.u]
    [metabase.api.common :as api]
    [metabase.legacy-mbql.normalize :as mbql.normalize]
@@ -14,6 +15,8 @@
    [metabase.warehouse-schema.models.field-values :as field-values]
    [toucan2.core :as t2]))
 
+(set! *warn-on-reflection* true)
+
 (defn get-current-user
   "Get information about the current user."
   []
@@ -295,6 +298,20 @@
         {:structured-output details}
         {:output (or details "report not found")}))))
 
+(defn get-document-details
+  "Get information about the document with ID `document-id`."
+  [{:keys [document-id]}]
+  (if (int? document-id)
+    (try
+      (if-let [doc (documents/get-document document-id)]
+        {:structured-output {:id (:id doc)
+                             :name (:name doc)
+                             :document (:document doc)}}
+        {:output "report not found"})
+      (catch Exception e
+        {:output (str "error fetching report: " (.getMessage e))}))
+    {:output "invalid document_id"}))
+
 (defn- execute-query
   [query-id legacy-query]
   (let [legacy-query (mbql.normalize/normalize legacy-query)

enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.clj

@@ -3,7 +3,6 @@
   (:require
    [buddy.core.hash :as buddy-hash]
    [buddy.sign.jwt :as jwt]
-   [clj-time.core :as time]
    [clojure.set :as set]
    [malli.core :as mc]
    [malli.transform :as mtx]
@@ -35,14 +34,6 @@
    [metabase.util.malli.registry :as mr]
    [metabase.util.malli.schema :as ms]))
 
-(defn- get-ai-service-token
-  [user-id metabot-id]
-  (let [secret (buddy-hash/sha256 (metabot-v3.settings/site-uuid-for-metabot-tools))
-        claims {:user user-id
-                :exp (time/plus (time/now) (time/seconds (metabot-v3.settings/metabot-ai-service-token-ttl)))
-                :metabot-id metabot-id}]
-    (jwt/encrypt claims secret {:alg :dir, :enc :a128cbc-hs256})))
-
 (defn- decode-ai-service-token
   [token]
   (try
@@ -55,7 +46,7 @@
 (defn handle-envelope
   "Executes the AI loop in the context of a new session. Returns the response of the AI service."
   [{:keys [metabot-id] :as e}]
-  (let [session-id (get-ai-service-token api/*current-user-id* metabot-id)]
+  (let [session-id (metabot-v3.client/get-ai-service-token api/*current-user-id* metabot-id)]
     (try
       (metabot-v3.client/request (assoc e :session-id session-id))
       (catch Exception ex
@@ -69,7 +60,7 @@
 (defn streaming-handle-envelope
   "Executes the AI loop in the context of a new session. Returns the response of the AI service."
   [{:keys [metabot-id] :as e}]
-  (let [session-id (get-ai-service-token api/*current-user-id* metabot-id)]
+  (let [session-id (metabot-v3.client/get-ai-service-token api/*current-user-id* metabot-id)]
     (metabot-v3.client/streaming-request (assoc e :session-id session-id))))
 
 (mr/def ::bucket
@@ -540,6 +531,24 @@
                          [:result_columns ::columns]]]]
    [:map [:output :string]]])
 
+(mr/def ::get-document-details-arguments
+  [:and
+   [:map
+    [:document_id                                         :int]]
+   [:map {:encode/tool-api-request
+          #(set/rename-keys % {:document_id         :document-id})}]])
+
+(mr/def ::get-document-details-result
+  [:or
+   [:map
+    {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
+    [:structured_output [:map
+                         {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
+                         [:id :int]
+                         [:name :string]
+                         [:document :string]]]]
+   [:map [:output :string]]])
+
 (mr/def ::get-table-details-arguments
   [:and
    [:map
@@ -777,6 +786,21 @@
               (assoc :conversation_id conversation_id))
       (metabot-v3.context/log :llm.log/be->llm))))
 
+(api.macros/defendpoint :post "/get-document-details" :- [:merge ::get-document-details-result ::tool-request]
+  "Get information about a given report."
+  [_route-params
+   _query-params
+   {:keys [arguments conversation_id] :as body} :- [:merge
+                                                    [:map [:arguments ::get-document-details-arguments]]
+                                                    ::tool-request]]
+  (metabot-v3.context/log (assoc body :api :get-document-details) :llm.log/llm->be)
+  (let [arguments (mc/encode ::get-document-details-arguments arguments (mtx/transformer {:name :tool-api-request}))]
+    (doto (-> (mc/decode ::get-document-details-result
+                         (metabot-v3.dummy-tools/get-document-details arguments)
+                         (mtx/transformer {:name :tool-api-response}))
+              (assoc :conversation_id conversation_id))
+      (metabot-v3.context/log :llm.log/be->llm))))
+
 (api.macros/defendpoint :post "/get-table-details" :- [:merge ::get-table-details-result ::tool-request]
   "Get information about a given table or model."
   [_route-params

enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj

@@ -25,7 +25,8 @@
   keys to remove from the model so that we'll be able to successfully load it. You can remove keys in vectors using :* to
   indicate that all items in that vector should have a key removed."
   {"Dashboard" #{:dashcards}
-   "Card"      #{:dashboard_id}})
+   "Document"  #{:document}
+   "Card"      #{:dashboard_id :document_id}})
 
 (defn- keys-to-strip [ingested]
   (let [model (-> ingested :serdes/meta last :model)]

enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj

@@ -16,6 +16,7 @@
    "Card"
    "Collection"
    "Dashboard"
+   "Document"
    "NativeQuerySnippet"
    "Timeline"])
 
@@ -57,6 +58,7 @@
    "DatabaseRouter"
    "DashboardBookmark"
    "DataPermissions"
+   "DocumentBookmark"
    "GroupTableAccessPolicy"
    "HTTPAction"
    "ImplicitAction"

enterprise/backend/src/metabase_enterprise/transforms/api.clj

@@ -0,0 +1,240 @@
+(ns metabase-enterprise.transforms.api
+  (:require
+   [clojure.set :as set]
+   [metabase-enterprise.transforms.api.transform-job]
+   [metabase-enterprise.transforms.api.transform-tag]
+   [metabase-enterprise.transforms.execute :as transforms.execute]
+   [metabase-enterprise.transforms.models.transform :as transform.model]
+   [metabase-enterprise.transforms.ordering :as transforms.ordering]
+   [metabase-enterprise.transforms.util :as transforms.util]
+   [metabase-enterprise.worker.core :as worker]
+   [metabase.api.common :as api]
+   [metabase.api.macros :as api.macros]
+   [metabase.api.routes.common :refer [+auth]]
+   [metabase.api.util.handlers :as handlers]
+   [metabase.driver.util :as driver.u]
+   [metabase.request.core :as request]
+   [metabase.util.i18n :refer [deferred-tru]]
+   [metabase.util.jvm :as u.jvm]
+   [metabase.util.log :as log]
+   [metabase.util.malli.registry :as mr]
+   [metabase.util.malli.schema :as ms]
+   [ring.util.response :as response]
+   [toucan2.core :as t2]))
+
+(comment metabase-enterprise.transforms.api.transform-job/keep-me
+         metabase-enterprise.transforms.api.transform-tag/keep-me)
+
+(set! *warn-on-reflection* true)
+
+(mr/def ::transform-source
+  [:map
+   [:type [:= "query"]]
+   [:query [:map [:database :int]]]])
+
+(mr/def ::transform-target
+  [:map
+   [:type [:enum "table" "view"]]
+   [:schema {:optional true} :string]
+   [:name :string]])
+
+(mr/def ::run-trigger
+  [:enum "none" "global-schedule"])
+
+(comment
+  ;; Examples
+  [{:id 1
+    :name "Gadget Products"
+    :source {:type "query"
+             :query {:database 1
+                     :type "native",
+                     :native {:query "SELECT * FROM PRODUCTS WHERE CATEGORY = 'Gadget'"
+                              :template-tags {}}}}
+    :target {:type "table"
+             :schema "transforms"
+             :name "gadget_products"}}])
+
+(defn- source-database-id
+  [transform]
+  (-> transform :source :query :database))
+
+(defn- check-database-feature
+  [transform]
+  (let [database (api/check-400 (t2/select-one :model/Database (source-database-id transform))
+                                (deferred-tru "The source database cannot be found."))
+        feature (transforms.util/required-database-feature transform)]
+    (api/check-400 (not (:is_sample database))
+                   (deferred-tru "Cannot run transforms on the sample database."))
+    (api/check-400 (driver.u/supports? (:engine database) feature database)
+                   (deferred-tru "The database does not support the requested transform target type."))))
+
+(api.macros/defendpoint :get "/"
+  "Get a list of all transforms available to the current user.
+  
+  Returns a collection of transform objects, each hydrated with their last run status
+  and associated tag IDs. Requires superuser permissions."
+  [_route-params
+   _query-params]
+  (api/check-superuser)
+  (t2/hydrate (t2/select :model/Transform) :last_run :transform_tag_ids))
+
+(api.macros/defendpoint :post "/"
+  "Create a new transform with the specified configuration.
+  
+  Creates a transform that defines how to extract data from a source query and materialize it
+  into a target table or view. Validates that the database supports the requested feature and
+  that the target table doesn't already exist. Returns the created transform with associated
+  tag IDs. Requires superuser permissions."
+  [_route-params
+   _query-params
+   body :- [:map
+            [:name :string]
+            [:description {:optional true} [:maybe :string]]
+            [:source ::transform-source]
+            [:target ::transform-target]
+            [:run_trigger {:optional true} ::run-trigger]
+            [:tag_ids {:optional true} [:sequential ms/PositiveInt]]]]
+  (api/check-superuser)
+  (check-database-feature body)
+  (when (transforms.util/target-table-exists? body)
+    (api/throw-403))
+  (let [tag-ids (:tag_ids body)
+        transform (t2/insert-returning-instance!
+                   :model/Transform (select-keys body [:name :description :source :target :run_trigger]))]
+    ;; Add tag associations if provided
+    (when (seq tag-ids)
+      (transform.model/update-transform-tags! (:id transform) tag-ids))
+    ;; Return with hydrated tag_ids
+    (t2/hydrate transform :transform_tag_ids)))
+
+(api.macros/defendpoint :get "/:id"
+  "Get details of a specific transform by ID."
+  [{:keys [id]} :- [:map
+                    [:id ms/PositiveInt]]]
+  (log/info "get transform" id)
+  (api/check-superuser)
+  (let [{:keys [target] :as transform} (api/check-404 (t2/select-one :model/Transform id))
+        database-id (source-database-id transform)
+        target-table (transforms.util/target-table database-id target :active true)]
+    (-> transform
+        (t2/hydrate :last_run :transform_tag_ids)
+        (assoc :table target-table))))
+
+(api.macros/defendpoint :get "/run"
+  "Get a paginated list of transform run history."
+  [_route-params
+   {:keys [sort_column sort_direction transform_ids statuses transform_tag_ids]} :-
+   [:map
+    [:sort_column {:optional true} [:enum "started_at" "ended_at"]]
+    [:sort_direction {:optional true} [:enum "asc" "desc"]]
+    [:transform_ids {:optional true} [:maybe (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]]
+    [:statuses {:optional true} [:maybe (ms/QueryVectorOf [:enum "started" "succeeded" "failed" "timeout"])]]
+    [:transform_tag_ids {:optional true} [:maybe (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]]]]
+  (log/info "get runs")
+  (api/check-superuser)
+  (letfn [(work-run->api-run [run]
+            (set/rename-keys run
+                             {:run_id :id
+                              :work_id :transform_id}))]
+    (update (worker/paged-runs {:work_type "transform"
+                                :work_ids transform_ids
+                                :work_tag_ids transform_tag_ids
+                                :statuses statuses
+                                :sort_column sort_column
+                                :sort_direction sort_direction
+                                :offset (request/offset)
+                                :limit (request/limit)})
+            :data #(mapv work-run->api-run %))))
+
+(api.macros/defendpoint :put "/:id"
+  "Update an existing transform configuration."
+  [{:keys [id]} :- [:map
+                    [:id ms/PositiveInt]]
+   _query-params
+   body :- [:map
+            [:name {:optional true} :string]
+            [:description {:optional true} [:maybe :string]]
+            [:source {:optional true} ::transform-source]
+            [:target {:optional true} ::transform-target]
+            [:run_trigger {:optional true} ::run-trigger]
+            [:tag_ids {:optional true} [:sequential ms/PositiveInt]]]]
+  (log/info "put transform" id)
+  (api/check-superuser)
+  (let [old (t2/select-one :model/Transform id)
+        new (merge old body)
+        target-fields #(-> % :target (select-keys [:schema :name]))]
+    (check-database-feature new)
+    (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]
+      (throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)
+                      {:status-code 400})))
+    (when (and (not= (target-fields old) (target-fields new))
+               (transforms.util/target-table-exists? new))
+      (api/throw-403)))
+  (t2/update! :model/Transform id (dissoc body :tag_ids))
+  ;; Update tag associations if provided
+  (when (contains? body :tag_ids)
+    (transform.model/update-transform-tags! id (:tag_ids body)))
+  (t2/hydrate (t2/select-one :model/Transform id) :transform_tag_ids))
+
+(api.macros/defendpoint :delete "/:id"
+  "Delete a transform by ID."
+  [{:keys [id]} :- [:map
+                    [:id ms/PositiveInt]]]
+  (log/info "delete transform" id)
+  (api/check-superuser)
+  (t2/delete! :model/Transform id)
+  nil)
+
+(api.macros/defendpoint :delete "/:id/table"
+  "Delete the target table for a transform."
+  [{:keys [id]} :- [:map
+                    [:id ms/PositiveInt]]]
+  (log/info "delete transform target table" id)
+  (api/check-superuser)
+  (transforms.util/delete-target-table-by-id! id)
+  nil)
+
+(api.macros/defendpoint :post "/:id/cancel"
+  "Cancel a running transform job."
+  [{:keys [id]} :- [:map
+                    [:id :string]]]
+  (log/info "canceling transform " id)
+  (api/check-superuser)
+  (let [run (api/check-404 (worker/running-run-for-work-id id "transform"))]
+    (if (:is_local run)
+      (worker/mark-cancel-started-run! (:run_id run))
+      (worker/cancel! (:run_id run))))
+  nil)
+
+(api.macros/defendpoint :post "/:id/run"
+  "Manually trigger a transform to run."
+  [{:keys [id]} :- [:map
+                    [:id ms/PositiveInt]]]
+  (log/info "run transform" id)
+  (api/check-superuser)
+  (let [transform (api/check-404 (t2/select-one :model/Transform id))
+        start-promise (promise)]
+    (u.jvm/in-virtual-thread*
+     (transforms.execute/run-mbql-transform! transform {:start-promise start-promise
+                                                        :run-method :manual}))
+    (when (instance? Throwable @start-promise)
+      (throw @start-promise))
+    (let [result @start-promise
+          run-id (when (and (vector? result) (= (first result) :started))
+                   (second result))]
+      (-> (response/response {:message (deferred-tru "Transform run started")
+                              :run_id run-id})
+          (assoc :status 202)))))
+
+(def ^{:arglists '([request respond raise])} routes
+  "`/api/ee/transform` routes."
+  (handlers/routes
+   (api.macros/ns-handler *ns* +auth)))
+
+(def ^{:arglists '([request respond raise])} transform-tag-routes
+  "`/api/ee/transform-tag` routes."
+  (api.macros/ns-handler 'metabase-enterprise.transforms.api.transform-tag +auth))
+
+(def ^{:arglists '([request respond raise])} transform-job-routes
+  "`/api/ee/transform-job` routes."
+  (api.macros/ns-handler 'metabase-enterprise.transforms.api.transform-job +auth))

enterprise/backend/src/metabase_enterprise/transforms/api/transform_job.clj

@@ -0,0 +1,161 @@
+(ns metabase-enterprise.transforms.api.transform-job
+  (:require
+   [metabase-enterprise.transforms.jobs :as transforms.jobs]
+   [metabase-enterprise.transforms.models.job-run :as transforms.job-run]
+   [metabase-enterprise.transforms.models.transform-job :as transform-job]
+   [metabase-enterprise.transforms.schedule :as transforms.schedule]
+   [metabase.api.common :as api]
+   [metabase.api.macros :as api.macros]
+   [metabase.api.routes.common :refer [+auth]]
+   [metabase.util.i18n :refer [deferred-tru]]
+   [metabase.util.jvm :as u.jvm]
+   [metabase.util.log :as log]
+   [metabase.util.malli.schema :as ms]
+   [toucan2.core :as t2])
+  (:import
+   (org.quartz CronExpression)))
+
+(set! *warn-on-reflection* true)
+
+(comment transform-job/keep-me)
+
+(api.macros/defendpoint :post "/"
+  "Create a new transform job."
+  [_route-params
+   _query-params
+   {:keys [name description schedule tag_ids]} :- [:map
+                                                   [:name ms/NonBlankString]
+                                                   [:description {:optional true} [:maybe ms/NonBlankString]]
+                                                   [:schedule ms/NonBlankString]
+                                                   [:tag_ids {:optional true} [:sequential ms/PositiveInt]]]]
+  (log/info "Creating transform job:" name "with schedule:" schedule)
+  (api/check-superuser)
+  ;; Validate cron expression
+  (api/check-400 (try
+                   (CronExpression/validateExpression schedule)
+                   true
+                   (catch Exception e
+                     false))
+                 (deferred-tru "Invalid cron expression: {0}" schedule))
+  ;; Validate tag IDs exist if provided
+  (when (seq tag_ids)
+    (let [existing-tags (set (t2/select-pks-vec :model/TransformTag :id [:in tag_ids]))]
+      (api/check-400 (= (set tag_ids) existing-tags)
+                     (deferred-tru "Some tag IDs do not exist"))))
+  (let [job (t2/insert-returning-instance! :model/TransformJob
+                                           {:name name
+                                            :description description
+                                            :schedule schedule})]
+    (transforms.schedule/initialize-job! job)
+    ;; Add tag associations if provided
+    (when (seq tag_ids)
+      (t2/insert! :transform_job_tags
+                  (for [tag-id tag_ids]
+                    {:job_id (:id job)
+                     :tag_id tag-id})))
+    ;; Return with hydrated tag_ids
+    (t2/hydrate job :tag_ids)))
+
+(api.macros/defendpoint :put "/:job-id"
+  "Update a transform job."
+  [{:keys [job-id]} :- [:map
+                        [:job-id ms/PositiveInt]]
+   _query-params
+   {:keys [name description schedule tag_ids]} :- [:map
+                                                   [:name {:optional true} ms/NonBlankString]
+                                                   [:description {:optional true} [:maybe ms/NonBlankString]]
+                                                   [:schedule {:optional true} ms/NonBlankString]
+                                                   [:tag_ids {:optional true} [:sequential ms/PositiveInt]]]]
+  (log/info "Updating transform job" job-id)
+  (api/check-superuser)
+  ;; Check job exists
+  (api/check-404 (t2/select-one :model/TransformJob :id job-id))
+  ;; Validate cron expression if provided
+  (when schedule
+    (api/check-400 (try
+                     (CronExpression/validateExpression schedule)
+                     true
+                     (catch Exception e
+                       false))
+                   (deferred-tru "Invalid cron expression: {0}" schedule)))
+  ;; Validate tag IDs if provided
+  (when tag_ids
+    (let [existing-tags (set (t2/select-pks-vec :model/TransformTag :id [:in tag_ids]))]
+      (api/check-400 (= (set tag_ids) existing-tags)
+                     (deferred-tru "Some tag IDs do not exist"))))
+  ;; Update the job
+  (let [updates (cond-> {}
+                  name (assoc :name name)
+                  (some? description) (assoc :description description)
+                  schedule (assoc :schedule schedule))]
+    (when (seq updates)
+      (t2/update! :model/TransformJob job-id updates)))
+  (when schedule
+    (transforms.schedule/update-job! job-id schedule))
+  ;; Update tag associations if provided
+  (when (some? tag_ids)
+    ;; Delete existing associations
+    (t2/delete! :transform_job_tags :job_id job-id)
+    ;; Add new associations
+    (when (seq tag_ids)
+      (t2/insert! :transform_job_tags
+                  (for [tag-id tag_ids]
+                    {:job_id job-id
+                     :tag_id tag-id}))))
+  ;; Return updated job with hydration
+  (-> (t2/select-one :model/TransformJob :id job-id)
+      (t2/hydrate :tag_ids)
+      (assoc :last_run nil)))
+
+(api.macros/defendpoint :delete "/:job-id"
+  "Delete a transform job."
+  [{:keys [job-id]} :- [:map [:job-id ms/PositiveInt]]]
+  (log/info "Deleting transform job" job-id)
+  (api/check-superuser)
+  (api/check-404 (t2/select-one :model/TransformJob :id job-id))
+  (t2/delete! :model/TransformJob :id job-id)
+  (transforms.schedule/delete-job! job-id)
+  api/generic-204-no-content)
+
+(api.macros/defendpoint :post "/:job-id/run"
+  "Run a transform job manually."
+  [{:keys [job-id]} :- [:map [:job-id ms/PositiveInt]]]
+  (log/info "Manual run of transform job" job-id)
+  (api/check-superuser)
+  (let [job (api/check-404 (t2/select-one :model/TransformJob :id job-id))]
+    (u.jvm/in-virtual-thread*
+     (try
+       (transforms.jobs/run-job! job-id {:run-method :manual})
+       (catch Throwable t
+         (log/error "Error executing transform job" job-id)
+         (log/error t)))))
+  {:message    "Job run started"
+   :job_run_id (str "stub-" job-id "-" (System/currentTimeMillis))})
+
+(api.macros/defendpoint :get "/:job-id"
+  "Get a transform job by ID."
+  [{:keys [job-id]} :- [:map
+                        [:job-id ms/PositiveInt]]]
+  (log/info "Getting transform job" job-id)
+  (api/check-superuser)
+  (let [job (api/check-404 (t2/select-one :model/TransformJob :id job-id))]
+    ;; Hydrate tag_ids and add last_run (null for now)
+    (-> job
+        (t2/hydrate :tag_ids)
+        transforms.job-run/add-last-run)))
+
+(api.macros/defendpoint :get "/"
+  "Get all transform jobs."
+  [_route-params
+   _query-params]
+  (log/info "Getting all transform jobs")
+  (api/check-superuser)
+  (let [jobs (t2/select :model/TransformJob {:order-by [[:created_at :desc]]})]
+    ;; Hydrate tag_ids for all jobs
+    ;; Note: last_run will be null until run tracking is implemented
+    (map #(assoc % :last_run nil)
+         (t2/hydrate jobs :tag_ids))))
+
+(def ^{:arglists '([request respond raise])} routes
+  "`/api/ee/transform-job` routes."
+  (api.macros/ns-handler *ns* +auth))

enterprise/backend/src/metabase_enterprise/transforms/api/transform_tag.clj

@@ -0,0 +1,62 @@
+(ns metabase-enterprise.transforms.api.transform-tag
+  (:require
+   [metabase-enterprise.transforms.models.transform-tag :as transform-tag]
+   [metabase.api.common :as api]
+   [metabase.api.macros :as api.macros]
+   [metabase.api.routes.common :refer [+auth]]
+   [metabase.api.util.handlers :as handlers]
+   [metabase.util.i18n :refer [deferred-tru]]
+   [metabase.util.log :as log]
+   [metabase.util.malli.schema :as ms]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(api.macros/defendpoint :post "/"
+  "Create a new transform tag."
+  [_route-params
+   _query-params
+   {:keys [name]} :- [:map
+                      [:name ms/NonBlankString]]]
+  (log/info "Creating transform tag:" name)
+  (api/check-superuser)
+  (api/check-400 (not (transform-tag/tag-name-exists? name))
+                 (deferred-tru "A tag with the name ''{0}'' already exists." name))
+  (t2/insert-returning-instance! :model/TransformTag {:name name}))
+
+(api.macros/defendpoint :put "/:tag-id"
+  "Update a transform tag."
+  [{:keys [tag-id]} :- [:map
+                        [:tag-id ms/PositiveInt]]
+   _query-params
+   {:keys [name]} :- [:map
+                      [:name ms/NonBlankString]]]
+  (log/info "Updating transform tag" tag-id "with name:" name)
+  (api/check-superuser)
+  (api/check-404 (t2/select-one :model/TransformTag :id tag-id))
+  (api/check-400 (not (transform-tag/tag-name-exists-excluding? name tag-id))
+                 (deferred-tru "A tag with the name ''{0}'' already exists." name))
+  (t2/update! :model/TransformTag tag-id {:name name})
+  (t2/select-one :model/TransformTag :id tag-id))
+
+(api.macros/defendpoint :delete "/:tag-id"
+  "Delete a transform tag. Removes it from all transforms and jobs."
+  [{:keys [tag-id]} :- [:map
+                        [:tag-id ms/PositiveInt]]]
+  (log/info "Deleting transform tag" tag-id)
+  (api/check-superuser)
+  (api/check-404 (t2/select-one :model/TransformTag :id tag-id))
+  (t2/delete! :model/TransformTag :id tag-id)
+  api/generic-204-no-content)
+
+(api.macros/defendpoint :get "/"
+  "Get a list of all transform tags."
+  [_route-params
+   _query-params]
+  (log/info "Getting all transform tags")
+  (api/check-superuser)
+  (t2/select :model/TransformTag {:order-by [[:name :asc]]}))
+
+(def ^{:arglists '([request respond raise])} routes
+  "`/api/ee/transform-tag` routes."
+  (api.macros/ns-handler *ns* +auth))

enterprise/backend/src/metabase_enterprise/transforms/core.clj

@@ -0,0 +1,2 @@
+(ns metabase-enterprise.transforms.core
+  "API namespace for the `metabase-enterprise.transform` module.")

enterprise/backend/src/metabase_enterprise/transforms/execute.clj

@@ -0,0 +1,123 @@
+(ns metabase-enterprise.transforms.execute
+  (:require
+   [clojure.core.async :as a]
+   [clojure.string :as str]
+   [medley.core :as m]
+   [metabase-enterprise.transforms.ordering :as transforms.ordering]
+   [metabase-enterprise.transforms.util :as transforms.util]
+   [metabase-enterprise.worker.core :as worker]
+   [metabase.driver :as driver]
+   [metabase.driver.util :as driver.u]
+   [metabase.lib.schema.common :as schema.common]
+   [metabase.query-processor.pipeline :as qp.pipeline]
+   [metabase.sync.core :as sync]
+   [metabase.util :as u]
+   [metabase.util.jvm :as u.jvm]
+   [metabase.util.log :as log]
+   [metabase.util.malli.registry :as mr]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(mr/def ::transform-details
+  [:map
+   [:transform-type [:enum {:decode/normalize schema.common/normalize-keyword} :view :table]]
+   [:connection-details :any]
+   [:query :string]
+   [:output-table [:keyword {:decode/normalize schema.common/normalize-keyword}]]])
+
+(mr/def ::transform-opts
+  [:map
+   [:overwrite? :boolean]])
+
+;; should be run in virtual thread (please :)
+(defn- run-transform-remote! [run-id driver transform-details opts]
+  (try
+    (log/trace "starting remote transform run" (pr-str run-id))
+    (worker/run-transform! run-id driver transform-details opts)
+    (catch Throwable t
+      (log/error t "Remote execution request failed; still syncing")))
+  ;; poll the server until it's not running
+  (u.jvm/poll {:timeout-ms (* 4 60 1000 1000)
+               :interval-ms (+ 2000 (- 500 (* 1000 (rand))))
+               :done? #(not= "running" (:status %))
+               :thunk #(worker/sync-single-run! run-id)}))
+
+(defn- sync-target!
+  ([transform-id run-id]
+   (let [{:keys [source target]} (t2/select-one :model/Transform transform-id)
+         db (get-in source [:query :database])
+         database (t2/select-one :model/Database db)]
+     (sync-target! target database run-id)))
+  ([target database _run-id]
+   ;; sync the new table (note that even a failed sync status means that the execution succeeded)
+   (log/info "Syncing target" (pr-str target) "for transform")
+   (transforms.util/activate-table! database target)))
+
+(defn- run-transform-local!
+  [run-id driver transform-details opts]
+  (worker/chan-start-timeout-vthread! run-id)
+  ;; local run is responsible for status
+  (try
+    (binding [qp.pipeline/*canceled-chan* (a/promise-chan)]
+      (worker/chan-start-run! run-id qp.pipeline/*canceled-chan*)
+      (driver/run-transform! driver transform-details opts))
+    (worker/succeed-started-run! run-id)
+    (catch Throwable t
+      (worker/fail-started-run! run-id {:message (.getMessage t)})
+      (throw t))
+    (finally
+      (worker/chan-end-run! run-id))))
+
+(defn run-transform!
+  "Run a compiled transform either locally or remotely."
+  [run-id driver transform-details opts]
+  (if (worker/run-remote?)
+    (run-transform-remote! run-id driver transform-details opts)
+    (run-transform-local!  run-id driver transform-details opts)))
+
+(defn run-mbql-transform!
+  "Run `transform` and sync its target table.
+
+  This is executing synchronously, but supports being kicked off in the background
+  by delivering the `start-promise` just before the start when the beginning of the execution has been booked
+  in the database."
+  ([transform] (run-mbql-transform! transform nil))
+  ([{:keys [id source target] :as transform} {:keys [run-method start-promise]}]
+   (try
+     (let [db (get-in source [:query :database])
+           {driver :engine :as database} (t2/select-one :model/Database db)
+           feature (transforms.util/required-database-feature transform)
+           run-id (str (u/generate-nano-id))
+           transform-details {:transform-type (keyword (:type target))
+                              :connection-details (driver/connection-details driver database)
+                              :query (transforms.util/compile-source source)
+                              :output-table (transforms.util/qualified-table-name driver target)}
+           opts {:overwrite? true}]
+       (when-not (driver.u/supports? driver feature database)
+         (throw (ex-info "The database does not support the requested transform target type."
+                         {:driver driver, :database database, :feature feature})))
+       ;; mark the execution as started and notify any observers
+       (try
+         (worker/start-run! run-id :transform id
+                            {:run_method run-method
+                             :is_local (not (worker/run-remote?))})
+         (catch java.sql.SQLException e
+           (if (= (.getSQLState e) "23505")
+             (throw (ex-info "Transform is already running"
+                             {:error :already-running
+                              :transform-id id}
+                             e))
+             (throw e))))
+       (when start-promise
+         (deliver start-promise [:started run-id]))
+       (log/info "Executing transform" id "with target" (pr-str target))
+       (run-transform! run-id driver transform-details opts)
+       (sync-target! target database run-id))
+     (catch Throwable t
+       (log/error t "Error executing transform")
+       (when start-promise
+         ;; if the start-promise has been delivered, this is a no-op,
+         ;; but we assume nobody would catch the exception anyway
+         (deliver start-promise t))
+       (throw t)))))

enterprise/backend/src/metabase_enterprise/transforms/init.clj

@@ -0,0 +1,6 @@
+(ns metabase-enterprise.transforms.init
+  (:require
+   [metabase-enterprise.transforms.execute]
+   [metabase-enterprise.transforms.models.job-run]
+   [metabase-enterprise.transforms.models.transform]
+   [metabase-enterprise.transforms.schedule]))

enterprise/backend/src/metabase_enterprise/transforms/jobs.clj

@@ -0,0 +1,101 @@
+(ns metabase-enterprise.transforms.jobs
+  (:require
+   [clojure.string :as str]
+   [clojurewerkz.quartzite.jobs :as jobs]
+   [clojurewerkz.quartzite.schedule.calendar-interval :as calendar-interval]
+   [clojurewerkz.quartzite.triggers :as triggers]
+   [metabase-enterprise.transforms.execute :as transforms.execute]
+   [metabase-enterprise.transforms.models.job-run :as transforms.job-run]
+   [metabase-enterprise.transforms.ordering :as transforms.ordering]
+   [metabase.task.core :as task]
+   [metabase.util :as u]
+   [metabase.util.log :as log]
+   [toucan2.core :as t2]))
+
+(defn- get-deps [ordering transform-ids]
+  (loop [found #{}
+         [current-transform & more-transforms] transform-ids]
+    (if current-transform
+      (recur (conj found current-transform)
+             (if (found current-transform)
+               more-transforms
+               (into more-transforms (get ordering current-transform))))
+      found)))
+
+(defn- get-plan [transform-ids]
+  (let [all-transforms (t2/select :model/Transform)
+        global-ordering (transforms.ordering/transform-ordering all-transforms)
+        relevant-ids (get-deps global-ordering transform-ids)
+        ordering (select-keys global-ordering relevant-ids)]
+    (when-let [cycle (transforms.ordering/find-cycle ordering)]
+      (let [id->name (into {} (map (juxt :id :name)) all-transforms)]
+        (throw (ex-info (str "Cyclic transform definitions detected: "
+                             (str/join " → " (map id->name cycle)))
+                        {:cycle cycle}))))
+    {:transforms-by-id (into {}
+                             (keep (fn [{:keys [id] :as transform}]
+                                     (when (relevant-ids id)
+                                       [id transform])))
+                             all-transforms)
+     :ordering ordering}))
+
+(defn- next-transform [{:keys [ordering transforms-by-id]} complete]
+  (-> (transforms.ordering/available-transforms ordering #{} complete)
+      first
+      transforms-by-id))
+
+(defn run-transforms! [run-id transform-ids-to-run {:keys [run-method start-promise]}]
+  (let [plan (get-plan transform-ids-to-run)]
+    (when start-promise
+      (deliver start-promise :started))
+    (loop [complete #{}]
+      (when-let [current-transform (next-transform plan complete)]
+        (log/info "Executing job transform" (pr-str (:id current-transform)))
+        (transforms.execute/run-mbql-transform! current-transform {:run-method run-method})
+        (transforms.job-run/add-run-activity! run-id)
+        (recur (conj complete (:id current-transform)))))))
+
+(defn run-job!
+  [job-id {:keys [run-method] :as opts}]
+  (let [transforms (t2/select-fn-set :transform_id
+                                     :transform_job_tags
+                                     {:select :transform_tags.transform_id
+                                      :from :transform_job_tags
+                                      :left-join [:transform_tags [:=
+                                                                   :transform_tags.tag_id
+                                                                   :transform_job_tags.tag_id]]
+                                      :where [:= :transform_job_tags.job_id job-id]})
+        run-id (str (u/generate-nano-id))]
+    (log/info "Executing transform job" (pr-str job-id) "with transforms" (pr-str transforms))
+    (transforms.job-run/start-run! run-id job-id run-method)
+    (try
+      (run-transforms! run-id transforms opts)
+      (transforms.job-run/succeed-started-run! run-id)
+      (catch Throwable t
+        (transforms.job-run/fail-started-run! run-id {:message (.getMessage t)})
+        (throw t)))))
+
+(def ^:private job-key "metabase-enterprise.transforms.jobs.timeout-job")
+
+(task/defjob  ^{:doc "Times out transform jobs when necesssary."
+                org.quartz.DisallowConcurrentExecution true}
+  SyncWorkerStatus [ctx]
+  (transforms.job-run/timeout-old-runs! 4 :hour))
+
+(defn- start-job! []
+  (when (not (task/job-exists? job-key))
+    (let [job (jobs/build
+               (jobs/of-type SyncWorkerStatus)
+               (jobs/with-identity (jobs/key job-key)))
+          trigger (triggers/build
+                   (triggers/with-identity (triggers/key job-key))
+                   (triggers/start-now)
+                   (triggers/with-schedule
+                    (calendar-interval/schedule
+                     (calendar-interval/with-interval-in-minutes 10)
+                     (calendar-interval/with-misfire-handling-instruction-do-nothing))))]
+      (task/schedule-task! job trigger))))
+
+(defmethod task/init! ::TimeoutJob [_]
+  (log/info "Scheduling transform job  timeout.")
+  (start-job!))

enterprise/backend/src/metabase_enterprise/transforms/models/job_run.clj

@@ -0,0 +1,140 @@
+(ns metabase-enterprise.transforms.models.job-run
+  (:require
+   [metabase.app-db.core :as mdb]
+   [metabase.driver.sql.query-processor :as sql.qp]
+   [metabase.models.interface :as mi]
+   [metabase.util :as u]
+   [methodical.core :as methodical]
+   [toucan2.core :as t2]
+   [toucan2.realize :as t2.realize]))
+
+(methodical/defmethod t2/table-name :model/TransformJobRun [_model] :transform_job_run)
+(methodical/defmethod t2/primary-keys :model/TransformJobRun [_model] [:run_id])
+
+(derive :model/TransformJobRun :metabase/model)
+(derive :model/TransformJobRun :hook/timestamped?)
+
+(t2/deftransforms :model/TransformJobRun
+  {:status mi/transform-keyword
+   :run_method mi/transform-keyword})
+
+(defn- latest-runs-query [job-ids]
+  {:with [[:ranked_runs
+           {:select [:*
+                     [[:over [[:row_number] {:partition-by :job_id, :order-by [[:start_time :desc]]}]] :rn]]
+            :from [:transform_job_run]
+            :where [:in :job_id job-ids]}]]
+   :select [:*]
+   :from [:ranked_runs]
+   :where [:= :rn [:inline 1]]})
+
+(defn latest-runs
+  "Return the latest runs for `job-ids`."
+  [job-ids]
+  (when (seq job-ids)
+    (into [] (map (comp t2.realize/realize #(dissoc % :rn)))
+          (t2/reducible-select :model/TransformJobRun (latest-runs-query job-ids)))))
+
+(defn add-last-run [job]
+  (assoc job
+         :last_run
+         (t2/select-one :model/TransformJobRun
+                        :job_id (:id job)
+                        {:order-by [[:start_time :desc]]})))
+
+(defn start-run!
+  "Start a run"
+  ([run-id job-id run-method]
+   (t2/insert! :model/TransformJobRun
+               {:run_id run-id
+                :job_id job-id
+                :run_method run-method
+                :status :started
+                :is_active true})))
+
+(defn add-run-activity!
+  "Notes that a run has had activity"
+  [run-id]
+  (t2/update! :model/TransformJobRun
+              :run_id run-id
+              :is_active true
+              {:updated_at :%now}))
+
+(defn succeed-started-run!
+  "Mark a started run as successfully completed."
+  ([run-id]
+   (succeed-started-run! run-id {}))
+  ([run-id properties]
+   (t2/update! :model/TransformJobRun
+               :run_id    run-id
+               :is_active true
+               (merge {:end_time :%now}
+                      properties
+                      {:status    :succeeded
+                       :is_active nil}))))
+
+(defn fail-started-run!
+  "Mark the started active run as failed and inactive."
+  [run-id properties]
+  (t2/update! :model/TransformJobRun
+              :run_id    run-id
+              :is_active true
+              (merge {:end_time :%now}
+                     properties
+                     {:status :failed
+                      :is_active nil})))
+
+(defn timeout-old-runs!
+  "Time out all active runs older than the specified age."
+  [age unit]
+  (t2/update! :model/TransformJobRun
+              :is_active true
+              :updated_at [:< (sql.qp/add-interval-honeysql-form (mdb/db-type) :%now (- age) unit)]
+              {:status :timeout
+               :end_time :%now
+               :is_active nil
+               :message "Timed out by metabase"}))
+
+(defn running-run-for-job-id
+  "Return a single active job run or nil."
+  [id]
+  (t2/select-one :model/TransformJobRun
+                 :job_id id
+                 :is_active true))
+
+(defn paged-runs
+  "Return a page of the list of the runs.
+
+  Follows the conventions used by the FE."
+  [{:keys [offset
+           limit
+           sort_column
+           sort_direction
+           job_id
+           status]}]
+  (let [offset (or offset 0)
+        limit  (or limit 20)
+        sort-direction (or (keyword sort_direction) :desc)
+        nulls-sort (if (= sort-direction :asc)
+                     :nulls-last
+                     :nulls-first)
+        sort-column (keyword sort_column)
+        order-by (case sort_column
+                   :started_at [[sort-column sort-direction]]
+                   :ended_at   [[sort-column sort-direction nulls-sort]]
+                   [[:start_time sort-direction]
+                    [:end_time   sort-direction nulls-sort]])
+        conditions (concat (when job_id
+                             [:job_id job_id])
+                           (when status
+                             [:= :status status])
+                           (when (= status "started")
+                             [:is_active true]))
+        conditions-with-sort-and-pagination (concat conditions [{:order-by order-by
+                                                                 :offset offset
+                                                                 :limit limit}])
+        runs (apply t2/select :model/TransformJobRun conditions-with-sort-and-pagination)]
+    {:data (t2/hydrate runs :transform)
+     :limit limit
+     :offset offset
+     :total (apply t2/count :model/TransformJobRun conditions)}))

enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj

@@ -0,0 +1,83 @@
+(ns metabase-enterprise.transforms.models.transform
+  (:require
+   [medley.core :as m]
+   [metabase-enterprise.worker.core :as worker]
+   [metabase.models.interface :as mi]
+   [methodical.core :as methodical]
+   [toucan2.core :as t2]))
+
+(methodical/defmethod t2/table-name :model/Transform [_model] :transform)
+
+(doseq [trait [:metabase/model :hook/entity-id :hook/timestamped?]]
+  (derive :model/Transform trait))
+
+(t2/deftransforms :model/Transform
+  {:source mi/transform-json
+   :target mi/transform-json
+   :run_trigger mi/transform-keyword})
+
+(mi/define-batched-hydration-method with-transform
+  :transform
+  "Add transform to a WorkRun"
+  [runs]
+  (if-not (seq runs)
+    runs
+    (let [work-ids (into #{} (map :work_id) runs)
+          id->transform (t2/select-pk->fn identity [:model/Transform :id :name] :id [:in work-ids])]
+      (for [run runs]
+        (assoc run :transform (get id->transform (:work_id run)))))))
+
+(mi/define-batched-hydration-method with-last-run
+  :last_run
+  "Add last_run to a transform"
+  [transforms]
+  (if-not (seq transforms)
+    transforms
+    (let [transform-ids (into #{} (map :id) transforms)
+          last-runs (m/index-by :work_id (worker/latest-runs :transform transform-ids))]
+      (for [transform transforms]
+        (assoc transform :last_run (get last-runs (:id transform)))))))
+
+(mi/define-batched-hydration-method transform-tag-ids
+  :transform_tag_ids
+  "Add tag_ids to a transform, preserving the order they were originally added"
+  [transforms]
+  (if-not (seq transforms)
+    transforms
+    (let [transform-ids (into #{} (map :id) transforms)
+          tag-associations (when (seq transform-ids)
+                             (t2/select [:transform_tags :transform_id :tag_id :id]
+                                        :transform_id [:in transform-ids]
+                                        {:order-by [[:id :asc]]}))
+          transform-id->tag-ids (reduce (fn [acc {:keys [transform_id tag_id]}]
+                                          (update acc transform_id (fnil conj []) tag_id))
+                                        {}
+                                        tag-associations)]
+      (for [transform transforms]
+        (assoc transform :tag_ids (vec (get transform-id->tag-ids (:id transform) [])))))))
+
+(defn update-transform-tags!
+  "Update the tags associated with a transform. Replaces all existing associations while preserving order.
+   Duplicate tag IDs are automatically deduplicated, keeping the first occurrence."
+  [transform-id tag-ids]
+  (when transform-id
+    (t2/with-transaction [_conn]
+      ;; Delete existing associations
+      (t2/delete! :transform_tags :transform_id transform-id)
+      ;; Add new associations, preserving the order of tag-ids
+      (when (seq tag-ids)
+        (let [;; Deduplicate while preserving order of first occurrence
+              deduped-tag-ids (distinct tag-ids)
+              ;; Check which tags actually exist
+              valid-tag-ids (into #{} (t2/select-fn-set :id :model/TransformTag :id [:in deduped-tag-ids]))
+              ;; Filter to only include valid ones, preserving order
+              ordered-valid-tag-ids (filter valid-tag-ids deduped-tag-ids)]
+          (when (seq ordered-valid-tag-ids)
+            (t2/insert! :transform_tags
+                        (for [tag-id ordered-valid-tag-ids]
+                          {:transform_id transform-id
+                           :tag_id tag-id}))))))))
+
+(defmethod worker/model->work-type :model/Transform
+  [_]
+  :transform)

enterprise/backend/src/metabase_enterprise/transforms/models/transform_job.clj

@@ -0,0 +1,38 @@
+(ns metabase-enterprise.transforms.models.transform-job
+  (:require
+   [medley.core :as m]
+   [metabase-enterprise.transforms.models.job-run :as transforms.job-run]
+   [metabase.models.interface :as mi]
+   [methodical.core :as methodical]
+   [toucan2.core :as t2]))
+
+(methodical/defmethod t2/table-name :model/TransformJob [_model] :transform_job)
+
+(doto :model/TransformJob
+  (derive :metabase/model)
+  (derive :hook/entity-id)
+  (derive :hook/timestamped?))
+
+(mi/define-batched-hydration-method tag-ids
+  :tag_ids
+  "Hydrate tag IDs for jobs"
+  [jobs]
+  (when (seq jobs)
+    (let [job-ids (map :id jobs)
+          tag-mappings (group-by :job_id
+                                 (t2/select [:transform_job_tags :job_id :tag_id]
+                                            :job_id [:in job-ids]))]
+      (for [job jobs]
+        (assoc job :tag_ids
+               (mapv :tag_id (get tag-mappings (:id job) [])))))))
+
+#_(mi/define-batched-hydration-method with-last-execution
+    :last_execution
+    "Add last_execution to a transform"
+    [jobs]
+    (if-not (seq jobs)
+      jobs
+      (let [job-ids (into #{} (map :id) jobs)
+            last-executions (m/index-by :work_id (transforms.job-run/latest-runs job-ids))]
+        (for [job jobs]
+          (assoc job :last_execution (get last-executions (:id job)))))))

enterprise/backend/src/metabase_enterprise/transforms/models/transform_tag.clj

@@ -0,0 +1,20 @@
+(ns metabase-enterprise.transforms.models.transform-tag
+  (:require
+   [methodical.core :as methodical]
+   [toucan2.core :as t2]))
+
+(methodical/defmethod t2/table-name :model/TransformTag [_model] :transform_tag)
+
+(doto :model/TransformTag
+  (derive :metabase/model)
+  (derive :hook/timestamped?))
+
+(defn tag-name-exists?
+  "Check if a tag with the given name already exists"
+  [tag-name]
+  (t2/exists? :model/TransformTag :name tag-name))
+
+(defn tag-name-exists-excluding?
+  "Check if a tag with the given name exists, excluding the specified ID"
+  [tag-name tag-id]
+  (t2/exists? :model/TransformTag :name tag-name :id [:not= tag-id]))

enterprise/backend/src/metabase_enterprise/transforms/ordering.clj

@@ -0,0 +1,128 @@
+(ns metabase-enterprise.transforms.ordering
+  (:require
+   [clojure.core.match]
+   [clojure.set :as set]
+   [clojure.string :as str]
+   [flatland.ordered.set :refer [ordered-set]]
+   [metabase-enterprise.transforms.util :as transforms.util]
+   [metabase.driver :as driver]
+   [metabase.lib.metadata :as lib.metadata]
+   [metabase.lib.util :as lib.util]
+   [metabase.query-processor.preprocess :as qp.preprocess]
+   [metabase.query-processor.store :as qp.store]
+   [metabase.util :as u]
+   [toucan2.core :as t2]))
+
+(defn- transform-deps [transform]
+  (let [query (-> (get-in transform [:source :query])
+                  transforms.util/massage-sql-query
+                  qp.preprocess/preprocess)]
+    (case (:type query)
+      :native (driver/native-query-deps (-> (qp.store/metadata-provider)
+                                            lib.metadata/database
+                                            :engine)
+                                        (get-in query [:native :query]))
+      :query (into #{}
+                   (keep #(clojure.core.match/match %
+                            [:source-table source] (when (int? source)
+                                                     source)
+                            _ nil))
+                   (tree-seq coll? seq query)))))
+
+(defn- transform-deps-for-db [db-id transforms]
+  (qp.store/with-metadata-provider db-id
+    (into {}
+          (map (juxt :id transform-deps))
+          transforms)))
+
+(defn- dependency-map [transforms]
+  (into {}
+        (map (juxt :id transform-deps))
+        transforms))
+
+(defn- output-table-map [transforms]
+  (let [table-map (into {}
+                        (map (fn [{:keys [schema name id]}]
+                               [[schema name] id]))
+                        (lib.metadata/tables (qp.store/metadata-provider)))]
+    (into {}
+          (keep (fn [transform]
+                  (when-let [output-table (table-map [(get-in transform [:target :schema])
+                                                      (get-in transform [:target :name])])]
+                    [output-table (:id transform)])))
+          transforms)))
+
+(defn transform-ordering
+  "Computes an 'ordering' of a given list of transforms.
+
+  The result is a map of transform id -> #{transform ids the transform depends on}. Dependencies are limited to just
+  the transforms in the original list -- if a transform depends on some transform not in the list, the 'extra'
+  dependency is ignored."
+  [transforms]
+  (let [transforms-by-db (->> transforms
+                              (map (fn [transform]
+                                     {(get-in transform [:source :query :database]) [transform]}))
+                              (apply merge-with into))
+
+        {:keys [output-tables dependencies]} (->> transforms-by-db
+                                                  (map (fn [[db-id db-transforms]]
+                                                         (qp.store/with-metadata-provider db-id
+                                                           {:output-tables (output-table-map db-transforms)
+                                                            :dependencies (dependency-map db-transforms)})))
+                                                  (apply merge-with merge))]
+    (update-vals dependencies #(into #{} (keep output-tables) %))))
+
+(defn find-cycle
+  "Finds a path containing a cycle in the directed graph `node->children`.
+
+  Optionally takes a set of starting nodes.  If starting nodes are specified, `node->children` can be any
+  function-equivalent.  Without starting nodes, `node->children` must specifically be a map."
+  ([node->children]
+   (find-cycle node->children (keys node->children)))
+  ([node->children starting-nodes]
+   (loop [stack (into [] (map #(vector % (ordered-set))) starting-nodes)
+          visited #{}]
+     (when-let [[node path] (peek stack)]
+       (cond
+         (contains? path node)
+         (into [] (drop-while (complement #{node})) (conj path node))
+
+         (contains? visited node)
+         (recur (pop stack) visited)
+
+         :else
+         (let [path' (conj path node)
+               stack' (into (pop stack)
+                            (map #(vector % path'))
+                            (node->children node))]
+           (recur stack' (conj visited node))))))))
+
+(defn get-transform-cycle [{transform-id :id :as to-check}]
+  (let [transforms (map (fn [{:keys [id] :as transform}]
+                          (if (= id transform-id)
+                            to-check
+                            transform))
+                        (t2/select :model/Transform))
+        transforms-by-id (into {}
+                               (map (juxt :id identity))
+                               transforms)
+        db-id (get-in to-check [:source :query :database])]
+    (qp.store/with-metadata-provider db-id
+      (let [output-tables (output-table-map (filter #(= (get-in % [:source :query :database]) db-id)
+                                                    transforms))
+            node->children #(->> % transforms-by-id transform-deps (keep output-tables))
+            id->name (comp :name transforms-by-id)
+            cycle (find-cycle node->children [transform-id])]
+        (when cycle
+          {:cycle-str (str/join " -> " (map id->name cycle))
+           :cycle cycle})))))
+
+(defn available-transforms
+  "Given an ordering (see transform-ordering), a set of running transform ids, and a set of completed transform ids,
+  computes which transforms are currently able to be run."
+  [ordering running complete]
+  (for [[transform-id deps] ordering
+        :when (and (not (or (running transform-id)
+                            (complete transform-id)))
+                   (empty? (set/difference deps complete)))]
+    transform-id))

enterprise/backend/src/metabase_enterprise/transforms/schedule.clj

@@ -0,0 +1,110 @@
+(ns metabase-enterprise.transforms.schedule
+  (:require
+   [clojurewerkz.quartzite.conversion :as conversion]
+   [clojurewerkz.quartzite.jobs :as jobs]
+   [clojurewerkz.quartzite.schedule.cron :as cron]
+   [clojurewerkz.quartzite.triggers :as triggers]
+   [metabase-enterprise.transforms.jobs :as transforms.jobs]
+   [metabase.driver :as driver]
+   [metabase.events.core :as events]
+   [metabase.query-processor.timezone :as qp.timezone]
+   [metabase.task-history.core :as task-history]
+   [metabase.task.core :as task]
+   [metabase.util.log :as log]
+   [methodical.core :as methodical]
+   [toucan2.core :as t2])
+  (:import
+   (java.util TimeZone)
+   (org.quartz CronTrigger TriggerKey)))
+
+(set! *warn-on-reflection* true)
+
+(defn- job-key [job-id] (jobs/key (str "metabase.task.transforms.schedule." job-id)))
+
+(defn- timezone
+  []
+  (or (driver/report-timezone)
+      (qp.timezone/system-timezone-id)
+      "UTC"))
+
+(defn- trigger-key ^TriggerKey [job-id]
+  (triggers/key (str "metabase.task.transforms.trigger." job-id)))
+
+(defn- build-trigger
+  ^CronTrigger [job-id schedule]
+  (triggers/build
+   (triggers/with-description (str "Transform Job Schedule " job-id))
+   (triggers/with-identity (trigger-key job-id))
+   (triggers/for-job (job-key job-id))
+   (triggers/start-now)
+   (triggers/with-schedule
+    (cron/schedule
+     (cron/cron-schedule schedule)
+     (cron/in-time-zone (TimeZone/getTimeZone ^String (timezone)))
+        ;; We want to fire the trigger once even if the previous triggers missed
+        ;; (potentially several times)
+     (cron/with-misfire-handling-instruction-fire-and-proceed)))
+    ;; higher than sync
+   (triggers/with-priority 6)))
+
+(defn- create-trigger!
+  "Creates the trigger for a transform job."
+  [job-id schedule]
+  (when schedule
+    (log/info "Creating trigger for transform job" job-id "with schedule" schedule)
+    (task/add-trigger! (build-trigger job-id schedule))))
+
+(defn- delete-trigger!
+  "Delete the trigger for a transform job."
+  ([job-id-or-trigger]
+   (if (number? job-id-or-trigger)
+     (if-let [trigger (first (task/existing-triggers (job-key job-id-or-trigger)
+                                                     (trigger-key job-id-or-trigger)))]
+       (delete-trigger! trigger)
+       (log/info "No trigger for this transform job exists"))
+     (do
+       (log/info "Deleting trigger for transform job with schedule" (:schedule job-id-or-trigger))
+       (task/delete-trigger! (-> job-id-or-trigger :key triggers/key))))))
+
+(task/defjob ^{:doc "Run transforms."
+               org.quartz.DisallowConcurrentExecution true}
+  RunTransforms
+  [context]
+  (let [job-id (-> (conversion/from-job-data context)
+                   (get "job-id"))]
+    (log/info "Executing scheduled run of transform job" job-id)
+    (task-history/with-task-history {:task "run-transforms"}
+      (transforms.jobs/run-job! job-id {:run-method :cron}))))
+
+(defn initialize-job! [{job-id :id :keys [schedule]}]
+  (log/info "Initializing schedule for transform job" job-id)
+  (let [job (jobs/build
+             (jobs/with-identity (job-key job-id))
+             (jobs/with-description (str "Run Transform job " job-id))
+             (jobs/using-job-data {"job-id" job-id})
+             (jobs/of-type RunTransforms)
+             (jobs/store-durably))
+        trigger (build-trigger job-id schedule)]
+    (task/schedule-task! job trigger)))
+
+(defn update-job!
+  "Update the trigger for a transform job."
+  [job-id schedule]
+  (log/info "Updating schedule for transform job" job-id "to" schedule)
+  (let [existing-trigger (first (task/existing-triggers (job-key job-id) (trigger-key job-id)))]
+    (if (not= schedule (:schedule existing-trigger))
+      (do
+        (when existing-trigger
+          (delete-trigger! existing-trigger))
+        (when schedule
+          (create-trigger! job-id schedule)))
+      (log/info "No changes to the trigger for transform job" job-id))))
+
+(defn delete-job! [job-id]
+  (log/info "Deleting schedule for transform job" job-id)
+  (task/delete-task! (job-key job-id) (trigger-key job-id)))
+
+(defmethod task/init! ::RunTransform [_]
+  (log/info "Initializing transform job execution jobs")
+  (->> (t2/select :model/TransformJob)
+       (run! initialize-job!)))

enterprise/backend/src/metabase_enterprise/transforms/util.clj

@@ -0,0 +1,88 @@
+(ns metabase-enterprise.transforms.util
+  (:require
+   [metabase.driver :as driver]
+   [metabase.query-processor.compile :as qp.compile]
+   [metabase.sync.core :as sync]
+   [metabase.util.log :as log]
+   [toucan2.core :as t2]))
+
+(defn qualified-table-name
+  "Return the name of the target table of a transform as a possibly qualified symbol."
+  [_driver {:keys [schema name]}]
+  (if schema
+    (keyword schema name)
+    (keyword name)))
+
+(defn target-table-exists?
+  "Test if the target table of a transform already exists."
+  [{:keys [source target] :as _transform}]
+  (let [db-id (-> source :query :database)
+        {driver :engine :as database} (t2/select-one :model/Database db-id)]
+    (driver/table-exists? driver database target)))
+
+(defn target-table
+  "Load the `target` table of a transform from the database specified by `database-id`."
+  [database-id target & kv-args]
+  (some-> (apply t2/select-one :model/Table
+                 :db_id database-id
+                 :schema (:schema target)
+                 :name (:name target)
+                 kv-args)
+          (t2/hydrate :db)))
+
+(defn- sync-table!
+  ([database target] (sync-table! database target nil))
+  ([database target {:keys [create?]}]
+   (when-let [table (or (target-table (:id database) target)
+                        (when create?
+                          (sync/create-table! database (select-keys target [:schema :name]))))]
+     (sync/sync-table! table)
+     table)))
+
+(defn activate-table!
+  "Activate table for `target` in `database` in the app db."
+  [database target]
+  (when-let [table (sync-table! database target {:create? true})]
+    ;; TODO this should probably be a function in the sync module
+    (when (not (:active table))
+      (t2/update! :model/Table (:id table) {:active true}))))
+
+(defn deactivate-table!
+  "Deactivate table for `target` in `database` in the app db."
+  [database target]
+  (when-let [table (sync-table! database target)]
+    ;; TODO this should probably be a function in the sync module
+    (t2/update! :model/Table (:id table) {:active false})))
+
+(defn delete-target-table!
+  "Delete the target table of a transform and sync it from the app db."
+  [{:keys [id target source], :as _transform}]
+  (when target
+    (let [target (update target :type keyword)
+          database-id (-> source :query :database)
+          {driver :engine :as database} (t2/select-one :model/Database database-id)]
+      (driver/drop-transform-target! driver database target)
+      (log/info "Deactivating  target " (pr-str target) "for transform" id)
+      (deactivate-table! database target))))
+
+(defn delete-target-table-by-id!
+  "Delete the target table of the transform specified by `transform-id`."
+  [transform-id]
+  (delete-target-table! (t2/select-one :model/Transform transform-id)))
+
+(defn massage-sql-query [query]
+  (assoc-in query [:middleware :disable-remaps?] true))
+
+(defn compile-source
+  "Compile the source query of a transform."
+  [{query-type :type :as source}]
+  (case query-type
+    "query" (:query (qp.compile/compile-with-inline-parameters (massage-sql-query (:query source))))))
+
+(defn required-database-feature
+  "Returns the database feature necessary to execute `transform`."
+  [transform]
+  (case (-> transform :target :type)
+    "table"             :transforms/table
+    "view"              :transforms/view
+    "materialized-view" :transforms/materialized-view))

enterprise/backend/src/metabase_enterprise/worker/api.clj

@@ -0,0 +1,64 @@
+(ns metabase-enterprise.worker.api
+  (:require
+   [clj-http.client :as http]
+   [metabase.config.core :as config]
+   [metabase.system.core :as system]
+   [metabase.util.json :as json]))
+
+(set! *warn-on-reflection* true)
+
+(defn- mb-id []
+  (system/site-uuid))
+
+(defn- json-body [{:keys [body]}]
+  (json/decode+kw body))
+
+(defn- worker-uri []
+  (config/config-str :mb-transform-worker-uri))
+
+(defn- worker-route [^String path]
+  (assert (worker-uri) "Env var MB_TRANSFORM_WORKER_URI must be set.")
+  (-> (worker-uri)
+      java.net.URI.
+      (.resolve path)
+      str))
+
+(defn run-remote?
+  "Should this metabase instance run work remotely?"
+  []
+  (boolean (worker-uri)))
+
+(defn run-transform!
+  "Run a transform on the remote worker."
+  [run-id driver transform-details opts]
+  (json-body (http/put (worker-route (str "/transform/" run-id))
+                       {:form-params {:driver driver
+                                      :transform-details transform-details
+                                      :opts opts
+                                      :mb-source (mb-id)}
+                        :content-type :json})))
+
+(defn get-status
+  "Get the status of a run."
+  [run-id]
+  (json-body (http/get (worker-route (str "/status/" run-id "?mb-source=" (mb-id)))
+                       {:content-type :json})))
+
+(defn cancel!
+  "cancel on the remote worker."
+  [run-id]
+  (:body (http/post (worker-route (str "/cancel/" run-id "?mb-source=" (mb-id))))))
+
+(defn health-check []
+  (:body (http/get (worker-route "/api/health"))))
+
+(comment
+
+  (health-check)
+
+  (alter-var-root #'environ.core/env assoc
+                  :mb-transform-worker-uri "http://localhost:3030")
+
+  (get-status "jjj")
+
+  (:status (ex-data *e)))

enterprise/backend/src/metabase_enterprise/worker/canceling.clj

@@ -0,0 +1,89 @@
+(ns metabase-enterprise.worker.canceling
+  (:require
+   [clojure.core.async :as a]
+   [metabase-enterprise.worker.models.worker-run :as worker-run]
+   [metabase-enterprise.worker.models.worker-run-cancelation :as wr.cancelation]
+   [metabase-enterprise.worker.tracking :as tracking]
+   [metabase.task.core :as task]
+   [metabase.util.jvm :as u.jvm]
+   [metabase.util.log :as log])
+  (:import
+   (java.util.concurrent Executors ScheduledExecutorService TimeUnit)))
+
+(set! *warn-on-reflection* true)
+
+(defonce ^:private ^ScheduledExecutorService scheduler
+  (Executors/newScheduledThreadPool 1))
+
+(defonce ^:private connections (atom {}))
+
+(defn chan-start-run! [run-id cancel-chan]
+  (swap! connections assoc run-id cancel-chan)
+  nil)
+
+(defn chan-end-run! [run-id]
+  (-> (swap-vals! connections dissoc run-id)
+      first ;; old value
+      (get run-id)))
+
+(defn chan-signal-cancel! [run-id]
+  (when-some [cancel-chan (chan-end-run! run-id)]
+    (a/put! cancel-chan :cancel!)
+    true))
+
+(defn chan-start-timeout-vthread! [run-id]
+  (u.jvm/in-virtual-thread*
+   (Thread/sleep (* 4 60 60 1000)) ;; 4 hours
+   (chan-signal-cancel! run-id)
+   (worker-run/timeout-run! run-id)))
+
+;; please do not export from module
+(defn chan-start-timeout-vthread-worker-instance! [run-id]
+  (u.jvm/in-virtual-thread*
+   (Thread/sleep (* 4 60 60 1000)) ;; 4 hours
+   (chan-signal-cancel! run-id)
+   ;; we don't store timeouts in worker db
+   ))
+
+(defn- cancel-run-mb-instance! [run-id]
+  (when (chan-signal-cancel! run-id)
+    (worker-run/cancel-run! run-id)))
+
+(defn- cancel-run-worker-instance! [run-id]
+  (when (chan-signal-cancel! run-id)
+    (tracking/track-cancel! run-id "Canceled by user")))
+
+(defn schedule-cancel-runs!
+  "Start a scheduled task (not quartzite) that cancels tasks marked in worker_db. Should run only in the worker server. Returns the j.u.concurrent.ScheduledFuture that can be canceled."
+  []
+  (.scheduleAtFixedRate scheduler
+                        #(do
+                           (try
+                             (log/trace "Checking for canceling items.")
+                             (run! (fn [{:keys [run_id]}]
+                                     (try
+                                       (cancel-run-worker-instance! run_id)
+                                       (catch Throwable t
+                                         (log/error t (str "Error canceling " run_id)))))
+                                   (wr.cancelation/reducible-canceled-local-runs))
+                             (catch Throwable t
+                               (log/error t "Error while canceling on worker.")))
+                           (try
+                             (tracking/cancel-old-cancelations!)
+                             (catch Throwable t
+                               (log/error t "Error canceling old runs."))))
+                        0 20 TimeUnit/SECONDS))
+
+(defmethod task/init! ::CancelRuns [_]
+  ;; does not use the Quartz scheduler, should only run on the mb instance
+  (.scheduleAtFixedRate scheduler
+                        #(try
+                           (log/trace "Checking for canceling items.")
+                           (run! (fn [{:keys [run_id]}]
+                                   (try
+                                     (cancel-run-mb-instance! run_id)
+                                     (catch Throwable t
+                                       (log/error t (str "Error canceling " run_id)))))
+                                 (wr.cancelation/reducible-canceled-local-runs))
+                           (catch Throwable t
+                             (log/error t "Error while canceling on worker."))) 0 20 TimeUnit/SECONDS))

enterprise/backend/src/metabase_enterprise/worker/core.clj

@@ -0,0 +1,46 @@
+(ns metabase-enterprise.worker.core
+  (:require
+   [metabase-enterprise.worker.api]
+   [metabase-enterprise.worker.canceling]
+   [metabase-enterprise.worker.models.worker-run]
+   [metabase-enterprise.worker.models.worker-run-cancelation]
+   [metabase-enterprise.worker.server]
+   [metabase-enterprise.worker.sync]
+   [metabase-enterprise.worker.tracking]
+   [potemkin :as p]))
+
+(p/import-vars
+ [metabase-enterprise.worker.api
+  cancel!
+  run-transform!
+  get-status
+  run-remote?]
+ [metabase-enterprise.worker.canceling
+  chan-end-run!
+  chan-signal-cancel!
+  chan-start-run!
+  chan-start-timeout-vthread!]
+ [metabase-enterprise.worker.models.worker-run
+  cancel-old-canceling-runs!
+  cancel-run!
+  fail-started-run!
+  inactive-runs
+  latest-runs
+  model->work-type
+  paged-runs
+  reducible-active-remote-runs
+  running-run-for-work-id
+  start-run!
+  succeed-started-run!
+  timeout-old-runs!
+  timeout-run!]
+ [metabase-enterprise.worker.models.worker-run-cancelation
+  mark-cancel-started-run!
+  reducible-canceled-local-runs
+  delete-cancelation!
+  delete-old-canceling-runs!]
+ [metabase-enterprise.worker.server
+  start!
+  stop!]
+ [metabase-enterprise.worker.sync
+  sync-single-run!])

enterprise/backend/src/metabase_enterprise/worker/init.clj

@@ -0,0 +1,3 @@
+(ns metabase-enterprise.worker.init
+  (:require
+   [metabase-enterprise.worker.canceling]))

enterprise/backend/src/metabase_enterprise/worker/models/worker_run.clj

@@ -0,0 +1,275 @@
+(ns metabase-enterprise.worker.models.worker-run
+  (:require
+   [metabase-enterprise.worker.models.worker-run-cancelation :as cancel]
+   [metabase.app-db.core :as mdb]
+   [metabase.driver.sql.query-processor :as sql.qp]
+   [metabase.models.interface :as mi]
+   [metabase.util :as u]
+   [methodical.core :as methodical]
+   [toucan2.core :as t2]
+   [toucan2.realize :as t2.realize]))
+
+(methodical/defmethod t2/table-name :model/WorkerRun [_model] :worker_run)
+
+(derive :model/WorkerRun :metabase/model)
+
+(t2/deftransforms :model/WorkerRun
+  {:work_type mi/transform-keyword
+   :status mi/transform-keyword
+   :run_method mi/transform-keyword})
+
+(defn- check-is-active [{:keys [status is_active] :as run}]
+  (when (= (keyword status) :started)
+    (when (not is_active)
+      (throw (ex-info "If WorkerRun has started status, is_active should be true"
+                      {:run run}))))
+  (when is_active
+    (when (not= (keyword status) :started)
+      (throw (ex-info "If WorkerRun is_active = true, should have started status"
+                      {:run run})))))
+
+#_(t2/define-before-insert :model/WorkerRun
+    [run]
+    (check-is-active run)
+    run)
+
+#_(t2/define-before-update :model/WorkerRun
+    [run]
+    (check-is-active run)
+    run)
+
+(defmulti model->work-type
+  "Convert the toucan model name to a keyword representing the type of work."
+  {:arglists '([model-name])}
+  identity)
+
+(defmethod model->work-type :default
+  [model-name]
+  (throw (ex-info "No implementation of model->work-type for this model name."
+                  {:model-name model-name})))
+
+(mi/define-simple-hydration-method add-worker-runs
+  :worker-runs
+  "Add worker-runs for a transform or other work. Must have :id field."
+  [work]
+  (t2/select :model/WorkerRun
+             :work_id (:id work)
+             :work_type (model->work-type (t2/model work))
+             {:order-by [[:start_time :desc] [:end_time :desc]]}))
+
+(defn- latest-runs-query [work-type work-ids]
+  {:with [[:ranked_runs
+           {:select [:*
+                     [[:over [[:row_number] {:partition-by :work_id, :order-by [[:start_time :desc]]}]] :rn]]
+            :from [:worker_run]
+            :where [:and
+                    [:= :work_type work-type]
+                    [:in :work_id work-ids]]}]]
+   :select [:*]
+   :from [:ranked_runs]
+   :where [:= :rn [:inline 1]]})
+
+(defn latest-runs
+  "Return the latest runs for `work-type` and `work-ids`."
+  [work-type work-ids]
+  (when (seq work-ids)
+    (into [] (map (comp t2.realize/realize #(dissoc % :rn)))
+          (t2/reducible-select :model/WorkerRun (latest-runs-query (name work-type) work-ids)))))
+
+(defn inactive-runs
+  "Return the runs with run IDs in `run-ids` that are not active."
+  [run-ids]
+  (when (seq run-ids)
+    (t2/select :model/WorkerRun
+               :run_id    [:in run-ids]
+               :is_active nil)))
+
+(defn start-run!
+  "Start a run"
+  ([run-id work-type work-id]
+   (start-run! run-id work-type work-id {}))
+  ([run-id work-type work-id properties]
+   (t2/insert! :model/WorkerRun
+               (assoc properties
+                      :run_id run-id
+                      :work_type work-type
+                      :work_id work-id
+                      :status :started
+                      :is_active true))))
+
+(defn succeed-started-run!
+  "Mark a started run as successfully completed."
+  ([run-id]
+   (succeed-started-run! run-id {}))
+  ([run-id properties]
+   (u/prog1 (t2/update! :model/WorkerRun
+                        :run_id    run-id
+                        :is_active true
+                        (merge {:end_time :%now}
+                               properties
+                               {:status    :succeeded
+                                :is_active nil}))
+     (cancel/delete-old-canceling-runs!))))
+
+(defn fail-started-run!
+  "Mark the started active run as failed and inactive."
+  [run-id properties]
+  (u/prog1 (t2/update! :model/WorkerRun
+                       :run_id    run-id
+                       :is_active true
+                       (merge {:end_time :%now}
+                              properties
+                              {:status :failed
+                               :is_active nil}))
+    (cancel/delete-old-canceling-runs!)))
+
+(defn reducible-active-remote-runs
+  "Return a reducible sequence of active remote runs."
+  []
+  (t2/reducible-select :model/WorkerRun :is_local false :is_active true))
+
+(defn cancel-run!
+  "Cancel a started run."
+  ([run-id]
+   (cancel-run! run-id {:message "Canceled by user"}))
+  ([run-id properties]
+   (u/prog1 (t2/update! :model/WorkerRun
+                        :run_id    run-id
+                        :is_active true
+                        (merge {:end_time :%now}
+                               properties
+                               {:status    :canceled
+                                :is_active nil}))
+     (cancel/delete-old-canceling-runs!))))
+
+(defn timeout-run!
+  "Mark a started run as timed out."
+  ([run-id]
+   (timeout-run! run-id {}))
+  ([run-id properties]
+   (u/prog1 (t2/update! :model/WorkerRun
+                        :run_id    run-id
+                        :is_active true
+                        (merge {:end_time :%now
+                                :message  "Timed out"}
+                               properties
+                               {:status    :timeout
+                                :is_active nil}))
+     (cancel/delete-old-canceling-runs!))))
+
+(defn timeout-old-runs!
+  "Time out all active runs older than the specified age."
+  [age unit]
+  (u/prog1 (t2/update! :model/WorkerRun
+                       :is_active true
+                       :start_time [:< (sql.qp/add-interval-honeysql-form (mdb/db-type) :%now (- age) unit)]
+                       {:status :timeout
+                        :end_time :%now
+                        :is_active nil
+                        :message "Timed out by metabase"})
+    (cancel/delete-old-canceling-runs!)))
+
+(defn cancel-old-canceling-runs!
+  "Cancel all canceling runs older than the specified age."
+  [age unit]
+  (u/prog1 (t2/update! :model/WorkerRun
+                       :is_active true
+                       :run_id [:in {:select :run_id
+                                     :from   :worker_run_cancelation
+                                     :where  [:<
+                                              :worker_run_cancelation.time
+                                              (sql.qp/add-interval-honeysql-form (mdb/db-type) :%now (- age) unit)]}]
+                       {:status :canceled
+                        :end_time :%now
+                        :is_active nil
+                        :message "Canceled by user but could not guarantee run stopped."})
+    (cancel/delete-old-canceling-runs!)))
+
+(defn running-run-for-work-id
+  "Return a single active work run or nil."
+  [id work-type]
+  (t2/select-one :model/WorkerRun
+                 :work_id id
+                 :work_type work-type
+                 :is_active true))
+
+;; This is kinda weird, having this code know anything about a Transform model; we may want to generalize
+;; the tags feature or else make the worker run model oblivious to tags.
+(defn paged-runs
+  "Return a page of the list of the runs.
+
+  Follows the conventions used by the FE."
+  [{:keys [offset
+           limit
+           sort_column
+           sort_direction
+           work_type
+           work_ids
+           work_tag_ids
+           statuses]}]
+  (let [offset (or offset 0)
+        limit  (or limit 20)
+        sort-direction (or (keyword sort_direction) :desc)
+        nulls-sort (if (= sort-direction :asc)
+                     :nulls-last
+                     :nulls-first)
+        sort-column (keyword sort_column)
+        order-by (case sort_column
+                   :started_at [[sort-column sort-direction]]
+                   :ended_at   [[sort-column sort-direction nulls-sort]]
+                   [[:start_time sort-direction]
+                    [:end_time   sort-direction nulls-sort]])
+        ;; Build WHERE clause conditions
+        where-conditions (cond-> []
+                           ;; work_type condition
+                           work_type
+                           (conj [:= :work_type work_type])
+
+                           ;; work_ids and work_tag_ids (intersection)
+                           (and (seq work_ids) (seq work_tag_ids))
+                           (conj [:and
+                                  [:in :work_id work_ids]
+                                  [:in :work_id {:select [:transform_id]
+                                                 :from [:transform_tags]
+                                                 :where [:in :tag_id work_tag_ids]}]])
+
+                           ;; Only work_ids
+                           (and (seq work_ids) (not (seq work_tag_ids)))
+                           (conj [:in :work_id work_ids])
+
+                           ;; Only work_tag_ids
+                           (and (seq work_tag_ids) (not (seq work_ids)))
+                           (conj [:in :work_id {:select [:transform_id]
+                                                :from [:transform_tags]
+                                                :where [:in :tag_id work_tag_ids]}])
+
+                           ;; statuses condition
+                           (seq statuses)
+                           (conj [:in :status statuses])
+
+                           ;; is_active condition for started status
+                           (and (seq statuses)
+                                (some #(= % "started") statuses))
+                           (conj [:= :is_active true]))
+
+        where-clause     (when (seq where-conditions)
+                           (if (= 1 (count where-conditions))
+                             (first where-conditions)
+                             (into [:and] where-conditions)))
+        query-options    (cond-> {:order-by order-by
+                                  :offset offset
+                                  :limit    limit}
+                           where-clause (assoc :where where-clause))
+        runs             (t2/select :model/WorkerRun query-options)
+        count-options    (cond-> {}
+                           where-clause (assoc :where where-clause))]
+
+    {:data (t2/hydrate runs :transform)
+     :limit limit
+     :offset offset
+     :total (t2/count :model/WorkerRun count-options)}))
+
+(comment
+  (t2/select :model/WorkerRun)
+  (cancel-run! "slo7dhL7zoclb0uI0Zchj")
+  -)

enterprise/backend/src/metabase_enterprise/worker/models/worker_run_cancelation.clj

@@ -0,0 +1,46 @@
+(ns metabase-enterprise.worker.models.worker-run-cancelation
+  (:require
+   [metabase.app-db.core :as mdb]
+   [metabase.driver.sql.query-processor :as sql.qp]
+   [metabase.models.interface :as mi]
+   [methodical.core :as methodical]
+   [toucan2.core :as t2]
+   [toucan2.realize :as t2.realize]))
+
+(methodical/defmethod t2/table-name :model/WorkerRunCancelation [_model] :worker_run_cancelation)
+
+(derive :model/WorkerRunCancelation :metabase/model)
+
+(defn mark-cancel-started-run!
+  "Mark a started run for cancelation."
+  ([run-id]
+   (mark-cancel-started-run! run-id {}))
+  ([run-id properties]
+   (t2/insert! :model/WorkerRunCancelation
+               (assoc properties
+                      :run_id run-id)
+               {:where [:exists {:select [1]
+                                 :from   [:worker_run]
+                                 :where  [:and
+                                          [:= :worker_run.run_id run-id]
+                                          :worker_run.is_active]}]})))
+
+(defn reducible-canceled-local-runs
+  "Return a reducible sequence of local canceled runs."
+  []
+  (t2/reducible-select :model/WorkerRunCancelation))
+
+(defn delete-cancelation!
+  "Delete a cancelation once it has been handled."
+  [run-id]
+  (t2/delete! :model/WorkerRunCancelation
+              :run-id run-id))
+
+(defn delete-old-canceling-runs!
+  "Delete cancelations that are no longer running."
+  []
+  (t2/delete! :model/WorkerRunCancelation
+              :run_id [:not-in {:select :wrc.run_id
+                                :from   [[:worker_run_cancelation :wrc]]
+                                :join   [[:worker_run :wr] [:= :wr.run_id :wrc.run_id]]
+                                :where  :wr.is_active}]))

enterprise/backend/src/metabase_enterprise/worker/server.clj

@@ -0,0 +1,151 @@
+(ns metabase-enterprise.worker.server
+  (:require
+   [clojure.core.async :as a]
+   [clojure.edn :as edn]
+   [clojure.string :as str]
+   [compojure.core :as compojure]
+   [compojure.route :as route]
+   [malli.core :as mc]
+   [malli.transform :as mtx]
+   [metabase-enterprise.transforms.core :as transforms]
+   [metabase-enterprise.worker.canceling :as canceling]
+   [metabase-enterprise.worker.tracking :as tracking]
+   [metabase.api.util.handlers :as handlers]
+   [metabase.config.core :as config]
+   [metabase.driver :as driver]
+   [metabase.lib.schema.common :as schema.common]
+   [metabase.query-processor.pipeline :as qp.pipeline]
+   [metabase.server.core :as server]
+   [metabase.util :as u]
+   [metabase.util.json :as json]
+   [metabase.util.jvm :as u.jvm]
+   [metabase.util.log :as log]
+   [metabase.util.malli.registry :as mr]
+   [metabase.util.yaml :as yaml]
+   [ring.adapter.jetty :as ring-jetty]
+   [ring.middleware.keyword-params :refer [wrap-keyword-params]]
+   [ring.middleware.params :refer [wrap-params]]
+   [ring.util.response :as response])
+  (:import (java.util.concurrent
+            Executors
+            ExecutorService
+            Future
+            ScheduledExecutorService
+            Semaphore
+            TimeUnit)
+           (org.eclipse.jetty.util.thread QueuedThreadPool)))
+
+(set! *warn-on-reflection* true)
+
+;; allow no more than 1000 running workers at a time
+(defonce ^:private ^Semaphore semaphore (Semaphore. 1000 true))
+
+(mr/def ::transform-api-request
+  [:map
+   [:driver [:keyword {:decode/normalize schema.common/normalize-keyword}]]
+   [:transform-details :metabase-enterprise.transforms.execute/transform-details]
+   [:opts :metabase-enterprise.transforms.execute/transform-opts]
+   [:mb-source :string]])
+
+(defn- handle-transform-put
+  [{:keys [params body]}]
+  (log/info "Handling transform PUT request for run" (pr-str (:run-id params)))
+  (if (.tryAcquire semaphore)
+    (let [{:keys [run-id]} params
+          {:keys [driver transform-details opts mb-source]} (mc/coerce
+                                                             ::transform-api-request
+                                                             body
+                                                             (mtx/transformer {:name :normalize}))]
+      (when (tracking/track-start! run-id mb-source)
+        (u.jvm/in-virtual-thread*
+         (try
+           (canceling/chan-start-timeout-vthread-worker-instance! run-id)
+           (binding [qp.pipeline/*canceled-chan* (a/promise-chan)]
+             (canceling/chan-start-run! run-id qp.pipeline/*canceled-chan*)
+             (driver/run-transform! driver
+                                    transform-details
+                                    opts))
+           (tracking/track-finish! run-id)
+           (catch Throwable t
+             (log/error t "Error executing transform")
+             (tracking/track-error! run-id (.getMessage t)))
+           (finally
+             (canceling/chan-end-run! run-id)
+             (.release semaphore)))))
+      (-> (response/response (tracking/get-status run-id mb-source))
+          (response/content-type "application/json")))
+
+    (-> (response/response "Too many requests")
+        (response/status 429))))
+
+(defn- handle-status-get
+  [run-id mb-source]
+  (log/info "Handling status GET request for run" (pr-str run-id))
+  (if-let [resp (tracking/get-status run-id mb-source)]
+    (response/response resp)
+    (-> (response/response "Not found")
+        (response/status 404))))
+
+(defn- handle-cancel-post
+  [run-id mb-source]
+  (log/info "Handling cancel POST request for run" (pr-str run-id))
+  (if (tracking/mark-cancel-started-run! run-id mb-source)
+    (-> (response/response "Canceling")
+        (response/status 202))
+    (-> (response/response "Not found")
+        (response/status 404))))
+
+(def ^:private routes
+  (compojure/routes
+   (compojure/GET "/api/health" [] "healthy")
+   (compojure/PUT "/transform/:run-id" request (handle-transform-put request))
+   (compojure/GET "/status/:run-id" [run-id mb-source] (handle-status-get run-id mb-source))
+   (compojure/POST "/cancel/:run-id" [run-id mb-source] (handle-cancel-post run-id mb-source))
+   (route/not-found "Page not found")))
+
+(def ^:private handler
+  (reduce (fn [handler middleware-fn]
+            (middleware-fn handler))
+          routes
+          [server/wrap-json-body
+           server/wrap-streamed-json-response
+           wrap-keyword-params
+           wrap-params]))
+
+(defn ^:private port []
+  (if-let [port (config/config-str :mb-worker-jetty-port)]
+    (Integer/parseInt port)
+    3030))
+
+(defonce ^:private instance (atom nil))
+(defonce ^:private cancel-runs (atom nil))
+
+(defn stop! []
+  (when @instance
+    (log/info "Stopping worker server")
+    (.stop ^QueuedThreadPool @instance)
+    (reset! instance nil))
+  (when @cancel-runs
+    (.cancel ^Future @cancel-runs true)
+    (reset! cancel-runs nil))
+  nil)
+
+(defn start!
+  ([] (start! {}))
+  ([opts]
+   (stop!)
+   (let [thread-pool (doto (new QueuedThreadPool)
+                       (.setVirtualThreadsExecutor (Executors/newVirtualThreadPerTaskExecutor)))
+         jetty-port (port)]
+     (log/info "Starting worker server on port" jetty-port)
+     (reset! instance (ring-jetty/run-jetty #'handler
+                                            {:port jetty-port
+                                             :thread-pool thread-pool
+                                             :join? (not (:dev opts))})))
+   (reset! cancel-runs (canceling/schedule-cancel-runs!))
+   nil))
+
+(comment
+
+  (start! {:dev true})
+  (stop!))

enterprise/backend/src/metabase_enterprise/worker/sync.clj

@@ -0,0 +1,86 @@
+(ns metabase-enterprise.worker.sync
+  (:require
+   [clojurewerkz.quartzite.jobs :as jobs]
+   [clojurewerkz.quartzite.schedule.calendar-interval :as calendar-interval]
+   [clojurewerkz.quartzite.triggers :as triggers]
+   [java-time.api :as t]
+   [metabase-enterprise.worker.api :as api]
+   [metabase-enterprise.worker.models.worker-run :as worker-run]
+   [metabase.task.core :as task]
+   [metabase.util.log :as log]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private job-key "metabase-enterprise.worker.sync")
+
+(defn sync-single-run!
+  "Fetch status from worker and sync to mb db"
+  [run-id]
+  (let [resp (api/get-status run-id)]
+    (case (:status resp)
+      "running"
+      :do-nothing
+      "canceled"
+      (worker-run/cancel-run! run-id {:end_time (t/instant (:end-time resp))
+                                      :message (:note resp)})
+      "success"
+      (worker-run/succeed-started-run! run-id {:end_time (t/instant (:end-time resp))})
+      "error"
+      (worker-run/fail-started-run! run-id {:end_time (t/instant (:end-time resp))
+                                            :message (:note resp)})
+      "timeout"
+      (worker-run/timeout-run! run-id {:end_time (t/instant (:end-time resp))
+                                       :message (:note resp)}))
+    resp))
+
+(defn- wrap-log-errors [f msg]
+  (fn [& args]
+    (try
+      (apply f args)
+      (catch Throwable t
+        (log/error t msg)
+        nil))))
+
+(defn- sync-worker-runs! [_ctx]
+  (log/trace "Syncing worker runs.")
+  (try
+    (run! (wrap-log-errors (comp sync-single-run! :run_id) "Error syncing task")
+          (worker-run/reducible-active-remote-runs))
+    (catch Throwable t
+      (log/error t "Error syncing worker runs.")))
+
+  (log/trace "Timing out old runs.")
+  (try
+    (worker-run/timeout-old-runs! 4 :hour)
+    (catch Throwable t
+      (log/error t "Error timing out old runs.")))
+
+  (log/trace "Canceling items that haven't been marked canceled.")
+  (try
+    (worker-run/cancel-old-canceling-runs! 1 :minute)
+    (catch Throwable t
+      (log/error t "Error canceling items not marked canceled."))))
+
+(task/defjob  ^{:doc "Syncs remote execution information with local table."
+                org.quartz.DisallowConcurrentExecution true}
+  SyncWorkerStatus [ctx]
+  (sync-worker-runs! ctx))
+
+(defn- start-job! []
+  (when (not (task/job-exists? job-key))
+    (let [job (jobs/build
+               (jobs/of-type SyncWorkerStatus)
+               (jobs/with-identity (jobs/key job-key)))
+          trigger (triggers/build
+                   (triggers/with-identity (triggers/key job-key))
+                   (triggers/start-now)
+                   (triggers/with-schedule
+                    (calendar-interval/schedule
+                     (calendar-interval/with-interval-in-minutes 10)
+                     (calendar-interval/with-misfire-handling-instruction-do-nothing))))]
+      (task/schedule-task! job trigger))))
+
+(defmethod task/init! ::SyncWorkerStatus [_]
+  (when (api/run-remote?)
+    (log/info "Scheduling worker sync.")
+    (start-job!)))

enterprise/backend/src/metabase_enterprise/worker/tracking.clj

@@ -0,0 +1,192 @@
+(ns metabase-enterprise.worker.tracking
+  (:require
+   [java-time.api :as t]
+   [metabase.config.core :as config]
+   [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute])
+  (:import
+   (java.sql
+    PreparedStatement)))
+
+(set! *warn-on-reflection* true)
+
+(defn- worker-db []
+  (assert (config/config-str :mb-worker-db) "Must have env var MB_WORKER_DB set to run this function.")
+  (config/config-str :mb-worker-db))
+
+(defn- run-query! [driver conn-str sql params]
+  (with-open [conn (java.sql.DriverManager/getConnection conn-str)
+              stmt (sql-jdbc.execute/statement-or-prepared-statement driver
+                                                                     conn
+                                                                     sql
+                                                                     params
+                                                                     nil)]
+    (let [rs (if (instance? PreparedStatement stmt)
+               (.executeQuery ^PreparedStatement stmt)
+               (.executeQuery stmt sql))
+          rsmeta (.getMetaData rs)]
+      (into [] (sql-jdbc.execute/reducible-rows driver rs rsmeta)))))
+
+(defn- run-update! [driver conn-str sql params]
+  (with-open [conn (java.sql.DriverManager/getConnection conn-str)
+              stmt (sql-jdbc.execute/statement-or-prepared-statement driver
+                                                                     conn
+                                                                     sql
+                                                                     params
+                                                                     nil)]
+    (if (instance? PreparedStatement stmt)
+      (.executeUpdate ^PreparedStatement stmt)
+      (.executeUpdate stmt sql))))
+
+(defn track-start!
+  [run-id mb-source]
+  (-> (run-update! :postgres (worker-db)
+                   "
+INSERT INTO worker_runs (run_id, source, status)
+SELECT ?, ?, 'running'
+WHERE NOT EXISTS (
+    SELECT 1 FROM worker_runs WHERE run_id = ?
+);"
+
+                   [run-id mb-source run-id])
+      (= 1))) ;; new?
+
+(defn set-status!
+  ([run-id status]
+   (set-status! run-id status ""))
+  ([run-id status note]
+   (run-update! :postgres (worker-db)
+                "UPDATE worker_runs
+                 SET status = ?, end_time = now(), note = ?
+                 WHERE run_id = ? AND status = 'running'"
+                [status note run-id])
+   :ok))
+
+(defn track-finish!
+  [run-id]
+  (set-status! run-id "success"))
+
+(defn track-error!
+  [run-id msg]
+  (set-status! run-id "error" msg))
+
+(defn track-cancel!
+  [run-id msg]
+  (set-status! run-id "canceled" msg))
+
+(defn mark-cancel-started-run! [run-id mb-source]
+  (-> (run-update! :postgres (worker-db)
+                   "
+INSERT INTO worker_runs_cancelation (run_id)
+SELECT ?
+WHERE EXISTS (
+  SELECT 1 FROM worker_runs WHERE run_id = ? AND status = 'running' AND source = ?
+) AND NOT EXISTS (
+  SELECT 1 FROM worker_runs_cancelation WHERE run_id = ?
+)
+"
+                   [run-id run-id mb-source run-id])
+      (= 1)))
+
+(def ^:private timeout-sec (* 4 60 60) #_sec)
+
+(defn- handle-timeout
+  [run]
+  (if (and (= "running" (:status run))
+           (> (:run-time run) timeout-sec))
+    (-> run
+        (assoc :status "timeout")
+        (assoc :end-time (t/plus (:start-time run) (t/duration 4 :hours)))
+        (assoc :note "Timed out.")
+        (dissoc :run-time))
+    (dissoc run :run-time)))
+
+(defn get-status
+  [run-id mb-source]
+  (->> (run-query! :postgres (worker-db)
+                   "SELECT run_id, status, start_time, end_time, note, EXTRACT(EPOCH FROM now() - start_time) as run_time
+                      FROM worker_runs
+                      WHERE run_id = ? AND source = ?"
+                   [run-id mb-source])
+       first ;; row
+       (zipmap [:run-id :status :start-time :end-time :note :run-time])
+       handle-timeout
+       not-empty))
+
+(defn timeout-old-tasks
+  []
+  (run-update! :postgres (worker-db)
+               "UPDATE worker_runs
+                  SET status = ?, end_time = NOW(), note = ?
+                  WHERE status = 'running' AND start_time < NOW() - INTERVAL '4 hours'"
+               ["timeout" "Timed out by worker"]))
+
+(defn cancel-old-cancelations!
+  []
+  ;; update the status of canceled runs > 1 minute old
+  (run-update! :postgres (worker-db)
+               "UPDATE worker_runs
+                  SET status = 'canceled', end_time = NOW(), note = ?
+                  WHERE status = 'running' AND 
+                        run_id IN (
+       SELECT run_id 
+       FROM worker_runs_cancelation
+       WHERE time < (NOW() - INTERVAL '1 minute')
+                                  )"
+               ["Canceled by user but could not guarantee run stopped"])
+  ;; delete cancelations of all runs that are not running
+  (run-update! :postgres (config/config-str :mb-worker-db)
+               "DELETE FROM worker_runs_cancelation
+                WHERE run_id NOT IN (
+SELECT wrc.run_id
+FROM   worker_runs_cancelation AS wrc
+JOIN   worker_runs AS wr ON wr.run_id = wrc.run_id
+WHERE  wr.status = 'running'
+)"
+               []))
+
+(comment
+
+  (let [driver :postgres
+        sql "SELECT * from worker_runs"]
+
+    (with-open [conn (java.sql.DriverManager/getConnection "jdbc:postgresql://localhost:5432/worker")]
+      (with-open [stmt (sql-jdbc.execute/statement-or-prepared-statement driver
+                                                                         conn
+                                                                         sql
+                                                                         []
+                                                                         nil)]
+        (let [rs (if (instance? PreparedStatement stmt)
+                   (.executeQuery ^PreparedStatement stmt)
+                   (.executeQuery stmt sql))
+              rsmeta (.getMetaData rs)]
+          (into [] (sql-jdbc.execute/reducible-rows driver rs rsmeta))))))
+
+  (let [driver :postgres
+        sql "insert into worker_runs (work_id, type, source, status) values (?, ?, ?, ?) RETURNING run_id"]
+
+    (with-open [conn (java.sql.DriverManager/getConnection "jdbc:postgresql://localhost:5432/worker")]
+      (with-open [stmt (sql-jdbc.execute/statement-or-prepared-statement driver
+                                                                         conn
+                                                                         sql
+                                                                         [1, "sql-transform", "mb-1", "running"]
+                                                                         nil)]
+        (let [rs (if (instance? PreparedStatement stmt)
+                   (.executeQuery ^PreparedStatement stmt)
+                   (.executeQuery stmt sql))
+              rsmeta (.getMetaData rs)]
+          (into [] (sql-jdbc.execute/reducible-rows driver rs rsmeta)))))))
+
+(comment
+
+  (alter-var-root #'environ.core/env assoc :mb-worker-db "jdbc:postgresql://localhost:5432/worker")
+
+  (def id (str (metabase.util/generate-nano-id)))
+  (track-start! id "mb-1")
+
+  (mark-cancel-started-run! id "mb-1")
+
+  (cancel-old-cancelations!)
+  (track-finish! id)
+  (track-error! id "oops")
+
+  (get-status id "mb-1"))

enterprise/backend/src/metabase_enterprise/worker_db/migrations.clj

@@ -0,0 +1,101 @@
+(ns metabase-enterprise.worker-db.migrations
+  "Database migration utilities for worker databases."
+  (:require
+   [clojure.core :exclude [run!]]
+   [clojure.java.io :as io]
+   [metabase.classloader.core :as classloader]
+   [metabase.config.core :as config]
+   [metabase.util.log :as log]
+   [metabase.util.malli :as mu]
+   [metabase.util.malli.schema :as ms])
+  (:import
+   (javax.sql DataSource)
+   (liquibase Contexts LabelExpression Liquibase)
+   (liquibase.database Database DatabaseFactory)
+   (liquibase.database.jvm JdbcConnection)
+   (liquibase.resource ClassLoaderResourceAccessor ResourceAccessor)))
+
+(set! *warn-on-reflection* true)
+
+(mu/defn- run-migrations-from-resource!
+  "Run Liquibase migrations from a resource file in the classpath.
+
+  Parameters:
+  - data-source: A javax.sql.DataSource for the target database
+  - resource-path: Path to the Liquibase changelog file in resources (e.g. \"migrations/my-db-changelog.yaml\")
+
+  Example:
+    (run-migrations-from-resource! my-datasource \"migrations/external-db-changelog.yaml\")"
+  [data-source :- (ms/InstanceOfClass DataSource)
+   resource-path :- :string]
+  (log/info "Running migrations from resource:" resource-path)
+
+  ;; Verify the resource exists
+  (when-not (io/resource resource-path)
+    (throw (ex-info (str "Migration resource not found: " resource-path)
+                    {:resource-path resource-path})))
+
+  ;; Create Liquibase instance manually with custom changelog
+  (with-open [^java.sql.Connection conn (.getConnection ^DataSource data-source)]
+    (let [^JdbcConnection liquibase-conn (JdbcConnection. conn)
+          ^Database database (.findCorrectDatabaseImplementation (DatabaseFactory/getInstance) liquibase-conn)
+          ^ResourceAccessor resource-accessor (ClassLoaderResourceAccessor. (classloader/the-classloader))
+          ^Liquibase lb (Liquibase. ^String resource-path resource-accessor database)]
+      ;; Run the migrations
+      (try
+        (log/info "Starting migration execution for" resource-path)
+        (.update lb (Contexts.) (LabelExpression.))
+        (log/info "Successfully completed migrations from" resource-path)
+        :success
+        (catch Exception e
+          (log/error e "Failed to run migrations from" resource-path)
+          (throw e))))))
+
+(mu/defn- run-migrations-from-resource-with-connection-string!
+  "Run Liquibase migrations from a resource file using a JDBC connection string.
+
+  Parameters:
+  - connection-string: JDBC connection string (e.g. \"jdbc:postgresql://localhost/mydb?user=myuser&password=mypass\")
+  - resource-path: Path to the Liquibase changelog file in resources (e.g. \"migrations/my-db-changelog.yaml\")
+
+  Example:
+    (run-migrations-from-resource-with-connection-string!
+      \"jdbc:postgresql://localhost/mydb?user=myuser&password=mypass\"
+      \"migrations/external-db-changelog.yaml\")"
+  [connection-string :- :string
+   resource-path :- :string]
+  (log/info "Running migrations from resource:" resource-path "with connection:" connection-string)
+
+  ;; Verify the resource exists
+  (when-not (io/resource resource-path)
+    (throw (ex-info (str "Migration resource not found: " resource-path)
+                    {:resource-path resource-path})))
+
+  ;; Create Liquibase instance manually with custom changelog and connection string
+  (with-open [^java.sql.Connection conn (java.sql.DriverManager/getConnection connection-string)]
+    (let [^JdbcConnection liquibase-conn (JdbcConnection. conn)
+          ^Database database (.findCorrectDatabaseImplementation (DatabaseFactory/getInstance) liquibase-conn)
+          ^ResourceAccessor resource-accessor (ClassLoaderResourceAccessor. (classloader/the-classloader))
+          ^Liquibase lb (Liquibase. ^String resource-path resource-accessor database)]
+      ;; Run the migrations
+      (try
+        (log/info "Starting migration execution for" resource-path)
+        (.update lb (Contexts.) (LabelExpression.))
+        (log/info "Successfully completed migrations from" resource-path)
+        :success
+        (catch Exception e
+          (log/error e "Failed to run migrations from" resource-path)
+          (throw e))))))
+
+(defn run!
+  "Run migrations for the worker."
+  []
+  (assert (config/config-str :mb-worker-db) "Cannot run worker migrations with MB_WORKER_DB env var.")
+  (let [worker-db-connection-string (config/config-str :mb-worker-db)]
+    (run-migrations-from-resource-with-connection-string! worker-db-connection-string
+                                                          "migrations/workers/worker_runs.yaml")))
+
+(comment
+
+  (run-migrations-from-resource-with-connection-string! "jdbc:postgresql://localhost:5432/worker"
+                                                        "migrations/workers/worker_runs.yaml"))

enterprise/backend/src/metabase_enterprise/workspaces/api.clj

@@ -0,0 +1,180 @@
+(ns metabase-enterprise.workspaces.api
+  "`/api/ee/workspace/` routes"
+  (:require
+   [clj-yaml.core :as yaml]
+   [clojure.java.io :as io]
+   [metabase-enterprise.workspaces.common :as w.common]
+   [metabase-enterprise.workspaces.isolation-manager :as isolation-manager]
+   [metabase-enterprise.workspaces.models.workspace :as m.workspace]
+   [metabase.api.common :as api]
+   [metabase.api.macros :as api.macros]
+   [metabase.api.routes.common :refer [+auth]]
+   [metabase.util.log :as log]
+   [metabase.util.malli :as mu]
+   [metabase.util.malli.registry :as mr]
+   [metabase.util.malli.schema :as ms]
+   [toucan2.core :as t2]))
+
+(comment
+  isolation-manager/keep-me ;;temp
+  m.workspace/keep-me)
+
+(set! *warn-on-reflection* true)
+
+(mu/defn- add-workspace-entity!
+  "Adds a workspace entity, such as a plan or transform, to the workspace.
+
+   Args:
+   - workspace-id: The workspace ID
+   - entity-key: The key in the workspace map (:plans, :transforms, etc.)
+   - new-entity: The new entity to add, which should be a map with the necessary fields"
+  [workspace-id
+   entity-key :- ::m.workspace/entity-column
+   new-entity]
+  (try (w.common/add-workspace-entity! workspace-id entity-key new-entity)
+       (catch Exception e
+         (case (:error (ex-data e))
+           :no-workspace (throw (ex-info "Workspace not found" {:status-code 404
+                                                                :error :no-workspace}))
+           (throw e)))))
+
+(defn update-workspace-entity-at-index!
+  [workspace-id entity-key index update-fn]
+  (try (w.common/update-workspace-entity-at-index! workspace-id entity-key index update-fn)
+       (catch Exception e
+         (case (:error (ex-data e))
+           :no-workspace [404 "Workspace not found"]
+           :no-index [404 (str "No " entity-key " found at index " index " there are only " (:item-count (ex-data e)))]
+           (throw e)))))
+
+(defn- delete-workspace-entity-at-index!
+  "Deletes an entity at a specific index in the workspace's entity collection.
+
+   Args:
+   - workspace-id: The workspace ID
+   - entity-key: The key in the workspace map (:plans, :transforms, etc.)
+   - index: The 0-based index of the entity to delete"
+  [workspace-id entity-key index]
+  (try (w.common/delete-workspace-entity-at-index! workspace-id entity-key index)
+       (catch Exception e
+         (case (:error (ex-data e))
+           :no-workspace (#'api/generic-404 "Workspace not found")
+           :no-index (#'api/generic-404 (str "No " entity-key " found at index " index " there are only " (:item-count (ex-data e))))
+           (throw e)))))
+
+;; GET /api/ee/workspace/:workspace-id
+(api.macros/defendpoint :get "/:workspace-id"
+  "Get a workspace by ID"
+  [{:keys [workspace-id]} :- [:map [:workspace-id ms/PositiveInt]]]
+  (-> (t2/select-one :model/Workspace :id workspace-id)
+      api/check-404
+      m.workspace/sort-workspace))
+
+;; POST /api/ee/workspace
+(api.macros/defendpoint :post "/"
+  "Create a new workspace.
+
+   Request body:
+   - name (required): Workspace name
+   - description (optional): Workspace description"
+  [_route-params
+   _query-params
+   {:keys [name description]}
+   :- [:map
+       [:name ms/NonBlankString]
+       [:description {:optional true} [:maybe :string]]]]
+  {:status 200
+   :body (w.common/create-workspace! name description api/*current-user-id*)})
+
+(defn- normalize-plan [plan]
+  (into {}
+        (-> plan
+            (update :steps #(mapv (fn [s] (update s :type keyword)) %)))))
+
+(defn- init-and-run-workspace! [plan]
+  (let [workspace (w.common/create-workspace! api/*current-user-id* (str (gensym "workspace")) "description")]
+    (when-not (mr/validate ::m.workspace/plan plan)
+      (log/info "Invalid plan data!"))
+    (let [_ (w.common/init-workspace workspace plan)
+          updated-ws (t2/select-one :model/Workspace :id (:id workspace))
+          steps (w.common/run-steps updated-ws plan)]
+      {:collection-id (:collection_id workspace)
+       :steps steps})))
+
+(comment
+  (require '[metabase.test :as mt])
+
+  (normalize-plan (yaml/parse-string plan))
+
+  (def pd {:transforms [{:name "transform_api_key",
+                         :description nil
+                         :source {:type "query", :query {:database 2, :type "query", :query {:source-table 33}}}
+                         :target {:type "table", :name "transform_api_key_table", :schema "public"}}
+                        {:name "transform_user",
+                         :description nil,
+                         :source {:type "query", :query {:database 2, :type "query", :query {:source-table 144}}},
+                         :target {:type "table", :name "transform_user_table", :schema "public"}}]
+           :steps [{:type :run-transform :name "transform_api_key"}
+                   {:type :run-transform :name "transform_user"}
+                   {:type :create-model
+                    :transform-name "transform_api_key"
+                    :model-name "Api Key Model"}
+                   {:type :create-model
+                    :transform-name "transform_user"
+                    :model-name "User Model"}
+                   {:type :create-xray
+                    :name "Api Key Xray Creator"
+                    :model-name "Api Key Model"}
+                   {:type :create-xray
+                    :name "User Xray Creator"
+                    :model-name "User Model"}]})
+
+  (do   (binding [api/*current-user-id* (mt/user->id :crowberto)
+                  api/*current-user-permissions-set* (atom #{"/"})]
+          (def w (:workspace (w.common/create-workspace! "name" "description" api/*current-user-id*))))
+
+        (w.common/init-workspace w pd)
+
+        (t2/select-one :model/Workspace :id (:id w))
+        (metabase.request.session/with-current-user 1
+          (w.common/run-steps
+           (t2/select-one :model/Workspace :id (:id w))
+           pd)))
+
+  (binding [api/*current-user-id* (mt/user->id :crowberto)
+            api/*current-user-permissions-set* (atom #{"/"})]
+    (init-and-run-workspace! pd)))
+
+;; POST /api/ee/workspace/create
+(api.macros/defendpoint :post "/create"
+  "Create a new workspace.
+
+   Request body:
+   - name (required): Workspace name
+   - description (optional): Workspace description"
+  [_route-params
+   _query-params
+   _
+   r]
+  (let [content-type (-> r :content-type)]
+    (when-not (= content-type "application/yaml")
+      (throw (ex-info "Must provide yaml" {:content-type content-type}))))
+  (try
+    (with-open [r (io/reader (:body r))]
+      (let [raw-plan (yaml/parse-stream r)
+            plan (normalize-plan raw-plan)]
+        (init-and-run-workspace! plan)))
+    (catch Exception e
+      (throw e))))
+
+(api.macros/defendpoint :delete "/:workspace-id"
+  "Delete a workspace."
+  [{:keys [workspace-id]} :- [:map
+                              [:workspace-id ms/PositiveInt]]]
+  (api/check-404 (t2/select-one :model/Workspace :id workspace-id))
+  (w.common/delete-workspace! workspace-id)
+  api/generic-204-no-content)
+
+(def ^{:arglists '([request respond raise])} routes
+  "`/api/ee/workspace/` routes for workspace management"
+  (api.macros/ns-handler *ns* +auth))

enterprise/backend/src/metabase_enterprise/workspaces/common.clj

@@ -0,0 +1,524 @@
+(ns metabase-enterprise.workspaces.common
+  (:require
+   [clj-yaml.core :as yaml]
+   [metabase-enterprise.transforms.execute :as transforms.execute]
+   [metabase-enterprise.workspaces.isolation-manager :as isolation-manager]
+   [metabase-enterprise.workspaces.models.workspace :as m.workspace]
+   [metabase.api-keys.core :as api-keys]
+   [metabase.api.common :as api]
+   [metabase.dashboards.models.dashboard :as dashboard]
+   [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]
+   [metabase.lib.core :as lib]
+   [metabase.lib.metadata :as lib.metadata]
+   [metabase.query-processor :as qp]
+   [metabase.util.log :as log]
+   [metabase.util.malli :as mu]
+   [metabase.util.secret :as u.secret]
+   [metabase.xrays.api.automagic-dashboards :as api.automagic-dashboards]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(mu/defn create-workspace! #_#_:- [:merge
+                                   ::m.workspace/workspace
+                                   [:map
+                                    [::api-key ::api-keys.schema/key.secret]]]
+  "Creates a workspace, and returns it"
+  [user-id        :- pos-int?
+   workspace-name :- ::m.workspace/workspace.name
+   description    :- ::m.workspace/workspace.description]
+  (let [collection-id  (t2/insert-returning-pk! :model/Collection
+                                                {:name        workspace-name
+                                                 :description description})
+        api-key        (api-keys/create-api-key-for-existing-user!
+                        user-id
+                        (str "Key for Workspace " (pr-str workspace-name))
+                        :api-key.scope/workspace)
+        ;; TODO: merge + use: create-single-collection-api-key!
+        workspace-data {:name            workspace-name
+                        :description     description
+                        :collection_id   collection-id
+                        :users           []
+                        :plans           []
+                        :transforms      []
+                        :activity_logs   []
+                        :permissions     []
+                        :documents       []
+                        :data_warehouses {}
+                        :api_key_id      (:id api-key)}]
+;;; TODO (Cam 8/14/25) -- this is temporary until we actually figure out the right way to pass this token around.
+    (log/infof "Generated API key for workspace: %s" (u.secret/expose (:key api-key)))
+    (-> (t2/insert-returning-instance! :model/Workspace workspace-data)
+        ;; TODO -- not sure where this should really go.
+        (assoc ::api-key (:key api-key))
+        m.workspace/sort-workspace)))
+
+(defn delete-workspace! [workspace-id]
+  ;; TODO: tear down the entire workspace, including its collection
+  (t2/delete! :model/Workspace :id workspace-id))
+
+(mu/defn add-workspace-entity!
+  "Adds a workspace entity, such as a plan or transform, to the workspace.
+
+   Args:
+   - workspace-id: The workspace ID
+   - entity-key: The key in the workspace map (:plans, :transforms, etc.)
+   - new-entity: The new entity to add, which should be a map with the necessary fields"
+  [workspace-id-or-workspace :- [:or :map :int]
+   entity-key :- ::m.workspace/entity-column
+   new-entity :- :map]
+  (let [{workspace-id :id
+         :as workspace} (if (map? workspace-id-or-workspace)
+                          workspace-id-or-workspace
+                          (t2/select-one :model/Workspace :id workspace-id-or-workspace))
+        _ (when-not workspace (throw (ex-info "Workspace not found" {:error :no-workspace})))
+        current-entities (vec (or (get workspace entity-key) []))
+        entity-with-created-at (assoc new-entity :created_at (str (java.time.Instant/now)))
+        updated-entities (conj current-entities entity-with-created-at)]
+    (t2/update! :model/Workspace workspace-id {entity-key updated-entities})))
+
+(mu/defn add-workspace-entites!
+  "Adds a workspace entity, such as a plan or transform, to the workspace.
+
+   Args:
+   - workspace-id: The workspace ID
+   - entity-key: The key in the workspace map (:plans, :transforms, etc.)
+   - new-entity: The new entity to add, which should be a map with the necessary fields"
+  [workspace-id-or-workspace :- [:or :map :int]
+   entity-key :- ::m.workspace/entity-column
+   new-entities :- [:sequential :any]]
+  (let [{workspace-id :id
+         :as workspace} (if (map? workspace-id-or-workspace)
+                          workspace-id-or-workspace
+                          (t2/select-one :model/Workspace :id workspace-id-or-workspace))
+        _ (when-not workspace (throw (ex-info "Workspace not found" {:error :no-workspace})))
+        current-entities (vec (or (get workspace entity-key) []))
+        entities-with-created-at (map #(assoc % :created_at (str (java.time.Instant/now)))
+                                      new-entities)
+        updated-entities (into current-entities entities-with-created-at)]
+    (t2/update! :model/Workspace workspace-id {entity-key updated-entities})))
+
+(defn divert-transform-target
+  "Returns the target schema that a transform should run in when exectued from the workspace."
+  [workspace]
+  ;; TODO
+  (isolation-manager/isolation-schema-name (:slug workspace "demo")))
+
+(defn link-transform! [workspace-id transform-id]
+  (let [workspace (t2/select-one :model/Workspace :id workspace-id)
+        _ (when-not workspace (throw (ex-info "Workspace not found" {:error :no-workspace})))
+        transform (t2/select-one :model/Transform :id transform-id)
+        _ (when-not transform (throw (ex-info "Transform not found" {:error :no-transform})))
+        copied-transform {:id (:id transform)
+                          :name (:name transform)
+                          :description (:description transform)
+                          :source (:source transform)
+                          :target (:target transform)
+                          :config (:config transform)
+                          :created_at (str (java.time.Instant/now))}]
+    (add-workspace-entity! workspace :transforms copied-transform)
+    (add-workspace-entity! workspace :activity_logs {::type ::linked-transform
+                                                     :transform copied-transform})))
+(defn insert-transform! [workspace-id transform]
+  (let [workspace (t2/select-one :model/Workspace :id workspace-id)
+        _ (when-not workspace (throw (ex-info "Workspace not found" {:error :no-workspace})))
+        transform (assoc transform :created_at (str (java.time.Instant/now)))]
+    (add-workspace-entity! workspace :transforms transform)
+    (add-workspace-entity! workspace :activity_logs {::type ::added-transform
+                                                     :transform transform})))
+
+(comment)
+
+(defn update-workspace-entity-at-index!
+  "Updates an entity at a specific index in the workspace's entity collection.
+
+   Args:
+   - workspace-id: The workspace ID
+   - entity-key: The key in the workspace map (:plans, :transforms, etc.)
+   - index: The 0-based index of the entity to update
+   - update-fn: Function that takes the current entity and returns the updated entity"
+  [workspace-id entity-key index update-fn]
+  (let [workspace (t2/select-one :model/Workspace :id workspace-id)
+        _ (when-not workspace (throw (ex-info "Workspace not found" {:error :no-workspace})))
+        current-items (vec (get workspace entity-key []))
+        item-count (count current-items)
+        _ (when (or (nil? index) (>= index item-count))
+            (throw (ex-info "Index out of bounds" {:error :no-index
+                                                   :item-count item-count})))
+        current-item (nth current-items index nil)
+        new-item (update-fn current-item)
+        updated-items (assoc current-items index new-item)]
+    (t2/update! :model/Workspace workspace-id {entity-key updated-items})))
+
+(mu/defn- drop-index
+  "Note, the insertion order only works when current-items is a vector.
+   Drops the item at the specified index from the vector of current items."
+  [current-items :- [:vector :any] index]
+  (into (subvec current-items 0 index)
+        (subvec current-items (inc index))))
+
+(defn delete-workspace-entity-at-index!
+  "Deletes an entity at a specific index in the workspace's entity collection.
+
+   Args:
+   - workspace-id: The workspace ID
+   - entity-key: The key in the workspace map (:plans, :transforms, etc.)
+   - index: The 0-based index of the entity to delete"
+  [workspace-id entity-key index]
+  (let [workspace (t2/select-one :model/Workspace :id workspace-id)
+        _ (when-not workspace (throw (ex-info "Workspace not found" {:error :no-workspace})))
+        current-items (vec (get workspace entity-key []))
+        _ (when (or (nil? index) (>= index (count current-items)))
+            (throw (ex-info "Index out of bounds" {:error :no-index
+                                                   :item-count (count current-items)})))
+        updated-items (drop-index current-items index)]
+    (t2/update! :model/Workspace workspace-id {entity-key updated-items})
+    (m.workspace/sort-workspace (t2/select-one :model/Workspace :id workspace-id))))
+
+(defn source-database
+  "Return the `source` table of a transform."
+  [transform]
+  (-> transform :source :query :database))
+
+#_(defn delete-isolations! [{data-warehouses :data_warehouses :as workspace}]
+    (if-let [db-ids (seq (map (comp parse-long name) (keys data-warehouses)))]
+      (let [dbs (t2/select [:model/Database :id :engine :details] :id [:in db-ids])]
+        (doseq [{:keys [engine details id]} dbs
+                :let [isolation-info (get data-warehouses (keyword (str id)))]]
+          (log/info "Deleting isolations for workspace" (:id workspace) "for db id:" id engine)
+          (let [deletion-info (isolation-manager/delete-isolation engine details (:slug workspace) isolation-info)]
+          ;; remove it from the workspace's "data_warehouses":
+            (t2/update! :model/Workspace (:id workspace)
+                        {:data_warehouses (dissoc data-warehouses (keyword (str id)))})
+            (add-workspace-entity! workspace :activity_logs {::type ::deleted-isolation
+                                                             :isolation deletion-info}))))
+      (log/warn "No data warehouses found to delete isolations for workspace" (:id workspace))))
+
+(defn create-xray
+  "Create an xray of a model"
+  [workspace model-name]
+  (let [model-id (:id (t2/select-one :model/Card
+                                     :type :model
+                                     :collection_id (:collection_id workspace)
+                                     :name model-name))
+        destination-collection-id (:collection_id workspace)
+        dashboard (api.automagic-dashboards/get-automagic-dashboard :model model-id nil)]
+    (dashboard/save-transient-dashboard! dashboard destination-collection-id)))
+
+;; decision: this is so bespoke to us maybe it goes here now. But perhaps the querying team can help us adjust these
+;; things with a proper api in the future
+(defn patch-transforms
+  "Patch transforms so they point at the new isolation mechanism. Right now, only impelemnted for schema based
+  isolations. So clickhouse doesn't yet work. But clickhouse might just work the same because schema and database are
+  perhaps the same in the way we compile queries for clickhouse."
+  [schema-name transforms]
+  (for [t transforms]
+    (assoc-in t [:target :schema] schema-name)))
+
+(defn create-isolations!
+  "Creates an isolation for each of the workspace's transforms.
+
+   This function will create an isolation schema for each transform's source database.
+   It will throw an error if the workspace has no transforms or if the source database is not found.
+
+  Logs the creation of isolation schemas in the activity_logs of the workspace"
+  [{:keys [data_warehouses] :as workspace} db-ids]
+  (when (seq data_warehouses)
+    (throw (ex-info "Data warehouses already exist. Please delete them before creating new isolations."
+                    {:workspace-id (:id workspace)})))
+  (when (seq db-ids)
+    (let [databases (t2/select :model/Database :id [:in db-ids])
+          isolation-info (into {}
+                               (for [{db-id :id :keys [engine details] :as db} databases]
+                                 (do
+                                   ;; Create isolation for each transform's source database
+                                   (log/info "Creating isolation for workspace" (:id workspace) " db id: " db-id)
+                                   ;; Create the isolation manager for the transform
+                                   [db-id (assoc (isolation-manager/create-isolation engine details (:slug workspace))
+                                                 ::db-engine engine)])))
+          isolation-info-no-results (update-vals isolation-info #(-> %
+                                                                     (dissoc :results)
+                                                                     (assoc :type :isolation-info)))]
+      ;; TODO make this smarter at inserting stuff in 1 round trip:
+      (add-workspace-entity! workspace :activity_logs isolation-info)
+      (t2/update! :model/Workspace (:id workspace)
+                  {:data_warehouses (merge (:data_warehouses workspace) isolation-info-no-results)}))))
+
+;; decision: this is so bespoke to us maybe it goes here now. But perhaps the querying team can help us adjust these
+;; things with a proper api in the future
+(defn- patch-transform
+  "Patch transforms so they point at the new isolation mechanism. Right now, only impelemnted for schema based
+  isolations. So clickhouse doesn't yet work. But clickhouse might just work the same because schema and database are
+  perhaps the same in the way we compile queries for clickhouse."
+  [transform schema-name]
+  (assoc-in transform [:target :schema] schema-name))
+
+(defn- run-transforms*
+  "Execute transforms. If the isolation manager hasn't run, then the data_warhouses key will be empty and this will
+  fail."
+  [{:keys [transforms data_warehouses] :as workspace}]
+
+  ;; we should keep engine around, its necessary for setup and teardown anyways. why aren't we keeping it
+  (let [hack (into {} (map (juxt :id :engine))
+                   (t2/select :model/Database :id
+                              [:in (map (comp parse-long name) (keys data_warehouses))]))
+        ;; this is not needed. But it's a pointer that datawarehouses is perhaps a misleading name? bad
+        ;; terminology. POC doesn't need an answer, but shipping does
+        patched (for [t transforms
+                      :let [db-id (source-database t)
+                            ;; todo: in clickhouse need to get database name
+                            schema-name (or (get-in data_warehouses [(keyword (str db-id)) :schema-name])
+                                            (throw (ex-info "Missing schema for database" {:db-id db-id
+                                                                                           :transform t})))]]
+                  (patch-transform t schema-name))]
+    (reduce (fn [acc t]
+              (let [f (fn [t]
+                        (try {:status :success
+                              :response (transforms.execute/run-mbql-transform! (assoc t :id (- (rand-int 50000)))
+                                                                                {:run-method :workspace})}
+                             (catch Exception e
+                               (log/error "Error running transform" {:transform t
+                                                                     :workspace (select-keys workspace [:id :slug])})
+                               {:status :error
+                                :message (ex-message e)
+                                :exdata (ex-data e)
+                                :transform t})))]
+                (conj acc (f t))))
+            []
+            patched)))
+
+(defn run-transform
+  "Execute transforms. If the isolation manager hasn't run, then the data_warhouses key will be empty and this will
+  fail."
+  [{:keys [transforms data_warehouses] :as workspace} transform-name]
+  (let [transform (first (filter #(= (:name %) transform-name) transforms))
+        _ (when-not transform
+            (throw (ex-info "Transform not found" {:error :no-transform
+                                                   :transform-name transform-name})))
+        source-db-id (source-database transform)
+        schema-name (or (get-in data_warehouses [(keyword (str source-db-id)) :schema-name])
+                        (throw (ex-info "Missing schema for database" {:db-id     source-db-id
+                                                                       :transform transform})))
+        ;; todo: in clickhouse need to get database name
+        patched     (patch-transform transform schema-name)]
+    (try {:status :success
+          :response (transforms.execute/run-mbql-transform!
+                     (assoc patched :id (- (rand-int 50000)))
+                     {:run-method :workspace})}
+         (catch Exception e
+           (log/error "Error running transform" {:transform transform
+                                                 :patched patched
+                                                 :workspace (select-keys workspace [:id :slug])})
+           {:status    :error
+            :message   (ex-message e)
+            :exdata    (ex-data e)
+            :transform transform
+            :patched patched}))))
+
+(comment
+  (t2/select-one :model/Workspace :id 6))
+
+(defn run-transforms
+  "Executes transforms in workspace. Will throw an error if there are no transforms or if the data_warehouses is empty,
+  which likely means that you ahve not created the isolation things."
+  [{:keys [data_warehouses transforms] :as workspace}]
+  (when (empty? data_warehouses)
+    (throw (ex-info "Data warehouses are not prepared. Please run the isolation manager." {:workspace-id (:id workspace)})))
+  (when (empty? transforms)
+    (throw (ex-info "Can't run transforms if there aren't any" {})))
+  ;; don't loev this api, but it's 4:58 on demo day and it's good for now
+  (let [transform-activity (run-transforms* workspace)]
+    ;; todo: assoc some type here?
+    (add-workspace-entites! workspace :activity_logs transform-activity)))
+
+(defn- create-models*
+  [{:keys [transforms data_warehouses collection_id] :as workspace}]
+  (let [hack (into {} (map (juxt :id :engine)) (t2/select :model/Database :id
+                                                          [:in (map (comp parse-long name) (keys data_warehouses))]))
+        ;; this is not needed. But it's a pointer that datawarehouses is perhaps a misleading name? bad
+        ;; terminology. POC doesn't need an answer, but shipping does
+        ;; maybe: db->isolation-info
+        transform-info (:data_warehouses workspace)
+        patched (for [t transforms
+                      :let [db-id (source-database t)
+                            ;; todo: in clickhouse need to get database name
+                            schema-name (or (get-in transform-info [(keyword (str db-id)) :schema-name])
+                                            (throw (ex-info "Missing schema for database" {:db-id db-id
+                                                                                           :transform t})))]]
+                  (patch-transform t schema-name))]
+    (reduce (fn [acc t]
+              (let [f (fn [t]
+                        (let [card {:name (format "Model for %s" (:name t))
+                                    :description (format "model: %s" (:description t))
+                                    :collection_id collection_id
+                                    :card_schema 22 ;;??
+                                    :database_id (source-database t)
+                                    :creator_id api/*current-user-id*
+                                    :query_type :native
+                                    :type :model
+                                    :dataset_query {:database (source-database t)
+                                                    :type :native,
+                                                    :native {:template-tags {},
+                                                             ;; this needs to be formatted correctly for different dbs
+                                                             :query (format "select * from %s.%s"
+                                                                            (-> t :target :schema)
+                                                                            (-> t :target :name))}}
+                                    :display :table
+                                    :visualization_settings {}}]
+                          (try {:status :success
+                                :step :create-model
+                                :transform t
+                                :response (t2/insert! :model/Card card)}
+                               (catch Exception e
+                                 (log/error "Error saving model" {:transform t
+                                                                  :workspace (select-keys workspace [:id :slug])
+                                                                  :card card})
+                                 {:status :error
+                                  :message (ex-message e)
+                                  :exdata (ex-data e)
+                                  :transform t
+                                  :card card}))))]
+                (conj acc (f t))))
+            []
+            patched)))
+
+(defn create-models
+  "Creates models from the transforms in a  workspace. Will throw an error if there are no transforms or if the data_warehouses is empty,
+  which likely means that you ahve not created the isolation things."
+  [{:keys [data_warehouses transforms] :as workspace}]
+  (when (empty? data_warehouses)
+    (throw (ex-info "Data warehouses are not prepared. Please run the isolation manager." {:workspace-id (:id workspace)})))
+  (when (empty? transforms)
+    (throw (ex-info "Can't run transforms if there aren't any" {})))
+  ;; don't loev this api, but it's 4:58 on demo day and it's good for now
+  (let [model-activity (create-models* workspace)]
+    ;; todo: assoc some type here?
+    (add-workspace-entites! workspace :activity_logs {::type ::create-models
+                                                      :model-activity model-activity})))
+
+(defn- create-model* [{:keys [collection_id] :as workspace} transform model-name]
+  (let [table-id (:id (t2/select-one [:model/Table :id]
+                                     :name (-> transform :target :name)
+                                     :schema (-> transform :target :schema)))
+        mp (lib.metadata.jvm/application-database-metadata-provider (source-database transform))
+        query (lib/query mp (lib.metadata/table mp table-id))
+        card {:name model-name
+              :description (format "model: %s" (:description transform))
+              :collection_id collection_id
+              :card_schema 22 ;;??
+              :database_id (source-database transform)
+              :creator_id api/*current-user-id*
+              :query_type :native
+              :table_id table-id
+              :type :model
+              :dataset_query query
+              :display :table
+              :visualization_settings {}}]
+    (try {:status :success
+          :step :create-model
+          :transform transform
+          :response (t2/insert! :model/Card card)}
+         (catch Exception e
+           (log/error "Error saving model" {:transform transform
+                                            :workspace (select-keys workspace [:id :slug])
+                                            :card card})
+           {:status :error
+            :message (ex-message e)
+            :exdata (ex-data e)
+            :transform transform
+            :card card}))))
+
+(defn create-model
+  "Creates models from the transforms in a  workspace. Will throw an error if there are no transforms or if the data_warehouses is empty,
+  which likely means that you ahve not created the isolation things."
+  [{:keys [data_warehouses transforms] :as workspace} transform-name model-name]
+  (when (empty? data_warehouses)
+    (throw (ex-info "Data warehouses are not prepared. Please run the isolation manager." {:workspace-id (:id workspace)})))
+  (when (empty? transforms)
+    (throw (ex-info "Can't run transforms if there aren't any" {})))
+  ;; don't loev this api, but it's 4:58 on demo day and it's good for now
+  (let [transform (first (filter #(= (:name %) transform-name) transforms))
+        schema-name (or (get-in data_warehouses [(keyword (str (source-database transform))) :schema-name])
+                        (throw (ex-info "Missing schema for database" {:db-id     (source-database transform)
+                                                                       :transform transform})))
+        _ (when-not transform
+            (throw (ex-info "Transform not found" {:error :no-transform
+                                                   :transform-name transform-name})))
+        model-activity (create-model* workspace (patch-transform transform schema-name) model-name)]
+    (add-workspace-entity! workspace :activity_logs {::type ::create-models
+                                                     :model-activity model-activity})))
+
+(defn init-workspace [workspace {:keys [transforms] :as plan-data}]
+  (log/info "Initializing workspace" (:id workspace) "with plan data:" plan-data)
+
+  (add-workspace-entity! (:id workspace) :plans plan-data)
+  ;; Insert transforms into the workspace so we can run them in the steps
+  (doseq [t transforms]
+    (log/info "Inserting transform" (:name t) "into workspace" (:id workspace))
+    (insert-transform! (:id workspace) t))
+  (let [source-db-ids (distinct (map source-database transforms))]
+    (create-isolations!
+     (t2/select-one :model/Workspace :id (:id workspace))
+     source-db-ids)))
+
+(defn run-step [workspace {:keys [type] :as step}]
+  (case type
+    :run-transform (run-transform workspace (:name step))
+    :create-model (create-model workspace
+                                (:transform-name step)
+                                (:model-name step))
+    :create-xray (create-xray workspace (:model-name step))))
+
+(defn run-steps [workspace {:keys [steps] :as _plan-data}]
+  (def pd _plan-data)
+  (log/info "Running steps for workspace" (:id workspace) "with steps:" steps)
+  (reduce (fn [acc step]
+            (log/info "Running step:" step)
+            (let [o (try {:step step
+                          :success true
+                          :output (run-step workspace step)}
+                         (catch Exception e
+                           (log/error e "Error running step" step)
+                           {:step step
+                            :message (.getMessage e)
+                            :ex-data (ex-data e)}))]
+              (conj acc o)))
+          []
+          steps))
+
+(comment
+  (require '[metabase.util.malli.registry :as mr])
+
+  (do (require '[metabase.test :as mt])
+      ;; will id alone work here? ^
+      ;; current user is used to generate the api key
+      (def w
+        (binding [api/*current-user-id* (mt/user->id :crowberto)
+                  api/*current-user-permissions-set* (atom #{"/"})]
+          (:workspace (create-workspace! "repl workspacex" nil api/*current-user-id*)))))
+
+  (def t (t2/select-one :model/Transform :name "transform_api"))
+
+  ;; (create-isolations! (t2/select-one :model/Workspace :slug (:slug w)))
+
+  ;; (run-transforms (t2/select-one :model/Workspace :slug (:slug w)))
+
+  ;; (make-models (t2/select-one :model/Workspace :slug (:slug w)))
+
+  ;; (require '[clj-yaml.core :as yaml])
+  ;; (println (yaml/generate-string
+  ;;           ))
+
+  #_:clj-kondo/ignore ;;nocommit
+  (require '[malli.core :as mc] '[malli.error :as me] '[malli.util :as mut] '[metabase.util.malli :as mu]
+           '[metabase.util.malli.describe :as umd] '[malli.provider :as mp] '[malli.generator :as mg]
+           '[malli.transform :as mtx] '[metabase.util.malli.registry :as mr] '[malli.json-schema :as mjs])
+
+  (mr/explain ::m.workspace/plan sample-plan)
+
+  (println (str "\n" (yaml/generate-string sample-plan {:dumper-options
+                                                        {:flow-style :block
+                                                         :default-style nil
+                                                         :line-width 80
+                                                         :indent 2
+                                                         :allow-non-plain-numbers? true}}))))

enterprise/backend/src/metabase_enterprise/workspaces/isolation_manager.clj

@@ -0,0 +1,541 @@
+(ns metabase-enterprise.workspaces.isolation-manager
+  "A namespace that will manage database isolation. Will operate on database connection details compatible with
+  clojure.java.jdbc. The goal is that satisfy a high level goal of creating an isolated workspace in a database that
+  an LLM agent can have read/write access to. This space will be adjacent to data in the rest of the database but
+  isolated to it. We must create three things:
+
+  1. a schema (for postgres, snowflake, redshift) or a database (for clickhouse) where data can be ETL'd into. The
+  schema should have some pattern to it that includes some identifying information of the workspace, and also a prefix
+  so that Metabase knows now to index or sync this schema. It must be very bespoke and at extremely low risk of being
+  confused with real customer data. The schema/database will have the following naming converion:
+
+             mb__isolation_7748c_test_workspace_123
+             |-----------| |---| |----------------|
+                  †         ††         †††
+
+  † mb__isolation: common prefix to all workspace instances. Indicates that metabase owns this and we can sync or not
+  as we like
+
+  †† 7748c: derived from instance-uuid. On my instance `7f43c806-7ef1-4308-8fab-cb41b177db91`
+                                                        7        7    4    8    c
+     Let's an instance know if this isolation unit was created by this instance or not.
+
+  ††† test_workspace_123: a mapping that should tie back to a workspace in some manner. This is largely opaque but
+  should indicate uniquely back to a workspace on this instance.
+
+  2. a user with read privileges to the original data and write access to the new schema
+
+  The public api is
+  (create-isolation driver, connection-details, workspace-id)
+
+  (delete-isolation driver, connection-details, workspace-id, [isolation-info])
+  Isolation info is optional but nice to have. The isolation mechanism should be deterministic from the connection
+  details and worksapce-id.
+
+  It is expected that a workspace can use databases it knows about to create isolations for it's work."
+  (:require
+   [clojure.java.jdbc :as jdbc]
+   [clojure.string :as str]
+   [metabase.driver.ddl.interface :as ddl.i]
+   [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
+   [metabase.driver.sql.util :as sql.u]
+   [metabase.system.core :as system]
+   [metabase.util.log :as log]
+   [metabase.util.malli :as mu]
+   [metabase.util.malli.registry :as mr]))
+
+(defn- instance-uuid-slug
+  "Create a slug from the site UUID, taking the first character of each section."
+  [site-uuid-string]
+  (apply str (map first (str/split site-uuid-string #"-"))))
+
+(defn isolation-schema-name
+  "Generate schema/database name for workspace isolation following mb__isolation_<slug>_<workspace-id> pattern."
+  [workspace-slug]
+  (when-not workspace-slug (throw (ex-info "workspace-slug is required" {:error :no-workspace-slug})))
+  (let [instance-slug (instance-uuid-slug (str (system/site-uuid)))
+        clean-workspace-id (str/replace (str workspace-slug) #"[^a-zA-Z0-9]" "_")]
+    (format "mb__isolation_%s_%s" instance-slug clean-workspace-id)))
+
+(defn isolation-user-name
+  "Generate username for workspace isolation."
+  [workspace-id user-type]
+  (let [instance-slug (instance-uuid-slug (str (system/site-uuid)))
+        clean-workspace-id (str/replace (str workspace-id) #"[^a-zA-Z0-9]" "_")]
+    (format "mb_iso_%s_%s_%s" instance-slug clean-workspace-id (name user-type))))
+
+(mr/def ::rule any?)
+(mr/def ::options :map)
+(mr/def ::steps
+  [:cat :keyword ::options [:* [:alt [:schema [:ref ::steps]] ::rule]]])
+
+(mu/defn- evaluate-steps
+  "Evaluate steps. Steps is a tree of steps matching the ::steps schema. F should be a function that takes in the
+  opaque value of a step. `f` must return a tuple `[:success _]` or `[:error ]`.
+
+  Error handling: by default an error will stop all evaluation. It will report which _steps_ are skipped, and which
+  _trees_ are also skipped. It will not descend into trees to enumerate steps once skipped. You can put
+  `{:error-strategy ::continue-on-error}` as options anywhere in the tree. This option affects subtrees. You can
+  shadow this back to the regular error strategy with any other value to prevent finding the continue on error
+  strategy."
+  [steps :- ::steps f]
+  (letfn [(subtree? [rule] (and (vector? rule) (keyword? (first rule))))
+          (error-strategy [stackframes]
+            (when (seq stackframes)
+              (if-some [strategy (-> stackframes peek :options :error-strategy)]
+                strategy
+                (recur (pop stackframes)))))]
+    (loop [stackframes [{:q       (into clojure.lang.PersistentQueue/EMPTY
+                                        (nthrest steps 2))
+                         :options (second steps)
+                         :tree    (first steps)}]
+           acc         [[:tree (first steps)]]
+           state       :running
+           gas         50]
+      (when (zero? gas) (throw (ex-info "ran out of gas" {:acc acc})))
+      (if-not (seq stackframes)
+        [acc state]
+        (let [{:keys [q] :as frame} (peek stackframes)
+              base-stack            (pop stackframes)
+              rule                  (peek q)]
+          ;; done with this stackframe
+          (cond (not (seq q))
+                (let [state' (if (= state :error)
+                               (let [strategy (error-strategy base-stack)]
+                                 (if (not= strategy ::continue-on-error)
+                                   state
+                                   :running))
+                               state)]
+                  (recur base-stack acc state' (dec gas)))
+
+                ;; scoot through
+                (= state :error)
+                (let [info   (if (subtree? rule)
+                               [:skipping-tree (first rule)]
+                               [:skipping-step rule])
+                      frame' (update frame :q pop)]
+                  (recur (conj base-stack frame') (conj acc info) state (dec gas)))
+
+                ;; hit a tree node
+                (subtree? rule)
+                (let [[tree-name options & sub-rules] rule
+
+                      new-stackframe {:q       (into clojure.lang.PersistentQueue/EMPTY
+                                                     sub-rules)
+                                      :options options
+                                      :tree    tree-name}
+                      stackframes    (conj base-stack
+                                           (update frame :q pop)
+                                           new-stackframe)]
+                  (recur stackframes
+                         (conj acc [:tree tree-name])
+                         state
+                         (dec gas)))
+                :else
+                (let [[status _response :as result] (f rule)]
+                  (when-not (#{:error :success} status)
+                    (throw (ex-info "Bad status from rule" {:rule rule :result result})))
+                  (let [switch-to-error? (and (= :error status)
+                                              (not= (error-strategy stackframes)
+                                                    ::continue-on-error))]
+                    (recur (conj base-stack (update frame :q pop))
+                           (conj acc result)
+                           (if switch-to-error? :error state)
+                           (dec gas))))))))))
+
+(defmulti ^:private create-isolation*
+  "Create database isolation for a workspace."
+  (fn [driver workspace] driver))
+
+;; todo: use (jdbc/metadata-result (.getSchemas metadata))
+
+(mu/defn- postgres-steps :- ::steps
+  "Return postgres steps. Needs the populator and read users (maps with user and password) and the schema name to be
+  populated. Will create the schema, uses, and setup privileges. Currently hardcoded to give the populator access to
+  the `public` schema. This will need to be updated to include the schemas from the source database."
+  [{:keys [populator schema-name]}]
+  [:schema
+   {}
+   (format "CREATE SCHEMA %s" schema-name)
+   [:users
+    {}
+    ;; Create populator user with random password
+    [:populator
+     {}
+     (format "CREATE USER %s WITH PASSWORD '%s'" (:user populator) (:password populator))
+     [:populator-privileges
+      {}
+      ;; Grant privileges to populator user
+
+      ;; source data hardcorded to public: FIX
+      (format "GRANT USAGE ON SCHEMA %s TO %s" "public" (:user populator))
+      ;; todo: don't hardcode this to public
+      (format "GRANT USAGE ON SCHEMA public TO %s" (:user populator))
+
+      (format "GRANT USAGE ON SCHEMA %s TO %s" schema-name (:user populator))
+      (format "GRANT SELECT ON ALL TABLES IN SCHEMA %s TO %s" "public" (:user populator))
+      (format "GRANT CREATE ON SCHEMA %s TO %s" schema-name (:user populator))
+      (format "GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA %s TO %s"
+              schema-name (:user populator))]]]])
+
+(mu/defn snowflake-steps :- ::steps
+  [{:keys [populator schema-name source-database-name]}]
+  (letfn [(q [d]
+            (try
+              (sql.u/quote-name :clickhouse :database (ddl.i/format-name :clickhouse d))
+              (catch Exception e
+                (log/errorf e "Error quoting %s" (pr-str d))
+                (throw e))))]
+    [:use-database
+     {}
+     (format "USE DATABASE \"%s\" " source-database-name)
+     [:schema
+      {}
+      (format "CREATE SCHEMA IF NOT EXISTS %s" schema-name)
+      ;; todo: snowflake extra users. They are difficult to get right with roles and quoting
+      #_[:usage-on-new-schema
+         {}
+         ;; hardcoded developer. ownership of new schema to developer conflicts with next bit
+         #_(format "GRANT OWNERSHIP ON SCHEMA %s to ROLE DEVELOPER" schema-name)
+         #_(format "GRANT USAGE ON SCHEMA %s to ROLE DEVELOPER" schema-name)]
+      #_[:users
+         {}
+       ;; Create populator user with random password
+         [:populator-role
+          {}
+          (format "CREATE ROLE \"%s\" " (:role populator))
+          [:populator
+           {}
+           (format "CREATE OR REPLACE USER %s PASSWORD = '%s' DEFAULT_ROLE='%s'"
+                   (:user populator)
+                   (:password populator)
+                   (:role populator))
+           [:populator-privileges
+            {}
+          ;; Grant privileges to populator user
+
+          ;; source data hardcorded to public: FIX
+            (format "GRANT OWNERSHIP ON SCHEMA %s TO \"%s\" " schema-name #_(:user populator) (:role populator))
+          ;; todo: don't hardcode this to public
+            (format "GRANT USAGE ON DATABASE \"%s\" TO \"%s\" " source-database-name #_(:user populator)  (:role populator))]
+           [:assign-role
+            {}
+            (format "GRANT ROLE \"%s\" to USER %s" (:role populator) (:user populator))]]]]]]))
+
+(comment
+  (let [spec (metabase.driver.sql-jdbc.connection/db->pooled-connection-spec (toucan2.core/select-one-fn :id :model/Database :id 2))]
+    (jdbc/query spec ["select current_role()"])))
+
+(defmethod create-isolation* :snowflake
+  [_driver {:keys [id connection-details]}]
+  ;; schema name is automatically uppercased by snowflake, but then if you quote it it doesn't find the uppercased
+  ;; version.
+  (let [schema-name (str/upper-case (isolation-schema-name id))
+        jdbc-spec (sql-jdbc.conn/connection-details->spec :snowflake connection-details)
+        populator {:user (isolation-user-name id :populator)
+                   :password (str (random-uuid))
+                   :role (str (random-uuid))}
+        steps (snowflake-steps {:populator populator
+                                :schema-name schema-name
+                                :source-database-name ((some-fn :db :dbname) connection-details)})]
+    (jdbc/with-db-transaction [tx jdbc-spec]
+      (let [results (evaluate-steps steps (fn [sql]
+                                            (try [:success sql (jdbc/execute! tx
+                                                                              (if (string? sql)
+                                                                                [sql]
+                                                                                sql))]
+                                                 (catch Exception e
+                                                   (log/error e)
+                                                   [:error sql (ex-message e)]))))]
+        {:isolation-type :schema
+         :schema-name schema-name
+         ;;:populator populator
+         :results results}))))
+
+(comment
+  (malli.core/validate ::steps
+                       (snowflake-steps {:populator            {:user "populator_user123" :password (str (random-uuid))}
+                                         :schema-name          (isolation-schema-name "workspace_1234")
+                                         :source-database-name "sales"}))
+  (postgres-steps {:populator {:user "populator" :password "populator-pw"}
+                   :schema-name "scratchpad"})
+  (malli.core/validate ::steps
+                       (postgres-steps {:populator {:user "populator" :password "populator-pw"}
+                                        :schema-name "scratchpad"}))
+
+  (evaluate-steps (postgres-steps {:populator {:user "populator" :password "populator-pw"}
+                                   :schema-name "scratchpad"})
+                  (fn [step] (if (= 0 (rand-int 10))
+                               [:error (format "error during: %s" step)]
+                               [:success (format "performed: %s" step)])))
+
+  (evaluate-steps [:schema
+                   {:error-strategy ::continue-on-error}
+                   "CREATE SCHEMA scratchpad"
+                   [:users
+                    {}
+                    [:populator
+                     {}
+                     "CREATE USER populator WITH PASSWORD 'populator-pw'"
+                     [:populator-privileges
+                      {}
+                      "GRANT USAGE ON SCHEMA public TO populator"
+                      "GRANT USAGE ON SCHEMA public TO populator"
+                      "GRANT USAGE ON SCHEMA scratchpad TO populator"
+                      "GRANT SELECT ON ALL TABLES IN SCHEMA public TO populator"
+                      "GRANT CREATE ON SCHEMA scratchpad TO populator"
+                      "GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA scratchpad TO populator"]]]]
+                  (fn [step] (if (= 0 (rand-int 10))
+                               [:error (format "error during: %s" step)]
+                               [:success (format "performed: %s" step)]))))
+
+(defmethod create-isolation* :postgres
+  [_driver {:keys [id connection-details]}]
+  (let [schema-name (isolation-schema-name id)
+        populator {:user (isolation-user-name id :populator)
+                   :password (str (random-uuid))}
+        jdbc-spec (sql-jdbc.conn/connection-details->spec :postgres connection-details)
+        steps (postgres-steps {:populator populator
+                               :schema-name schema-name})]
+    (jdbc/with-db-transaction [tx jdbc-spec]
+      ;; Create schema
+
+      (let [results (evaluate-steps steps (fn [sql]
+                                            (try [:success
+                                                  sql
+                                                  (jdbc/execute! tx [sql])]
+                                                 (catch Exception e
+                                                   [:error sql (ex-message e)]))))]
+        ;; Return isolation info
+        {:isolation-type :schema
+         :schema-name schema-name
+         :populator populator
+         :results results}))))
+
+;; todo: create an evaluator of steps. steps will perhaps for a dag, preconditions, checks, failures. ideally a test
+;; would have a list of all steps, randomly execute one, and then execute all steps and ensure that we can proceed to
+;; the end. ie, any particular step tha tis already satisfied (schema already exists, user already exists) is ok and
+;; we can carry on.
+
+(mu/defn- clickhouse-steps :- ::steps
+  [{:keys [populator database-name source-database-name]}]
+  (letfn [(q [d]
+            (try
+              (sql.u/quote-name :clickhouse :database (ddl.i/format-name :clickhouse d))
+              (catch Exception e
+                (log/errorf e "Error quoting %s" (pr-str d))
+                (throw e))))]
+    [:database
+     {}
+     (format "CREATE DATABASE %s" (q database-name))
+     [:users
+      {}
+      [:populator
+       {}
+       (format "CREATE USER %s IDENTIFIED WITH sha256_password BY '%s'"
+               (:user populator) (:password populator))
+       [:populator-privileges
+        {}
+        (format "GRANT SELECT ON %s.* TO %s" (q source-database-name)  (:user populator))
+        (format "GRANT CREATE ON %s.* TO %s" (q database-name) (:user populator))
+        (format "GRANT INSERT ON %s.* TO %s" (q database-name) (:user populator))
+        (format "GRANT SELECT ON %s.* TO %s" (q database-name) (:user populator))
+        (format "GRANT DROP ON %s.* TO %s" (q database-name) (:user populator))
+        (format "GRANT SHOW DATABASES ON *.* TO %s" (:user populator))]]]]))
+
+(comment
+
+  (malli.core/validate ::steps
+                       (clickhouse-steps {:populator {:user "populator" :password "populator-pw"}
+                                          :database-name "scratchpad"
+                                          :source-database-name "foo"})))
+
+(defmethod create-isolation* :clickhouse
+  [_driver {:keys [id connection-details]}]
+  (let [database-name (isolation-schema-name id)
+        populator {:user (isolation-user-name id :populator) :password (str (random-uuid))}
+        jdbc-spec (sql-jdbc.conn/connection-details->spec :clickhouse connection-details)
+        steps (clickhouse-steps {:populator populator :database-name database-name
+                                 :source-database-name ((some-fn :db :dbname) connection-details)})]
+
+    (jdbc/with-db-transaction [tx jdbc-spec]
+      (let [results (evaluate-steps steps (fn [sql]
+                                            (try [:success sql (jdbc/execute! tx [sql])]
+                                                 (catch Exception e
+                                                   [:error sql (ex-message e)]))))]
+        {:isolation-type :database
+         :database-name database-name
+         :populator populator
+         :results results}))))
+
+(comment
+  (require '[toucan2.core :as t2])
+
+  ;; future work to check for existing schemas
+  (let [metadata (.getMetaData (:connection tx))]
+    (let [schemas (into [] (map :table_schem) (jdbc/result-set-seq (.getSchemas metadata)))]
+      {:schemas schemas
+       :isolations (filter #(String/.startsWith % "mb__isolation_") schemas)
+       :our-isolations (filter #(String/.startsWith % "mb__isolation_7748c") schemas)
+       :already-exists? (some #{schema-name} schemas)
+       :driver _driver}))
+
+  ;; we're a bit sloppy with db vs dbname?
+  (let [workspace-id "workspace_manual_01"
+        connection-details {:scan-all-databases false, :ssl false, :password "password", :destination-database false,
+                            :port 8123, :advanced-options false, :dbname "sales_data", :host "localhost",
+                            :tunnel-enabled false, :user "default"}]
+
+    (create-isolation
+     :postgres
+     (:connection-details (t2/select-one-fn :details :model/Database 2))
+     workspace-id))
+
+  (delete-isolation :postgres (:connection-details (t2/select-one-fn :details :model/Database 2)) "workspace_manual_01")
+
+  {:isolation-type :database,
+   :database-name "mb__isolation_7748c_workspace_manual_01",
+   :populator {:user "mb_iso_7748c_workspace_manual_01_populator",
+               :password "9d8285d5-94c3-4e87-a98a-1c47c9fb727b"}}
+
+  (delete-isolation :clickhouse (:connection-details workspace) (:id workspace))
+  {:deleted-database "mb__isolation_7748c_workspace_manual_01",
+   :deleted-users ["mb_iso_7748c_workspace_manual_01_populator"]})
+
+(defn create-isolation
+  "Create database isolation for a single database.
+
+  Creates an isolated environment in the specified database where LLM agents can operate safely.
+  The isolation includes:
+  - A schema (PostgreSQL) or database (ClickHouse) with workspace-specific naming
+  - A populator user with read access to original data + write access to isolated area
+
+  Args:
+    engine             - Database engine keyword (:postgres, :clickhouse, etc.)
+    connection-details - Database connection details (compatible with Metabase format)
+    workspace-slug     - Unique workspace identifier for naming isolation resources
+
+  Returns:
+    Map containing isolation details:
+      :schema-name/:database-name - Name of created isolation area
+      :populator                  - map with user and password for populator (read original +
+                                    write isolated)
+
+  Example:
+    (create-isolation :postgres db-connection-details \"workspace-123\")
+       ;; postgres isolations by schema
+       {:isolation-type :schema
+        :schema-name    \"mb__isolation_7748c_test_workspace_123\"
+        :populator      {:user \"mb_iso_7748c_test_workspace_123_populator\"
+                         :password \"d632f61c-398a-467d-856c-d8411665bd0f\"}}
+
+       ;; clickhouse isolates by database
+      {:isolation-type :database
+       :database-name  \"mb__isolation_7748c_testing_workspace216479\"
+       :populator      {:user     \"mb_iso_7748c_testing_workspace216479_populator\"
+                        :password \"a9445a75-621f-4211-82e1-9f96ee797dd6\"}}"
+  [engine connection-details workspace-id]
+  (-> (create-isolation* engine {:id workspace-id :connection-details connection-details})
+      (assoc :engine engine)))
+
+(defmulti ^:private delete-isolation*
+  "Delete database isolation for a workspace. This is deliberately not public. Use `delete-isolation` as an entrypoint."
+  (fn [driver workspace] driver))
+
+(defmethod delete-isolation* :postgres
+  [_driver {:keys [id connection-details isolation-info]}]
+  (let [schema-name    (or (:schema-name isolation-info)
+                           (str/upper-case (isolation-schema-name id)))
+        populator-user (or (-> isolation-info :populator :user)
+                           (isolation-user-name id :populator))
+        jdbc-spec      (sql-jdbc.conn/connection-details->spec :postgres connection-details)
+        teardown-steps [:cleanup
+                        {:error-strategy ::continue-on-error}
+                        [:remove-privileges
+                         {}
+                         (format "REVOKE ALL PRIVILEGES ON ALL TABLES IN  SCHEMA %s FROM %s" "public" populator-user)
+                         (format "REVOKE ALL PRIVILEGES ON SCHEMA %s FROM %s" "public" populator-user)]
+                        [:remove-new-schema
+                         {}
+                         ;; Drop schema (CASCADE to drop all objects in it).
+                         (format "DROP SCHEMA IF EXISTS %s CASCADE" schema-name)]
+                        [:drop-user
+                         {}
+                         ;; Drop users
+                         (format "DROP USER IF EXISTS %s" populator-user)]]]
+    (jdbc/with-db-transaction [tx jdbc-spec]
+      (let [results (evaluate-steps teardown-steps (fn [sql]
+                                                     (try [:success sql (jdbc/execute! tx [sql])]
+                                                          (catch Exception e
+                                                            [:error sql (ex-message e)]))))]
+        {:deleted-schema schema-name
+         :deleted-users  [populator-user]
+         :results        results}))))
+
+(defmethod delete-isolation* :clickhouse
+  [_driver {:keys [id connection-details isolation-info]}]
+  (let [database-name (or (:database-name isolation-info)
+                          (isolation-schema-name id))
+        populator-user (or (-> isolation-info :populator :user)
+                           (isolation-user-name id :populator))
+        jdbc-spec (sql-jdbc.conn/connection-details->spec :clickhouse connection-details)
+        steps [:cleanup
+               {:error-strategy ::continue-on-error}
+               [:drop-database {}
+                (format "DROP DATABASE IF EXISTS %s" database-name)]
+               [:remove-users {}
+                (format "DROP USER IF EXISTS %s" populator-user)]]]
+    (jdbc/with-db-transaction [tx jdbc-spec]
+      (let [results (evaluate-steps steps (fn [sql]
+                                            (try [:success sql (jdbc/execute! tx [sql])]
+                                                 (catch Exception e
+                                                   [:error sql (ex-message e)]))))]
+        {:deleted-database database-name
+         :deleted-users [populator-user]
+         :results results}))))
+
+(defmethod delete-isolation* :snowflake
+  [_driver {:keys [id connection-details isolation-info] :as x}]
+  (let [schema-name    (or (:schema-name isolation-info)
+                           (isolation-schema-name id))
+        populator-user (or (-> isolation-info :populator :user)
+                           (isolation-user-name id :populator))
+        jdbc-spec      (sql-jdbc.conn/connection-details->spec :snowflake connection-details)
+        steps          [:cleanup {:error-strategy ::continue-on-error}
+                        [:drop-schema {} (format "DROP SCHEMA %s" schema-name)]
+                        [:drop-user {} (format "DROP USER %s" populator-user)]
+                        [:drop-role {} (format "DROP ROLE %s" (-> isolation-info :populator :role))]]]
+    (jdbc/with-db-transaction [tx jdbc-spec]
+      (let [results (evaluate-steps steps (fn [sql]
+                                            (try [:success sql (jdbc/execute! tx [sql])]
+                                                 (catch Exception e
+                                                   [:error sql (ex-message e)]))))]
+        {:deleted-schema schema-name :deleted-users [populator-user] :results results}))))
+
+(defn delete-isolation
+  "Delete database isolation for a single database.
+
+  Cleans up all isolation resources created for the specified database:
+  - Drops the isolated schema/database
+  - Drops the created user (populator)
+  - Revokes any granted privileges
+
+  Args:
+    engine             - Database engine keyword (:postgres, :clickhouse, etc.)
+    connection-details - Database connection details (compatible with Metabase format)
+    workspace-id       - Unique workspace identifier used when creating isolation
+    isolation-info     - (Optional) Result from create-isolation for more efficient cleanup
+
+  Returns:
+    Map containing cleanup details:
+      :deleted-schema/:deleted-database - Name of dropped isolation area
+      :deleted-users                   - List of dropped usernames
+
+  Example:
+    (delete-isolation :postgres db-connection-details \"workspace-123\" isolation-result)"
+  ([engine connection-details workspace-id]
+   (delete-isolation* engine {:id workspace-id :connection-details connection-details}))
+  ([engine connection-details workspace-id isolation-info]
+   (delete-isolation* engine {:id workspace-id
+                              :connection-details connection-details
+                              :isolation-info isolation-info})))

enterprise/backend/src/metabase_enterprise/workspaces/models/workspace.clj

@@ -0,0 +1,282 @@
+(ns metabase-enterprise.workspaces.models.workspace
+  "Workspaces are a container that an LLM can use to store and manage state. They are the answer to 'How do we allow LLMs to do stuff, in a safe way?'.
+
+  It has locked down permissions and
+   is used to store information that the LLM can use to answer questions or perform tasks. It is not a database, but
+   rather a way to manage state in a way that is secure and controlled.
+
+  There's also a concept of a workspace having a user and a workspace having a database. The user has limited access to the database so that they can only
+   access the data that is relevant to them. The workspace is a way to manage this access and control what the user can do with the data.
+
+  Workspace users are not the same as Metabase users. They do not have a permission group, They are a separate concept
+  that is used to manage access to the workspace and the data within it. Creating a workspace should also create a user.
+
+  Workspaces are independent of one another by design. A workspace can be deleted without affecting other workspaces.
+  And they cannot be used together. This is to ensure that the data in one workspace cannot be accessed by another
+  workspace user. "
+  (:require
+   [babashka.fs :as fs]
+   [clj-yaml.core :as yaml]
+   [clojure.string :as str]
+   [java-time.api :as t]
+   [malli.generator :as mg]
+   [malli.util :as mut]
+   [metabase-enterprise.transforms.ordering :as transforms.ordering]
+   [metabase.models.interface :as mi]
+   [metabase.query-processor.store :as qp.store]
+   [metabase.util :as u]
+   [metabase.util.malli :as mu]
+   [metabase.util.malli.registry :as mr]
+   [metabase.util.malli.schema :as ms]
+   [methodical.core :as methodical]
+   [toucan2.core :as t2]))
+
+(mr/def ::plan.step
+  [:multi {:dispatch :type}
+   [:run-transform [:map [:name :string]]]
+   [:create-model [:map
+                   [:transform-name
+                    {:description "The name of the transform to create a model from"} :string]
+                   [:model-name {:description "The name of the model to create"} :string]]]
+   [:create-xray [:map
+                  [:name :string]
+                  [:model-name
+                   {:description "The name of the model to create an xray dashboard from"} :string]]]])
+
+(mr/def ::transform
+  [:map
+   [:name :string]
+   [:description :nil]
+   [:source :map]
+   [:target :map]
+   [:config {:optional true} :any]])
+
+(mr/def ::plan
+  [:map
+   [:transforms [:sequential ::transform]]
+   [:steps [:sequential ::plan.step]]])
+
+;;;;;;;;;;; column checking:
+
+(mr/def ::created_at
+  [:string {:gen/fmap #(str (t/instant (hash %)))}])
+
+;; maybe another table?
+(mr/def ::activity-log [:map
+                        [:name [:string {:min 1}]]
+                        ;; needs a plan?
+                        [:steps [:sequential
+                                 [:map
+                                  [:step-id [:int {:min 0}]]
+                                  [:description :string]
+                                  [:status [:enum :pending :running :completed :failed]]
+                                  [:outcome {:optional true} [:enum :success :error :warning]]
+                                  [:error-message {:optional true} :string]
+                                  [:start-time [:string {:description "When the step started"}]]
+                                  [:end-time {:optional true} [:maybe {:description "When the step ended"} :string]]
+                                  [:created_at ::created_at]
+                                  [:updated-at [:string {:description "When the step was last updated"}]]]]]])
+
+;; look at transforms, make it match
+
+;; delete me:
+(mr/def ::document :any)
+(mr/def ::user :any)
+(mr/def ::permission :any)
+
+(mr/def ::workspace.name
+  [:string {:min 1}])
+
+(mr/def ::workspace.description
+  [:maybe :string])
+
+(mr/def ::workspace
+  [:map
+   [:id              {:optional true} [:maybe [:int {:min 1}]]]
+   [:name            [:ref ::workspace.name]]
+   [:slug            {:optional true} [:maybe [:string {:min 1}]]]
+   [:description     {:optional true} [:ref ::workspace.description]]
+   [:created_at      [:schema
+                      {:description "The date and time the workspace was created"}
+                      (ms/InstanceOfClass java.time.OffsetDateTime)]]
+   [:updated_at      [:schema
+                      {:description "The date and time the workspace was last updated"}
+                      (ms/InstanceOfClass java.time.OffsetDateTime)]]
+   [:plan            ::plan]
+   [:activity_logs   [:sequential ::activity-log]] ;; This should maybe be another table:
+   [:transforms      [:sequential ::transform]]
+   [:documents       [:sequential ::document]]
+   [:users           [:sequential ::user]]
+   [:data_warehouses [:map-of {:description "data warehouse id -> isolation info"} :int :map]]
+   [:collection_id   pos-int?]
+   [:api_key_id      pos-int?]
+   [:attributes      {:optional true} [:maybe [:map-of :string :any]]]])
+
+(def entity-cols
+  "Columns that are considered entities in the workspace model. They are stored as json on the workspace table."
+  [:plans :activity_logs :transforms :documents :users :data_warehouses :permissions])
+
+(mr/def ::entity-column (into [:enum] entity-cols))
+
+(methodical/defmethod t2/table-name :model/Workspace [_model] :workspace)
+
+(defn- generate-slug
+  "Generates a unique slug for the workspace based on its name.
+   Uses `gensym` to ensure uniqueness, especially useful for testing or when names might collide."
+  [{:keys [name]}]
+  (str (str/replace name " " "_") "_" (gensym "")))
+
+(t2/define-before-insert :model/Workspace
+  [workspace]
+  (assoc workspace :slug (generate-slug workspace)))
+
+(t2/define-before-delete :model/Workspace
+  [{api-key-id :api_key_id, :as workspace}]
+  (println "api-key-id:" api-key-id) ; NOCOMMIT
+  (u/prog1 workspace
+    (t2/delete! :model/ApiKey :id api-key-id)))
+
+(t2/deftransforms :model/Workspace
+  {:plans           mi/transform-json
+   :activity_logs   mi/transform-json
+   :transforms      mi/transform-json
+   :documents       mi/transform-json
+   :users           mi/transform-json
+   :data_warehouses mi/transform-json
+   :permissions     mi/transform-json
+   :attributes      mi/transform-json})
+
+(doto :model/Workspace
+  (derive :metabase/model)
+  (derive :hook/timestamped?))
+
+(defn- sort-toplevel-keys
+  "Sorts top-level keys in the order they're defined in the schema."
+  [k1 k2]
+  (let [order (zipmap (mut/keys (mr/resolve-schema ::workspace)) (range))
+        a (get order k1)
+        b (get order k2)]
+    (if (and a b)
+      (compare a b)
+      (compare (name k1) (name k2)))))
+
+(defn- created-at-sort
+  "Sort a collection of workspaces by their `created_at` timestamp, newest first.
+   Handles nil or invalid created_at values by treating them as oldest."
+  [xs]
+  (vec (sort-by (fn [item]
+                  (try
+                    (when-let [created_at (get item :created_at)]
+                      (t/instant created_at))
+                    (catch Exception _
+                      ;; Return a very old instant for invalid dates
+                      (t/instant "1970-01-01T00:00:00Z"))))
+                #(compare %2 %1) ; Reverse comparison for newest first
+                xs)))
+
+(defn sort-workspace
+  "Required for a stable diff in a yaml view"
+  [workspace]
+  (-> (into (sorted-map-by sort-toplevel-keys) workspace)
+      (update :documents (fnil (comp vec sort) []))
+      (update :plans (fnil created-at-sort []))
+      (update :transforms (fnil created-at-sort []))
+      (update :users (fnil created-at-sort []))
+      (update :permissions (fnil created-at-sort []))))
+
+(defn transform-table-dependencies
+  "Identifies the database tables that a transform depends on.
+
+   For workspace transforms, this function takes a transform object and returns
+   a set of table IDs that the transform's source query depends on.
+
+   Args:
+   - transform: A transform object with :source containing query information
+
+   Returns:
+   - A set of table IDs (integers) that the transform depends on"
+  [transform]
+  (when-let [db-id (get-in transform [:source :query :database])]
+    (qp.store/with-metadata-provider db-id
+      (#'transforms.ordering/transform-deps transform))))
+
+(defn workspace-transform-dependencies
+  "Analyzes all transforms in a workspace and returns their table dependencies.
+
+   Args:
+   - workspace: A workspace object containing transforms
+
+   Returns:
+   - A map of transform name -> set of table IDs that transform depends on"
+  [workspace]
+  (into {}
+        (keep (fn [transform]
+                (when-let [deps (transform-table-dependencies transform)]
+                  [(:name transform) deps])))
+        (:transforms workspace [])))
+
+(defn transform-table-dependencies-detailed
+  "Returns detailed information about the tables a transform depends on.
+
+   Args:
+   - transform: A transform object with :source containing query information
+
+   Returns:
+   - A seq of maps containing table details: {:id, :name, :schema, :display_name, etc.}"
+  [transform]
+  (when-let [table-ids (transform-table-dependencies transform)]
+    (when (seq table-ids)
+      (t2/select [:model/Table :id :name :schema :display_name :description :db_id]
+                 :id [:in table-ids]))))
+
+(defn workspace-transform-dependencies-detailed
+  "Returns detailed table dependency information for all transforms in a workspace.
+
+   Args:
+   - workspace: A workspace object containing transforms
+
+   Returns:
+   - A map of transform name -> seq of detailed table information"
+  [workspace]
+  (into {}
+        (keep (fn [transform]
+                (when-let [deps (transform-table-dependencies-detailed transform)]
+                  [(:name transform) deps])))
+        (:transforms workspace [])))
+
+(comment
+  (require '[clojure.walk :as walk])
+
+  (require '[malli.core :as mc] '[malli.error :as me] '[malli.util :as mut] '[metabase.util.malli :as mu]
+           '[metabase.util.malli.describe :as umd] '[malli.provider :as mp] '[malli.generator :as mg]
+           '[malli.transform :as mtx] '[metabase.util.malli.registry :as mr] '[malli.json-schema :as mjs])
+
+  (defn super-get [m k]
+    (let [found (volatile! [])]
+      (walk/postwalk
+       (fn [x]
+         (when (and (coll? x)
+                    (= 2 (count x))
+                    (= k (first x)))
+           (vswap! found conj (second x)))
+         x)
+       m)
+      @found))
+
+  (transform-table-dependencies (:transforms (sort-workspace (t2/select-one :model/Workspace 254))))
+  ;; correct order:
+  ;; => ("2025-08-08T21:16:50.420896Z"
+  ;;     "2025-08-08T21:13:40.567837Z"
+  ;;     "2025-08-08T21:07:45.052178Z"
+  ;;     "2025-08-08T21:03:43.194484Z"
+  ;;     "2025-08-08T20:07:45.709135Z")
+
+;; => ("2025-08-08T20:07:45.709135Z"
+;;     "2025-08-08T21:16:50.420896Z"
+;;     "2025-08-08T21:13:40.567837Z"
+;;     "2025-08-08T21:07:45.052178Z"
+;;     "2025-08-08T21:03:43.194484Z")
+  )
+
+(mu/defn- ->yaml [workspace]
+  (yaml/generate-string (sort-workspace workspace) :dumper-options {:flow-style :block}))

enterprise/backend/test/metabase_enterprise/bookmarks/api_test.clj

@@ -0,0 +1,79 @@
+(ns metabase-enterprise.bookmarks.api-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase.models.interface :as mi]
+   [metabase.test :as mt]
+   [metabase.util :as u]
+   [toucan2.core :as t2]))
+
+(defn bookmark-models [user-id & models]
+  (doseq [model models]
+    (cond
+      (mi/instance-of? :model/Card model)
+      (t2/insert! :model/CardBookmark
+                  {:user_id user-id
+                   :card_id (u/the-id model)})
+
+      (mi/instance-of? :model/Document model)
+      (t2/insert! :model/DocumentBookmark
+                  {:user_id user-id
+                   :document_id (u/the-id model)})
+
+      :else
+      (throw (ex-info "Unknown type" {:user-id user-id :model model})))))
+
+(deftest document-bookmarks-test
+  (testing "Document bookmarks with enterprise features"
+    (mt/with-premium-features #{:documents}
+      (mt/with-temp [:model/Collection {coll-id :id} {:name "Test Collection"}
+                     :model/Document document {:name "Test Document" :collection_id coll-id}]
+        (testing "can bookmark a document when enterprise enabled"
+          (is (= (u/the-id document)
+                 (->> (mt/user-http-request :rasta :post 200 (str "bookmark/document/" (u/the-id document)))
+                      :document_id))))
+
+        (testing "document appears in bookmark list"
+          (let [result (mt/user-http-request :rasta :get 200 "bookmark")
+                document-bookmark (first (filter #(= (:type %) "document") result))]
+            (is (some? document-bookmark))
+            (is (= "Test Document" (:name document-bookmark)))
+            (is (= (u/the-id document) (:item_id document-bookmark)))))
+
+        (testing "can delete document bookmark"
+          (mt/user-http-request :rasta :delete 204 (str "bookmark/document/" (u/the-id document)))
+          (is (empty? (filter #(= (:type %) "document")
+                              (mt/user-http-request :rasta :get 200 "bookmark")))))
+
+        (testing "document bookmarks are included in ordering"
+          (mt/with-temp [:model/Card card {:name "Test Card"}]
+            (mt/with-model-cleanup [:model/BookmarkOrdering]
+              (bookmark-models (mt/user->id :rasta) document card)
+              (mt/user-http-request :rasta :put 204 "bookmark/ordering"
+                                    {:orderings [{:type "document" :item_id (u/the-id document)}
+                                                 {:type "card" :item_id (u/the-id card)}]})
+              (is (= ["document" "card"]
+                     (map :type (mt/user-http-request :rasta :get 200 "bookmark"))))))))))
+
+  (testing "Document bookmarks without enterprise features"
+    (mt/with-premium-features #{}
+      (mt/with-temp [:model/Collection {coll-id :id} {:name "Test Collection"}
+                     :model/Document document {:name "Test Document" :collection_id coll-id}]
+        (testing "cannot bookmark documents when enterprise disabled"
+          (mt/user-http-request :rasta :post 402 (str "bookmark/document/" (u/the-id document))))
+
+        (testing "document model not included in allowed models"
+          (let [response (mt/user-http-request :rasta :post 402 (str "bookmark/document/" (u/the-id document)))]
+            (is (=? {:message #"Documents is a paid feature.*"}
+                    response))))))))
+
+(deftest document-bookmarks-archived-test
+  (testing "Document bookmarks on archived documents"
+    (mt/with-premium-features #{:documents}
+      (mt/with-temp [:model/Collection {coll-id :id} {:name "Test Collection"}
+                     :model/Document archived-document {:name "Archived Document"
+                                                        :collection_id coll-id
+                                                        :archived true}]
+        (bookmark-models (mt/user->id :rasta) archived-document)
+        (testing "archived documents don't appear in bookmark list"
+          (is (empty? (filter #(= (:type %) "document")
+                              (mt/user-http-request :rasta :get 200 "bookmark")))))))))

enterprise/backend/test/metabase_enterprise/documents/api/collection_test.clj

@@ -0,0 +1,294 @@
+(ns metabase-enterprise.documents.api.collection-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase.collections.models.collection :as collection]
+   [metabase.test :as mt]))
+
+(deftest document-cards-do-not-appear-in-collection-items
+  (testing "GET /api/collection/:id/items excludes cards with document_id"
+    (mt/with-premium-features #{:documents}
+      (mt/with-temp [:model/Collection {coll-id :id} {}
+                     :model/Document {doc-id :id} {}
+                     :model/Card {normal-card-id :id} {:collection_id coll-id}
+                     :model/Card _ {:collection_id coll-id
+                                    :document_id doc-id}
+                     :model/Dashboard {dash-id :id} {:collection_id coll-id}]
+        (testing "Normal cards and dashboards appear, but in_document cards do not"
+          (is (= #{[normal-card-id "card"]
+                   [dash-id "dashboard"]}
+                 (set (map (juxt :id :model)
+                           (:data (mt/user-http-request :rasta :get 200
+                                                        (str "collection/" coll-id "/items"))))))))
+        (testing "Even with show_dashboard_questions=true, in_document cards do not appear"
+          (is (= #{[normal-card-id "card"]
+                   [dash-id "dashboard"]}
+                 (set (map (juxt :id :model)
+                           (:data (mt/user-http-request :rasta :get 200
+                                                        (str "collection/" coll-id "/items?show_dashboard_questions=true"))))))))))))
+
+(deftest document-cards-do-not-appear-in-root-items
+  (testing "GET /api/collection/root/items excludes cards with document_id"
+    (mt/with-premium-features #{:documents}
+      (mt/with-temp [:model/Document {doc-id :id} {}
+                     :model/Card {normal-card-id :id} {:collection_id nil
+                                                       :name "Normal Root Card"}
+                     :model/Card {in-document-card-id :id} {:collection_id nil
+                                                            :document_id doc-id
+                                                            :name "In Document Root Card"}
+                     :model/Dashboard {dash-id :id} {:collection_id nil
+                                                     :name "Root Dashboard"}]
+        (testing "Normal cards and dashboards in root appear, but in_document cards do not"
+          (let [items (mt/user-http-request :rasta :get 200 "collection/root/items")
+                root-test-items (filter #(#{normal-card-id dash-id in-document-card-id} (:id %))
+                                        (:data items))]
+            (is (= #{[normal-card-id "card"]
+                     [dash-id "dashboard"]}
+                   (set (map (juxt :id :model) root-test-items))))))
+        (testing "Even with show_dashboard_questions=true, in_document cards do not appear"
+          (let [items (mt/user-http-request :rasta :get 200 "collection/root/items?show_dashboard_questions=true")
+                root-test-items (filter #(#{normal-card-id dash-id in-document-card-id} (:id %))
+                                        (:data items))]
+            (is (= #{[normal-card-id "card"]
+                     [dash-id "dashboard"]}
+                   (set (map (juxt :id :model) root-test-items))))))))))
+
+(deftest document-cards-do-not-appear-in-root-items-without-premium
+  (testing "GET /api/collection/root/items excludes cards with document_id when documents feature is disabled"
+    (mt/with-premium-features #{}
+      (mt/with-temp [:model/Card {normal-card-id :id} {:collection_id nil
+                                                       :name "Normal Root Card"}
+                     :model/Dashboard {dash-id :id} {:collection_id nil
+                                                     :name "Root Dashboard"}]
+        (testing "Normal cards and dashboards in root appear, no documents"
+          (let [items (mt/user-http-request :rasta :get 200 "collection/root/items")
+                root-test-items (filter #(#{normal-card-id dash-id} (:id %))
+                                        (:data items))]
+            (is (= #{[normal-card-id "card"]
+                     [dash-id "dashboard"]}
+                   (set (map (juxt :id :model) root-test-items))))))))))
+
+(deftest documents-appear-in-collection-items
+  (testing "GET /api/collection/:id/items includes documents"
+    (mt/with-premium-features #{:documents}
+      (mt/with-temp [:model/Collection {coll-id :id} {}
+                     :model/Document {doc-id :id} {:collection_id coll-id
+                                                   :name "Test Document"}
+                     :model/Card {card-id :id} {:collection_id coll-id}
+                     :model/Dashboard {dash-id :id} {:collection_id coll-id}]
+        (testing "Documents appear alongside cards and dashboards"
+          (is (= #{[doc-id "document"]
+                   [card-id "card"]
+                   [dash-id "dashboard"]}
+                 (set (map (juxt :id :model)
+                           (:data (mt/user-http-request :rasta :get 200
+                                                        (str "collection/" coll-id "/items"))))))))))))
+
+(deftest documents-do-not-appear-in-collection-items-without-premium
+  (testing "GET /api/collection/:id/items excludes documents when documents feature is disabled"
+    (mt/with-premium-features #{}
+      (mt/with-temp [:model/Collection {coll-id :id} {}
+                     :model/Card {card-id :id} {:collection_id coll-id}
+                     :model/Document _ {:collection_id coll-id
+                                        :name "Test Document"}
+                     :model/Dashboard {dash-id :id} {:collection_id coll-id}]
+        (testing "Only cards and dashboards appear, no documents"
+          (is (= #{[card-id "card"]
+                   [dash-id "dashboard"]}
+                 (set (map (juxt :id :model)
+                           (:data (mt/user-http-request :rasta :get 200
+                                                        (str "collection/" coll-id "/items"))))))))))))
+
+(deftest documents-appear-in-root-items
+  (testing "GET /api/collection/root/items includes documents"
+    (mt/with-premium-features #{:documents}
+      (mt/with-temp [:model/Document {doc-id :id} {:collection_id nil
+                                                   :name "Root Document"}
+                     :model/Card {card-id :id} {:collection_id nil
+                                                :name "Root Card"}
+                     :model/Dashboard {dash-id :id} {:collection_id nil
+                                                     :name "Root Dashboard"}]
+        (testing "Documents appear alongside cards and dashboards in root"
+          (let [items (mt/user-http-request :rasta :get 200 "collection/root/items")
+                root-test-items (filter #(#{doc-id card-id dash-id} (:id %))
+                                        (:data items))]
+            (is (= #{[doc-id "document"]
+                     [card-id "card"]
+                     [dash-id "dashboard"]}
+                   (set (map (juxt :id :model) root-test-items))))))))))
+
+(deftest documents-do-not-appear-in-root-items-without-premium
+  (testing "GET /api/collection/root/items excludes documents when documents feature is disabled"
+    (mt/with-premium-features #{}
+      (mt/with-temp [:model/Card {card-id :id} {:collection_id nil
+                                                :name "Root Card"}
+                     :model/Document _ {:collection_id nil
+                                        :name "Test Document"}
+                     :model/Dashboard {dash-id :id} {:collection_id nil
+                                                     :name "Root Dashboard"}]
+        (testing "Only cards and dashboards appear in root, no documents"
+          (let [items (mt/user-http-request :rasta :get 200 "collection/root/items")
+                root-test-items (filter #(#{card-id dash-id} (:id %))
+                                        (:data items))]
+            (is (= #{[card-id "card"]
+                     [dash-id "dashboard"]}
+                   (set (map (juxt :id :model) root-test-items))))))))))
+
+(deftest archived-documents-appear-in-trash-items
+  (testing "GET /api/collection/trash/items includes documents with archived_directly true"
+    (mt/with-premium-features #{:documents}
+      (mt/with-temp [:model/Document {archived-doc-id :id} {:collection_id nil
+                                                            :name "Archived Document"
+                                                            :archived true
+                                                            :archived_directly true}
+                     :model/Document {normal-doc-id :id} {:collection_id nil
+                                                          :name "Normal Document"
+                                                          :archived_directly false}
+                     :model/Card {archived-card-id :id} {:collection_id nil
+                                                         :name "Archived Card"
+                                                         :archived true
+                                                         :archived_directly true}
+                     :model/Dashboard {archived-dash-id :id} {:collection_id nil
+                                                              :name "Archived Dashboard"
+                                                              :archived true
+                                                              :archived_directly true}]
+        (testing "Archived documents appear alongside other archived items in trash"
+          (let [items (mt/user-http-request :rasta :get 200 (format "collection/%d/items" (collection/trash-collection-id)))
+                trash-test-items (filter #(#{archived-doc-id normal-doc-id archived-card-id archived-dash-id} (:id %))
+                                         (:data items))]
+            (is (= #{[archived-doc-id "document" true true]
+                     [archived-card-id "card" true true]
+                     [archived-dash-id "dashboard" true true]}
+                   (set (map (juxt :id :model :can_delete :can_restore) trash-test-items))))
+            (testing "Non-archived documents do not appear in trash"
+              (is (not (some #(= normal-doc-id (:id %)) trash-test-items))))))))))
+
+(deftest archived-documents-do-not-appear-in-trash-items-without-premium
+  (testing "GET /api/collection/trash/items excludes documents when documents feature is disabled"
+    (mt/with-premium-features #{}
+      (mt/with-temp [:model/Document _ {:collection_id nil
+                                        :name "Archived Document"
+                                        :archived true
+                                        :archived_directly true}
+                     :model/Card {archived-card-id :id} {:collection_id nil
+                                                         :name "Archived Card"
+                                                         :archived true
+                                                         :archived_directly true}
+                     :model/Dashboard {archived-dash-id :id} {:collection_id nil
+                                                              :name "Archived Dashboard"
+                                                              :archived true
+                                                              :archived_directly true}]
+        (testing "Only archived cards and dashboards appear in trash, no documents"
+          (let [items (mt/user-http-request :rasta :get 200 (format "collection/%d/items" (collection/trash-collection-id)))
+                trash-test-items (:data items)]
+            (is (= #{[archived-card-id "card"]
+                     [archived-dash-id "dashboard"]}
+                   (set (map (juxt :id :model) trash-test-items))))))))))
+
+(deftest document-pinning-collection-items
+  (testing "GET /api/collection/:id/items supports pinned_state parameter for documents"
+    (mt/with-premium-features #{:documents}
+      (mt/with-temp [:model/Collection {coll-id :id} {}
+                     :model/Document {pinned-doc-id :id} {:collection_id coll-id
+                                                          :name "Pinned Document"
+                                                          :collection_position 1}
+                     :model/Document {unpinned-doc-id :id} {:collection_id coll-id
+                                                            :name "Unpinned Document"
+                                                            :collection_position nil}
+                     :model/Card {pinned-card-id :id} {:collection_id coll-id
+                                                       :name "Pinned Card"
+                                                       :collection_position 2}
+                     :model/Card {unpinned-card-id :id} {:collection_id coll-id
+                                                         :name "Unpinned Card"
+                                                         :collection_position nil}]
+        (testing "pinned_state=is_pinned returns only pinned documents and cards"
+          (let [items (:data (mt/user-http-request :rasta :get 200
+                                                   (str "collection/" coll-id "/items?pinned_state=is_pinned")))
+                item-ids (set (map :id items))]
+            (is (contains? item-ids pinned-doc-id))
+            (is (contains? item-ids pinned-card-id))
+            (is (not (contains? item-ids unpinned-doc-id)))
+            (is (not (contains? item-ids unpinned-card-id)))))
+
+        (testing "pinned_state=is_not_pinned returns only unpinned documents and cards"
+          (let [items (:data (mt/user-http-request :rasta :get 200
+                                                   (str "collection/" coll-id "/items?pinned_state=is_not_pinned")))
+                item-ids (set (map :id items))]
+            (is (not (contains? item-ids pinned-doc-id)))
+            (is (not (contains? item-ids pinned-card-id)))
+            (is (contains? item-ids unpinned-doc-id))
+            (is (contains? item-ids unpinned-card-id))))
+
+        (testing "pinned_state=all returns all documents and cards"
+          (let [items (:data (mt/user-http-request :rasta :get 200
+                                                   (str "collection/" coll-id "/items?pinned_state=all")))
+                item-ids (set (map :id items))]
+            (is (contains? item-ids pinned-doc-id))
+            (is (contains? item-ids pinned-card-id))
+            (is (contains? item-ids unpinned-doc-id))
+            (is (contains? item-ids unpinned-card-id))))))))
+
+(deftest document-pinning-root-items
+  (testing "GET /api/collection/root/items supports pinned_state parameter for documents"
+    (mt/with-premium-features #{:documents}
+      (mt/with-temp [:model/Document {pinned-doc-id :id} {:collection_id nil
+                                                          :name "Pinned Root Document"
+                                                          :collection_position 1}
+                     :model/Document {unpinned-doc-id :id} {:collection_id nil
+                                                            :name "Unpinned Root Document"
+                                                            :collection_position nil}
+                     :model/Card {pinned-card-id :id} {:collection_id nil
+                                                       :name "Pinned Root Card"
+                                                       :collection_position 2}
+                     :model/Card {unpinned-card-id :id} {:collection_id nil
+                                                         :name "Unpinned Root Card"
+                                                         :collection_position nil}]
+        (testing "pinned_state=is_pinned returns only pinned root documents and cards"
+          (let [items (:data (mt/user-http-request :rasta :get 200 "collection/root/items?pinned_state=is_pinned"))
+                test-item-ids (set (filter #{pinned-doc-id unpinned-doc-id pinned-card-id unpinned-card-id}
+                                           (map :id items)))]
+            (is (contains? test-item-ids pinned-doc-id))
+            (is (contains? test-item-ids pinned-card-id))
+            (is (not (contains? test-item-ids unpinned-doc-id)))
+            (is (not (contains? test-item-ids unpinned-card-id)))))
+
+        (testing "pinned_state=is_not_pinned returns only unpinned root documents and cards"
+          (let [items (:data (mt/user-http-request :rasta :get 200 "collection/root/items?pinned_state=is_not_pinned"))
+                test-item-ids (set (filter #{pinned-doc-id unpinned-doc-id pinned-card-id unpinned-card-id}
+                                           (map :id items)))]
+            (is (not (contains? test-item-ids pinned-doc-id)))
+            (is (not (contains? test-item-ids pinned-card-id)))
+            (is (contains? test-item-ids unpinned-doc-id))
+            (is (contains? test-item-ids unpinned-card-id))))))))
+
+(deftest mixed-pinned-unpinned-documents-collection-view
+  (testing "Integration test for mixed pinned/unpinned documents in collection view"
+    (mt/with-premium-features #{:documents}
+      (mt/with-temp [:model/Collection {coll-id :id} {}
+                     :model/Document {doc1-id :id} {:collection_id coll-id
+                                                    :name "Document A"
+                                                    :collection_position 1}
+                     :model/Document _ {:collection_id coll-id
+                                        :name "Document B"
+                                        :collection_position nil}
+                     :model/Document {doc3-id :id} {:collection_id coll-id
+                                                    :name "Document C"
+                                                    :collection_position 3}]
+        (testing "Default view includes all documents with collection_position field"
+          (let [items (:data (mt/user-http-request :rasta :get 200
+                                                   (str "collection/" coll-id "/items")))
+                docs (filter #(= "document" (:model %)) items)
+                doc-positions (map :collection_position docs)]
+            (is (= 3 (count docs)))
+            ;; Verify collection_position is included in the response
+            (is (some #(= 1 %) doc-positions))
+            (is (some nil? doc-positions))
+            (is (some #(= 3 %) doc-positions))))
+
+        (testing "Pinned documents have higher collection_position values and appear before unpinned"
+          (let [items (:data (mt/user-http-request :rasta :get 200
+                                                   (str "collection/" coll-id "/items?pinned_state=is_pinned")))
+                pinned-docs (filter #(= "document" (:model %)) items)]
+            (is (= 2 (count pinned-docs)))
+            ;; Verify all pinned docs have non-nil collection_position
+            (is (every? #(some? (:collection_position %)) pinned-docs))
+            (is (= #{doc1-id doc3-id} (set (map :id pinned-docs))))))))))

enterprise/backend/test/metabase_enterprise/documents/models/dashboard_card_test.clj

@@ -0,0 +1,68 @@
+(ns backend.test.metabase-enterprise.documents.models.dashboard-card-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase.dashboards.models.dashboard-card :as dashboard-card]
+   [metabase.test :as mt]
+   [toucan2.core :as t2]))
+
+(deftest prevent-in-report-cards-from-being-added-to-dashboards-test
+  (testing "Cannot add a card with document_id to dashboard"
+    (mt/with-temp [:model/Dashboard dashboard   {}
+                   :model/Document  document    {}
+                   :model/Card      normal-card  {:name "Normal Card" :type :question}
+                   :model/Card      report-card  {:name "Report Card" :type :question :document_id (:id document)}]
+      (testing "Adding a normal card works fine"
+        (let [result (dashboard-card/create-dashboard-cards!
+                      [{:dashboard_id (:id dashboard)
+                        :card_id      (:id normal-card)
+                        :size_x       4
+                        :size_y       3
+                        :row          0
+                        :col          0}])]
+          (is (= 1 (count result)))
+          (is (= (:id normal-card) (:card_id (first result))))))
+
+      (testing "Adding an in_document card throws an exception"
+        (is (thrown-with-msg?
+             clojure.lang.ExceptionInfo
+             #"Cards with 'document_id' cannot be added to dashboards"
+             (dashboard-card/create-dashboard-cards!
+              [{:dashboard_id (:id dashboard)
+                :card_id      (:id report-card)
+                :size_x       4
+                :size_y       3
+                :row          1
+                :col          0}]))))))
+
+  (testing "Cannot add multiple cards if any of them has a document_id"
+    (mt/with-temp [:model/Dashboard dashboard    {}
+                   :model/Document  document    {}
+                   :model/Card      normal-card1  {:name "Normal Card 1" :type :question}
+                   :model/Card      normal-card2  {:name "Normal Card 2" :type :model}
+                   :model/Card      report-card1 {:name "Report Card" :type :question :document_id (:id document)}]
+      (testing "Batch creation fails if any card is in_document type"
+        (is (thrown-with-msg?
+             clojure.lang.ExceptionInfo
+             #"Cards with 'document_id' cannot be added to dashboards"
+             (dashboard-card/create-dashboard-cards!
+              [{:dashboard_id (:id dashboard)
+                :card_id      (:id normal-card1)
+                :size_x       4
+                :size_y       3
+                :row          0
+                :col          0}
+               {:dashboard_id (:id dashboard)
+                :card_id      (:id report-card1)
+                :size_x       4
+                :size_y       3
+                :row          0
+                :col          4}
+               {:dashboard_id (:id dashboard)
+                :card_id      (:id normal-card2)
+                :size_x       4
+                :size_y       3
+                :row          0
+                :col          8}])))
+
+        ;; Verify that no cards were added due to transaction rollback
+        (is (= 0 (t2/count :model/DashboardCard :dashboard_id (:id dashboard))))))))

enterprise/backend/test/metabase_enterprise/documents/models/document_test.clj

@@ -0,0 +1,726 @@
+(ns metabase-enterprise.documents.models.document-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase-enterprise.documents.models.document :as document]
+   [metabase.collections.models.collection :as collection]
+   [metabase.models.serialization :as serdes]
+   [metabase.permissions.core :as perms]
+   [metabase.test :as mt]
+   [metabase.test.fixtures :as fixtures]
+   [toucan2.core :as t2]))
+
+(use-fixtures :once (fixtures/initialize :db :test-users))
+
+(deftest sync-document-cards-collection-matching-cards-test
+  (testing "should only update cards with matching document_id"
+    (mt/with-temp [:model/Collection {collection-id :id} {:name "Target Collection"}
+                   :model/Document {document-id :id} {:name "Test Document"}
+                   :model/Card {card1-id :id} {:name "Document Card 1"
+                                               :document_id document-id
+                                               :collection_id nil
+                                               :dataset_query (mt/mbql-query venues)}
+                   :model/Card {card2-id :id} {:name "Document Card 2"
+                                               :document_id document-id
+                                               :collection_id nil
+                                               :dataset_query (mt/mbql-query venues)}
+                   :model/Card {other-card-id :id} {:name "Other Card"
+                                                    :type :question
+                                                    :collection_id nil
+                                                    :dataset_query (mt/mbql-query venues)}]
+      (let [updated-count (document/sync-document-cards-collection! document-id collection-id)]
+        ;; Should update exactly 2 cards
+        (is (= 2 updated-count))
+
+        ;; Verify the document cards were updated
+        (let [updated-card1 (t2/select-one :model/Card :id card1-id)
+              updated-card2 (t2/select-one :model/Card :id card2-id)
+              other-card (t2/select-one :model/Card :id other-card-id)]
+          (is (= collection-id (:collection_id updated-card1)))
+          (is (= collection-id (:collection_id updated-card2)))
+          ;; Other card should not be affected
+          (is (nil? (:collection_id other-card))))))))
+
+(deftest sync-document-cards-collection-nil-document-document-id-test
+  (testing "should not affect cards with nil document_id"
+    (mt/with-temp [:model/Collection {collection-id :id} {:name "Target Collection"}
+                   :model/Document {document-id :id} {:name "Test Document"}
+                   :model/Card {nil-document-card-id :id} {:name "Nil Document Card"
+                                                           :document_id nil
+                                                           :collection_id nil
+                                                           :dataset_query (mt/mbql-query venues)}]
+      (let [updated-count (document/sync-document-cards-collection! document-id collection-id)]
+        ;; Should update 0 cards since no cards have matching document_id
+        (is (= 0 updated-count))
+
+        ;; Verify the card with nil document_id was not affected
+        (let [unchanged-card (t2/select-one :model/Card :id nil-document-card-id)]
+          (is (nil? (:collection_id unchanged-card))))))))
+
+(deftest sync-document-cards-collection-empty-result-sets-test
+  (testing "should handle empty result sets gracefully"
+    (mt/with-temp [:model/Collection {collection-id :id} {:name "Target Collection"}
+                   :model/Document {document-id :id} {:name "Empty Document"}]
+      ;; No cards associated with this document
+      (let [updated-count (document/sync-document-cards-collection! document-id collection-id)]
+        ;; Should return 0 for no updates
+        (is (= 0 updated-count))))))
+
+(deftest document-collection-sync-hook-triggers-on-collection-change-test
+  (testing "Hook is called when document collection_id changes"
+    (mt/with-temp
+      [:model/Collection {old-collection-id :id} {:name "Old Collection"}
+       :model/Collection {new-collection-id :id} {:name "New Collection"}
+       :model/Document {document-id :id} {:collection_id old-collection-id
+                                          :name "Test Document"}
+       :model/Card {card1-id :id} {:name "Card 1"
+                                   :document_id document-id
+                                   :collection_id old-collection-id}
+       :model/Card {card2-id :id} {:name "Card 2"
+                                   :document_id document-id
+                                   :collection_id old-collection-id}]
+
+      ;; Update the document's collection_id
+      (t2/update! :model/Document document-id {:collection_id new-collection-id})
+
+      ;; Verify that associated cards were updated to match the new collection
+      (is (= new-collection-id (:collection_id (t2/select-one :model/Card :id card1-id))))
+      (is (= new-collection-id (:collection_id (t2/select-one :model/Card :id card2-id)))))))
+
+(deftest document-collection-sync-hook-handles-nil-collections-test
+  (testing "Hook correctly handles nil collection values"
+    (mt/with-temp
+      [:model/Collection {collection-id :id} {:name "Test Collection"}
+       :model/Document {document-id :id} {:collection_id collection-id
+                                          :name "Test Document"}
+       :model/Card {card-id :id} {:name "Card"
+                                  :document_id document-id
+                                  :collection_id collection-id}]
+
+      ;; Move document to no collection (nil)
+      (t2/update! :model/Document document-id {:collection_id nil})
+
+      ;; Verify that the card's collection_id was updated to nil
+      (is (nil? (:collection_id (t2/select-one :model/Card :id card-id)))))))
+
+(deftest document-collection-sync-hook-only-affects-cards-test
+  (testing "Hook only updates cards with  matching document_id"
+    (mt/with-temp
+      [:model/Collection {old-collection-id :id} {:name "Old Collection"}
+       :model/Collection {new-collection-id :id} {:name "New Collection"}
+       :model/Document {document-id :id} {:collection_id old-collection-id
+                                          :name "Test Document"}
+       ;; Card that should be updated (correct type and document_id)
+       :model/Card {in-document-card-id :id} {:name "In-Document Card"
+                                              :type :question
+                                              :document_id document-id
+                                              :collection_id old-collection-id}
+       ;; Card that should be updated (wrong type)
+       :model/Card {question-card-id :id} {:name "Question Card"
+                                           :type :model
+                                           :collection_id old-collection-id}
+       ;; Card that should NOT be updated (no document_id)
+       :model/Card {regular-card-id :id} {:name "Regular Card"
+                                          :collection_id old-collection-id}]
+
+      ;; Update the document's collection_id
+      (t2/update! :model/Document document-id {:collection_id new-collection-id})
+
+      ;; Verify only the correct card was updated
+      (is (= new-collection-id (:collection_id (t2/select-one :model/Card :id in-document-card-id))))
+
+      ;; Verify other cards were NOT updated
+      (is (= old-collection-id (:collection_id (t2/select-one :model/Card :id question-card-id))))
+      (is (= old-collection-id (:collection_id (t2/select-one :model/Card :id regular-card-id)))))))
+
+(deftest personal-collection-edge-cases-test
+  (testing "Personal collection handling"
+    (binding [collection/*allow-deleting-personal-collections* true]
+      (mt/with-temp [:model/User {user-id :id} {:first_name "Test" :last_name "User" :email "test@example.com"}
+                     :model/Collection {personal-collection-id :id} {:name "Personal Collection"
+                                                                     :personal_owner_id user-id}
+                     :model/Collection {regular-collection-id :id} {:name "Regular Collection"}
+                     :model/Document {document-id :id} {:collection_id regular-collection-id
+                                                        :name "Personal Test Document"}
+                     :model/Card {card-id :id} {:name "Personal Card"
+                                                :document_id document-id
+                                                :collection_id regular-collection-id
+                                                :dataset_query (mt/mbql-query venues)}]
+
+        (testing "moving document to personal collection moves associated cards"
+          (mt/with-current-user user-id
+            ;; As the personal collection owner, update should succeed
+            (t2/update! :model/Document document-id {:collection_id personal-collection-id})
+
+            ;; Verify both document and card moved to personal collection
+            (is (= personal-collection-id (:collection_id (t2/select-one :model/Document :id document-id))))
+            (is (= personal-collection-id (:collection_id (t2/select-one :model/Card :id card-id))))))
+
+        (testing "moving document from personal collection works"
+          (mt/with-current-user user-id
+            ;; Move back to regular collection
+            (t2/update! :model/Document document-id {:collection_id regular-collection-id})
+
+            ;; Verify both document and card moved back
+            (is (= regular-collection-id (:collection_id (t2/select-one :model/Document :id document-id))))
+            (is (= regular-collection-id (:collection_id (t2/select-one :model/Card :id card-id))))))))))
+
+(deftest validate-collection-move-permissions-allows-move-with-both-permissions-test
+  (testing "allows move when user has write permissions for both collections"
+    (mt/with-non-admin-groups-no-root-collection-perms
+      (mt/with-temp [:model/Collection {old-collection-id :id} {:name "Old Collection"}
+                     :model/Collection {new-collection-id :id} {:name "New Collection"}
+                     :model/User {user-id :id} {}]
+        (mt/with-current-user user-id
+          ;; Grant write permissions to both collections
+          (perms/grant-collection-readwrite-permissions! (perms/all-users-group) old-collection-id)
+          (perms/grant-collection-readwrite-permissions! (perms/all-users-group) new-collection-id)
+
+          ;; Should not throw any exception
+          (is (true? (document/validate-collection-move-permissions old-collection-id new-collection-id))))))))
+
+(deftest validate-collection-move-permissions-throws-403-missing-old-permission-test
+  (testing "throws 403 when user lacks write permission for old collection"
+    (mt/with-non-admin-groups-no-root-collection-perms
+      (mt/with-temp [:model/Collection {old-collection-id :id} {:name "Old Collection"}
+                     :model/Collection {new-collection-id :id} {:name "New Collection"}
+                     :model/User {user-id :id} {}]
+        (mt/with-current-user user-id
+          ;; Grant write permission only to new collection
+          (perms/grant-collection-readwrite-permissions! (perms/all-users-group) new-collection-id)
+
+          ;; Should throw 403 exception
+          (is (thrown-with-msg? clojure.lang.ExceptionInfo #"You do not have curate permissions for this Collection."
+                                (document/validate-collection-move-permissions old-collection-id new-collection-id))))))))
+
+(deftest validate-collection-move-permissions-throws-403-missing-new-permission-test
+  (testing "throws 403 when user lacks write permission for new collection"
+    (mt/with-non-admin-groups-no-root-collection-perms
+      (mt/with-temp [:model/Collection {old-collection-id :id} {:name "Old Collection"}
+                     :model/Collection {new-collection-id :id} {:name "New Collection"}
+                     :model/User {user-id :id} {}]
+        (mt/with-current-user user-id
+          ;; Grant write permission only to old collection
+          (perms/grant-collection-readwrite-permissions! (perms/all-users-group) old-collection-id)
+
+          ;; Should throw 403 exception
+          (is (thrown-with-msg? clojure.lang.ExceptionInfo #"You do not have curate permissions for this Collection."
+                                (document/validate-collection-move-permissions old-collection-id new-collection-id))))))))
+
+(deftest validate-collection-move-permissions-throws-403-no-permissions-test
+  (testing "throws 403 when user lacks write permissions for both collections"
+    (mt/with-non-admin-groups-no-root-collection-perms
+      (mt/with-temp [:model/Collection {old-collection-id :id} {:name "Old Collection"}
+                     :model/Collection {new-collection-id :id} {:name "New Collection"}
+                     :model/User {user-id :id} {}]
+        (mt/with-current-user user-id
+          ;; No permissions granted
+
+          ;; Should throw 403 exception
+          (is (thrown-with-msg? clojure.lang.ExceptionInfo #"You do not have curate permissions for this Collection."
+                                (document/validate-collection-move-permissions old-collection-id new-collection-id))))))))
+
+(deftest validate-collection-move-permissions-allows-move-from-root-test
+  (testing "allows move when old collection is nil (moving from root)"
+    (mt/with-non-admin-groups-no-root-collection-perms
+      (mt/with-temp [:model/Collection {new-collection-id :id} {:name "New Collection"}
+                     :model/User {user-id :id} {}]
+        (mt/with-current-user user-id
+          ;; Grant write permission to new collection
+          (perms/grant-collection-readwrite-permissions! (perms/all-users-group) new-collection-id)
+
+          ;; Should not throw any exception when old collection is nil
+          (is (true? (document/validate-collection-move-permissions nil new-collection-id))))))))
+
+(deftest validate-collection-move-permissions-allows-move-to-root-test
+  (testing "allows move when new collection is nil (moving to root)"
+    (mt/with-non-admin-groups-no-root-collection-perms
+      (mt/with-temp [:model/Collection {old-collection-id :id} {:name "Old Collection"}
+                     :model/User {user-id :id} {}]
+        (mt/with-current-user user-id
+          ;; Grant write permission to old collection
+          (perms/grant-collection-readwrite-permissions! (perms/all-users-group) old-collection-id)
+
+          ;; Should not throw any exception when new collection is nil
+          (is (nil? (document/validate-collection-move-permissions old-collection-id nil))))))))
+
+(deftest validate-collection-move-permissions-allows-move-both-nil-test
+  (testing "allows move when both collections are nil"
+    (mt/with-temp [:model/User {user-id :id} {}]
+      (mt/with-current-user user-id
+        ;; Should not throw any exception when both collections are nil
+        (is (nil? (document/validate-collection-move-permissions nil nil)))))))
+
+(deftest validate-collection-move-permissions-throws-400-nonexistent-collection-test
+  (testing "throws 400 when new collection does not exist"
+    (mt/with-non-admin-groups-no-root-collection-perms
+      (mt/with-temp [:model/Collection {old-collection-id :id} {:name "Old Collection"}
+                     :model/User {user-id :id} {}]
+        (mt/with-current-user user-id
+          ;; Grant write permission to old collection
+          (perms/grant-collection-readwrite-permissions! (perms/all-users-group) old-collection-id)
+
+          ;; Use a non-existent collection ID
+          (let [non-existent-id 999999]
+            (is (thrown-with-msg? clojure.lang.ExceptionInfo #"You do not have curate permissions for this Collection."
+                                  (document/validate-collection-move-permissions old-collection-id non-existent-id)))))))))
+
+(deftest validate-collection-move-permissions-throws-400-archived-collection-test
+  (testing "throws 400 when new collection is archived"
+    (mt/with-non-admin-groups-no-root-collection-perms
+      (mt/with-temp [:model/Collection {old-collection-id :id} {:name "Old Collection"}
+                     :model/Collection {archived-collection-id :id} {:name "Archived Collection" :archived true}
+                     :model/User {user-id :id} {}]
+        (mt/with-current-user user-id
+          ;; Grant write permissions to both collections
+          (perms/grant-collection-readwrite-permissions! (perms/all-users-group) old-collection-id)
+          (perms/grant-collection-readwrite-permissions! (perms/all-users-group) archived-collection-id)
+
+          ;; Should throw 400 exception for archived collection
+          (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Invalid"
+                                (document/validate-collection-move-permissions old-collection-id archived-collection-id))))))))
+
+(deftest validate-collection-move-permissions-superuser-test
+  (testing "superuser can move between any collections"
+    (mt/with-temp [:model/Collection {old-collection-id :id} {:name "Old Collection"}
+                   :model/Collection {new-collection-id :id} {:name "New Collection"}]
+      (mt/with-current-user (mt/user->id :crowberto)
+        ;; Superuser should be able to move without explicit permissions
+        (is (true? (document/validate-collection-move-permissions old-collection-id new-collection-id)))))))
+
+(deftest hydrate-document-creator-test
+  (testing "hydrates document creator correctly"
+    (mt/with-temp [:model/User {user-id :id} {:first_name "John"
+                                              :last_name "Doe"
+                                              :email "john.doe@example.com"}
+                   :model/Document {document-id :id} {:name "Test Document"
+                                                      :creator_id user-id}]
+      (let [hydrated-doc (t2/hydrate (t2/select-one :model/Document :id document-id) :creator)]
+        (testing "creator is hydrated with correct user data"
+          (is (some? (:creator hydrated-doc)))
+          (is (= user-id (get-in hydrated-doc [:creator :id])))
+          (is (= "John" (get-in hydrated-doc [:creator :first_name])))
+          (is (= "Doe" (get-in hydrated-doc [:creator :last_name])))
+          (is (= "john.doe@example.com" (get-in hydrated-doc [:creator :email]))))))))
+
+(deftest hydrate-multiple-documents-creator-test
+  (testing "hydrates creators for multiple documents efficiently"
+    (mt/with-temp [:model/User {user1-id :id} {:first_name "Alice"
+                                               :last_name "Smith"
+                                               :email "alice@example.com"}
+                   :model/User {user2-id :id} {:first_name "Bob"
+                                               :last_name "Jones"
+                                               :email "bob@example.com"}
+                   :model/Document {doc1-id :id} {:name "Document 1"
+                                                  :creator_id user1-id}
+                   :model/Document {doc2-id :id} {:name "Document 2"
+                                                  :creator_id user2-id}
+                   :model/Document {doc3-id :id} {:name "Document 3"
+                                                  :creator_id user1-id}] ; Same creator as doc1
+      (let [documents (t2/select :model/Document :id [:in [doc1-id doc2-id doc3-id]])
+            hydrated-docs (t2/hydrate documents :creator)]
+        (testing "all documents have their creators hydrated"
+          (is (= 3 (count hydrated-docs)))
+          (doseq [doc hydrated-docs]
+            (is (some? (:creator doc)))
+            (is (some? (get-in doc [:creator :id])))
+            (is (some? (get-in doc [:creator :first_name])))
+            (is (some? (get-in doc [:creator :last_name])))
+            (is (some? (get-in doc [:creator :email])))))
+
+        (testing "creators are correctly matched to documents"
+          (let [doc1 (first (filter #(= doc1-id (:id %)) hydrated-docs))
+                doc2 (first (filter #(= doc2-id (:id %)) hydrated-docs))
+                doc3 (first (filter #(= doc3-id (:id %)) hydrated-docs))]
+            (is (= user1-id (get-in doc1 [:creator :id])))
+            (is (= "Alice" (get-in doc1 [:creator :first_name])))
+
+            (is (= user2-id (get-in doc2 [:creator :id])))
+            (is (= "Bob" (get-in doc2 [:creator :first_name])))
+
+            (is (= user1-id (get-in doc3 [:creator :id])))
+            (is (= "Alice" (get-in doc3 [:creator :first_name])))))))))
+
+(deftest document-collection-position-field-handling-test
+  (testing "Document model supports collection_position field"
+    (mt/with-temp [:model/Collection {collection-id :id} {:name "Test Collection"}
+                   :model/Document {document-id :id} {:name "Positioned Document"
+                                                      :collection_id collection-id
+                                                      :collection_position 5}]
+      (let [document (t2/select-one :model/Document :id document-id)]
+        (testing "collection_position is stored and retrieved correctly"
+          (is (= 5 (:collection_position document)))))
+
+      (testing "collection_position can be updated"
+        (t2/update! :model/Document document-id {:collection_position 10})
+        (let [updated-document (t2/select-one :model/Document :id document-id)]
+          (is (= 10 (:collection_position updated-document)))))
+
+      (testing "collection_position can be set to nil"
+        (t2/update! :model/Document document-id {:collection_position nil})
+        (let [updated-document (t2/select-one :model/Document :id document-id)]
+          (is (nil? (:collection_position updated-document))))))))
+
+;;; ------------------------------------------------- Serialization Tests -------------------------------------------
+
+(deftest document-serdes-spec-test
+  (testing "Document serialization spec includes all required fields"
+    (let [spec (serdes/make-spec "Document" {})]
+      (is (= [:archived :archived_directly :content_type :entity_id :name :collection_position]
+             (:copy spec)))
+      (is (= [:view_count :last_viewed_at] (:skip spec)))
+      (is (contains? (:transform spec) :created_at))
+      (is (contains? (:transform spec) :document))
+      (is (contains? (:transform spec) :updated_at))
+      (is (contains? (:transform spec) :collection_id))
+      (is (contains? (:transform spec) :creator_id))
+      (testing "foreign key transformers are properly configured"
+        (is (get-in spec [:transform :collection_id ::serdes/fk]))
+        (is (get-in spec [:transform :creator_id ::serdes/fk]))))))
+
+(deftest document-serdes-dependencies-test
+  (testing "Document dependencies method works correctly"
+    (testing "with collection and creator"
+      (let [document {:collection_id 123
+                      :creator_id 456
+                      :serdes/meta [{:model "Document" :id "test-entity-id"}]}
+            deps (serdes/dependencies document)]
+        (is (= #{[{:model "Collection" :id 123}]}
+               deps))))))
+
+(deftest document-serdes-smartlink-single-reference-test
+  (testing "single smartLink card reference"
+    (let [document {:collection_id 123
+                    :creator_id 456
+                    :document {:type "doc"
+                               :content [{:type "paragraph"
+                                          :content [{:type "smartLink"
+                                                     :attrs {:entityId [{:model "Card" :id 789}]
+                                                             :model "card"}}]}]}
+                    :content_type "application/json+vnd.prose-mirror"
+                    :serdes/meta [{:model "Document" :id "test-entity-id"}]}
+          deps (serdes/dependencies document)]
+      (is (contains? deps [{:model "Card" :id 789}]))
+      (is (contains? deps [{:model "Collection" :id 123}])))))
+
+(deftest document-serdes-smartlink-multiple-references-test
+  (testing "multiple smartLink references of different types"
+    (let [document {:collection_id 123
+                    :creator_id 456
+                    :document {:type "doc"
+                               :content [{:type "paragraph"
+                                          :content [{:type "smartLink"
+                                                     :attrs {:entityId [{:model "Card" :id 789}]
+                                                             :model "card"}}
+                                                    {:type "smartLink"
+                                                     :attrs {:entityId [{:model "Dashboard" :id 456}]
+                                                             :model "dashboard"}}
+                                                    {:type "smartLink"
+                                                     :attrs {:entityId [{:model "Table" :id 321}]
+                                                             :model "table"}}]}]}
+                    :content_type "application/json+vnd.prose-mirror"
+                    :serdes/meta [{:model "Document" :id "test-entity-id"}]}
+          deps (serdes/dependencies document)]
+      (is (contains? deps [{:model "Card" :id 789}]))
+      (is (contains? deps [{:model "Dashboard" :id 456}]))
+      (is (contains? deps [{:model "Table" :id 321}]))
+      (is (contains? deps [{:model "Collection" :id 123}])))))
+
+(deftest document-serdes-smartlink-nested-structure-test
+  (testing "nested smartLink in complex prose mirror structure"
+    (let [document {:collection_id 123
+                    :creator_id 456
+                    :document {:type "doc"
+                               :content [{:type "paragraph"
+                                          :content [{:type "text" :text "Some text"}]}
+                                         {:type "bulletList"
+                                          :content [{:type "listItem"
+                                                     :content [{:type "paragraph"
+                                                                :content [{:type "smartLink"
+                                                                           :attrs {:entityId [{:model "Card" :id 999}]
+                                                                                   :model "card"}}]}]}]}]}
+                    :content_type "application/json+vnd.prose-mirror"
+                    :serdes/meta [{:model "Document" :id "test-entity-id"}]}
+          deps (serdes/dependencies document)]
+      (is (contains? deps [{:model "Card" :id 999}]))
+      (is (contains? deps [{:model "Collection" :id 123}])))))
+
+(deftest document-serdes-smartlink-no-smartlinks-test
+  (testing "document with no smartLinks"
+    (let [document {:collection_id 123
+                    :creator_id 456
+                    :document {:type "doc"
+                               :content [{:type "paragraph"
+                                          :content [{:type "text" :text "Plain text only"}]}]}
+                    :content_type "application/json+vnd.prose-mirror"
+                    :serdes/meta [{:model "Document" :id "test-entity-id"}]}
+          deps (serdes/dependencies document)]
+      (is (= #{[{:model "Collection" :id 123}]}
+             deps)))))
+
+(deftest document-serdes-smartlink-unknown-model-test
+  (testing "unknown smartLink model type is ignored"
+    (let [document {:collection_id 123
+                    :creator_id 456
+                    :document {:type "doc"
+                               :content [{:type "paragraph"
+                                          :content [{:type "smartLink"
+                                                     :attrs {:entityId [{:model "Unknown" :id 789}]
+                                                             :model "unknown-model"}}
+                                                    {:type "smartLink"
+                                                     :attrs {:entityId [{:model "Card" :id 456}]
+                                                             :model "card"}}]}]}
+                    :content_type "application/json+vnd.prose-mirror"
+                    :serdes/meta [{:model "Document" :id "test-entity-id"}]}
+          deps (serdes/dependencies document)]
+      (is (contains? deps [{:model "Card" :id 456}]))
+      (is (not (some #(= (:model (first %)) "unknown-model") deps))))))
+
+(deftest document-serdes-smartlink-missing-entity-id-test
+  (testing "smartLink with missing entityId is ignored"
+    (let [document {:collection_id 123
+                    :creator_id 456
+                    :document {:type "doc"
+                               :content [{:type "paragraph"
+                                          :content [{:type "smartLink"
+                                                     :attrs {:model "card"}}
+                                                    {:type "smartLink"
+                                                     :attrs {:entityId [{:model "Card" :id 456}]
+                                                             :model "card"}}]}]}
+                    :content_type "application/json+vnd.prose-mirror"
+                    :serdes/meta [{:model "Document" :id "test-entity-id"}]}
+          deps (serdes/dependencies document)]
+      (is (contains? deps [{:model "Card" :id 456}]))
+      (is (= 2 (count deps)))))) ; collection and one valid card
+
+(deftest document-serdes-smartlink-duplicate-references-test
+  (testing "duplicate smartLink references are deduplicated"
+    (let [document {:collection_id 123
+                    :creator_id 456
+                    :document {:type "doc"
+                               :content [{:type "paragraph"
+                                          :content [{:type "smartLink"
+                                                     :attrs {:entityId [{:model "Card" :id 789}]
+                                                             :model "card"}}
+                                                    {:type "smartLink"
+                                                     :attrs {:entityId [{:model "Card" :id 789}]
+                                                             :model "card"}}]}]}
+                    :content_type "application/json+vnd.prose-mirror"
+                    :serdes/meta [{:model "Document" :id "test-entity-id"}]}
+          deps (serdes/dependencies document)]
+      (is (contains? deps [{:model "Card" :id 789}]))
+      (is (= 2 (count deps)))))) ; collection and one unique card
+
+(deftest document-serdes-smartlink-empty-content-test
+  (testing "empty document content"
+    (let [document {:collection_id 123
+                    :creator_id 456
+                    :document {:type "doc"
+                               :content []}
+                    :content_type "application/json+vnd.prose-mirror"
+                    :serdes/meta [{:model "Document" :id "test-entity-id"}]}
+          deps (serdes/dependencies document)]
+      (is (= #{[{:model "Collection" :id 123}]}
+             deps)))))
+
+(deftest document-serdes-smartlink-missing-model-test
+  (testing "smartLink with missing model is ignored"
+    (let [document {:collection_id 123
+                    :creator_id 456
+                    :document {:type "doc"
+                               :content [{:type "paragraph"
+                                          :content [{:type "smartLink"
+                                                     :attrs {:entityId [{:model "Card" :id 789}]}}
+                                                    {:type "smartLink"
+                                                     :attrs {:entityId [{:model "Card" :id 456}]
+                                                             :model "card"}}]}]}
+                    :content_type "application/json+vnd.prose-mirror"
+                    :serdes/meta [{:model "Document" :id "test-entity-id"}]}
+          deps (serdes/dependencies document)]
+      (is (contains? deps [{:model "Card" :id 456}]))
+      (is (= 2 (count deps)))))) ; collection and one valid card
+
+(deftest document-serdes-smartlink-nil-attrs-test
+  (testing "smartLink with nil attrs is ignored"
+    (let [document {:collection_id 123
+                    :creator_id 456
+                    :document {:type "doc"
+                               :content [{:type "paragraph"
+                                          :content [{:type "smartLink"
+                                                     :attrs nil}
+                                                    {:type "smartLink"
+                                                     :attrs {:entityId [{:model "Card" :id 456}]
+                                                             :model "card"}}]}]}
+                    :content_type "application/json+vnd.prose-mirror"
+                    :serdes/meta [{:model "Document" :id "test-entity-id"}]}
+          deps (serdes/dependencies document)]
+      (is (contains? deps [{:model "Card" :id 456}]))
+      (is (= 2 (count deps)))))) ; collection and one valid card
+
+(deftest document-serdes-smartlink-mixed-content-test
+  (testing "mix of smartLinks and other node types"
+    (let [document {:collection_id 123
+                    :creator_id 456
+                    :document {:type "doc"
+                               :content [{:type "paragraph"
+                                          :content [{:type "text" :text "Check out this "}
+                                                    {:type "smartLink"
+                                                     :attrs {:entityId [{:model "Card" :id 789}]
+                                                             :model "card"}}
+                                                    {:type "text" :text " and this "}
+                                                    {:type "smartLink"
+                                                     :attrs {:entityId [{:model "Dashboard" :id 456}]
+                                                             :model "dashboard"}}]}
+                                         {:type "heading"
+                                          :attrs {:level 2}
+                                          :content [{:type "text" :text "Section with table"}]}
+                                         {:type "table"
+                                          :content [{:type "tableRow"
+                                                     :content [{:type "tableCell"
+                                                                :content [{:type "paragraph"
+                                                                           :content [{:type "smartLink"
+                                                                                      :attrs {:entityId [{:model "Table" :id 321}]
+                                                                                              :model "table"}}]}]}]}]}]}
+                    :content_type "application/json+vnd.prose-mirror"
+                    :serdes/meta [{:model "Document" :id "test-entity-id"}]}
+          deps (serdes/dependencies document)]
+      (is (contains? deps [{:model "Card" :id 789}]))
+      (is (contains? deps [{:model "Dashboard" :id 456}]))
+      (is (contains? deps [{:model "Table" :id 321}]))
+      (is (contains? deps [{:model "Collection" :id 123}]))
+      (is (= 4 (count deps)))))) ; 3 smartLinks + collection
+
+(deftest document-serdes-smartlink-non-prose-mirror-test
+  (testing "non-prose-mirror content type documents don't extract smartLinks"
+    (let [document {:collection_id 123
+                    :creator_id 456
+                    :document {:type "doc"
+                               :content [{:type "smartLink"
+                                          :attrs {:entityId [{:model "Card" :id 456}]
+                                                  :model "card"}}]}
+                    :content_type "application/json" ; Not prose-mirror
+                    :serdes/meta [{:model "Document" :id "test-entity-id"}]}
+          deps (serdes/dependencies document)]
+      (is (contains? deps [{:model "Collection" :id 123}])))))
+
+(deftest document-serdes-descendants-embedded-cards-test
+  (testing "Document descendants includes embedded cards"
+    (mt/with-temp [:model/Document {document-id :id} {:document {:type "doc"
+                                                                 :content [{:type "cardEmbed"
+                                                                            :attrs {:id 456}}
+                                                                           {:type "cardEmbed"
+                                                                            :attrs {:id 789}}]}
+                                                      :content_type "application/json+vnd.prose-mirror"}]
+      (let [descendants (serdes/descendants "Document" document-id)]
+        (is (= {["Card" 456] {"Document" document-id}
+                ["Card" 789] {"Document" document-id}}
+               descendants))))))
+
+(deftest document-serdes-descendants-smart-links-test
+  (testing "Document descendants includes smart links"
+    (mt/with-temp [:model/Document {document-id :id} {:document {:type "doc"
+                                                                 :content [{:type "smartLink"
+                                                                            :attrs {:id 456
+                                                                                    :model "card"}}
+                                                                           {:type "smartLink"
+                                                                            :attrs {:id 789
+                                                                                    :model "dashboard"}}]}
+                                                      :content_type "application/json+vnd.prose-mirror"}]
+      (let [descendants (serdes/descendants "Document" document-id)]
+        (is (= {["Card" 456] {"Document" document-id}
+                ["Dashboard" 789] {"Document" document-id}}
+               descendants))))))
+
+(deftest document-serdes-descendants-mixed-content-test
+  (testing "Document descendants includes both embedded cards and smart links"
+    (mt/with-temp [:model/Document {document-id :id} {:document {:type "doc"
+                                                                 :content [{:type "cardEmbed"
+                                                                            :attrs {:id 111}}
+                                                                           {:type "smartLink"
+                                                                            :attrs {:id 222
+                                                                                    :model "card"}}
+                                                                           {:type "smartLink"
+                                                                            :attrs {:id 333
+                                                                                    :model "table"}}]}
+                                                      :content_type "application/json+vnd.prose-mirror"}]
+      (let [descendants (serdes/descendants "Document" document-id)]
+        (is (= {["Card" 111] {"Document" document-id}
+                ["Card" 222] {"Document" document-id}
+                ["Table" 333] {"Document" document-id}}
+               descendants))))))
+
+(deftest document-serdes-descendants-empty-document-test
+  (testing "Document descendants handles document with no embedded content"
+    (mt/with-temp [:model/Document {document-id :id} {:document {:type "doc"
+                                                                 :content [{:type "paragraph"
+                                                                            :content [{:type "text" :text "Plain text only"}]}]}
+                                                      :content_type "application/json+vnd.prose-mirror"}]
+      (let [descendants (serdes/descendants "Document" document-id)]
+        (is (= {} descendants))))))
+
+(deftest document-serdes-descendants-nonexistent-document-test
+  (testing "Document descendants returns nil for non-existent document"
+    (let [descendants (serdes/descendants "Document" 99999999)]
+      (is (nil? descendants)))))
+
+(deftest document-serdes-descendants-unknown-smart-link-model-test
+  (testing "Document descendants ignores smart links with unknown model types"
+    (mt/with-temp [:model/Document {document-id :id} {:document {:type "doc"
+                                                                 :content [{:type "smartLink"
+                                                                            :attrs {:id 456
+                                                                                    :model "card"}}
+                                                                           {:type "smartLink"
+                                                                            :attrs {:id 789
+                                                                                    :model "unknown-model"}}]}
+                                                      :content_type "application/json+vnd.prose-mirror"}]
+      (let [descendants (serdes/descendants "Document" document-id)]
+        ;; Should only include the known model type
+        (is (= {["Card" 456] {"Document" document-id}}
+               descendants))
+        ;; Should not include unknown model
+        (is (not (contains? descendants ["Unknown" 789])))))))
+
+(deftest document-serdes-descendants-duplicate-references-test
+  (testing "Document descendants handles duplicate references correctly"
+    (mt/with-temp [:model/Document {document-id :id} {:document {:type "doc"
+                                                                 :content [{:type "cardEmbed"
+                                                                            :attrs {:id 456}}
+                                                                           {:type "smartLink"
+                                                                            :attrs {:id 456
+                                                                                    :model "card"}}]}
+                                                      :content_type "application/json+vnd.prose-mirror"}]
+      (let [descendants (serdes/descendants "Document" document-id)]
+        ;; Should merge duplicate references - same card referenced twice
+        (is (= {["Card" 456] {"Document" document-id}}
+               descendants))
+        ;; Should only have one entry for the card
+        (is (= 1 (count descendants)))))))
+
+(deftest document-serdes-descendants-non-prose-mirror-test
+  (testing "Document descendants handles non-prose-mirror documents"
+    (mt/with-temp [:model/Document {document-id :id} {:document {:some "other format"}
+                                                      :content_type "application/json"}] ; Not prose-mirror
+      (let [descendants (serdes/descendants "Document" document-id)]
+        (is (nil? descendants))))))
+
+(deftest document-serdes-descendants-all-model-types-test
+  (testing "Document descendants correctly maps all supported smart link model types"
+    (mt/with-temp [:model/Document {document-id :id} {:document {:type "doc"
+                                                                 :content [{:type "smartLink"
+                                                                            :attrs {:id 111
+                                                                                    :model "card"}}
+                                                                           {:type "smartLink"
+                                                                            :attrs {:id 222
+                                                                                    :model "dashboard"}}
+                                                                           {:type "smartLink"
+                                                                            :attrs {:id 333
+                                                                                    :model "table"}}]}
+                                                      :content_type "application/json+vnd.prose-mirror"}]
+      (let [descendants (serdes/descendants "Document" document-id)]
+        (is (= {["Card" 111] {"Document" document-id}
+                ["Dashboard" 222] {"Document" document-id}
+                ["Table" 333] {"Document" document-id}}
+               descendants))
+        ;; Verify all supported model types are included
+        (is (contains? descendants ["Card" 111]))
+        (is (contains? descendants ["Dashboard" 222]))
+        (is (contains? descendants ["Table" 333]))))))

enterprise/backend/test/metabase_enterprise/documents/prose_mirror_test.clj

@@ -0,0 +1,319 @@
+(ns metabase-enterprise.documents.prose-mirror-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase-enterprise.documents.prose-mirror :as prose-mirror]))
+
+(deftest ^:parallel update-ast-basic-test
+  (testing "updates nodes that match predicate"
+    (let [doc {:document {:type "doc"
+                          :content [{:type "paragraph"
+                                     :content [{:type "text"
+                                                :text "Hello"}]}
+                                    {:type "paragraph"
+                                     :content [{:type "text"
+                                                :text "World"}]}]}
+               :content_type prose-mirror/prose-mirror-content-type}
+          predicate #(and (map? %) (= (:type %) "text"))
+          updater #(assoc % :text (str (:text %) "!"))
+          result (prose-mirror/update-ast doc predicate updater)]
+      (is (= "Hello!" (get-in result [:document :content 0 :content 0 :text])))
+      (is (= "World!" (get-in result [:document :content 1 :content 0 :text]))))))
+
+(deftest ^:parallel update-ast-non-matching-test
+  (testing "leaves non-matching nodes unchanged"
+    (let [doc {:document {:type "doc"
+                          :content [{:type "paragraph"
+                                     :content [{:type "text"
+                                                :text "Hello"}]}
+                                    {:type "heading"
+                                     :content [{:type "text"
+                                                :text "Title"}]}]}
+               :content_type prose-mirror/prose-mirror-content-type}
+          predicate #(and (map? %) (= (:type %) "paragraph"))
+          updater #(assoc % :modified true)
+          result (prose-mirror/update-ast doc predicate updater)]
+      (is (true? (get-in result [:document :content 0 :modified])))
+      (is (nil? (get-in result [:document :content 1 :modified]))))))
+
+(deftest ^:parallel update-ast-empty-document-test
+  (testing "handles empty document"
+    (let [doc {:document {}
+               :content_type prose-mirror/prose-mirror-content-type}
+          predicate #(= (:type %) "text")
+          updater #(assoc % :updated true)
+          result (prose-mirror/update-ast doc predicate updater)]
+      (is (= {} (:document result))))))
+
+(deftest ^:parallel update-ast-nested-structures-test
+  (testing "handles nested structures"
+    (let [doc {:document {:type "doc"
+                          :content [{:type "table"
+                                     :content [{:type "table_row"
+                                                :content [{:type "table_cell"
+                                                           :content [{:type "paragraph"
+                                                                      :content [{:type "text"
+                                                                                 :text "Cell"}]}]}]}]}]}
+               :content_type prose-mirror/prose-mirror-content-type}
+          predicate #(and (map? %) (= (:type %) "text"))
+          updater #(update % :text str "-updated")
+          result (prose-mirror/update-ast doc predicate updater)]
+      (is (= "Cell-updated"
+             (get-in result [:document :content 0 :content 0 :content 0 :content 0 :content 0 :text]))))))
+
+(deftest ^:parallel update-ast-all-node-types-test
+  (testing "predicate receives all node types including non-maps"
+    (let [doc {:document {:type "doc"
+                          :content ["string-node"
+                                    42
+                                    {:type "paragraph"}]}
+               :content_type prose-mirror/prose-mirror-content-type}
+          visited-nodes (atom [])
+          predicate (fn [node]
+                      (swap! visited-nodes conj node)
+                      false)
+          updater identity]
+      (prose-mirror/update-ast doc predicate updater)
+      (is (some string? @visited-nodes))
+      (is (some number? @visited-nodes))
+      (is (some map? @visited-nodes)))))
+
+(deftest ^:parallel update-ast-updater-function-test
+  (testing "updater function is applied correctly"
+    (let [doc {:document {:type "doc"
+                          :attrs {:level 1}}
+               :content_type prose-mirror/prose-mirror-content-type}
+          predicate #(and (map? %) (contains? % :attrs))
+          updater #(update-in % [:attrs :level] inc)
+          result (prose-mirror/update-ast doc predicate updater)]
+      (is (= 2 (get-in result [:document :attrs :level]))))))
+
+(deftest ^:parallel update-ast-invalid-content-type-test
+  (testing "throws exception for invalid content-type"
+    (let [doc {:document {:type "doc"}
+               :content_type "text/plain"}
+          predicate identity
+          updater identity]
+      (is (thrown-with-msg?
+           clojure.lang.ExceptionInfo
+           #"Document does not have the prose mirror content-type"
+           (prose-mirror/update-ast doc predicate updater))))))
+
+(deftest ^:parallel update-ast-exception-data-test
+  (testing "exception contains correct data"
+    (let [doc {:document {:type "doc"}
+               :content_type "application/json"}
+          predicate identity
+          updater identity]
+      (try
+        (prose-mirror/update-ast doc predicate updater)
+        (is false "Should have thrown exception")
+        (catch clojure.lang.ExceptionInfo e
+          (let [data (ex-data e)]
+            (is (= "application/json" (:content-type data)))
+            (is (= 400 (:status-code data)))))))))
+
+(deftest ^:parallel update-ast-preserve-fields-test
+  (testing "preserves other document fields"
+    (let [doc {:document {:type "doc"}
+               :content_type prose-mirror/prose-mirror-content-type
+               :id 123
+               :name "Test Doc"
+               :other-field "value"}
+          predicate (constantly false)
+          updater identity
+          result (prose-mirror/update-ast doc predicate updater)]
+      (is (= 123 (:id result)))
+      (is (= "Test Doc" (:name result)))
+      (is (= "value" (:other-field result))))))
+
+(deftest ^:parallel update-ast-nil-document-test
+  (testing "handles nil document"
+    (let [doc {:document nil
+               :content_type prose-mirror/prose-mirror-content-type}
+          predicate identity
+          updater identity
+          result (prose-mirror/update-ast doc predicate updater)]
+      (is (nil? (:document result))))))
+
+(deftest ^:parallel update-ast-multiple-updates-test
+  (testing "multiple updates on same node"
+    (let [doc {:document {:type "text" :text "hello"}
+               :content_type prose-mirror/prose-mirror-content-type}
+          predicate #(and (map? %) (= (:type %) "text"))
+          updater #(-> %
+                       (update :text str "!")
+                       (assoc :modified true))
+          result (prose-mirror/update-ast doc predicate updater)]
+      (is (= "hello!" (get-in result [:document :text])))
+      (is (true? (get-in result [:document :modified]))))))
+
+(deftest ^:parallel update-ast-content-type-constant-test
+  (testing "works with valid prose-mirror content-type constant"
+    (let [doc {:document {:type "doc"}
+               :content_type prose-mirror/prose-mirror-content-type}
+          predicate (constantly false)
+          updater identity]
+      (is (some? (prose-mirror/update-ast doc predicate updater))))))
+
+(deftest ^:parallel collect-ast-basic-test
+  (testing "collects values from matching nodes"
+    (let [doc {:document {:type "doc"
+                          :content [{:type "paragraph"
+                                     :content [{:type "text"
+                                                :text "Hello"}]}
+                                    {:type "paragraph"
+                                     :content [{:type "text"
+                                                :text "World"}]}]}
+               :content_type prose-mirror/prose-mirror-content-type}
+          collector #(when (and (map? %) (= (:type %) "text"))
+                       (:text %))
+          result (prose-mirror/collect-ast doc collector)]
+      (is (= ["Hello" "World"] (vec result))))))
+
+(deftest ^:parallel collect-ast-empty-results-test
+  (testing "returns empty sequence when no matches"
+    (let [doc {:document {:type "doc"
+                          :content [{:type "paragraph"
+                                     :content [{:type "text"
+                                                :text "Hello"}]}]}
+               :content_type prose-mirror/prose-mirror-content-type}
+          collector #(when (= (:type %) "nonexistent") %)
+          result (prose-mirror/collect-ast doc collector)]
+      (is (empty? result)))))
+
+(deftest ^:parallel collect-ast-nil-filtering-test
+  (testing "removes nils from results"
+    (let [doc {:document {:type "doc"
+                          :content [{:type "paragraph"
+                                     :content [{:type "text"
+                                                :text "Hello"}]}
+                                    {:type "heading"
+                                     :content [{:type "text"
+                                                :text "Title"}]}]}
+               :content_type prose-mirror/prose-mirror-content-type}
+          collector #(when (and (map? %) (= (:type %) "text") (= (:text %) "Hello"))
+                       (:text %))
+          result (prose-mirror/collect-ast doc collector)]
+      (is (= ["Hello"] (vec result))))))
+
+(deftest ^:parallel collect-ast-nested-structures-test
+  (testing "works with nested structures"
+    (let [doc {:document {:type "doc"
+                          :content [{:type "table"
+                                     :content [{:type "table_row"
+                                                :content [{:type "table_cell"
+                                                           :content [{:type "paragraph"
+                                                                      :content [{:type "text"
+                                                                                 :text "Cell1"}]}]}
+                                                          {:type "table_cell"
+                                                           :content [{:type "paragraph"
+                                                                      :content [{:type "text"
+                                                                                 :text "Cell2"}]}]}]}]}]}
+               :content_type prose-mirror/prose-mirror-content-type}
+          collector #(when (and (map? %) (= (:type %) "text"))
+                       (:text %))
+          result (prose-mirror/collect-ast doc collector)]
+      (is (= ["Cell1" "Cell2"] (vec result))))))
+
+(deftest ^:parallel collect-ast-traversal-order-test
+  (testing "collector receives all nodes in tree traversal order"
+    (let [doc {:document {:type "doc"
+                          :content [{:type "paragraph"
+                                     :content [{:type "text"
+                                                :text "First"}]}
+                                    {:type "paragraph"
+                                     :content [{:type "text"
+                                                :text "Second"}]}]}
+               :content_type prose-mirror/prose-mirror-content-type}
+          visited-types (atom [])
+          collector (fn [node]
+                      (when (map? node)
+                        (swap! visited-types conj (:type node)))
+                      nil)]
+      (doall (prose-mirror/collect-ast doc collector))
+      (is (= ["doc" "paragraph" "text" "paragraph" "text"] @visited-types)))))
+
+(deftest ^:parallel collect-ast-lazy-sequence-test
+  (testing "returns lazy sequence"
+    (let [doc {:document {:type "doc"
+                          :content [{:type "text" :text "test"}]}
+               :content_type prose-mirror/prose-mirror-content-type}
+          collector #(when (map? %) (:type %))
+          result (prose-mirror/collect-ast doc collector)]
+      (is (instance? clojure.lang.LazySeq result)))))
+
+(deftest ^:parallel collect-ast-empty-document-test
+  (testing "handles empty document"
+    (let [doc {:document {}
+               :content_type prose-mirror/prose-mirror-content-type}
+          collector #(when (map? %) (:type %))
+          result (prose-mirror/collect-ast doc collector)]
+      (is (empty? result)))))
+
+(deftest ^:parallel collect-ast-nil-document-test
+  (testing "handles nil document"
+    (let [doc {:document nil
+               :content_type prose-mirror/prose-mirror-content-type}
+          collector identity
+          result (prose-mirror/collect-ast doc collector)]
+      (is (empty? result)))))
+
+(deftest ^:parallel collect-ast-complex-objects-test
+  (testing "collector can return complex objects"
+    (let [doc {:document {:type "doc"
+                          :content [{:type "paragraph"
+                                     :attrs {:id "p1"}
+                                     :content [{:type "text"
+                                                :text "Hello"}]}]}
+               :content_type prose-mirror/prose-mirror-content-type}
+          collector #(when (and (map? %) (= (:type %) "paragraph"))
+                       {:id (get-in % [:attrs :id])
+                        :type (:type %)})
+          result (prose-mirror/collect-ast doc collector)]
+      (is (= [{:id "p1" :type "paragraph"}] (vec result))))))
+
+(deftest ^:parallel collect-ast-invalid-content-type-test
+  (testing "throws exception for invalid content-type"
+    (let [doc {:document {:type "doc"}
+               :content_type "text/plain"}
+          collector identity]
+      (is (thrown-with-msg?
+           clojure.lang.ExceptionInfo
+           #"Document does not have the prose mirror content-type"
+           (prose-mirror/collect-ast doc collector))))))
+
+(deftest ^:parallel collect-ast-exception-data-test
+  (testing "exception contains correct data for collect-ast"
+    (let [doc {:document {:type "doc"}
+               :content_type "application/xml"}
+          collector identity]
+      (try
+        (doall (prose-mirror/collect-ast doc collector))
+        (is false "Should have thrown exception")
+        (catch clojure.lang.ExceptionInfo e
+          (let [data (ex-data e)]
+            (is (= "application/xml" (:content-type data)))
+            (is (= 400 (:status-code data)))))))))
+
+(deftest ^:parallel collect-ast-card-embed-test
+  (testing "works with card embed nodes"
+    (let [doc {:document {:type "doc"
+                          :content [{:type prose-mirror/card-embed-type
+                                     :attrs {:cardId 123}}]}
+               :content_type prose-mirror/prose-mirror-content-type}
+          collector #(when (and (map? %) (= (:type %) prose-mirror/card-embed-type))
+                       (get-in % [:attrs :cardId]))
+          result (prose-mirror/collect-ast doc collector)]
+      (is (= [123] (vec result))))))
+
+(deftest ^:parallel collect-ast-order-preservation-test
+  (testing "preserves order of collection"
+    (let [doc {:document {:type "doc"
+                          :content [{:type "text" :text "first"}
+                                    {:type "text" :text "second"}
+                                    {:type "text" :text "third"}]}
+               :content_type prose-mirror/prose-mirror-content-type}
+          collector #(when (and (map? %) (= (:type %) "text"))
+                       (:text %))
+          result (prose-mirror/collect-ast doc collector)]
+      (is (= ["first" "second" "third"] (vec result))))))

enterprise/backend/test/metabase_enterprise/documents/recent_views_test.clj

@@ -0,0 +1,204 @@
+(ns metabase-enterprise.documents.recent-views-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase-enterprise.documents.recent-views :as recent-views]
+   [metabase.activity-feed.core :as activity-feed]
+   [metabase.events.core :as events]
+   [metabase.test :as mt]
+   [toucan2.core :as t2]))
+
+(defn- most-recent-view
+  [user-id model-id model-type]
+  (t2/select-one [:model/RecentViews :user_id :model :model_id]
+                 :user_id  user-id
+                 :model_id model-id
+                 :model    model-type
+                 {:order-by [[:id :desc]]}))
+
+(deftest card-query-test
+  (mt/with-test-user :rasta
+    (testing "document cards should not be counted"
+      (mt/with-temp [:model/Document {doc-id :id} {}
+                     :model/Card card-3 {:creator_id (mt/user->id :rasta)
+                                         :document_id doc-id}]
+        (events/publish-event! :event/card-query {:card-id (:id card-3)
+                                                  :user-id (mt/user->id :rasta)
+                                                  :context :question})
+        (is (nil? (most-recent-view (mt/user->id :rasta) (:id card-3) "card")))))))
+
+(deftest legacy-card-read-test
+  (testing "in_document cards should not be counted even with context :question"
+    (mt/with-temp [:model/Document {doc-id :id} {}
+                   :model/Card card {:creator_id (mt/user->id :rasta)
+                                     :document_id doc-id}]
+      (mt/with-test-user :rasta
+        (events/publish-event! :event/card-read {:object-id (:id card)
+                                                 :user-id (mt/user->id :rasta)
+                                                 :context :question})
+        (is (nil? (most-recent-view (mt/user->id :rasta) (:id card) "card")))))))
+
+(deftest select-documents-for-recents-empty-input-test
+  (mt/with-premium-features #{:documents}
+    (testing "returns empty vector when given empty document-ids"
+      (is (= [] (recent-views/select-documents-for-recents [])))
+      (is (= [] (recent-views/select-documents-for-recents nil))))))
+
+(deftest select-documents-for-recents-with-collection-test
+  (mt/with-premium-features #{:documents}
+    (testing "returns documents with collection information"
+      (mt/with-temp [:model/Collection {coll-id :id} {:name "Test Collection"
+                                                      :authority_level "official"
+                                                      :archived false}
+                     :model/Document {doc1-id :id} {:name "Document 1"
+                                                    :collection_id coll-id
+                                                    :archived false}
+                     :model/Document {doc2-id :id} {:name "Document 2"
+                                                    :collection_id coll-id
+                                                    :archived false}]
+        (let [results (recent-views/select-documents-for-recents [doc1-id doc2-id])]
+          (is (= 2 (count results)))
+
+          ;; Check first document
+          (let [doc1 (first (filter #(= (:id %) doc1-id) results))]
+            (is (= "Document 1" (:name doc1)))
+            (is (= false (:archived doc1)))
+            (is (= coll-id (:entity-coll-id doc1)))
+            (is (= coll-id (:collection_id doc1)))
+            (is (= "Test Collection" (:collection_name doc1)))
+            (is (= "official" (:collection_authority_level doc1))))
+
+          ;; Check second document
+          (let [doc2 (first (filter #(= (:id %) doc2-id) results))]
+            (is (= "Document 2" (:name doc2)))
+            (is (= false (:archived doc2)))))))))
+
+(deftest select-documents-for-recents-without-collection-test
+  (mt/with-premium-features #{:documents}
+    (testing "returns documents without collection when collection_id is nil"
+      (mt/with-temp [:model/Document {doc-id :id} {:name "Orphan Document"
+                                                   :collection_id nil
+                                                   :archived false}]
+        (let [results (recent-views/select-documents-for-recents [doc-id])]
+          (is (= 1 (count results)))
+          (let [doc (first results)]
+            (is (= "Orphan Document" (:name doc)))
+            (is (= false (:archived doc)))
+            (is (nil? (:entity-coll-id doc)))
+            (is (nil? (:collection_id doc)))
+            (is (nil? (:collection_name doc)))
+            (is (nil? (:collection_authority_level doc)))))))))
+
+(deftest select-documents-for-recents-archived-collections-test
+  (mt/with-premium-features #{:documents}
+    (testing "excludes documents from archived collections via left join"
+      (mt/with-temp [:model/Collection {archived-coll-id :id} {:name "Archived Collection"
+                                                               :authority_level "official"
+                                                               :archived true}
+                     :model/Collection {active-coll-id :id} {:name "Active Collection"
+                                                             :authority_level "official"
+                                                             :archived false}
+                     :model/Document {doc1-id :id} {:name "Document in Archived Collection"
+                                                    :collection_id archived-coll-id
+                                                    :archived false}
+                     :model/Document {doc2-id :id} {:name "Document in Active Collection"
+                                                    :collection_id active-coll-id
+                                                    :archived false}]
+        (let [results (recent-views/select-documents-for-recents [doc1-id doc2-id])]
+          (is (= 2 (count results)))
+
+          ;; Document from archived collection should have nil collection info
+          (let [doc1 (first (filter #(= (:id %) doc1-id) results))]
+            (is (= "Document in Archived Collection" (:name doc1)))
+            (is (= archived-coll-id (:entity-coll-id doc1)))
+            (is (nil? (:collection_id doc1)))
+            (is (nil? (:collection_name doc1)))
+            (is (nil? (:collection_authority_level doc1))))
+
+          ;; Document from active collection should have collection info
+          (let [doc2 (first (filter #(= (:id %) doc2-id) results))]
+            (is (= "Document in Active Collection" (:name doc2)))
+            (is (= active-coll-id (:entity-coll-id doc2)))
+            (is (= active-coll-id (:collection_id doc2)))
+            (is (= "Active Collection" (:collection_name doc2)))
+            (is (= "official" (:collection_authority_level doc2)))))))))
+
+(deftest select-documents-for-recents-archived-documents-test
+  (mt/with-premium-features #{:documents}
+    (testing "includes archived documents in results"
+      (mt/with-temp [:model/Collection {coll-id :id} {:name "Test Collection"
+                                                      :archived false}
+                     :model/Document {doc-id :id} {:name "Archived Document"
+                                                   :collection_id coll-id
+                                                   :archived true}]
+        (let [results (recent-views/select-documents-for-recents [doc-id])]
+          (is (= 1 (count results)))
+          (let [doc (first results)]
+            (is (= "Archived Document" (:name doc)))
+            (is (true? (:archived doc)))))))))
+
+(deftest select-documents-for-recents-nonexistent-ids-test
+  (mt/with-premium-features #{:documents}
+    (testing "handles non-existent document IDs gracefully"
+      (let [results (recent-views/select-documents-for-recents [999999 888888])]
+        (is (= [] results))))))
+
+(deftest select-documents-for-recents-mixed-ids-test
+  (mt/with-premium-features #{:documents}
+    (testing "handles mixed existing and non-existing document IDs"
+      (mt/with-temp [:model/Document {doc-id :id} {:name "Existing Document"
+                                                   :archived false}]
+        (let [results (recent-views/select-documents-for-recents [doc-id 999999])]
+          (is (= 1 (count results)))
+          (is (= "Existing Document" (:name (first results)))))))))
+
+(deftest select-documents-for-recents-duplicates-test
+  (mt/with-premium-features #{:documents}
+    (testing "preserves order and handles duplicates in document-ids"
+      (mt/with-temp [:model/Document {doc1-id :id} {:name "Document 1" :archived false}
+                     :model/Document {doc2-id :id} {:name "Document 2" :archived false}]
+        (let [results (recent-views/select-documents-for-recents [doc1-id doc2-id doc1-id])]
+          ;; Should return unique documents even if IDs are duplicated
+          (is (= 2 (count results)))
+          (is (= #{doc1-id doc2-id} (set (map :id results)))))))))
+
+(deftest recents-api-with-documents-premium-feature-test
+  (testing "/recents API returns documents when :document premium feature is enabled"
+    (mt/with-premium-features #{:documents}
+      (mt/with-temp [:model/Collection {coll-id :id} {:name "Test Collection"}
+                     :model/Document {doc-id :id} {:name "Test Document"
+                                                   :collection_id coll-id
+                                                   :archived false}]
+        ;; Add document to recent views
+        (activity-feed/update-users-recent-views! (mt/user->id :rasta) :model/Document doc-id :view)
+
+        ;; Call the API
+        (let [response (mt/user-http-request :rasta :get 200 "activity/recents" :context [:views])]
+          (is (some #(and (= (:model %) "document")
+                          (= (:id %) doc-id)
+                          (= (:name %) "Test Document"))
+                    (:recents response))
+              "Document should be present in recents when premium feature is enabled"))))))
+
+(deftest select-documents-for-recents-without-premium-feature-test
+  (testing "returns empty vector when :documents premium feature is disabled"
+    (mt/with-premium-features #{}
+      (mt/with-temp [:model/Document {doc-id :id} {:name "Test Document"
+                                                   :archived false}]
+        (let [results (recent-views/select-documents-for-recents [doc-id])]
+          (is (= [] results)
+              "Should return empty vector when premium feature is disabled"))))))
+
+(deftest recents-api-without-documents-premium-feature-test
+  (testing "/recents API does not return documents when :document premium feature is disabled"
+    (mt/with-premium-features #{}
+      (mt/with-temp [:model/Collection {coll-id :id} {:name "Test Collection"}
+                     :model/Document {doc-id :id} {:name "Test Document"
+                                                   :collection_id coll-id
+                                                   :archived false}]
+        ;; Add document to recent views
+        (activity-feed/update-users-recent-views! (mt/user->id :rasta) :model/Document doc-id :view)
+
+        ;; Call the API
+        (let [response (mt/user-http-request :rasta :get 200 "activity/recents" :context [:views])]
+          (is (not-any? #(= (:model %) :document) (:recents response))
+              "No documents should be present in recents when premium feature is disabled"))))))

enterprise/backend/test/metabase_enterprise/documents/revisions/impl_test.clj

@@ -0,0 +1,103 @@
+(ns metabase-enterprise.documents.revisions.impl-test
+  "Unit tests for Document revision serialization and reversion functionality."
+  (:require
+   [clojure.test :refer :all]
+   [metabase-enterprise.documents.revisions.impl]
+   [metabase.revisions.models.revision :as revision]
+   [metabase.test :as mt]
+   [toucan2.core :as t2]))
+
+(deftest serialize-instance-document-test
+  (testing "Document revision serialization excludes metadata fields"
+    (let [document {:id 123
+                    :name "Test Document"
+                    :document {:type "doc" :content [{:type "paragraph" :content [{:type "text" :text "Hello World"}]}]}
+                    :creator_id 456
+                    :created_at #t "2023-01-01T00:00:00Z"
+                    :updated_at #t "2023-01-02T00:00:00Z"
+                    :collection_id 789
+                    :archived false}
+          serialized (revision/serialize-instance :model/Document 123 document)]
+      (testing "includes Document-specific fields"
+        (is (= "Test Document" (:name serialized)))
+        (is (= {:type "doc" :content [{:type "paragraph" :content [{:type "text" :text "Hello World"}]}]}
+               (:document serialized)))
+        (is (= false (:archived serialized))))
+      (testing "excludes metadata fields"
+        (is (not (contains? serialized :id)))
+        (is (not (contains? serialized :creator_id)))
+        (is (not (contains? serialized :created_at)))
+        (is (not (contains? serialized :updated_at)))
+        (is (not (contains? serialized :collection_id)))))))
+
+(deftest serialize-instance-document-with-id-test
+  (testing "Document revision serialization works with different IDs"
+    (let [document {:id 123
+                    :name "Test Document"
+                    :document {:type "doc" :content []}
+                    :creator_id 456
+                    :created_at #t "2023-01-01T00:00:00Z"
+                    :updated_at #t "2023-01-02T00:00:00Z"
+                    :collection_id 789}]
+      (testing "works with matching ID"
+        (let [serialized (revision/serialize-instance :model/Document 123 document)]
+          (is (= "Test Document" (:name serialized)))
+          (is (not (contains? serialized :id)))))
+      (testing "works with different ID"
+        (let [serialized (revision/serialize-instance :model/Document 999 document)]
+          (is (= "Test Document" (:name serialized)))
+          (is (not (contains? serialized :id))))))))
+
+(deftest serialize-instance-document-minimal-test
+  (testing "Document revision serialization works with minimal document"
+    (let [document {:id 1
+                    :name "Minimal Doc"
+                    :document {:type "doc"}
+                    :creator_id 2
+                    :created_at #t "2023-01-01T00:00:00Z"
+                    :updated_at #t "2023-01-01T00:00:00Z"
+                    :collection_id nil}
+          serialized (revision/serialize-instance :model/Document 1 document)]
+      (is (= "Minimal Doc" (:name serialized)))
+      (is (= {:type "doc"} (:document serialized)))
+      (is (nil? (:collection_id serialized)) "collection_id should be excluded even when nil")
+      (is (not (contains? serialized :id)))
+      (is (not (contains? serialized :creator_id))))))
+
+(deftest revert-to-revision-document-test
+  (testing "Document reversion uses default implementation"
+    (mt/with-temp [:model/Document {doc-id :id, :as document} {:name "Original Document"
+                                                               :document {:type "doc" :content [{:type "paragraph" :content [{:type "text" :text "Original content"}]}]}
+                                                               :creator_id (mt/user->id :crowberto)}]
+      (let [original-serialized (revision/serialize-instance :model/Document doc-id document)
+            _ (t2/update! :model/Document doc-id {:name "Updated Document"
+                                                  :document {:type "doc" :content [{:type "paragraph" :content [{:type "text" :text "Updated content"}]}]}})
+            updated-document (t2/select-one :model/Document :id doc-id)]
+
+        (testing "document was updated"
+          (is (= "Updated Document" (:name updated-document)))
+          (is (= {:type "doc" :content [{:type "paragraph" :content [{:type "text" :text "Updated content"}]}]}
+                 (:document updated-document))))
+
+        (testing "reversion restores original state"
+          (revision/revert-to-revision! :model/Document doc-id (mt/user->id :crowberto) original-serialized)
+          (let [reverted-document (t2/select-one :model/Document :id doc-id)]
+            (is (= "Original Document" (:name reverted-document)))
+            (is (= {:type "doc" :content [{:type "paragraph" :content [{:type "text" :text "Original content"}]}]}
+                   (:document reverted-document)))
+            (testing "preserves metadata fields not in serialization"
+              (is (= doc-id (:id reverted-document)))
+              (is (= (mt/user->id :crowberto) (:creator_id reverted-document)))
+              (is (:created_at reverted-document))
+              (is (:updated_at reverted-document)))))))))
+
+(deftest excluded-columns-test
+  (testing "excluded columns constant contains expected fields"
+    (let [excluded-columns @#'metabase-enterprise.documents.revisions.impl/excluded-columns-for-document-revision]
+      (is (contains? excluded-columns :id))
+      (is (contains? excluded-columns :creator_id))
+      (is (contains? excluded-columns :created_at))
+      (is (contains? excluded-columns :updated_at))
+      (is (contains? excluded-columns :collection_id))
+      (is (contains? excluded-columns :collection_position))
+      (is (= 8 (count excluded-columns)) "Should exclude exactly 5 metadata fields"))))

enterprise/backend/test/metabase_enterprise/documents/revisions/integration_test.clj

@@ -0,0 +1,270 @@
+(ns metabase-enterprise.documents.revisions.integration-test
+  "Integration tests for Document revision history including API endpoints, events, and permissions."
+  (:require
+   [clojure.string :as str]
+   [clojure.test :refer :all]
+   [metabase-enterprise.documents.revisions.impl]
+   [metabase.config.core :as config]
+   [metabase.events.core :as events]
+   [metabase.revisions.models.revision :as revision]
+   [metabase.test :as mt]
+   [metabase.util :as u]
+   [toucan2.core :as t2]))
+
+(def ^:private rasta-revision-info
+  (delay {:id (mt/user->id :rasta)
+          :first_name "Rasta"
+          :last_name "Toucan"
+          :common_name "Rasta Toucan"}))
+
+(defn- create-document-revision!
+  "Helper function to create a Document revision manually for testing."
+  [document-id is-creation? user]
+  (revision/push-revision!
+   {:object (t2/select-one :model/Document :id document-id)
+    :entity :model/Document
+    :id document-id
+    :user-id (mt/user->id user)
+    :is-creation? is-creation?}))
+
+(defn- get-document-revisions
+  "Helper function to get revisions for a Document via API."
+  [document-id]
+  (mt/user-http-request :rasta :get 200 (format "revision/document/%d" document-id)))
+
+(deftest document-create-event-triggers-revision-test
+  (testing "Document creation triggers revision creation via events"
+    (mt/with-premium-features #{:documents}
+      (mt/with-temp [:model/Document {doc-id :id, :as document} {:name "Test Document"
+                                                                 :document {:type "doc" :content [{:type "paragraph" :content [{:type "text" :text "Hello World"}]}]}
+                                                                 :creator_id (mt/user->id :crowberto)}]
+        (events/publish-event! :event/document-create {:object document :user-id (mt/user->id :crowberto)})
+        (let [revision (t2/select-one :model/Revision :model "Document" :model_id doc-id)]
+          (is revision "Revision should be created")
+          (is (= "Document" (:model revision)))
+          (is (= doc-id (:model_id revision)))
+          (is (= (mt/user->id :crowberto) (:user_id revision)))
+          (is (:is_creation revision))
+          (is (not (:is_reversion revision)))
+          (testing "serialized object excludes metadata fields"
+            (let [object (:object revision)]
+              (is (= "Test Document" (:name object)))
+              (is (= {:type "doc" :content [{:type "paragraph" :content [{:type "text" :text "Hello World"}]}]} (:document object)))
+              (is (not (contains? object :id)))
+              (is (not (contains? object :creator_id)))
+              (is (not (contains? object :created_at)))
+              (is (not (contains? object :updated_at)))
+              (is (not (contains? object :collection_id))))))))))
+
+(deftest document-update-event-triggers-revision-test
+  (testing "Document update triggers revision creation via events"
+    (mt/with-premium-features #{:documents}
+      (mt/with-temp [:model/Document {doc-id :id, :as document} {:name "Original Document"
+                                                                 :document {:type "doc" :content [{:type "paragraph" :content [{:type "text" :text "Original content"}]}]}
+                                                                 :creator_id (mt/user->id :crowberto)}]
+        (let [updated-document (assoc document
+                                      :name "Updated Document"
+                                      :document {:type "doc" :content [{:type "paragraph" :content [{:type "text" :text "Updated content"}]}]})]
+          (events/publish-event! :event/document-update {:object updated-document :user-id (mt/user->id :rasta)})
+          (let [revision (t2/select-one :model/Revision :model "Document" :model_id doc-id :user_id (mt/user->id :rasta))]
+            (is revision "Revision should be created for update")
+            (is (= "Document" (:model revision)))
+            (is (= doc-id (:model_id revision)))
+            (is (= (mt/user->id :rasta) (:user_id revision)))
+            (is (not (:is_creation revision)))
+            (is (not (:is_reversion revision)))
+            (testing "serialized object contains updated data"
+              (let [object (:object revision)]
+                (is (= "Updated Document" (:name object)))
+                (is (= {:type "doc" :content [{:type "paragraph" :content [{:type "text" :text "Updated content"}]}]} (:document object)))))))))))
+
+(deftest document-revision-api-endpoints-test
+  (testing "Document revision API endpoints work correctly"
+    (mt/with-premium-features #{:documents}
+      (mt/with-temp [:model/Document {doc-id :id} {:name "Test Document"
+                                                   :document {:type "doc" :content [{:type "paragraph" :content [{:type "text" :text "Initial content"}]}]}
+                                                   :creator_id (mt/user->id :rasta)}]
+        (create-document-revision! doc-id true :rasta)
+
+        (testing "GET /api/revision/document/:id returns Document revisions"
+          (let [revisions (get-document-revisions doc-id)]
+            (is (= 1 (count revisions)))
+            (let [revision (first revisions)]
+              (is (:is_creation revision))
+              (is (not (:is_reversion revision)))
+              (is (= "created this." (:description revision)))
+              (is (=? @rasta-revision-info (:user revision)))
+              (is (= config/mb-version-string (:metabase_version revision))))))
+
+        (testing "Multiple revisions are returned in correct order"
+          (t2/update! :model/Document doc-id {:name "Updated Document"})
+          (create-document-revision! doc-id false :rasta)
+
+          (let [revisions (get-document-revisions doc-id)]
+            (is (= 2 (count revisions)))
+            (let [[latest-revision creation-revision] revisions]
+              (is (not (:is_creation latest-revision)))
+              (is (:is_creation creation-revision))
+              (is (= "created this." (:description creation-revision)))
+              (is (not= "created this." (:description latest-revision))))))))))
+
+(deftest document-reversion-api-test
+  (testing "Document reversion via API works correctly"
+    (mt/with-premium-features #{:documents}
+      (mt/with-temp [:model/Document {doc-id :id} {:name "Original Document"
+                                                   :document {:type "doc" :content [{:type "paragraph" :content [{:type "text" :text "Original content"}]}]}
+                                                   :creator_id (mt/user->id :rasta)}]
+        (create-document-revision! doc-id true :rasta)
+
+        (testing "Document can be updated and reverted"
+          (t2/update! :model/Document doc-id {:name "Updated Document"
+                                              :document {:type "doc" :content [{:type "paragraph" :content [{:type "text" :text "Updated content"}]}]}})
+          (create-document-revision! doc-id false :rasta)
+
+          (let [revisions (revision/revisions :model/Document doc-id)
+                [_ {original-revision-id :id}] revisions]
+
+            (testing "Revert via API endpoint"
+              (let [revert-response (mt/user-http-request :rasta :post "revision/revert"
+                                                          {:entity :document
+                                                           :id doc-id
+                                                           :revision_id original-revision-id})]
+                (is (:is_reversion revert-response))
+                (is (= "reverted to an earlier version." (:description revert-response)))
+                (is (=? @rasta-revision-info (:user revert-response)))))
+
+            (testing "Document state is restored"
+              (let [reverted-document (t2/select-one :model/Document :id doc-id)]
+                (is (= "Original Document" (:name reverted-document)))
+                (is (= {:type "doc" :content [{:type "paragraph" :content [{:type "text" :text "Original content"}]}]}
+                       (:document reverted-document)))))
+
+            (testing "Reversion creates new revision entry"
+              (let [final-revisions (get-document-revisions doc-id)]
+                (is (= 3 (count final-revisions)))
+                (let [latest-revision (first final-revisions)]
+                  (is (:is_reversion latest-revision))
+                  (is (= "reverted to an earlier version." (:description latest-revision))))))))))))
+
+(deftest document-revision-permission-checks-test
+  (testing "Document revision API respects permissions"
+    (mt/with-premium-features #{:documents}
+      (mt/with-non-admin-groups-no-root-collection-perms
+        (mt/with-temp [:model/Collection collection {:name "Private collection"}
+                       :model/Document document {:collection_id (u/the-id collection)
+                                                 :name "Private document"
+                                                 :document {:type "doc" :content []}
+                                                 :creator_id (mt/user->id :crowberto)}]
+          (create-document-revision! (:id document) true :crowberto)
+          (t2/update! :model/Document (:id document) {:name "Updated Private Document"})
+          (create-document-revision! (:id document) false :crowberto)
+
+          (let [document-id (u/the-id document)
+                [_ {prev-rev-id :id}] (revision/revisions :model/Document document-id)]
+
+            (testing "User without permissions cannot access revisions"
+              (is (= "You don't have permissions to do that."
+                     (mt/user-http-request :rasta :get 403 (format "revision/document/%d" document-id)))))
+
+            (testing "User without permissions cannot revert"
+              (is (= "You don't have permissions to do that."
+                     (mt/user-http-request :rasta :post 403 "revision/revert"
+                                           {:entity :document
+                                            :id document-id
+                                            :revision_id prev-rev-id}))))))))))
+
+(deftest document-revision-cross-boundary-integration-test
+  (testing "Document revisions integrate properly with OSS revision system"
+    (mt/with-premium-features #{:documents}
+      (mt/with-temp [:model/Document {doc-id :id} {:name "Cross-boundary Document"
+                                                   :document {:type "doc" :content [{:type "paragraph" :content [{:type "text" :text "Enterprise content"}]}]}
+                                                   :creator_id (mt/user->id :rasta)}]
+
+        (testing "Document revisions work with OSS revision/revisions function"
+          (create-document-revision! doc-id true :rasta)
+          (let [revisions (revision/revisions :model/Document doc-id)]
+            (is (seq revisions))
+            (is (= 1 (count revisions)))
+            (let [revision (first revisions)]
+              (is (= doc-id (:model_id revision)))
+              (is (= "Document" (:model revision))))))
+
+        (testing "Document revisions work with OSS revision API routes"
+          (let [revisions-response (mt/user-http-request :rasta :get (format "revision/document/%d" doc-id))]
+            (is (seq revisions-response))
+            (is (every? #(contains? % :is_creation) revisions-response))
+            (is (every? #(contains? % :description) revisions-response))))
+
+        (testing "Document reversion integrates with OSS revert endpoint"
+          (t2/update! :model/Document doc-id {:name "Modified Document"})
+          (create-document-revision! doc-id false :rasta)
+
+          (let [[_ {original-rev-id :id}] (revision/revisions :model/Document doc-id)
+                revert-response (mt/user-http-request :rasta :post "revision/revert"
+                                                      {:entity :document
+                                                       :id doc-id
+                                                       :revision_id original-rev-id})]
+            (is (:is_reversion revert-response))
+            (let [reverted-doc (t2/select-one :model/Document :id doc-id)]
+              (is (= "Cross-boundary Document" (:name reverted-doc))))))))))
+
+(deftest document-revision-event-handler-integration-test
+  (testing "Document revision events are published and handled correctly"
+    (mt/with-premium-features #{:documents}
+      (let [original-event events/publish-event!
+            events-received (atom [])]
+        (with-redefs [events/publish-event! (fn [topic event]
+                                              (swap! events-received conj [topic event])
+                                              (original-event topic event))]
+          (mt/with-temp [:model/Document {doc-id :id, :as document} {:name "Event Test Document"
+                                                                     :document {:type "doc" :content []}
+                                                                     :creator_id (mt/user->id :rasta)}]
+
+            (testing "Create event triggers revision creation"
+              (events/publish-event! :event/document-create {:object document :user-id (mt/user->id :rasta)})
+              (let [revision (t2/select-one :model/Revision :model "Document" :model_id doc-id)]
+                (is revision)
+                (is (:is_creation revision))))
+
+            (testing "Update event triggers revision creation"
+              (let [updated-document (assoc document :name "Updated Event Document")]
+                (events/publish-event! :event/document-update {:object updated-document :user-id (mt/user->id :rasta)})
+                (let [revisions (t2/select :model/Revision :model "Document" :model_id doc-id {:order-by [[:id :desc]]})]
+                  (is (= 2 (count revisions)))
+                  (let [latest-revision (first revisions)]
+                    (is (not (:is_creation latest-revision)))
+                    (is (= "Updated Event Document" (get-in latest-revision [:object :name])))))))
+
+            (testing "Events were published correctly"
+              (is (>= (count @events-received) 2))
+              (is (some #(= :event/document-create (first %)) @events-received))
+              (is (some #(= :event/document-update (first %)) @events-received)))))))))
+
+(deftest document-revision-diff-and-descriptions-test
+  (testing "Document revision diffs and descriptions work correctly"
+    (mt/with-premium-features #{:documents}
+      (mt/with-temp [:model/Document {doc-id :id} {:name "Original Title"
+                                                   :document {:type "doc" :content [{:type "paragraph" :content [{:type "text" :text "Original content"}]}]}
+                                                   :creator_id (mt/user->id :rasta)}]
+        (create-document-revision! doc-id true :rasta)
+
+        (testing "Name change produces correct diff and description"
+          (t2/update! :model/Document doc-id {:name "Updated Title"})
+          (create-document-revision! doc-id false :rasta)
+
+          (let [revisions (get-document-revisions doc-id)
+                update-revision (first revisions)]
+            (is (= "Original Title" (get-in update-revision [:diff :before :name])))
+            (is (= "Updated Title" (get-in update-revision [:diff :after :name])))
+            (is (str/includes? (:description update-revision) "renamed"))))
+
+        (testing "Content change produces diff"
+          (t2/update! :model/Document doc-id {:document {:type "doc" :content [{:type "paragraph" :content [{:type "text" :text "New content"}]}]}})
+          (create-document-revision! doc-id false :rasta)
+
+          (let [revisions (get-document-revisions doc-id)
+                content-revision (first revisions)]
+            (is (= {:content [{:content [{:text "Original content"}]}]}
+                   (get-in content-revision [:diff :before :document])))
+            (is (= {:content [{:content [{:text "New content"}]}]}
+                   (get-in content-revision [:diff :after :document])))))))))

enterprise/backend/test/metabase_enterprise/documents/search_test.clj

@@ -0,0 +1,176 @@
+(ns metabase-enterprise.documents.search-test
+  (:require
+   [clojure.test :refer :all]
+   [java-time.api :as t]
+   [metabase.events.core :as events]
+   [metabase.permissions.core :as perms]
+   [metabase.permissions.models.permissions-group :as perms-group]
+   [metabase.search.test-util :as search.tu]
+   [metabase.test :as mt]
+   [metabase.test.fixtures :as fixtures]
+   [metabase.util :as u]
+   [toucan2.core :as t2]))
+
+(defn- text->prose-mirror-ast
+  "Convert plain text to a ProseMirror AST structure."
+  [text]
+  (if (empty? text)
+    {:type "doc" :content []}
+    {:type "doc"
+     :content [{:type "paragraph"
+                :content [{:type "text"
+                           :text text}]}]}))
+
+(use-fixtures :once (fixtures/initialize :db :test-users))
+
+(defn card-with-name-and-query
+  ([]
+   (card-with-name-and-query (mt/random-name)))
+  ([card-name]
+   {:name card-name
+    :display "scalar"
+    :dataset_query (mt/mbql-query venues)
+    :visualization_settings {:global {:title nil}}}))
+
+(deftest card-search-exclusion-test
+  (testing "Cards with document_id are excluded from search indexing"
+    (let [card-name (mt/random-name)
+          regular-card-name (str card-name "-regular")]
+      (search.tu/with-temp-index-table
+        (mt/with-temp [:model/Document document {:name "Test Document"
+                                                 :document (text->prose-mirror-ast "")}
+                       :model/Card document-card (-> (card-with-name-and-query card-name)
+                                                     (assoc :document_id (u/the-id document)))
+                       :model/Card regular-card (card-with-name-and-query regular-card-name)]
+
+          (testing "Search API excludes document cards"
+            (let [results (mt/user-http-request :crowberto :get 200 "search" :q card-name :models "card")]
+              (is (not (some #(= (:id %) (u/the-id document-card)) (:data results)))
+                  "Document card should not appear in search results"))
+            (let [regular-results (mt/user-http-request :crowberto :get 200 "search" :q regular-card-name :models "card")]
+              (is (some #(= (:id %) (u/the-id regular-card)) (:data regular-results))
+                  "Regular card should appear in search results"))))))))
+
+(deftest document-permission-filtering-test
+  (testing "Document search respects collection-based permissions"
+    (mt/with-premium-features #{:documents}
+      (mt/with-non-admin-groups-no-root-collection-perms
+        (search.tu/with-temp-index-table
+          (mt/with-temp [:model/Collection {coll-id :id} {}
+                         :model/Collection {private-coll-id :id} {}
+                         :model/Document {doc-in-public-coll :id} {:name "Public Document"
+                                                                   :collection_id coll-id
+                                                                   :document (text->prose-mirror-ast "content")}
+                         :model/Document {doc-in-private-coll :id} {:name "Private Document"
+                                                                    :collection_id private-coll-id
+                                                                    :document (text->prose-mirror-ast "content")}
+                         :model/Document {doc-archived :id} {:name "Archived Document"
+                                                             :collection_id coll-id
+                                                             :archived true
+                                                             :document (text->prose-mirror-ast "content")}]
+            ;; Give user read access to first collection only
+            (perms/grant-collection-read-permissions! (perms-group/all-users) coll-id)
+
+            (testing "Regular user sees only documents in accessible collections"
+              (let [results (mt/user-http-request :rasta :get 200 "search" :q "Document" :models "document")]
+                (is (= #{doc-in-public-coll}
+                       (set (map :id (:data results))))
+                    "Should only see document in accessible collection")))
+
+            (testing "Regular user cannot see archived documents (no write perms)"
+              (let [results (mt/user-http-request :rasta :get 200 "search" :q "Archived" :archived true :models "document")]
+                (is (empty? (:data results))
+                    "Should not see archived document without write permissions")))
+
+            (testing "Admin can see all documents including archived"
+              (let [regular-results (mt/user-http-request :crowberto :get 200 "search" :q "Document" :models "document")
+                    archived-results (mt/user-http-request :crowberto :get 200 "search" :q "Archived" :archived true :models "document")]
+                (is (= #{doc-in-public-coll doc-in-private-coll}
+                       (set (map :id (:data regular-results))))
+                    "Admin should see all non-archived documents")
+                (is (= #{doc-archived}
+                       (set (map :id (:data archived-results))))
+                    "Admin should see archived documents")))
+
+            (testing "User with write permissions can see archived documents"
+              ;; Give user write access to first collection
+              (perms/grant-collection-readwrite-permissions! (perms-group/all-users) coll-id)
+              (let [archived-results (mt/user-http-request :rasta :get 200 "search" :q "Archived" :archived true :models "document")]
+                (is (= #{doc-archived}
+                       (set (map :id (:data archived-results))))
+                    "User with write permissions should see archived documents in accessible collections")))))))))
+
+(deftest document-search-without-premium-feature-test
+  (testing "Documents are not returned from search when documents premium feature is disabled"
+    (mt/with-premium-features #{}
+      (search.tu/with-temp-index-table
+        (mt/with-temp [:model/Collection {coll-id :id} {}
+                       :model/Document {doc-id :id} {:name "Test Document"
+                                                     :collection_id coll-id
+                                                     :document (text->prose-mirror-ast "searchable content")}
+                       :model/Card {card-id :id} (-> (card-with-name-and-query "Test Card")
+                                                     (assoc :collection_id coll-id))]
+
+          (testing "Search for documents returns empty results"
+            (let [results (mt/user-http-request :crowberto :get 200 "search" :q "Test Document" :models "document")]
+              (is (empty? (:data results))
+                  "Should not return any documents when premium feature is disabled")))
+
+          (testing "Search for cards still works normally"
+            (let [results (mt/user-http-request :crowberto :get 200 "search" :q "Test Card" :models "card")]
+              (is (= #{card-id}
+                     (set (map :id (:data results))))
+                  "Should still return cards when documents feature is disabled")))
+
+          (testing "General search excludes documents but includes other models"
+            (let [results (mt/user-http-request :crowberto :get 200 "search" :q "Test")]
+              (is (some #(= (:id %) card-id) (:data results))
+                  "Should include cards in general search")
+              (is (not (some #(= (:id %) doc-id) (:data results)))
+                  "Should not include documents in general search when feature is disabled"))))))))
+
+(deftest document-view-tracking-integration-test
+  (testing "Document view tracking integrates with search"
+    (mt/with-premium-features #{:documents}
+      (mt/with-temp [:model/Document {doc-id :id} {:name "Viewed Document"
+                                                   :document (text->prose-mirror-ast "content")
+                                                   :view_count 0}]
+
+        (testing "Document has initial state"
+          (let [doc (t2/select-one :model/Document :id doc-id)]
+            (is (= 0 (:view_count doc)))
+            ;; last_viewed_at has a default timestamp from migration, not nil
+            (is (some? (:last_viewed_at doc)))))
+
+        (testing "Publishing document-read event works without errors"
+          ;; The actual view count increment is batched and asynchronous
+          ;; So we test that the event publishes successfully
+          (is (some? (events/publish-event! :event/document-read
+                                            {:object-id doc-id
+                                             :user-id (mt/user->id :crowberto)}))
+              "Document read event should publish successfully"))
+
+        (testing "Document with higher view count appears in search"
+          ;; Manually set view count to test search integration
+          (t2/update! :model/Document doc-id {:view_count 5
+                                              :last_viewed_at (t/offset-date-time)})
+          (search.tu/with-temp-index-table
+            (let [results (mt/user-http-request :crowberto :get 200 "search" {:q "Viewed" :models "document"})
+                  doc-results (filter #(= "document" (:model %)) (:data results))]
+              (when (seq doc-results)
+                (let [doc-result (first doc-results)]
+                  (is (= doc-id (:id doc-result)))
+                  ;; View count affects scoring but isn't directly in result
+                  (is (some #(= "view-count" (:name %)) (:scores doc-result))))))))
+
+        (testing "Document with recent view appears in search results"
+          (let [recent-time (t/minus (t/offset-date-time) (t/minutes 5))]
+            (t2/update! :model/Document doc-id {:last_viewed_at recent-time})
+            (search.tu/with-temp-index-table
+              (let [results (mt/user-http-request :crowberto :get 200 "search" {:q "Viewed" :models "document"})
+                    doc-results (filter #(= "document" (:model %)) (:data results))]
+                (when (seq doc-results)
+                  (let [doc-result (first doc-results)]
+                    (is (= doc-id (:id doc-result)))
+                    ;; User recency affects scoring
+                    (is (some #(= "user-recency" (:name %)) (:scores doc-result)))))))))))))

enterprise/backend/test/metabase_enterprise/documents/view_log_test.clj

@@ -0,0 +1,149 @@
+(ns metabase-enterprise.documents.view-log-test
+  (:require
+   [clojure.test :refer :all]
+   [java-time.api :as t]
+   [metabase-enterprise.documents.view-log :as documents.view-log]
+   [metabase.events.core :as events]
+   [metabase.test :as mt]
+   [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(defn latest-view
+  "Returns the most recent view for a given user and document ID"
+  [user-id document-id]
+  (t2/select-one :model/ViewLog
+                 :user_id user-id
+                 :model "document"
+                 :model_id document-id
+                 {:order-by [[:id :desc]]}))
+
+(deftest document-read-ee-test
+  (mt/with-premium-features #{:audit-app :documents}
+    (mt/with-temp [:model/Collection collection {}
+                   :model/User user {}
+                   :model/Document document {:collection_id (:id collection)
+                                             :name "Test Document"
+                                             :document "{\"type\":\"doc\",\"content\":[]}"
+                                             :creator_id (:id user)}]
+      (testing "A basic document read event is recorded in EE"
+        (events/publish-event! :event/document-read {:object-id (:id document) :user-id (:id user)})
+        (is (partial=
+             {:user_id (:id user)
+              :model "document"
+              :model_id (:id document)
+              :has_access true
+              :context nil}
+             (latest-view (:id user) (:id document))))))))
+
+(deftest document-read-oss-no-view-logging-test
+  (mt/with-premium-features #{:documents} ; documents feature but not audit-app
+    (mt/with-temp [:model/Collection collection {}
+                   :model/User user {}
+                   :model/Document document {:collection_id (:id collection)
+                                             :name "Test Document"
+                                             :document "{\"type\":\"doc\",\"content\":[]}"
+                                             :creator_id (:id user)}]
+      (testing "A basic document read event is not recorded without audit-app"
+        (events/publish-event! :event/document-read {:object-id (:id document) :user-id (:id user)})
+        (is (nil? (latest-view (:id user) (:id document)))
+            "view log entries should not be made without audit-app feature")))))
+
+(deftest document-read-view-count-test
+  (mt/test-helpers-set-global-values!
+    (mt/with-temporary-setting-values [synchronous-batch-updates true]
+      (mt/with-premium-features #{:documents}
+        (mt/with-temp [:model/Collection collection {}
+                       :model/User user {}
+                       :model/Document document {:collection_id (:id collection)
+                                                 :name "Test Document"
+                                                 :document "{\"type\":\"doc\",\"content\":[]}"
+                                                 :creator_id (:id user)
+                                                 :view_count 0}]
+          (testing "Document read events are recorded by a document's view_count"
+            (is (= 0 (:view_count document))
+                "view_count should be 0 before the event is published")
+            (events/publish-event! :event/document-read {:object-id (:id document) :user-id (:id user)})
+            (is (= 1 (t2/select-one-fn :view_count :model/Document (:id document))))
+            (events/publish-event! :event/document-read {:object-id (:id document) :user-id (:id user)})
+            (is (= 2 (t2/select-one-fn :view_count :model/Document (:id document))))))))))
+
+(deftest update-document-last-viewed-at-test
+  (let [now (-> (t/offset-date-time)
+                (.withNano 0))
+        one-hour-ago (t/minus now (t/hours 1))
+        two-hours-ago (t/minus now (t/hours 2))]
+    (testing "update with multiple documents of the same IDs will set timestamp to the latest"
+      (mt/with-premium-features #{:documents}
+        (mt/with-temp [:model/Collection collection {}
+                       :model/User user {}
+                       :model/Document document {:collection_id (:id collection)
+                                                 :name "Test Document"
+                                                 :document "{\"type\":\"doc\",\"content\":[]}"
+                                                 :creator_id (:id user)
+                                                 :last_viewed_at two-hours-ago}]
+          (#'documents.view-log/update-document-last-viewed-at!* [{:id (:id document) :timestamp one-hour-ago}
+                                                                  {:id (:id document) :timestamp two-hours-ago}])
+          (is (= one-hour-ago
+                 (-> (t2/select-one-fn :last_viewed_at :model/Document (:id document))
+                     t/offset-date-time
+                     (.withNano 0)))))))
+
+    (testing "if the existing last_viewed_at is greater than the updating values, do not override it"
+      (mt/with-premium-features #{:documents}
+        (mt/with-temp [:model/Collection collection {}
+                       :model/User user {}
+                       :model/Document document {:collection_id (:id collection)
+                                                 :name "Test Document"
+                                                 :document "{\"type\":\"doc\",\"content\":[]}"
+                                                 :creator_id (:id user)
+                                                 :last_viewed_at now}]
+          (#'documents.view-log/update-document-last-viewed-at!* [{:id (:id document) :timestamp one-hour-ago}])
+          (is (= now
+                 (-> (t2/select-one-fn :last_viewed_at :model/Document (:id document))
+                     t/offset-date-time
+                     (.withNano 0)))))))))
+
+(deftest document-event-derivation-test
+  (testing "Document events are properly derived from base events"
+    (is (isa? :metabase-enterprise.documents.view-log/document-read :metabase/event))
+    (is (isa? :event/document-read :metabase-enterprise.documents.view-log/document-read))))
+
+(deftest document-read-error-handling-test
+  (mt/with-premium-features #{:documents}
+    (testing "Document read event handles missing document gracefully"
+      ;; This should not throw an exception
+      (is (some? (events/publish-event! :event/document-read {:object-id 999999 :user-id (mt/user->id :rasta)}))))))
+
+(deftest document-statistics-lock-test
+  (testing "Document statistics lock is properly defined"
+    (is (= :metabase-enterprise.documents.view-log/document-statistics-lock
+           @#'documents.view-log/document-statistics-lock))))
+
+(deftest document-read-updates-recent-views-test
+  (mt/with-premium-features #{:audit-app :documents}
+    (mt/with-temp [:model/Collection collection {}
+                   :model/User user {}
+                   :model/Document document {:collection_id (:id collection)
+                                             :name "Test Document"
+                                             :document "{\"type\":\"doc\",\"content\":[]}"
+                                             :creator_id (:id user)}]
+      (testing "Document read event updates user's recent views"
+        ;; Verify no recent views initially
+        (is (empty? (t2/select :model/RecentViews
+                               :user_id (:id user)
+                               :model "document"
+                               :model_id (:id document))))
+
+        ;; Publish document read event
+        (events/publish-event! :event/document-read {:object-id (:id document) :user-id (:id user)})
+
+        ;; Verify recent view was created
+        (let [recent-view (t2/select-one :model/RecentViews
+                                         :user_id (:id user)
+                                         :model "document"
+                                         :model_id (:id document))]
+          (is (some? recent-view) "Recent view should be created")
+          (is (= (:id user) (:user_id recent-view)))
+          (is (= "document" (:model recent-view)))
+          (is (= (:id document) (:model_id recent-view))))))))

enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj

@@ -4,6 +4,7 @@
    [malli.core :as mc]
    [malli.transform :as mtx]
    [medley.core :as m]
+   [metabase-enterprise.metabot-v3.client :as metabot-v3.client]
    [metabase-enterprise.metabot-v3.tools.api :as metabot-v3.tools.api]
    [metabase-enterprise.metabot-v3.tools.create-dashboard-subscription :as metabot-v3.tools.create-dashboard-subscription]
    [metabase-enterprise.metabot-v3.tools.filters :as metabot-v3.tools.filters]
@@ -36,7 +37,7 @@
   ([] (ai-session-token :rasta (str (random-uuid))))
   ([metabot-id] (ai-session-token :rasta metabot-id))
   ([user metabot-id]
-   (-> user mt/user->id (#'metabot-v3.tools.api/get-ai-service-token metabot-id))))
+   (-> user mt/user->id (#'metabot-v3.client/get-ai-service-token metabot-id))))
 
 (deftest create-dashboard-subscription-test
   (mt/with-premium-features #{:metabot-v3}

enterprise/backend/test/metabase_enterprise/models/entity_id_test.clj

@@ -45,6 +45,7 @@
     :model/DashboardBookmark
     :model/DataPermissions
     :model/DatabaseRouter
+    :model/DocumentBookmark
     :model/CollectionPermissionGraphRevision
     :model/DashboardCardSeries
     :model/LoginHistory

enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj

@@ -1832,6 +1832,70 @@
             (is (= #{[{:model "Collection" :id model-eid}] [{:model "Card" :id card-eid}]}
                    (set (serdes/dependencies ser))))))))))
 
+(deftest document-test
+  (mt/with-empty-h2-app-db!
+    (ts/with-temp-dpc [:model/Collection collection {}
+                       :model/User user {:first_name "Mark"
+                                         :last_name  "Knopfler"
+                                         :email      "mark@direstrai.ts"}
+                       :model/Document document {:name "Test Document"
+                                                 :document {}
+                                                 :content_type "application/json+vnd.prose-mirror"
+                                                 :collection_id (:id collection)
+                                                 :creator_id (:id user)}
+                       :model/Card card {:name "Dependent Card"
+                                         :document_id (u/the-id document)}
+                       :model/Card linked-card {:name "Linked Card"}
+                       :model/Dashboard dashboard {:name "Smart Linked Dashboard"}
+                       :model/Table table {:name "linked_table"}]
+
+      (t2/update! :model/Document :id (u/the-id document) {:document {:type "doc"
+                                                                      :content [{:type "cardEmbed"
+                                                                                 :attrs {:id (u/the-id card)}}
+                                                                                {:type "smartLink"
+                                                                                 :attrs {:entityId (u/the-id linked-card)
+                                                                                         :model "card"}}
+                                                                                {:type "smartLink"
+                                                                                 :attrs {:entityId (u/the-id table)
+                                                                                         :model "table"}}
+                                                                                {:type "smartLink"
+                                                                                 :attrs {:entityId (u/the-id dashboard)
+                                                                                         :model "dashboard"}}]}})
+      (testing "document extraction"
+        (let [ser (ts/extract-one "Document" (u/the-id document))]
+          (is (=? {:serdes/meta [{:model "Document" :id (:entity_id document)}]
+                   :name "Test Document"
+                   :entity_id (:entity_id document)
+                   :document {:type "doc"
+                              :content [{:type "cardEmbed"
+                                         :attrs {:id [{:model "Card" :id (:entity_id card)}]}}
+                                        {:type "smartLink"
+                                         :attrs {:entityId [{:model "Card" :id (:entity_id linked-card)}]
+                                                 :model "card"}}
+                                        {:type "smartLink"
+                                         :attrs {:entityId (serdes/generate-path "Table" table)
+                                                 :model "table"}}
+                                        {:type "smartLink"
+                                         :attrs {:entityId [{:model "Dashboard" :id (:entity_id dashboard)}]
+                                                 :model "dashboard"}}]}
+                   :archived false
+                   :archived_directly false
+                   :creator_id (:email user)
+                   :collection_id (:entity_id collection)
+                   :content_type "application/json+vnd.prose-mirror"
+                   :updated_at string?
+                   :created_at string?}
+                  ser))
+          (is (not (contains? ser :id)))
+
+          (testing "depends on its collection, cardEmbeds and smarkLinks "
+            (is (= #{[{:model "Collection" :id (:entity_id collection)}]
+                     [{:model "Card" :id (:entity_id card)}]
+                     [{:model "Dashboard" :id (:entity_id dashboard)}]
+                     [{:model "Card" :id (:entity_id linked-card)}]
+                     (serdes/generate-path "Table" table)}
+                   (set (serdes/dependencies ser))))))))))
+
 (deftest visualizer-dashboard-card-settings-test
   (testing "visualizer settings transform entity IDs <-> card IDs"
     (let [card-entity-id "WcMlLFNVcy0iO49mKW3WH"

enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj

@@ -0,0 +1,111 @@
+(ns metabase-enterprise.workspaces.api-test
+  "Tests for workspace API endpoints"
+  (:require
+   [clojure.test :refer :all]
+   [java-time.api :as t]
+   [malli.error :as me]
+   [malli.util :as mut]
+   [metabase-enterprise.workspaces.models.workspace :as m.workspace]
+   [metabase.test :as mt]
+   [metabase.test.fixtures :as fixtures]
+   [metabase.util.malli.registry :as mr]
+   [toucan2.core :as t2]))
+
+(use-fixtures :once (fixtures/initialize :db :test-users))
+
+(deftest ^:parallel workspace-validation-test
+  (testing "Workspace validation with updated Malli schema structure"
+    (let [valid-workspace {:id              1
+                           :name            "Test"
+                           :description     "Test workspace"
+                           :created_at      (t/offset-date-time "2023-08-06T15:00:00Z")
+                           :updated_at      (t/offset-date-time "2023-08-06T15:00:00Z")
+                           :plans           []
+                           :transforms      []
+                           :activity_logs   []
+                           :permissions     []
+                           :users           []
+                           :data_warehouses {1 {}}
+                           :documents       []
+                           :api_key_id      1
+                           :collection_id   2}]
+      (is (nil? (me/humanize (mr/explain ::m.workspace/workspace valid-workspace)))))))
+
+(deftest ^:parallel create-workspace-test
+  (testing "Creating a workspace with valid data using mt/with-temp"
+    (mt/with-temp [:model/Collection {collection-id :id} {:name "API Workspace Collection"}
+                   :model/ApiKey     {api-key-id :id}    {:user_id                             (mt/user->id :rasta)
+                                                          :creator_id                          (mt/user->id :rasta)
+                                                          :updated_by_id                       (mt/user->id :rasta)
+                                                          :name                                "Car Key"
+                                                          :metabase.api-keys.core/unhashed-key "mb_123456789"}
+                   :model/Workspace {workspace-id :id :as wksp} {:name          "Test Workspace"
+                                                                 :description   "A test workspace"
+                                                                 :collection_id collection-id
+                                                                 :api_key_id    api-key-id}]
+      (is (pos? workspace-id))
+      (is (= "Test Workspace"
+             (:name (t2/select-one :model/Workspace :id workspace-id))))
+      (is (= (mut/keys (mr/resolve-schema ::m.workspace/workspace))
+             (keys (m.workspace/sort-workspace wksp)))))))
+
+ ;;; API Endpoint Tests
+
+(defn- test-workspace-data
+  "Generate test workspace data with a valid collection."
+  ([]
+   (test-workspace-data {}))
+  ([overrides]
+   (merge {:name "Test Workspace"
+           :description "A test workspace"
+           :collection_id 1 ; Will be replaced with actual collection ID in tests
+           :users []
+           :plans []
+           :transforms []
+           :documents []
+           :data_warehouses []
+           :permissions []
+           :activity_logs []}
+          overrides)))
+
+(deftest ^:parallel api-get-workspaces-test
+  (testing "GET /api/ee/workspace/ - list workspaces"
+    (mt/with-temp [:model/Collection {collection-1-id :id} {:name "API Workspace Collection"}
+                   :model/Collection {collection-2-id :id} {:name "API Workspace Collection 2"}
+                   :model/ApiKey     {api-key-1-id :id}    {:user_id                             (mt/user->id :rasta)
+                                                            :creator_id                          (mt/user->id :rasta)
+                                                            :updated_by_id                       (mt/user->id :rasta)
+                                                            :name                                "Key Lime"
+                                                            :metabase.api-keys.core/unhashed-key "mb_123456789"}
+                   :model/ApiKey     {api-key-2-id :id}    {:user_id                             (mt/user->id :rasta)
+                                                            :creator_id                          (mt/user->id :rasta)
+                                                            :updated_by_id                       (mt/user->id :rasta)
+                                                            :name                                "Skeleton Key"
+                                                            :metabase.api-keys.core/unhashed-key "mb_987654321"}
+                   :model/Workspace _ {:name          "API Workspace 1"
+                                       :collection_id collection-1-id
+                                       :api_key_id    api-key-1-id}
+                   :model/Workspace _ {:name          "API Workspace 2"
+                                       :collection_id collection-2-id
+                                       :api_key_id    api-key-2-id}]
+      (let [result (mt/user-http-request :crowberto :get 200 "ee/workspace/")]
+        (is (>= (count result) 2))
+        (is (some #(= "API Workspace 1" (:name %)) result))
+        (is (some #(= "API Workspace 2" (:name %)) result))))))
+
+(deftest ^:parallel api-get-workspace-by-id-test
+  (testing "GET /api/ee/workspace/:workspace-id - get specific workspace"
+    (mt/with-temp [:model/Collection {collection-id :id} {:name "Test Workspace Collection"}
+                   :model/ApiKey     {api-key-id :id}    {:user_id                             (mt/user->id :rasta)
+                                                          :creator_id                          (mt/user->id :rasta)
+                                                          :updated_by_id                       (mt/user->id :rasta)
+                                                          :name                                "Keystone Light"
+                                                          :metabase.api-keys.core/unhashed-key "mb_123456789"}
+                   :model/Workspace  {workspace-id :id} {:name          "Test Workspace"
+                                                         :collection_id collection-id
+                                                         :api_key_id    api-key-id}]
+      (testing "should get existing workspace"
+        (let [result (mt/user-http-request :crowberto :get 200 (format "ee/workspace/%s" workspace-id))]
+          (is (= workspace-id (:id result)))))
+      (testing "should return 404 for non-existent workspace"
+        (mt/user-http-request :crowberto :get 404 "ee/workspace/888888")))))

enterprise/backend/test/metabase_enterprise/workspaces/common_test.clj

@@ -0,0 +1,96 @@
+(ns metabase-enterprise.workspaces.common-test
+  (:require
+   [clojure.set :as set]
+   [clojure.test :refer :all]
+   [metabase-enterprise.workspaces.common :as workspaces.common]
+   [metabase.api-keys.core :as api-key]
+   [metabase.api.common :as api]
+   [metabase.permissions.core :as perms]
+   [metabase.request.core :as request]
+   [metabase.server.middleware.session :as mw.session]
+   [metabase.test :as mt]
+   [metabase.test.fixtures :as fixtures]
+   [metabase.util.secret :as u.secret]
+   [toucan2.core :as t2]))
+
+(use-fixtures :once (fixtures/initialize :db :test-users :web-server))
+
+(deftest create-workspace!-test
+  (mt/with-temp [:model/User {user-id :id} {}]
+    (mt/with-model-cleanup [:model/Workspace]
+      (let [workspace (workspaces.common/create-workspace!
+                       user-id
+                       (mt/random-name)
+                       "Dark Workspace")]
+        (testing "Should have created a collection"
+          (let [collection-id (:collection_id workspace)]
+            (is (pos-int? collection-id))
+            (is (t2/exists? :model/Collection collection-id))))
+        (testing "should have created an API key"
+          (let [api-key-id (:api_key_id workspace)]
+            (is (pos-int? api-key-id))
+            (is (t2/exists? :model/ApiKey api-key-id)))
+          (testing "Should return raw API Key"
+            (let [api-key (::workspaces.common/api-key workspace)]
+              (is (u.secret/secret? api-key))
+              (is (=? #"^mb_.{44}$"
+                      (u.secret/expose api-key))))))))))
+
+(deftest delete-workspace-test
+  (let [workspace-exists?  (fn [workspace]
+                             (t2/exists? :model/Workspace (:id workspace)))
+        collection-exists? (fn [workspace]
+                             (t2/exists? :model/Collection (:collection_id workspace)))
+        api-key-exists?    (fn [workspace]
+                             (t2/exists? :model/ApiKey (:api_key_id workspace)))
+        workspace          (workspaces.common/create-workspace!
+                            (mt/user->id :rasta)
+                            (mt/random-name)
+                            "Dark Workspace")]
+    (is (workspace-exists? workspace))
+    (is (collection-exists? workspace))
+    (is (api-key-exists? workspace))
+    (t2/delete! :model/Workspace (:id workspace))
+    (testing "After deleting the workspace..."
+      (is (not (workspace-exists? workspace)))
+      (testing "Collection should still exist"
+        (is (collection-exists? workspace)))
+      (testing "API key should be deleted"
+        (is (not (api-key-exists? workspace))))
+      (testing "User should still be active"
+        (is (t2/select-one-fn :is_active :model/User (mt/user->id :rasta)))))))
+
+(deftest single-collection-api-key-e2e-test
+  (mt/with-temp [:model/Collection {original-collection-id :id} {}]
+    (let [perms-set             (fn [current-user-info]
+                                  (request/do-with-current-user
+                                   current-user-info
+                                   (fn [] @api/*current-user-permissions-set*)))
+          has-collection-perms? (fn [perms-set collection-id]
+                                  (contains? perms-set (perms/collection-readwrite-path collection-id)))
+          original-perms-set    (perms-set {:metabase-user-id (mt/user->id :rasta)})]
+      (testing "Sanity check: original perms should have perms for collections"
+        (is (has-collection-perms? original-perms-set original-collection-id)))
+      (testing "with API key"
+        (mt/with-model-cleanup [:model/Workspace]
+          (let [workspace                 (workspaces.common/create-workspace!
+                                           (mt/user->id :crowberto)
+                                           (mt/random-name)
+                                           "Dark Workspace")
+                workspace-collection-id   (:collection_id workspace)
+                _                         (is (> workspace-collection-id original-collection-id)
+                                              "Should be a NEW collection")
+                api-key                   (-> (::workspaces.common/api-key workspace)
+                                              u.secret/expose)
+                api-key-current-user-info (#'mw.session/merge-current-user-info {:headers {"x-api-key" api-key}})]
+            (is (=? {:workspace/collection-id workspace-collection-id
+                     :workspace/attributes    nil}
+                    api-key-current-user-info))
+            (let [api-key-perms-set (perms-set api-key-current-user-info)]
+              (testing "Should not have unrestricted admin perms"
+                (is (not (contains? api-key-perms-set "/"))))
+              (is (set/subset? api-key-perms-set
+                               (-> original-perms-set
+                                   (conj (perms/collection-readwrite-path workspace-collection-id)))))
+              (is (has-collection-perms? api-key-perms-set workspace-collection-id))
+              (is (not (has-collection-perms? api-key-perms-set original-collection-id))))))))))

enterprise/backend/test/metabase_enterprise/workspaces/isolation_manager_test.clj

@@ -0,0 +1,235 @@
+(ns ^:mb/driver-tests metabase-enterprise.workspaces.isolation-manager-test
+  "Tests for metabase-enterprise.workspaces.isolation-manager."
+  (:require
+   [clojure.java.jdbc :as jdbc]
+   [clojure.string :as str]
+   [clojure.test :refer :all]
+   [mb.hawk.assert-exprs.approximately-equal :as =?]
+   [metabase-enterprise.workspaces.isolation-manager :as isolation-manager]
+   [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
+   [metabase.query-processor.compile :as qp.compile]
+   [metabase.test :as mt]
+   [metabase.driver :as driver]))
+
+(deftest create-isolation-test
+  (mt/test-drivers #{:postgres :clickhouse}
+    (let [db (mt/db)
+          result (isolation-manager/create-isolation (:engine db) (:details db) "test-workspace-123")]
+      (try
+        (is (=? {:isolation-type (=?/malli [:enum :database :schema])
+                 :populator {:user (=?/malli :string) :password (=?/malli :string)}}
+                result))
+        (if (= :database (:isolation-type result))
+            (is (not (str/blank? (:database-name result))))
+            (is (not (str/blank? (:schema-name result)))))
+        (if (= :schema (:isolation-type result))
+          (is (str/starts-with? (:schema-name result) "mb__isolation_"))
+          (is (str/starts-with? (:database-name result) "mb__isolation_")))
+        (is (str/includes? (-> result :populator :user) "populator"))
+        (finally
+          (isolation-manager/delete-isolation (:engine db) (:details db) "test-workspace-123" result))))))
+
+(def driver-creates
+  {:postgres "create table %s.%s (id int, stuff text)"
+   :clickhouse "CREATE TABLE %s.%s (id Int32, stuff String) ENGINE = MergeTree() ORDER BY id"
+   :snowflake "CREATE TABLE \"%s\".\"%s\" (id INT, stuff TEXT)"})
+
+(def driver-drops
+  {:postgres   "DROP table %s.%s"
+   :clickhouse "DROP table %s.%s"
+   :snowflake  "DROP table \"%s\".\"%s\" "})
+
+(deftest e2e
+  (mt/test-drivers #{:postgres :clickhouse :snowflake}
+    (let [db (mt/db)
+          driver (:engine db)
+          details (merge (:details db))
+          workspace-id (str (gensym "testing-workspace"))
+          isolation-info (isolation-manager/create-isolation (:engine db)
+                                                             details
+                                                             workspace-id)]
+      (letfn [(run-query! [user-info read|write sql]
+                ;; user-info from snowflake is nil intentionally. that does not create a new user
+                (let [spec (sql-jdbc.conn/connection-details->spec (:engine db)
+                                                                   (merge details user-info))]
+                  (try {:success true
+                        :results (with-open [conn (jdbc/get-connection spec)]
+                                   (driver/set-database-used! driver conn db)
+                                   (if (= read|write :read)
+                                     (jdbc/query {:connection conn} sql)
+                                     (jdbc/execute! {:connection conn} sql)))}
+                       (catch Exception e
+                         {:error true
+                          :message  (ex-message e)
+                          #_#_:e e
+                          :query sql :user-info user-info}))))]
+        (try
+          (let [{:keys [query]} (qp.compile/compile (mt/mbql-query venues {:limit 10}))
+                {:keys [populator isolation-type]} isolation-info
+                isolation (isolation-info (if (= :schema isolation-type) :schema-name :database-name))
+                temp-table (str (gensym "some_table"))]
+            (testing "populator can query original data"
+              (is (=? {:success true} (run-query! populator :read query))))
+            (testing "populator can populate new schema"
+              (is (=? {:success true} (run-query! populator :write
+                                                  (format (driver-creates driver) isolation temp-table)))))
+            (testing "populator can delete table in new schema"
+              (is (=? {:success true} (run-query! populator :write (format (driver-drops driver) isolation temp-table)))))
+            (testing "populator can recreate dropped table"
+              (is (=? {:success true} (run-query! populator :write (format (driver-creates (:engine db)) isolation temp-table))))))
+          (finally
+            (isolation-manager/delete-isolation (:engine db) details workspace-id
+                                                isolation-info)))))))
+
+(deftest evaluator-test
+  (let [steps [:overall
+               {}
+               [:subgoal1 {} "action1" "action2"]
+               [:subgoal2 {}
+                "action3"
+                [:subsubgoal {} "action4"]]]]
+    (testing "general evaluation works"
+      (let [result (#'isolation-manager/evaluate-steps steps (fn [x] [:success x]))]
+        (is (= [[[:tree :overall]
+                 [:tree :subgoal1]
+                 [:success "action1"]
+                 [:success "action2"]
+                 [:tree :subgoal2]
+                 [:success "action3"]
+                 [:tree :subsubgoal]
+                 [:success "action4"]]
+                :running] result ))))
+    (testing "errors prevent more work"
+      (let [result (#'isolation-manager/evaluate-steps steps (fn [x]
+                                                               (if (= x "action2")
+                                                                 [:error "action2 is failed"]
+                                                                 [:success x])))]
+        (is (= [[[:tree :overall]
+                 [:tree :subgoal1]
+                 [:success "action1"]
+                 [:error "action2 is failed"]
+                 [:skipping-tree :subgoal2]]
+                :error]
+               result)))))
+  (testing "errors can be marked recoverable"
+    (let [steps [:cleanup {:error-strategy ::isolation-manager/continue-on-error}
+                 [:remove-privileges {} "revoke privileges"]
+                 [:remove-schema {} "drop schema"]
+                 [:remove-user {} "drop user"]]
+          result (#'isolation-manager/evaluate-steps steps
+                                                     (fn [x]
+                                                       [:error x "failed"]))]
+      (is (= [[[:tree :cleanup]
+               [:tree :remove-privileges]
+               [:error "revoke privileges" "failed"]
+               [:tree :remove-schema]
+               [:error "drop schema" "failed"]
+               [:tree :remove-user]
+               [:error "drop user" "failed"]]
+              :running] result))))
+  (testing "can error one subtree and then continue"
+    (let [steps [:overall-with-continue
+                 {:error-strategy ::isolation-manager/continue-on-error}
+                 [:subgoal1-with-fail {:error-strategy ::isolation-manager/fail} "action1" "action2"]
+                 [:subgoal2 {}
+                  "action3"
+                  [:subsubgoal {} "action4"]]]
+          results (#'isolation-manager/evaluate-steps steps
+                                                      (fn [x]
+                                                        (if (= x "action1")
+                                                          [:error x]
+                                                          [:success x])))]
+      (is (= [[[:tree :overall-with-continue]
+               [:tree :subgoal1-with-fail] [:error "action1"] [:skipping-step "action2"]
+               [:tree :subgoal2] [:success "action3"]
+               [:tree :subsubgoal] [:success "action4"]]
+              :running]
+             results))))
+  (testing "can error one subtree and then continue"
+    (let [steps [:overall-with-continue
+                 {:error-strategy ::isolation-manager/continue-on-error}
+                 [:subgoal1-with-fail {:error-strategy ::isolation-manager/fail}
+                  "action1" "action2"
+                  [:subtree-should-be-skipped {} "not-important1" "not-important2"]]
+                 [:subgoal2 {}
+                  "action3"
+                  [:subsubgoal {} "action4"]]]
+          results (#'isolation-manager/evaluate-steps steps
+                                                      (fn [x]
+                                                        (if (= x "action1")
+                                                          [:error x]
+                                                          [:success x])))]
+      (is (= [[[:tree :overall-with-continue]
+               [:tree :subgoal1-with-fail]
+               [:error "action1"] [:skipping-step "action2"]
+               ;; tree maintains the failure strategy
+               [:skipping-tree :subtree-should-be-skipped]
+               ;; but popping back out continues according to main strategy
+               [:tree :subgoal2] [:success "action3"]
+               [:tree :subsubgoal] [:success "action4"]]
+              :running]
+             results))))
+  (testing "rules can be opaque"
+    (let [steps [:compute {}
+                 [+ 1]
+                 [+ 1]
+                 [:multiply {}
+                  [* 10] [* 5]]
+                 [:divide {:error-strategy ::isolation-manager/continue-on-error}
+                  [/ 2] [/ 0]]
+                 [:subtract
+                  {}
+                  [- 10]]]]
+      (let [result (atom 0)
+            results (#'isolation-manager/evaluate-steps steps (fn [[op x]]
+                                                                (try
+                                                                  [:success (swap! result op x)]
+                                                                  (catch Exception _e
+                                                                    [:error :failed]))))]
+        (is (=? [[[:tree :compute]
+                  [:success 1]
+                  [:success 2]
+                  [:tree :multiply]
+                  [:success 20]
+                  [:success 100]
+                  [:tree :divide]
+                  [:success 50]
+                  [:error :failed]
+                  [:tree :subtract]
+                  [:success 40]]
+                 :running]
+                results)
+            (= 50 @result))))))
+
+
+(comment
+  (run-tests)
+  (mt/set-test-drivers! #{:postgres :clickhouse :snowflake})
+  (mt/set-test-drivers! #{:snowflake})
+  (mt/set-test-drivers! #{:postgres})
+  (mt/set-test-drivers! #{:clickhouse})
+  (do (run-test e2e)
+      (run-test create-isolation-test))
+
+  {:isolation-type :schema
+   :schema-name "mb__isolation_7748c_test_workspace_123"
+   :populator {:user "mb_iso_7748c_test_workspace_123_populator"
+               :password "d632f61c-398a-467d-856c-d8411665bd0f"}}
+  (mt/test-drivers #{:postgres}
+    (let [db (mt/db)]
+      (isolation-manager/delete-isolation (:engine db) (:details db) "test-workspace-123")
+      (tap> (isolation-manager/create-isolation (:engine db) (:details db) "test-workspace-123"))))
+  (mt/test-drivers #{:postgres}
+    (let [db (mt/db)
+          engine (:engine db)
+          connection-details (:details db)
+
+          ;; Simple API call
+          result (isolation-manager/create-isolation engine connection-details "demo-workspace")]
+      (try
+        (tap> result)
+        (def result result)
+        (finally
+          (isolation-manager/delete-isolation engine connection-details "demo-workspace" result)
+          (println "Cleaned up demo")))))
+  )

enterprise/frontend/src/embedding-sdk/types/collection.ts

@@ -33,6 +33,7 @@ export type MetabaseCollectionItem = {
     | "model"
     | "question"
     | "metric"
+    | "in_document"
     | null;
   "last-edit-info"?: {
     email: string;

enterprise/frontend/src/metabase-enterprise/api/document.ts

@@ -0,0 +1,52 @@
+import type {
+  Card,
+  CreateDocumentRequest,
+  Document,
+  DocumentContent,
+  DocumentId,
+} from "metabase-types/api";
+
+import { EnterpriseApi } from "./api";
+import { idTag, listTag } from "./tags";
+
+export const documentApi = EnterpriseApi.injectEndpoints({
+  endpoints: (builder) => ({
+    getDocument: builder.query<Document, { id: DocumentId }>({
+      query: ({ id }) => ({
+        method: "GET",
+        url: `/api/ee/document/${id}`,
+      }),
+      providesTags: (result, error, { id }) =>
+        !error ? [idTag("document", id)] : [],
+    }),
+    createDocument: builder.mutation<Document, CreateDocumentRequest>({
+      query: (body) => ({
+        method: "POST",
+        url: "/api/ee/document",
+        body,
+      }),
+      invalidatesTags: (_, error) => (error ? [] : [listTag("document")]), // TODO: invalidate parent collection?
+    }),
+    updateDocument: builder.mutation<
+      Document,
+      Pick<Document, "id" | "name"> & {
+        document?: DocumentContent;
+        cards?: Record<number, Card>;
+      }
+    >({
+      query: (document) => ({
+        method: "PUT",
+        url: `/api/ee/document/${document.id}`,
+        body: document,
+      }),
+      invalidatesTags: (_, error, { id }) =>
+        !error ? [listTag("document"), idTag("document", id)] : [],
+    }),
+  }),
+});
+
+export const {
+  useGetDocumentQuery,
+  useCreateDocumentMutation,
+  useUpdateDocumentMutation,
+} = documentApi;

enterprise/frontend/src/metabase-enterprise/api/index.ts

@@ -3,11 +3,16 @@ export * from "./ai-sql-generation";
 export * from "./audit-info";
 export * from "./api";
 export * from "./billing-info";
-export * from "./metabot";
 export * from "./content-translation";
-export * from "./gdrive";
 export * from "./db-routing";
+export * from "./gdrive";
+export * from "./metabot";
+export * from "./document";
 export * from "./saml";
 export * from "./scim";
 export * from "./tags";
+export * from "./transform";
+export * from "./transform-job";
+export * from "./transform-tag";
 export * from "./upload-management";
+export * from "./workspace";

enterprise/frontend/src/metabase-enterprise/api/metabot.ts

@@ -1,6 +1,8 @@
 import type {
   DeleteSuggestedMetabotPromptRequest,
   MetabotApiEntity,
+  MetabotDocumentNodeRequest,
+  MetabotDocumentNodeResponse,
   MetabotEntity,
   MetabotId,
   MetabotInfo,
@@ -111,6 +113,16 @@ export const metabotApi = EnterpriseApi.injectEndpoints({
       invalidatesTags: (_, error, metabot_id) =>
         !error ? [idTag("metabot-prompt-suggestions", metabot_id)] : [],
     }),
+    metabotDocumentNode: builder.query<
+      MetabotDocumentNodeResponse,
+      MetabotDocumentNodeRequest
+    >({
+      query: (params) => ({
+        method: "POST",
+        url: "/api/ee/metabot-v3/document/generate-content",
+        params,
+      }),
+    }),
   }),
 });
 
@@ -122,4 +134,5 @@ export const {
   useGetSuggestedMetabotPromptsQuery,
   useDeleteSuggestedMetabotPromptMutation,
   useRegenerateSuggestedMetabotPromptsMutation,
+  useLazyMetabotDocumentNodeQuery,
 } = metabotApi;

enterprise/frontend/src/metabase-enterprise/api/tags.ts

@@ -1,11 +1,26 @@
 import type { TagDescription } from "@reduxjs/toolkit/query";
 
+import { TAG_TYPES } from "metabase/api/tags";
+import type {
+  Transform,
+  TransformJob,
+  TransformRun,
+  TransformTag,
+} from "metabase-types/api";
+
 export const ENTERPRISE_TAG_TYPES = [
+  ...TAG_TYPES,
   "scim",
   "metabot",
   "metabot-entities-list",
   "metabot-prompt-suggestions",
   "gsheets-status",
+  "transform",
+  "transform-tag",
+  "transform-job",
+  "transform-run",
+  "document",
+  "workspace",
 ] as const;
 
 export type EnterpriseTagType = (typeof ENTERPRISE_TAG_TYPES)[number];
@@ -35,3 +50,63 @@ export function invalidateTags(
 ): TagDescription<EnterpriseTagType>[] {
   return !error ? tags : [];
 }
+
+export function provideTransformTags(
+  transform: Transform,
+): TagDescription<EnterpriseTagType>[] {
+  return [idTag("transform", transform.id)];
+}
+
+export function provideTransformListTags(
+  transforms: Transform[],
+): TagDescription<EnterpriseTagType>[] {
+  return [listTag("transform"), ...transforms.flatMap(provideTransformTags)];
+}
+
+export function provideTransformRunTags(
+  run: TransformRun,
+): TagDescription<EnterpriseTagType>[] {
+  return [
+    idTag("transform-run", run.id),
+    ...(run.transform ? provideTransformTags(run.transform) : []),
+  ];
+}
+
+export function provideTransformRunListTags(
+  runs: TransformRun[],
+): TagDescription<EnterpriseTagType>[] {
+  return [listTag("transform-run"), ...runs.flatMap(provideTransformRunTags)];
+}
+
+export function provideTransformTagTags(
+  tag: TransformTag,
+): TagDescription<EnterpriseTagType>[] {
+  return [idTag("transform-tag", tag.id)];
+}
+
+export function provideTransformTagListTags(
+  tags: TransformTag[],
+): TagDescription<EnterpriseTagType>[] {
+  return [listTag("transform-tag"), ...tags.flatMap(provideTransformTagTags)];
+}
+
+export function provideTransformJobTags(
+  job: TransformJob,
+): TagDescription<EnterpriseTagType>[] {
+  return [idTag("transform-job", job.id)];
+}
+
+export function provideTransformJobListTags(
+  jobs: TransformJob[],
+): TagDescription<EnterpriseTagType>[] {
+  return [listTag("transform-job"), ...jobs.flatMap(provideTransformJobTags)];
+}
+
+export function provideWorkspaceTags(
+  workspaces: Array<{ id: number }>,
+): TagDescription<EnterpriseTagType>[] {
+  return [
+    listTag("workspace"),
+    ...workspaces.map(workspace => idTag("workspace", workspace.id)),
+  ];
+}

enterprise/frontend/src/metabase-enterprise/api/transform-job.ts

@@ -0,0 +1,109 @@
+import type {
+  CreateTransformJobRequest,
+  TransformJob,
+  TransformJobId,
+  UpdateTransformJobRequest,
+} from "metabase-types/api";
+
+import { EnterpriseApi } from "./api";
+import {
+  idTag,
+  invalidateTags,
+  listTag,
+  provideTransformJobListTags,
+  provideTransformJobTags,
+  tag,
+} from "./tags";
+
+export const transformJobApi = EnterpriseApi.injectEndpoints({
+  endpoints: (builder) => ({
+    listTransformJobs: builder.query<TransformJob[], void>({
+      query: (params) => ({
+        method: "GET",
+        url: "/api/ee/transform-job",
+        params,
+      }),
+      providesTags: (jobs = []) => provideTransformJobListTags(jobs),
+    }),
+    getTransformJob: builder.query<TransformJob, TransformJobId>({
+      query: (id) => ({
+        method: "GET",
+        url: `/api/ee/transform-job/${id}`,
+      }),
+      providesTags: (job) => (job ? provideTransformJobTags(job) : []),
+    }),
+    runTransformJob: builder.mutation<void, TransformJobId>({
+      query: (id) => ({
+        method: "POST",
+        url: `/api/ee/transform-job/${id}/run`,
+      }),
+      invalidatesTags: (_, error, id) =>
+        invalidateTags(error, [
+          idTag("transform-job", id),
+          tag("transform"),
+          tag("table"),
+        ]),
+    }),
+    createTransformJob: builder.mutation<
+      TransformJob,
+      CreateTransformJobRequest
+    >({
+      query: (body) => ({
+        method: "POST",
+        url: "/api/ee/transform-job",
+        body,
+      }),
+      invalidatesTags: (_, error) =>
+        invalidateTags(error, [listTag("transform-job")]),
+    }),
+    updateTransformJob: builder.mutation<
+      TransformJob,
+      UpdateTransformJobRequest
+    >({
+      query: ({ id, ...body }) => ({
+        method: "PUT",
+        url: `/api/ee/transform-job/${id}`,
+        body,
+      }),
+      invalidatesTags: (_, error, { id }) =>
+        invalidateTags(error, [idTag("transform-job", id)]),
+      onQueryStarted: async (
+        { id, ...patch },
+        { dispatch, queryFulfilled },
+      ) => {
+        const patchResult = dispatch(
+          transformJobApi.util.updateQueryData(
+            "getTransformJob",
+            id,
+            (draft) => {
+              Object.assign(draft, patch);
+            },
+          ),
+        );
+        try {
+          await queryFulfilled;
+        } catch {
+          patchResult.undo();
+        }
+      },
+    }),
+    deleteTransformJob: builder.mutation<void, TransformJobId>({
+      query: (id) => ({
+        method: "DELETE",
+        url: `/api/ee/transform-job/${id}`,
+      }),
+      invalidatesTags: (_, error) =>
+        invalidateTags(error, [listTag("transform-job")]),
+    }),
+  }),
+});
+
+export const {
+  useListTransformJobsQuery,
+  useGetTransformJobQuery,
+  useLazyGetTransformJobQuery,
+  useRunTransformJobMutation,
+  useCreateTransformJobMutation,
+  useUpdateTransformJobMutation,
+  useDeleteTransformJobMutation,
+} = transformJobApi;

enterprise/frontend/src/metabase-enterprise/api/transform-tag.ts

@@ -0,0 +1,66 @@
+import type {
+  CreateTransformTagRequest,
+  TransformTag,
+  TransformTagId,
+  UpdateTransformTagRequest,
+} from "metabase-types/api";
+
+import { EnterpriseApi } from "./api";
+import {
+  idTag,
+  invalidateTags,
+  listTag,
+  provideTransformTagListTags,
+} from "./tags";
+
+export const transformTagApi = EnterpriseApi.injectEndpoints({
+  endpoints: (builder) => ({
+    listTransformTags: builder.query<TransformTag[], void>({
+      query: (params) => ({
+        method: "GET",
+        url: "/api/ee/transform-tag",
+        params,
+      }),
+      providesTags: (tags = []) => provideTransformTagListTags(tags),
+    }),
+    createTransformTag: builder.mutation<
+      TransformTag,
+      CreateTransformTagRequest
+    >({
+      query: (body) => ({
+        method: "POST",
+        url: "/api/ee/transform-tag",
+        body,
+      }),
+      invalidatesTags: (_, error) =>
+        invalidateTags(error, [listTag("transform-tag")]),
+    }),
+    updateTransformTag: builder.mutation<
+      TransformTag,
+      UpdateTransformTagRequest
+    >({
+      query: ({ id, ...body }) => ({
+        method: "PUT",
+        url: `/api/ee/transform-tag/${id}`,
+        body,
+      }),
+      invalidatesTags: (_, error, { id }) =>
+        invalidateTags(error, [idTag("transform-tag", id)]),
+    }),
+    deleteTransformTag: builder.mutation<void, TransformTagId>({
+      query: (id) => ({
+        method: "DELETE",
+        url: `/api/ee/transform-tag/${id}`,
+      }),
+      invalidatesTags: (_, error) =>
+        invalidateTags(error, [listTag("transform-tag")]),
+    }),
+  }),
+});
+
+export const {
+  useListTransformTagsQuery,
+  useCreateTransformTagMutation,
+  useUpdateTransformTagMutation,
+  useDeleteTransformTagMutation,
+} = transformTagApi;

enterprise/frontend/src/metabase-enterprise/api/transform.ts

@@ -0,0 +1,121 @@
+import type {
+  CreateTransformRequest,
+  ListTransformRunsRequest,
+  ListTransformRunsResponse,
+  Transform,
+  TransformId,
+  UpdateTransformRequest,
+} from "metabase-types/api";
+
+import { EnterpriseApi } from "./api";
+import {
+  idTag,
+  invalidateTags,
+  listTag,
+  provideTransformListTags,
+  provideTransformRunListTags,
+  provideTransformTags,
+  tag,
+} from "./tags";
+
+export const transformApi = EnterpriseApi.injectEndpoints({
+  endpoints: (builder) => ({
+    listTransforms: builder.query<Transform[], void>({
+      query: (params) => ({
+        method: "GET",
+        url: "/api/ee/transform",
+        params,
+      }),
+      providesTags: (transforms = []) => provideTransformListTags(transforms),
+    }),
+    listTransformRuns: builder.query<
+      ListTransformRunsResponse,
+      ListTransformRunsRequest
+    >({
+      query: (params) => ({
+        method: "GET",
+        url: "/api/ee/transform/run",
+        params,
+      }),
+      providesTags: (response) =>
+        response ? provideTransformRunListTags(response.data) : [],
+    }),
+    getTransform: builder.query<Transform, TransformId>({
+      query: (id) => ({
+        method: "GET",
+        url: `/api/ee/transform/${id}`,
+      }),
+      providesTags: (transform) =>
+        transform ? provideTransformTags(transform) : [],
+    }),
+    runTransform: builder.mutation<void, TransformId>({
+      query: (id) => ({
+        method: "POST",
+        url: `/api/ee/transform/${id}/run`,
+      }),
+      invalidatesTags: (_, error, id) =>
+        invalidateTags(error, [idTag("transform", id), tag("table")]),
+    }),
+    createTransform: builder.mutation<Transform, CreateTransformRequest>({
+      query: (body) => ({
+        method: "POST",
+        url: "/api/ee/transform",
+        body,
+      }),
+      invalidatesTags: (_, error) =>
+        invalidateTags(error, [listTag("transform")]),
+    }),
+    updateTransform: builder.mutation<Transform, UpdateTransformRequest>({
+      query: ({ id, ...body }) => ({
+        method: "PUT",
+        url: `/api/ee/transform/${id}`,
+        body,
+      }),
+      invalidatesTags: (_, error, { id }) =>
+        invalidateTags(error, [idTag("transform", id)]),
+      onQueryStarted: async (
+        { id, ...patch },
+        { dispatch, queryFulfilled },
+      ) => {
+        const patchResult = dispatch(
+          transformApi.util.updateQueryData("getTransform", id, (draft) => {
+            Object.assign(draft, patch);
+          }),
+        );
+        try {
+          await queryFulfilled;
+        } catch {
+          patchResult.undo();
+        }
+      },
+    }),
+    deleteTransform: builder.mutation<void, TransformId>({
+      query: (id) => ({
+        method: "DELETE",
+        url: `/api/ee/transform/${id}`,
+      }),
+      invalidatesTags: (_, error) =>
+        invalidateTags(error, [listTag("transform")]),
+    }),
+    deleteTransformTarget: builder.mutation<void, TransformId>({
+      query: (id) => ({
+        method: "DELETE",
+        url: `/api/ee/transform/${id}/table`,
+      }),
+      invalidatesTags: (_, error, id) =>
+        invalidateTags(error, [idTag("transform", id), listTag("table")]),
+    }),
+  }),
+});
+
+export const {
+  useListTransformsQuery,
+  useListTransformRunsQuery,
+  useGetTransformQuery,
+  useLazyGetTransformQuery,
+  useRunTransformMutation,
+  useCreateTransformMutation,
+  useUpdateTransformMutation,
+  useDeleteTransformMutation,
+  useDeleteTransformTargetMutation,
+} = transformApi;

enterprise/frontend/src/metabase-enterprise/api/workspace.ts

@@ -0,0 +1,299 @@
+import type { CollectionId } from "metabase-types/api";
+
+import { EnterpriseApi } from "./api";
+import { idTag, listTag, provideWorkspaceTags } from "./tags";
+
+export interface Plan {
+  title: string;
+  description?: string;
+  content: any;
+  created_at: string;
+}
+
+export interface Transform {
+  id: string;
+  name: string;
+  description: string;
+  source: any;
+  target: any;
+  config?: any;
+  created_at: string;
+}
+
+export interface WorkspaceUser {
+  id: string;
+  name: string;
+  email: string;
+  type: string;
+  created_at: string;
+}
+
+export interface DataWarehouse {
+  id: string;
+  data_warehouse_id?: number;
+  name: string;
+  type: "read-only" | "read-write";
+  credentials: any;
+  created_at: string;
+}
+
+export interface Permission {
+  id: string;
+  table: string;
+  permission: "read" | "write";
+  created_at: string;
+}
+
+export interface Workspace {
+  id: number;
+  collection_id: CollectionId;
+  name: string;
+  description?: string | null;
+  plans?: Plan[];
+  activity_logs?: any[];
+  transforms?: Transform[];
+  documents?: any[];
+  users?: WorkspaceUser[];
+  data_warehouses?: DataWarehouse[];
+  permissions?: Permission[];
+  created_at?: string;
+  updated_at?: string;
+}
+
+export interface CreateWorkspaceRequest {
+  name: string;
+  collection_id: CollectionId;
+  description?: string;
+}
+
+export interface UpdateWorkspaceRequest {
+  id: number;
+  name: string;
+  description?: string;
+}
+
+export interface ListWorkspacesResponse {
+  data?: Workspace[];
+}
+
+export const workspaceApi = EnterpriseApi.injectEndpoints({
+  endpoints: (builder) => ({
+    listWorkspaces: builder.query<Workspace[], void>({
+      query: () => ({
+        method: "GET",
+        url: `/api/ee/workspace`,
+      }),
+      providesTags: (workspaces = []) => provideWorkspaceTags(workspaces),
+    }),
+    getWorkspace: builder.query<Workspace, number>({
+      query: (id) => ({
+        method: "GET",
+        url: `/api/ee/workspace/${id}`,
+      }),
+      providesTags: (workspace) =>
+        workspace ? [idTag("workspace", workspace.id)] : [],
+    }),
+    createWorkspace: builder.mutation<Workspace, CreateWorkspaceRequest>({
+      query: (body) => ({
+        method: "POST",
+        url: `/api/ee/workspace`,
+        body,
+      }),
+      invalidatesTags: [listTag("workspace")],
+    }),
+    updateWorkspace: builder.mutation<Workspace, UpdateWorkspaceRequest>({
+      query: ({ id, ...body }) => ({
+        method: "PUT",
+        url: `/api/ee/workspace/${id}`,
+        body,
+      }),
+      invalidatesTags: (_, __, { id }) => [
+        idTag("workspace", id),
+        listTag("workspace"),
+      ],
+    }),
+    deleteWorkspace: builder.mutation<void, number>({
+      query: (id) => ({
+        method: "DELETE",
+        url: `/api/ee/workspace/${id}`,
+      }),
+      invalidatesTags: (_, __, id) => [
+        idTag("workspace", id),
+        listTag("workspace"),
+      ],
+    }),
+    updatePlan: builder.mutation<
+      Workspace,
+      { workspaceId: number; planIndex: number; title: string; description?: string; content: any }
+    >({
+      query: ({ workspaceId, planIndex, ...body }) => ({
+        method: "PUT",
+        url: `/api/ee/workspace/${workspaceId}/plan/${planIndex}`,
+        body,
+      }),
+      invalidatesTags: (_, __, { workspaceId }) => [
+        idTag("workspace", workspaceId),
+        listTag("workspace"),
+      ],
+    }),
+    deletePlan: builder.mutation<
+      void,
+      { workspaceId: number; planIndex: number }
+    >({
+      query: ({ workspaceId, planIndex }) => ({
+        method: "DELETE",
+        url: `/api/ee/workspace/${workspaceId}/plan/${planIndex}`,
+      }),
+      invalidatesTags: (_, __, { workspaceId }) => [
+        idTag("workspace", workspaceId),
+        listTag("workspace"),
+      ],
+    }),
+    // Transform endpoints (workspace-specific)
+    updateWorkspaceTransform: builder.mutation<
+      Workspace,
+      { workspaceId: number; transformIndex: number; name: string; description: string; source: any; target: any; config?: any }
+    >({
+      query: ({ workspaceId, transformIndex, ...body }) => ({
+        method: "PUT",
+        url: `/api/ee/workspace/${workspaceId}/transform/${transformIndex}`,
+        body,
+      }),
+      invalidatesTags: (_, __, { workspaceId }) => [
+        idTag("workspace", workspaceId),
+        listTag("workspace"),
+      ],
+    }),
+    deleteWorkspaceTransform: builder.mutation<
+      void,
+      { workspaceId: number; transformIndex: number }
+    >({
+      query: ({ workspaceId, transformIndex }) => ({
+        method: "DELETE",
+        url: `/api/ee/workspace/${workspaceId}/transform/${transformIndex}`,
+      }),
+      invalidatesTags: (_, __, { workspaceId }) => [
+        idTag("workspace", workspaceId),
+        listTag("workspace"),
+      ],
+    }),
+    // User endpoints
+    updateUser: builder.mutation<
+      Workspace,
+      { workspaceId: number; userIndex: number; user_id: number; name: string; email: string; type: string }
+    >({
+      query: ({ workspaceId, userIndex, ...body }) => ({
+        method: "PUT",
+        url: `/api/ee/workspace/${workspaceId}/user/${userIndex}`,
+        body,
+      }),
+      invalidatesTags: (_, __, { workspaceId }) => [
+        idTag("workspace", workspaceId),
+        listTag("workspace"),
+      ],
+    }),
+    deleteUser: builder.mutation<
+      void,
+      { workspaceId: number; userIndex: number }
+    >({
+      query: ({ workspaceId, userIndex }) => ({
+        method: "DELETE",
+        url: `/api/ee/workspace/${workspaceId}/user/${userIndex}`,
+      }),
+      invalidatesTags: (_, __, { workspaceId }) => [
+        idTag("workspace", workspaceId),
+        listTag("workspace"),
+      ],
+    }),
+    // Data warehouse endpoints
+    updateDataWarehouse: builder.mutation<
+      Workspace,
+      { workspaceId: number; dataWarehouseIndex: number; data_warehouses_id: number; name: string; type: "read-only" | "read-write"; credentials: any }
+    >({
+      query: ({ workspaceId, dataWarehouseIndex, ...body }) => ({
+        method: "PUT",
+        url: `/api/ee/workspace/${workspaceId}/data_warehouse/${dataWarehouseIndex}`,
+        body,
+      }),
+      invalidatesTags: (_, __, { workspaceId }) => [
+        idTag("workspace", workspaceId),
+        listTag("workspace"),
+      ],
+    }),
+    deleteDataWarehouse: builder.mutation<
+      void,
+      { workspaceId: number; dataWarehouseIndex: number }
+    >({
+      query: ({ workspaceId, dataWarehouseIndex }) => ({
+        method: "DELETE",
+        url: `/api/ee/workspace/${workspaceId}/data_warehouse/${dataWarehouseIndex}`,
+      }),
+      invalidatesTags: (_, __, { workspaceId }) => [
+        idTag("workspace", workspaceId),
+        listTag("workspace"),
+      ],
+    }),
+    // Permission endpoints
+    updatePermission: builder.mutation<
+      Workspace,
+      { workspaceId: number; permissionIndex: number; table: string; permission: "read" | "write" }
+    >({
+      query: ({ workspaceId, permissionIndex, ...body }) => ({
+        method: "PUT",
+        url: `/api/ee/workspace/${workspaceId}/permission/${permissionIndex}`,
+        body,
+      }),
+      invalidatesTags: (_, __, { workspaceId }) => [
+        idTag("workspace", workspaceId),
+        listTag("workspace"),
+      ],
+    }),
+    deletePermission: builder.mutation<
+      void,
+      { workspaceId: number; permissionIndex: number }
+    >({
+      query: ({ workspaceId, permissionIndex }) => ({
+        method: "DELETE",
+        url: `/api/ee/workspace/${workspaceId}/permission/${permissionIndex}`,
+      }),
+      invalidatesTags: (_, __, { workspaceId }) => [
+        idTag("workspace", workspaceId),
+        listTag("workspace"),
+      ],
+    }),
+    // Link existing transform to workspace
+    linkTransformToWorkspace: builder.mutation<
+      Workspace,
+      { workspaceId: number; transformId: number }
+    >({
+      query: ({ workspaceId, transformId }) => ({
+        method: "POST",
+        url: `/api/ee/workspace/${workspaceId}/transform/link`,
+        body: { transform_id: transformId },
+      }),
+      invalidatesTags: (_, __, { workspaceId }) => [
+        idTag("workspace", workspaceId),
+        listTag("workspace"),
+      ],
+    }),
+  }),
+});
+
+export const {
+  useListWorkspacesQuery,
+  useGetWorkspaceQuery,
+  useCreateWorkspaceMutation,
+  useUpdateWorkspaceMutation,
+  useDeleteWorkspaceMutation,
+  useUpdatePlanMutation,
+  useDeletePlanMutation,
+  useUpdateWorkspaceTransformMutation,
+  useDeleteWorkspaceTransformMutation,
+  useUpdateUserMutation,
+  useDeleteUserMutation,
+  useUpdateDataWarehouseMutation,
+  useDeleteDataWarehouseMutation,
+  useUpdatePermissionMutation,
+  useDeletePermissionMutation,
+  useLinkTransformToWorkspaceMutation,
+} = workspaceApi;

enterprise/frontend/src/metabase-enterprise/documents/actions.ts

@@ -0,0 +1,8 @@
+import { push } from "react-router-redux";
+
+import type { Dispatch } from "metabase-types/store";
+
+export const navigateBackToDocument =
+  (documentId: number) => (dispatch: Dispatch) => {
+    dispatch(push(`/document/${documentId}`));
+  };

enterprise/frontend/src/metabase-enterprise/documents/components/DocumentBackButton/DocumentBackButton.tsx

@@ -0,0 +1,66 @@
+import { type HTMLAttributes, useCallback } from "react";
+import { Link } from "react-router";
+import { t } from "ttag";
+
+import { useDispatch } from "metabase/lib/redux";
+import { ActionIcon, type ActionIconProps, Icon, Tooltip } from "metabase/ui";
+import { navigateBackToDocument } from "metabase-enterprise/documents/actions";
+import { useDocumentsSelector } from "metabase-enterprise/documents/redux-utils";
+import {
+  getDocument,
+  getShowNavigateBackToDocumentButton,
+} from "metabase-enterprise/documents/selectors";
+
+export type DocumentBackButtonProps = {
+  noLink?: boolean;
+  onClick?: () => void;
+  documentOverride?: {
+    id: number;
+    name: string;
+  };
+} & ActionIconProps &
+  HTMLAttributes<HTMLButtonElement>;
+
+export function DocumentBackButton({
+  noLink,
+  onClick,
+  documentOverride,
+  ...actionIconProps
+}: DocumentBackButtonProps) {
+  const documentState = useDocumentsSelector(getDocument);
+  const showButton = useDocumentsSelector(getShowNavigateBackToDocumentButton);
+  const document = documentOverride ?? documentState;
+  const dispatch = useDispatch();
+
+  const handleClick = useCallback(() => {
+    if (document) {
+      dispatch(navigateBackToDocument(document.id));
+    }
+
+    onClick?.();
+  }, [document, dispatch, onClick]);
+
+  if (!document || !showButton) {
+    return null;
+  }
+
+  const label = t`Back to ${document.name}`;
+
+  return (
+    <Tooltip label={label}>
+      <ActionIcon
+        variant="outline"
+        radius="xl"
+        size="2.625rem"
+        color="border"
+        aria-label={label}
+        onClick={handleClick}
+        component={noLink ? undefined : Link}
+        to={`/document/${document.id}`}
+        {...actionIconProps}
+      >
+        <Icon c="brand" name="arrow_left" />
+      </ActionIcon>
+    </Tooltip>
+  );
+}

enterprise/frontend/src/metabase-enterprise/documents/components/DocumentBackButton/index.ts

@@ -0,0 +1 @@
+export { DocumentBackButton } from "./DocumentBackButton";

enterprise/frontend/src/metabase-enterprise/documents/components/DocumentPage.module.css

@@ -0,0 +1,148 @@
+.documentPage {
+  display: flex;
+  flex-direction: column;
+  height: calc(100vh - var(--nav-sidebar-height, 60px));
+  background-color: var(--mb-color-bg-white);
+}
+
+.header {
+  display: flex;
+  align-items: flex-start;
+  justify-content: space-between;
+  padding: 1rem 0;
+  background-color: var(--mb-color-bg-white);
+  gap: 1rem;
+  max-width: 900px;
+  margin: 0 auto;
+  width: 100%;
+}
+
+.titleInput {
+  flex: 1;
+  max-width: 600px;
+  border: none;
+  outline: none;
+  background: transparent;
+  font-size: 2rem;
+  font-weight: 700;
+  line-height: 1.2;
+  color: var(--mb-color-text-dark);
+  font-family: inherit;
+  padding: 0;
+  margin: 0;
+}
+
+.titleInput::placeholder {
+  color: var(--mb-color-text-light);
+  opacity: 0.7;
+}
+
+.titleInput:focus {
+  outline: none;
+}
+
+.contentArea {
+  display: flex;
+  flex: 1;
+  overflow: hidden;
+}
+
+.mainContent {
+  flex: 1;
+  display: flex;
+  flex-direction: column;
+  overflow-y: auto;
+  padding: 0 2rem;
+}
+
+.documentContainer {
+  width: 100%;
+  padding: 0 2rem 4rem;
+  min-height: 100%;
+}
+
+@media print {
+  body:has(.documentContainer) {
+    visibility: hidden;
+
+    /* side nav */
+    aside {
+      display: none;
+    }
+  }
+
+  .documentContainer {
+    position: absolute;
+    visibility: visible;
+    top: 0;
+    right: 0;
+    z-index: 1000;
+
+    .header {
+      padding: 0;
+    }
+
+    :global(.node-cardEmbed) {
+      break-inside: avoid;
+    }
+
+    [data-hide-on-print] {
+      visibility: hidden;
+    }
+  }
+}
+
+.sidebar {
+  width: 360px;
+  flex-shrink: 0;
+  background-color: var(--mb-color-bg-light);
+  border-left: 1px solid var(--mb-color-border);
+  overflow-y: auto;
+}
+
+.cardEmbed {
+  padding: 0.5rem;
+  border-radius: 4px;
+  cursor: pointer;
+  transition: background-color 0.2s;
+}
+
+.cardEmbed:hover {
+  background-color: var(--mb-color-bg-medium);
+}
+
+.highlighted {
+  animation: highlight 2s ease-out;
+}
+
+@keyframes highlight {
+  0% {
+    background-color: var(--mb-color-brand-light);
+    box-shadow: 0 0 0 4px var(--mb-color-brand-light);
+  }
+
+  100% {
+    background-color: transparent;
+    box-shadow: none;
+  }
+}
+
+.metadataItem {
+  display: flex;
+  align-items: center;
+  gap: 0.25rem;
+  color: var(--mb-color-text-secondary);
+}
+
+.hidable {
+  visibility: hidden;
+  width: 0;
+  opacity: 0;
+  transition: opacity 200ms;
+
+  &.visible {
+    visibility: visible;
+    width: auto;
+    opacity: 1;
+  }
+}

enterprise/frontend/src/metabase-enterprise/documents/components/DocumentPage.tsx

@@ -0,0 +1,478 @@
+import { useDisclosure } from "@mantine/hooks";
+import type { Editor as TiptapEditor } from "@tiptap/core";
+import cx from "classnames";
+import dayjs from "dayjs";
+import { useCallback, useEffect, useState } from "react";
+import type { Route } from "react-router";
+import { push, replace } from "react-router-redux";
+import { usePrevious, useUnmount } from "react-use";
+import useBeforeUnload from "react-use/lib/useBeforeUnload";
+import { t } from "ttag";
+import _ from "underscore";
+
+import { skipToken } from "metabase/api";
+import { canonicalCollectionId } from "metabase/collections/utils";
+import DateTime, {
+  getFormattedTime,
+} from "metabase/common/components/DateTime";
+import { LeaveRouteConfirmModal } from "metabase/common/components/LeaveConfirmModal";
+import { CollectionPickerModal } from "metabase/common/components/Pickers/CollectionPicker";
+import { useToast } from "metabase/common/hooks";
+import { useCallbackEffect } from "metabase/common/hooks/use-callback-effect";
+import { useDispatch } from "metabase/lib/redux";
+import { extractEntityId } from "metabase/lib/urls";
+import { setErrorPage } from "metabase/redux/app";
+import {
+  ActionIcon,
+  Box,
+  Button,
+  Flex,
+  Icon,
+  Menu,
+  Text,
+  TextInput,
+  Tooltip,
+} from "metabase/ui";
+import {
+  useCreateDocumentMutation,
+  useGetDocumentQuery,
+  useUpdateDocumentMutation,
+} from "metabase-enterprise/api";
+import type { Card, RegularCollectionId } from "metabase-types/api";
+
+import {
+  clearDraftCards,
+  closeSidebar,
+  openVizSettingsSidebar,
+  resetDocuments,
+  setCurrentDocument,
+} from "../documents.slice";
+import { useDocumentState, useRegisterDocumentMetabotContext } from "../hooks";
+import { useDocumentsSelector } from "../redux-utils";
+import {
+  getDraftCards,
+  getSelectedEmbedIndex,
+  getSelectedQuestionId,
+} from "../selectors";
+
+import styles from "./DocumentPage.module.css";
+import { Editor } from "./Editor";
+import { EmbedQuestionSettingsSidebar } from "./EmbedQuestionSettingsSidebar";
+
+export const DocumentPage = ({
+  params: { entityId },
+  route,
+}: {
+  params: { entityId?: string };
+  location?: { query?: { version?: string } };
+  route: Route;
+}) => {
+  const dispatch = useDispatch();
+  const selectedQuestionId = useDocumentsSelector(getSelectedQuestionId);
+  const selectedEmbedIndex = useDocumentsSelector(getSelectedEmbedIndex);
+  const draftCards = useDocumentsSelector(getDraftCards);
+  const [editorInstance, setEditorInstance] = useState<TiptapEditor | null>(
+    null,
+  );
+  const [hasUnsavedEditorChanges, setHasUnsavedEditorChanges] = useState(false);
+  const [createDocument, { isLoading: isCreating }] =
+    useCreateDocumentMutation();
+  const [updateDocument, { isLoading: isUpdating }] =
+    useUpdateDocumentMutation();
+  const [
+    isShowingCollectionPicker,
+    { open: showCollectionPicker, close: hideCollectionPicker },
+  ] = useDisclosure(false);
+  const [sendToast] = useToast();
+
+  const documentId = entityId === "new" ? "new" : extractEntityId(entityId);
+  const previousDocumentId = usePrevious(documentId);
+  const [isNavigationScheduled, scheduleNavigation] = useCallbackEffect();
+  const isNewDocument = documentId === "new";
+
+  let {
+    data: documentData,
+    isLoading: isDocumentLoading,
+    error,
+  } = useGetDocumentQuery(
+    documentId && !isNewDocument ? { id: documentId } : skipToken,
+  );
+  if (documentId !== documentData?.id) {
+    documentData = undefined;
+  }
+
+  const canWrite = isNewDocument ? true : documentData?.can_write;
+
+  useEffect(() => {
+    if (error) {
+      dispatch(setErrorPage(error));
+    }
+  }, [dispatch, error]);
+
+  const {
+    documentTitle,
+    setDocumentTitle,
+    documentContent,
+    setDocumentContent,
+    updateCardEmbeds,
+  } = useDocumentState(documentData);
+
+  // This is important as it will affect collection breadcrumbs in the appbar
+  useUnmount(() => {
+    dispatch(resetDocuments());
+  });
+
+  useRegisterDocumentMetabotContext();
+  useBeforeUnload(() => {
+    // warn if you try to navigate away with unsaved changes
+    return hasUnsavedChanges();
+  });
+
+  // Reset state when document changes
+  useEffect(() => {
+    if (documentId !== previousDocumentId) {
+      setHasUnsavedEditorChanges(false);
+      if (isNewDocument && previousDocumentId !== "new") {
+        setDocumentTitle("");
+        setDocumentContent(null);
+        dispatch(resetDocuments());
+      }
+    }
+  }, [
+    documentId,
+    previousDocumentId,
+    isNewDocument,
+    setDocumentTitle,
+    setDocumentContent,
+    dispatch,
+  ]);
+
+  // Reset dirty state when document content loads from API
+  useEffect(() => {
+    if (documentContent && !isNewDocument) {
+      setHasUnsavedEditorChanges(false);
+    }
+  }, [documentContent, isNewDocument]);
+
+  useEffect(() => {
+    // Set current document when document loads (includes collection_id and all other data)
+    if (documentData && !isNewDocument) {
+      dispatch(setCurrentDocument(documentData));
+    } else if (isNewDocument) {
+      dispatch(setCurrentDocument(null));
+    }
+  }, [documentData, documentId, dispatch, isNewDocument]);
+
+  const hasUnsavedChanges = useCallback(() => {
+    const currentTitle = documentTitle.trim();
+    const originalTitle = documentData?.name || "";
+    const titleChanged = currentTitle !== originalTitle;
+
+    // Check if there are any draft cards
+    const hasDraftCards = Object.keys(draftCards).length > 0;
+
+    // For new documents, show Save if there's title or editor changes or draft cards
+    if (isNewDocument) {
+      return (
+        currentTitle.length > 0 || hasUnsavedEditorChanges || hasDraftCards
+      );
+    }
+
+    // For existing documents, use simple change tracking
+    return titleChanged || hasUnsavedEditorChanges || hasDraftCards;
+  }, [
+    documentTitle,
+    isNewDocument,
+    documentData,
+    hasUnsavedEditorChanges,
+    draftCards,
+  ]);
+
+  const isSaving = isCreating || isUpdating;
+  const showSaveButton = hasUnsavedChanges() && canWrite && !isSaving;
+
+  const handleContentChanged = useCallback(() => {
+    if (!editorInstance) {
+      return;
+    }
+
+    // Compare current content with original content
+    const currentContent = editorInstance.getJSON();
+    const originalContent = documentContent;
+
+    // For new documents, any content means changes
+    if (isNewDocument) {
+      setHasUnsavedEditorChanges(!editorInstance.isEmpty);
+      return;
+    }
+
+    // For existing documents, compare with original content
+    const hasChanges = !_.isEqual(currentContent, originalContent);
+    setHasUnsavedEditorChanges(hasChanges);
+  }, [editorInstance, documentContent, isNewDocument]);
+
+  const handleSave = useCallback(
+    async (collectionId: RegularCollectionId | null = null) => {
+      if (!editorInstance || isSaving) {
+        return;
+      }
+
+      try {
+        const cardsToSave: Record<number, Card> = {};
+        const processedCardIds = new Set<number>();
+
+        editorInstance.state.doc.descendants((node: any) => {
+          if (node.type.name === "cardEmbed") {
+            const cardId = node.attrs.id;
+            if (!processedCardIds.has(cardId)) {
+              processedCardIds.add(cardId);
+
+              if (cardId < 0 && draftCards[cardId]) {
+                cardsToSave[cardId] = draftCards[cardId];
+              }
+            }
+          }
+        });
+
+        const documentAst = editorInstance ? editorInstance.getJSON() : null;
+        const name =
+          documentTitle ||
+          t`Untitled document - ${dayjs().local().format("MMMM D, YYYY")}`;
+
+        const newDocumentData: any = {
+          name,
+          document: documentAst,
+          cards: Object.keys(cardsToSave).length > 0 ? cardsToSave : undefined,
+        };
+
+        const result = await (documentData?.id
+          ? updateDocument({ ...newDocumentData, id: documentData.id }).then(
+              (response) => {
+                if (response.data) {
+                  scheduleNavigation(() => {
+                    dispatch(push(`/document/${response.data.id}`));
+                  });
+                }
+                return response.data;
+              },
+            )
+          : createDocument({
+              ...newDocumentData,
+              collection_id: collectionId || undefined,
+            }).then((response) => {
+              if (response.data) {
+                scheduleNavigation(() => {
+                  dispatch(replace(`/document/${response.data.id}`));
+                });
+              }
+              return response.data;
+            }));
+
+        if (result) {
+          sendToast({
+            message: documentData?.id ? t`Document saved` : t`Document created`,
+          });
+          dispatch(clearDraftCards());
+          // Mark document as clean
+          setHasUnsavedEditorChanges(false);
+        }
+      } catch (error) {
+        console.error("Failed to save document:", error);
+        sendToast({ message: t`Error saving document`, icon: "warning" });
+      }
+    },
+    [
+      editorInstance,
+      isSaving,
+      documentTitle,
+      draftCards,
+      documentData?.id,
+      updateDocument,
+      createDocument,
+      scheduleNavigation,
+      dispatch,
+      sendToast,
+    ],
+  );
+
+  useEffect(() => {
+    const handleKeyDown = (event: KeyboardEvent) => {
+      // Save shortcut: Cmd+S (Mac) or Ctrl+S (Windows/Linux)
+      if ((event.metaKey || event.ctrlKey) && event.key === "s") {
+        event.preventDefault();
+        if (!hasUnsavedChanges() || !canWrite) {
+          return;
+        }
+
+        isNewDocument ? showCollectionPicker() : handleSave();
+      }
+    };
+
+    document.addEventListener("keydown", handleKeyDown);
+
+    return () => {
+      document.removeEventListener("keydown", handleKeyDown);
+    };
+  }, [
+    hasUnsavedChanges,
+    handleSave,
+    isNewDocument,
+    showCollectionPicker,
+    canWrite,
+  ]);
+
+  const handleQuestionSelect = useCallback(
+    (cardId: number | null, embedIndex?: number | null) => {
+      if (
+        cardId !== null &&
+        embedIndex !== null &&
+        embedIndex !== undefined &&
+        embedIndex >= 0 &&
+        selectedEmbedIndex !== null
+      ) {
+        // Only update the selected embed index if the sidebar is already open
+        dispatch(openVizSettingsSidebar({ embedIndex }));
+      }
+    },
+    [dispatch, selectedEmbedIndex],
+  );
+
+  const handlePrintDocument = useCallback(() => {
+    window.print();
+  }, []);
+
+  return (
+    <Box className={styles.documentPage}>
+      <Box className={styles.contentArea}>
+        <Box className={styles.mainContent}>
+          <Box className={styles.documentContainer}>
+            <Box className={styles.header} mt="xl" pt="xl">
+              <Flex direction="column" w="100%">
+                <TextInput
+                  aria-label={t`Document Title`}
+                  autoFocus={isNewDocument}
+                  value={documentTitle}
+                  onChange={(event) =>
+                    setDocumentTitle(event.currentTarget.value)
+                  }
+                  flex={1}
+                  placeholder={t`New document`}
+                  readOnly={!canWrite}
+                  styles={{
+                    input: {
+                      border: "none",
+                      padding: 0,
+                      fontSize: "2rem",
+                      fontWeight: "bold",
+                    },
+                  }}
+                />
+                {documentData && (
+                  <Flex gap="md">
+                    <Text className={styles.metadataItem}>
+                      <Icon name="person" />
+                      {documentData.creator.common_name}{" "}
+                    </Text>
+                    <Tooltip
+                      label={getFormattedTime(
+                        documentData.updated_at,
+                        "default",
+                        { local: true },
+                      )}
+                    >
+                      <Text className={styles.metadataItem}>
+                        <Icon name="clock" />
+                        <DateTime value={documentData.updated_at} unit="day" />
+                      </Text>
+                    </Tooltip>
+                  </Flex>
+                )}
+              </Flex>
+              <Flex gap="md" align="center">
+                <Box
+                  className={cx(styles.hidable, {
+                    [styles.visible]: showSaveButton,
+                  })}
+                >
+                  <Button
+                    onClick={() => {
+                      isNewDocument ? showCollectionPicker() : handleSave();
+                    }}
+                    variant="filled"
+                    data-hide-on-print
+                  >
+                    {t`Save`}
+                  </Button>
+                </Box>
+                <Menu position="bottom-end">
+                  <Menu.Target>
+                    <ActionIcon
+                      variant="subtle"
+                      size="md"
+                      aria-label={t`More options`}
+                      data-hide-on-print
+                    >
+                      <Icon name="ellipsis" />
+                    </ActionIcon>
+                  </Menu.Target>
+                  <Menu.Dropdown>
+                    <Menu.Item
+                      leftSection={<Icon name="document" />}
+                      onClick={handlePrintDocument}
+                    >
+                      {t`Print Document`}
+                    </Menu.Item>
+                  </Menu.Dropdown>
+                </Menu>
+              </Flex>
+            </Box>
+            <Editor
+              onEditorReady={setEditorInstance}
+              onCardEmbedsChange={updateCardEmbeds}
+              onQuestionSelect={handleQuestionSelect}
+              initialContent={documentContent}
+              onContentChanged={handleContentChanged}
+              editable={canWrite}
+              isLoading={isDocumentLoading}
+            />
+          </Box>
+        </Box>
+
+        {selectedQuestionId &&
+          selectedEmbedIndex !== null &&
+          editorInstance && (
+            <Box className={styles.sidebar}>
+              <EmbedQuestionSettingsSidebar
+                cardId={selectedQuestionId}
+                onClose={() => dispatch(closeSidebar())}
+                editorInstance={editorInstance}
+              />
+            </Box>
+          )}
+
+        {isShowingCollectionPicker && (
+          <CollectionPickerModal
+            title={t`Where should we save this document?`}
+            onClose={hideCollectionPicker}
+            value={{ id: "root", model: "collection" }}
+            options={{
+              showPersonalCollections: true,
+              showRootCollection: true,
+            }}
+            onChange={(collection) => {
+              handleSave(canonicalCollectionId(collection.id));
+              hideCollectionPicker();
+            }}
+          />
+        )}
+        <LeaveRouteConfirmModal
+          // `key` remounts this modal when navigating between different documents or to a new document.
+          // The `route` doesn't change in that scenario which prevents the modal from closing when you confirm you want to discard your changes.
+          key={documentId}
+          isEnabled={hasUnsavedChanges() && !isNavigationScheduled}
+          route={route}
+        />
+      </Box>
+    </Box>
+  );
+};

enterprise/frontend/src/metabase-enterprise/documents/components/Editor/Editor.module.css

@@ -0,0 +1,228 @@
+.editor {
+  height: 100%;
+  display: flex;
+  flex-direction: column;
+}
+
+.editorContent {
+  flex: 1;
+  cursor: text;
+  position: relative;
+}
+
+.editorContent :global(.ProseMirror) {
+  padding: 1.5rem 0 2rem 0;
+  min-height: 100%;
+  outline: none;
+  max-width: 900px;
+  margin: 0 auto;
+  color: var(--mb-color-text-dark);
+  font-size: 1rem;
+  line-height: 1.5;
+}
+
+/* Apply typography styles only to elements NOT inside .editor-style-boundary */
+.editorContent
+  :global(.ProseMirror)
+  :where(:global(h1), :global(h2), :global(h3)):not(
+    :global(.editor-style-boundary) *
+  ) {
+  font-weight: 700;
+  color: var(--mb-color-text-dark);
+}
+
+.editorContent
+  :global(.ProseMirror)
+  :global(h1):not(:global(.editor-style-boundary) *) {
+  font-size: 1.75rem;
+  margin: 1.5rem 0 1rem;
+  line-height: 1.3;
+}
+
+.editorContent
+  :global(.ProseMirror)
+  :global(h2):not(:global(.editor-style-boundary) *) {
+  font-size: 1.375rem;
+  margin: 1.25rem 0 0.75rem;
+  line-height: 1.3;
+}
+
+.editorContent
+  :global(.ProseMirror)
+  :global(h3):not(:global(.editor-style-boundary) *) {
+  font-size: 1.125rem;
+  margin: 1rem 0 0.5rem;
+  line-height: 1.4;
+}
+
+.editorContent
+  :global(.ProseMirror)
+  :global(p):not(:global(.editor-style-boundary) *) {
+  margin: 0 0 1rem 0;
+  line-height: 1.5;
+  font-size: 1rem;
+}
+
+.editorContent
+  :global(.ProseMirror)
+  :where(:global(ul), :global(ol)):not(:global(.editor-style-boundary) *) {
+  margin: 0 0 1rem 0;
+  padding-left: 1.5rem;
+}
+
+.editorContent
+  :global(.ProseMirror)
+  :global(ul):not(:global(.editor-style-boundary) *) {
+  list-style-type: disc;
+}
+
+.editorContent
+  :global(.ProseMirror)
+  :global(ol):not(:global(.editor-style-boundary) *) {
+  list-style-type: decimal;
+}
+
+.editorContent
+  :global(.ProseMirror)
+  :global(li):not(:global(.editor-style-boundary) *) {
+  margin: 0 0 0.25rem 0;
+  line-height: 1.5;
+  display: list-item;
+}
+
+.editorContent
+  :global(.ProseMirror)
+  :global(ul ul):not(:global(.editor-style-boundary) *) {
+  list-style-type: circle;
+  margin-top: 0.25rem;
+}
+
+.editorContent
+  :global(.ProseMirror)
+  :global(ul ul ul):not(:global(.editor-style-boundary) *) {
+  list-style-type: square;
+}
+
+.editorContent
+  :global(.ProseMirror)
+  :where(:global(li > ul), :global(li > ol)):not(
+    :global(.editor-style-boundary) *
+  ) {
+  margin-top: 0.25rem;
+}
+
+.editorContent
+  :global(.ProseMirror)
+  :global(li p):not(:global(.editor-style-boundary) *) {
+  line-height: 1.5;
+  margin-bottom: 0.25rem;
+}
+
+.editorContent
+  :global(.ProseMirror)
+  :global(code):not(:global(.editor-style-boundary) *) {
+  background-color: var(--mb-base-color-orion-5);
+  padding: 0.125rem 0.375rem;
+  border-radius: var(--mantine-radius-xs);
+  font-family: var(--mb-default-monospace-font-family, monospace);
+  font-size: 0.875rem;
+  color: var(--mb-color-text-dark);
+  border: 1px solid var(--mb-base-color-orion-10);
+}
+
+.editorContent
+  :global(.ProseMirror)
+  :global(pre):not(:global(.editor-style-boundary) *) {
+  background-color: var(--mb-base-color-gray-10);
+  color: var(--mb-color-text-dark);
+  padding: 1rem;
+  border-radius: var(--mantine-radius-sm);
+  overflow-x: auto;
+  margin: 0 0 1rem 0;
+  font-size: 0.875rem;
+  border: 1px solid var(--mb-base-color-gray-20);
+}
+
+.editorContent
+  :global(.ProseMirror)
+  :global(pre code):not(:global(.editor-style-boundary) *) {
+  background: none;
+  padding: 0;
+  color: inherit;
+  font-size: inherit;
+  border: none;
+}
+
+.editorContent
+  :global(.ProseMirror)
+  :global(blockquote):not(:global(.editor-style-boundary) *) {
+  border-left: 3px solid var(--mb-base-color-orion-20);
+  margin: 0 0 1rem 0;
+  color: var(--mb-color-text-medium);
+  font-style: italic;
+  background-color: var(--mb-base-color-orion-5);
+  padding: 1rem;
+  padding-left: 1.25rem;
+  border-radius: var(--mantine-radius-xs);
+}
+
+.editorContent
+  :global(.ProseMirror)
+  :global(hr):not(:global(.editor-style-boundary) *) {
+  border: none;
+  border-top: 1px solid var(--mb-color-border);
+  margin: 1.5rem 0;
+}
+
+.editorContent :global(.ProseMirror:focus) {
+  outline: none;
+}
+
+.editorContent :global(p.is-empty:first-child:before) {
+  color: var(--mb-color-text-light);
+  content: attr(data-placeholder);
+  float: left;
+  height: 0;
+  pointer-events: none;
+  font-size: 1rem;
+}
+
+.img {
+  max-width: 80%;
+  display: block;
+  margin: 1rem auto;
+
+  &:global(.ProseMirror-selectednode) {
+    box-shadow: 0 0 0 1px var(--mb-color-brand);
+    border-radius: var(--mantine-radius-xs);
+  }
+}
+
+.suggestionPopup {
+  width: 320px;
+  min-height: 50px;
+  max-height: 400px;
+  overflow: hidden;
+  background: var(--mb-color-bg-white);
+  border: 1px solid var(--mb-color-border);
+  border-radius: var(--mantine-radius-md);
+  box-shadow: 0 2px 8px var(--mb-color-shadow);
+}
+
+.suggestionScroll {
+  max-height: 400px;
+  overflow-y: auto;
+}
+
+.suggestionMenuItem {
+  cursor: pointer;
+  background-color: transparent;
+}
+
+.suggestionMenuItem[data-selected] {
+  background-color: var(--mb-color-bg-light);
+}
+
+.suggestionMenuFooter {
+  border-top: 1px solid var(--mb-color-border);
+}

enterprise/frontend/src/metabase-enterprise/documents/components/Editor/Editor.stories.tsx

@@ -0,0 +1,52 @@
+import type { Store } from "@reduxjs/toolkit";
+import type { StoryFn } from "@storybook/react/*";
+import _ from "underscore";
+
+import { mockSettings } from "__support__/settings";
+import { createMockEntitiesState } from "__support__/store";
+import { Api } from "metabase/api";
+import { MetabaseReduxProvider } from "metabase/lib/redux";
+import { publicReducers } from "metabase/reducers-public";
+import { getStore } from "metabase/store";
+import type { State } from "metabase-types/store";
+import { createMockState } from "metabase-types/store/mocks";
+
+import { Editor } from "./Editor";
+import Data from "./data/data.json";
+
+const storeInitialState = createMockState({
+  settings: mockSettings(),
+  entities: createMockEntitiesState({}),
+});
+const publicReducerNames = Object.keys(publicReducers);
+const initialState = _.pick(storeInitialState, ...publicReducerNames) as State;
+const reducers = publicReducers;
+
+const storeMiddleware = [Api.middleware];
+
+const store = getStore(
+  reducers,
+  initialState,
+  storeMiddleware,
+) as unknown as Store<State>;
+
+const ReduxDecorator = (Story: StoryFn) => {
+  return (
+    <MetabaseReduxProvider store={store}>
+      <Story />
+    </MetabaseReduxProvider>
+  );
+};
+
+const DefaultTemplate = () => <Editor content={Data.markdownTest} />;
+
+export default {
+  title: "Components/Documents",
+  component: Editor,
+  decorators: [ReduxDecorator],
+  layout: "fullscreen",
+};
+
+export const Default = {
+  render: DefaultTemplate,
+};

enterprise/frontend/src/metabase-enterprise/documents/components/Editor/Editor.tsx

@@ -0,0 +1,239 @@
+import Image from "@tiptap/extension-image";
+import Link from "@tiptap/extension-link";
+import { Placeholder } from "@tiptap/extension-placeholder";
+import type { EditorState } from "@tiptap/pm/state";
+import type { JSONContent, Editor as TiptapEditor } from "@tiptap/react";
+import { EditorContent, useEditor } from "@tiptap/react";
+import StarterKit from "@tiptap/starter-kit";
+import cx from "classnames";
+import type React from "react";
+import { useEffect, useMemo } from "react";
+import { t } from "ttag";
+
+import { DND_IGNORE_CLASS_NAME } from "metabase/common/components/dnd";
+import CS from "metabase/css/core/index.css";
+import { useSelector } from "metabase/lib/redux";
+import { getSetting } from "metabase/selectors/settings";
+import { Box, Loader } from "metabase/ui";
+import { useDocumentsStore } from "metabase-enterprise/documents/redux-utils";
+import { getMentionsCache } from "metabase-enterprise/documents/selectors";
+import type { DocumentsStoreState } from "metabase-enterprise/documents/types";
+import { getMentionsCacheKey } from "metabase-enterprise/documents/utils/mentionsUtils";
+
+import styles from "./Editor.module.css";
+import { EditorBubbleMenu } from "./EditorBubbleMenu";
+import { CardEmbed } from "./extensions/CardEmbed";
+import { CommandExtension } from "./extensions/Command/CommandExtension";
+import { CommandSuggestion } from "./extensions/Command/CommandSuggestion";
+import { DisableMetabotSidebar } from "./extensions/DisableMetabotSidebar";
+import { MentionExtension } from "./extensions/Mention/MentionExtension";
+import { MentionSuggestion } from "./extensions/Mention/MentionSuggestion";
+import { MetabotNode, type PromptSerializer } from "./extensions/MetabotEmbed";
+import { MetabotMentionExtension } from "./extensions/MetabotMention/MetabotMentionExtension";
+import { MetabotMentionSuggestion } from "./extensions/MetabotMention/MetabotSuggestion";
+import { SmartLinkEmbed } from "./extensions/SmartLink";
+import { Markdown } from "./extensions/markdown/index";
+import { createSuggestionRenderer } from "./extensions/suggestionRenderer";
+import { useCardEmbedsTracking, useQuestionSelection } from "./hooks";
+import type { CardEmbedRef } from "./types";
+
+const getMetabotPromptSerializer =
+  (getState: () => DocumentsStoreState): PromptSerializer =>
+  (node) => {
+    const payload: ReturnType<PromptSerializer> = { instructions: "" };
+    return node.content.content.reduce((acc, child) => {
+      // Serialize @ mentions in the metabot prompt
+      if (child.type.name === SmartLinkEmbed.name) {
+        const { model, entityId } = child.attrs;
+        const key = getMentionsCacheKey({ model, entityId });
+        const value = getMentionsCache(getState())[key];
+        if (!value) {
+          return acc;
+        }
+        acc.instructions += `[${value.name}](${key})`;
+        if (!acc.references) {
+          acc.references = {};
+        }
+        acc.references[key] = value.name;
+      } else {
+        acc.instructions += child.textContent;
+      }
+      return acc;
+    }, payload);
+  };
+
+const isMetabotBlock = (state: EditorState): boolean =>
+  state.selection.$head.parent.type.name === "metabot";
+
+interface EditorProps {
+  onEditorReady?: (editor: TiptapEditor) => void;
+  onCardEmbedsChange?: (refs: CardEmbedRef[]) => void;
+  initialContent?: JSONContent | null;
+  onChange?: (content: JSONContent) => void;
+  onContentChanged?: () => void; // Simple dirty flag callback
+  onQuestionSelect?: (cardId: number | null) => void;
+  editable?: boolean;
+  isLoading?: boolean;
+}
+
+export const Editor: React.FC<EditorProps> = ({
+  onEditorReady,
+  onCardEmbedsChange,
+  initialContent,
+  onChange,
+  onContentChanged,
+  editable = true,
+  onQuestionSelect,
+  isLoading = false,
+}) => {
+  const siteUrl = useSelector((state) => getSetting(state, "site-url"));
+  const { getState } = useDocumentsStore();
+
+  const extensions = useMemo(
+    () => [
+      StarterKit,
+      Image.configure({
+        inline: false,
+        HTMLAttributes: {
+          class: styles.img,
+        },
+      }),
+      SmartLinkEmbed.configure({
+        HTMLAttributes: {
+          class: "smart-link",
+        },
+        siteUrl,
+      }),
+      Link.configure({
+        HTMLAttributes: {
+          class: CS.link,
+        },
+      }),
+      Placeholder.configure({
+        placeholder: t`Start writing, press "/" to open command palette, or "@" to insert a link...`,
+      }),
+      Markdown,
+      CardEmbed,
+      MetabotNode.configure({
+        serializePrompt: getMetabotPromptSerializer(getState),
+      }),
+      DisableMetabotSidebar,
+      MentionExtension.configure({
+        suggestion: {
+          allow: ({ state }) => !isMetabotBlock(state),
+          render: createSuggestionRenderer(MentionSuggestion),
+        },
+      }),
+      CommandExtension.configure({
+        suggestion: {
+          allow: ({ state }) => !isMetabotBlock(state),
+          render: createSuggestionRenderer(CommandSuggestion),
+        },
+      }),
+      MetabotMentionExtension.configure({
+        suggestion: {
+          allow: ({ state }) => isMetabotBlock(state),
+          render: createSuggestionRenderer(MetabotMentionSuggestion),
+        },
+      }),
+    ],
+    [siteUrl, getState],
+  );
+
+  const editor = useEditor(
+    {
+      extensions,
+      content: initialContent || "",
+      autofocus: false,
+      immediatelyRender: false,
+      onUpdate: ({ editor }) => {
+        const currentContent = editor.getJSON();
+        onChange?.(currentContent);
+
+        // Simple content changed notification
+        if (!editor.isEmpty) {
+          onContentChanged?.();
+        }
+      },
+    },
+    [],
+  );
+
+  // Handle content updates when initialContent changes
+  useEffect(() => {
+    if (editor && initialContent !== undefined) {
+      // Use Promise.resolve() to avoid flushSync warning
+      Promise.resolve().then(() => {
+        editor.commands.setContent(initialContent || "");
+      });
+    }
+  }, [editor, initialContent]);
+
+  // Notify parent when editor is ready
+  useEffect(() => {
+    if (editor && onEditorReady) {
+      onEditorReady(editor);
+    }
+  }, [editor, onEditorReady]);
+
+  // Update editor editable state
+  useEffect(() => {
+    if (editor) {
+      editor.setEditable(editable);
+    }
+  }, [editor, editable]);
+
+  useCardEmbedsTracking(editor, onCardEmbedsChange);
+  useQuestionSelection(editor, onQuestionSelect);
+
+  if (!editor) {
+    return null;
+  }
+
+  if (isLoading) {
+    return (
+      <Box className={cx(styles.editor, DND_IGNORE_CLASS_NAME)}>
+        <Loader />
+      </Box>
+    );
+  }
+
+  return (
+    <Box className={cx(styles.editor, DND_IGNORE_CLASS_NAME)}>
+      <Box
+        className={styles.editorContent}
+        onClick={(e) => {
+          // Focus editor when clicking on empty space
+          const target = e.target as HTMLElement;
+          if (
+            target.classList.contains(styles.editorContent) ||
+            target.classList.contains("ProseMirror")
+          ) {
+            const clickY = e.clientY;
+            const proseMirrorElement = target.querySelector(".ProseMirror");
+
+            if (proseMirrorElement) {
+              const proseMirrorRect =
+                proseMirrorElement.getBoundingClientRect();
+              const isClickBelowContent = clickY > proseMirrorRect.bottom;
+
+              if (isClickBelowContent) {
+                // Only move to end if clicking below the actual content
+                editor.commands.focus("end");
+              } else {
+                // Just focus without changing cursor position for clicks in padding areas
+                editor.commands.focus();
+              }
+            } else {
+              // Fallback: just focus without position change
+              editor.commands.focus();
+            }
+          }
+        }}
+      >
+        <EditorContent data-testid="document-content" editor={editor} />
+        <EditorBubbleMenu editor={editor} />
+      </Box>
+    </Box>
+  );
+};

enterprise/frontend/src/metabase-enterprise/documents/components/Editor/EditorBubbleMenu.tsx

@@ -0,0 +1,154 @@
+import type { EditorState } from "@tiptap/pm/state";
+import type { Editor as TiptapEditor } from "@tiptap/react";
+// eslint-disable-next-line import/no-unresolved
+import { BubbleMenu } from "@tiptap/react/menus";
+import type React from "react";
+import { t } from "ttag";
+
+import { Flex } from "metabase/ui";
+
+import { FormatButton } from "./FormatButton";
+
+interface EditorBubbleMenuProps {
+  editor: TiptapEditor;
+}
+
+export const EditorBubbleMenu: React.FC<EditorBubbleMenuProps> = ({
+  editor,
+}) => {
+  return (
+    <BubbleMenu
+      editor={editor}
+      shouldShow={({
+        editor,
+        state,
+        from,
+        to,
+      }: {
+        editor: TiptapEditor;
+        state: EditorState;
+        from: number;
+        to: number;
+      }) => {
+        // Don't show bubble menu if nothing is selected
+        const { empty } = state.selection;
+        if (empty) {
+          return false;
+        }
+
+        // Don't show for code blocks
+        if (editor.isActive("codeBlock")) {
+          return false;
+        }
+
+        // Don't show bubble menu for certain node types
+        const disallowedNodes = ["cardEmbed", "metabot", "smartLink", "image"];
+
+        // Check if any disallowed node is active
+        for (const nodeName of disallowedNodes) {
+          if (editor.isActive(nodeName)) {
+            return false;
+          }
+        }
+
+        // Check if selection contains any disallowed nodes
+        let hasDisallowedNode = false;
+        state.doc.nodesBetween(from, to, (node) => {
+          if (
+            disallowedNodes.includes(node.type.name) ||
+            node.type.name === "codeBlock"
+          ) {
+            hasDisallowedNode = true;
+            return false; // Stop traversing
+          }
+        });
+
+        return !hasDisallowedNode;
+      }}
+    >
+      <Flex
+        gap={4}
+        bg="white"
+        p="2px"
+        style={{
+          border: "1px solid var(--mb-color-border)",
+          borderRadius: "6px",
+          boxShadow: "0 2px 12px var(--mb-color-shadow)",
+        }}
+      >
+        <FormatButton
+          isActive={editor.isActive("bold")}
+          onClick={() => editor.chain().focus().toggleBold().run()}
+          tooltip={t`Bold`}
+          icon="text_bold"
+        />
+        <FormatButton
+          isActive={editor.isActive("italic")}
+          onClick={() => editor.chain().focus().toggleItalic().run()}
+          tooltip={t`Italic`}
+          icon="text_italic"
+        />
+        <FormatButton
+          isActive={editor.isActive("strike")}
+          onClick={() => editor.chain().focus().toggleStrike().run()}
+          tooltip={t`Strikethrough`}
+          icon="text_strike"
+        />
+        <FormatButton
+          isActive={editor.isActive("code")}
+          onClick={() => editor.chain().focus().toggleCode().run()}
+          tooltip={t`Inline code`}
+          icon="format_code"
+        />
+        <FormatButton
+          isActive={editor.isActive("heading", { level: 1 })}
+          onClick={() =>
+            editor.chain().focus().toggleHeading({ level: 1 }).run()
+          }
+          tooltip={t`Heading 1`}
+          text="H1"
+        />
+        <FormatButton
+          isActive={editor.isActive("heading", { level: 2 })}
+          onClick={() =>
+            editor.chain().focus().toggleHeading({ level: 2 }).run()
+          }
+          tooltip={t`Heading 2`}
+          text="H2"
+        />
+        <FormatButton
+          isActive={editor.isActive("heading", { level: 3 })}
+          onClick={() =>
+            editor.chain().focus().toggleHeading({ level: 3 }).run()
+          }
+          tooltip={t`Heading 3`}
+          text="H3"
+        />
+        <FormatButton
+          isActive={editor.isActive("bulletList")}
+          onClick={() => editor.chain().focus().toggleBulletList().run()}
+          tooltip={t`Bullet list`}
+          icon="list"
+        />
+        <FormatButton
+          isActive={editor.isActive("orderedList")}
+          onClick={() => editor.chain().focus().toggleOrderedList().run()}
+          tooltip={t`Numbered list`}
+          icon="ordered_list"
+        />
+        <FormatButton
+          isActive={editor.isActive("blockquote")}
+          onClick={() => editor.chain().focus().toggleBlockquote().run()}
+          tooltip={t`Quote`}
+          icon="quote"
+        />
+        <FormatButton
+          isActive={editor.isActive("codeBlock")}
+          onClick={() => editor.chain().focus().toggleCodeBlock().run()}
+          tooltip={t`Code block`}
+          icon="code_block"
+        />
+      </Flex>
+    </BubbleMenu>
+  );
+};

enterprise/frontend/src/metabase-enterprise/documents/components/Editor/FormatButton.tsx

@@ -0,0 +1,49 @@
+import type React from "react";
+
+import type { IconName } from "metabase/ui";
+import { Icon, Tooltip, UnstyledButton } from "metabase/ui";
+
+interface FormatButtonProps {
+  isActive: boolean;
+  onClick: () => void;
+  tooltip: string;
+  icon?: IconName;
+  text?: string;
+}
+
+export const FormatButton: React.FC<FormatButtonProps> = ({
+  isActive,
+  onClick,
+  tooltip,
+  icon,
+  text,
+}) => {
+  return (
+    <Tooltip label={tooltip} position="top">
+      <UnstyledButton
+        style={{
+          width: "32px",
+          height: "32px",
+          borderRadius: "4px",
+          display: "flex",
+          alignItems: "center",
+          justifyContent: "center",
+          color: isActive
+            ? "var(--mb-color-brand)"
+            : "var(--mb-color-text-dark)",
+          backgroundColor: isActive
+            ? "var(--mb-color-bg-light)"
+            : "transparent",
+          fontSize: text ? "14px" : undefined,
+          fontWeight: text ? "bold" : undefined,
+          border: "none",
+          cursor: "pointer",
+          transition: "all 0.15s ease",
+        }}
+        onClick={onClick}
+      >
+        {icon ? <Icon name={icon} size={16} /> : text}
+      </UnstyledButton>
+    </Tooltip>
+  );
+};

enterprise/frontend/src/metabase-enterprise/documents/components/Editor/constants.ts

@@ -0,0 +1,2 @@
+// Class name used to prevent editor typography styles from affecting embedded content
+export const EDITOR_STYLE_BOUNDARY_CLASS = "editor-style-boundary";