hack

* Closes EMB-853 for cartesian charts

* tests

* cleanup

.clj-kondo/config.edn

@@ -489,6 +489,8 @@
   metabase.util.malli/defn-                                                             schema.core/defn
   metabase.util.malli/fn                                                                schema.core/fn
   metabase.util.namespaces/import-fns                                                   potemkin/import-vars
+  metabase.util.performance/doseq                                                       clojure.core/doseq
+  metabase.util.performance/for                                                         clojure.core/for
   metabase.util/with-timer-ms                                                           clojure.core/fn
   metabuild-common.core/with-open-jar-file-system                                       clojure.core/let
   metabuild-common.files/with-open-jar-file-system                                      clojure.core/let

.github/workflows/e2e-embedding-sdk-compatibility-tests.yml

@@ -179,7 +179,11 @@ jobs:
           name: metabase-ee-${{ github.event.pull_request.head.sha || github.sha }}-uberjar
 
       - name: Build Embedding SDK package
-        run: yarn build-embedding-sdk-package
+        run: |
+          # Remove the matchers to not report false positives on every PR
+          # unrelated to embedding (EMB-765)
+          echo "::remove-matcher owner=tsc::"
+          yarn build-embedding-sdk-package
 
       - name: Prepare Embedding SDK artifact
         uses: actions/upload-artifact@v4

.github/workflows/embedding-sdk.yml

@@ -43,7 +43,11 @@ jobs:
         with:
           m2-cache-key: "cljs"
       - name: Build Embedding SDK package
-        run: yarn build-embedding-sdk-package
+        run: |
+          # Remove the matchers to not report false positives on every PR
+          # unrelated to embedding (EMB-765)
+          echo "::remove-matcher owner=tsc::"
+          yarn build-embedding-sdk-package
       - name: Prepare Embedding SDK artifact
         uses: actions/upload-artifact@v4
         with:

deps.edn

@@ -108,7 +108,7 @@
                                                           org.bouncycastle/bcprov-jdk18on]}
   metabase/throttle                         {:mvn/version "1.0.2"}              ; Tools for throttling access to API endpoints and other code pathways
   methodical/methodical                     {:mvn/version "1.0.127"}            ; drop-in replacements for Clojure multimethods and adds several advanced features
-  metosin/malli                             {:mvn/version "0.19.1"}             ; Data-driven Schemas for Clojure/Script and babashka
+  metosin/malli                             {:mvn/version "0.19.2"}             ; Data-driven Schemas for Clojure/Script and babashka
   nano-id/nano-id                           {:mvn/version "1.1.0"}              ; NanoID generator for generating entity_ids
   net.cgrand/macrovich                      {:mvn/version "0.2.2"}              ; utils for writing macros for both Clojure & ClojureScript
   net.clojars.wkok/openai-clojure           {:mvn/version "0.23.0"

docs/embedding/static-embedding.md

@@ -174,7 +174,8 @@ This key is shared across all static embeds. Whoever has access to this key coul
 Dashboards are a fixed aspect ratio, so if you'd like to ensure they're automatically sized vertically to fit their contents you can use the [iFrame Resizer](https://github.com/davidjbradshaw/iframe-resizer) script. Metabase serves a copy for convenience:
 
 ```html
-<script src="https://metabase.example.com/app/iframeResizer.js"></script>
+<script src="{your-metabase-url}/app/iframeResizer.js"></script>
+
 <iframe
   src="https://metabase.example.com/embed/dashboard/TOKEN"
   onload="iFrameResize({}, this)"

docs/exploration-and-organization/delete-and-restore.md

@@ -1,10 +1,11 @@
 ---
 title: Delete and restore
+summary: You can move questions, dashboards, models, and metrics to Trash where they can be restored or permanently deleted. Deleting items affects related content like dashboards and subscriptions. Other entities in Metabase can be deleted or archived, but won't show up in Trash.
 ---
 
 # Delete and restore
 
-Sometimes your questions, dashboards, models, or collections outlive their usefulness. You can send outdated items to **Trash**.
+Sometimes your questions, dashboards, models, metrics, or collections outlive their usefulness. You can send outdated items to **Trash**.
 
 ![Move to trash](./images/move-to-trash.png)
 
@@ -104,7 +105,7 @@ What happens to related items when you delete a collection?
 | ---------------------------------------------- | -------- | ------------------- | -------- |
 | All items and subcollections in the collection | In Trash | N/A                 | Restored |
 
-> You can't permanently delete collections
+> You can't permanently delete collections.
 
 ## Permanently deleting items
 
@@ -117,7 +118,7 @@ To permanently delete an item:
 3. Click on the checkbox next to the item to select it;
 4. Select "Permanently delete". If you click this button, you won't be able to recover the item. It'll be lost to the void.
 
-> You can't permanently delete collections
+> You can't permanently delete collections.
 
 ## Deleting and restoring events and timelines
 
@@ -131,13 +132,11 @@ Snippets can be archived and unarchived. See [Archiving snippets](../questions/n
 
 ## Deleting segments
 
-Segments can be retired. See [Retiring Segments](../data-modeling/segments.md#editing-and-retiring-segments).
-
-You won't see retired Segments in Trash.
+Segments can be retired. See [Retiring Segments](../data-modeling/segments.md#editing-and-retiring-segments). You won't see retired Segments in Trash.
 
 ## Deleting subscriptions and alerts
 
-See [Deleting a subscription](../dashboards/subscriptions.md#deleting-a-subscription) and [Deleting alerts](../questions/alerts.md#editing-and-deleting-alerts).
+See [Deleting a subscription](../dashboards/subscriptions.md#deleting-a-subscription) and [Deleting alerts](../questions/alerts.md#editing-deleting-and-unsubscribing-from-alerts).
 
 ## Deleting databases
 

docs/questions/alerts.md

@@ -29,13 +29,16 @@ Only admins and people with [settings access](../permissions/application.md#sett
 To create an alert:
 
 1. Save your question.
-2. Click on the sharing icon in the top-right of the screen.
+2. Click on the **three dots** in the top-right of the screen.
 3. Select **Create an alert**.
-4. Select what you want to be alerted about. The alert options will depend on the question's visualization. See [types of alerts](#types-of-alerts).
-5. Select when you want Metabase to check the results. Metabase can check for results by the minute, hourly, daily, weekly, monthly, or on a custom schedule that you set using the Quartz [cron syntax](https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html).
-6. Select the alert's destination: [email](../configuring-metabase/email.md), [Slack](../configuring-metabase/slack.md), or to a [webhook](../configuring-metabase/webhooks.md) (only admins and people with [settings access](../permissions/application.md#settings-access) can create and send to webhooks).
-7. Configure any other options (like [only sending the alert once](#send-a-one-time-alert)).
-8. Click **Done**.
+4. Select what you want to be alerted about (options depend on the question type):
+    - [When a question returns a result](#results-alerts) - for any question.
+    - [When a time series crosses a goal line](#goal-line-alerts) - for a line, bar, or area chart displaying a time series.
+    - [When a progress bar reaches or goes below its goal](#progress-bar-alerts) - for progress bars.
+6. Select when you want Metabase to check the results: by the minute, hourly, daily, weekly, monthly, or on a custom schedule that you set using the Quartz [cron syntax](https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html).
+7. Select the alert's destination: [email](../configuring-metabase/email.md), [Slack](../configuring-metabase/slack.md), or to a [webhook](../configuring-metabase/webhooks.md) (only admins and people with [settings access](../permissions/application.md#settings-access) can create and send to webhooks).
+8. Configure any other options (like [only sending the alert once](#send-a-one-time-alert)).
+9. Click **Done**.
 
 ## Send a one-time alert
 
@@ -47,12 +50,6 @@ To test the alert, first make sure that the question returns results (if the que
 
 Then hit the **Send now** button to trigger the alert.
 
-## Types of alerts
-
-- [Results alerts](#results-alerts): when a question returns any result.
-- [Goal line alerts](#goal-line-alerts) when a time series crosses a goal line.
-- [Progress bar alerts](#progress-bar-alerts): when a progress bar reaches or goes below its goal.
-
 ## Results alerts
 
 Metabase can send you an alert on a schedule when one of your questions returns _any_ result. This kind of alert is also particularly useful if you have a question that doesn't _usually_ return any results, but you want to know when the question _does_ return results.
@@ -69,27 +66,22 @@ _Requires a [bar, line, or area chart](./visualizations/line-bar-and-area-charts
 
 Goal line alerts are useful when you're doing things like tracking daily active users (DAU) and you want to know when you reach a certain number of DAU, or when you're tracking orders per week and you want to know whenever the number of orders ever goes below a certain threshold.
 
-To start, you'll need a line, area, or bar chart displaying a number over time.
+To create an alert when a time series crosses a goal line:
 
-Next, you need to set up a goal line on your chart. Open up the visualization settings by clicking the **gear** icon in the bottom-left. Then click on the **Display** tab, and turn on the **Show goal** setting. Choose a value for your goal (and optionally a label) and click Done.
+1. Create a line, area, or bar chart displaying a number over time.
+2. Open up the visualization settings by clicking the **gear** icon in the bottom-left.
+3. In the **Display** tab, turn on the **Show goal** setting.
+4. Choose a value for your goal (and optionally a label) and click Done.
+5. Save the question.
+6. Click the **three dots** icon in top and select "Create alert"
 
-You can choose:
+    You can choose:
+    
+    - Whether you want Metabase to alert you when the time series goes above the goal line or when it goes below the goal line.
+    - Whether you want Metabase to alert you every time the time series crosses a goal line, or only the first time it crosses the goal line.
+    - How often you want Metabase to check to see if the goal line has been crossed.
 
-- Whether you want Metabase to alert you when the time series goes above the goal line or when it goes below the goal line.
-- Whether you want Metabase to alert you every time the time series crosses a goal line, or only the first time it crosses the goal line.
-- How often you want Metabase to check to see if the goal line has been crossed.
-
-Click Done, and your alert will be all set up!
-
-If you need to edit or unsubscribe from the alert you set up, just click that same icon. You'll see the Edit and Unsubscribe buttons. This is also where you'll see alerts about this question that admins might have added you to.
-
-Metabase will email you when:
-
-- You set up an alert
-- You've been unsubscribed from an alert
-- One of your alerts has stopped working
-- You unsubscribed from an alert
-- An admin added you to an alert
+7. Select the schedule and recepients for the alert, and click "Done"
 
 ## Progress bar alerts
 
@@ -97,11 +89,11 @@ _Requires the [progress bar visualization](./visualizations/progress-bar.md)._
 
 If you want to set up an alert when a single number reaches a goal, you can use a progress bar visualization.
 
-1. Create a question that returns a single number as its result,
-2. Choose the Progress Bar chart type,
-3. In Visualization settings, select a goal value
-4. Save your question,
-5. Create an alert by clicking on the Sharing icon in top right
+1. Create a question that returns a single number as its result.
+2. Choose the Progress Bar chart type.
+3. In Visualization settings, select a goal value.
+4. Save your question.
+5. Create an alert by clicking the **three dots** in top right.
 
 You'll see the options for when you want to get alerts about this progress bar:
 
@@ -109,9 +101,9 @@ You'll see the options for when you want to get alerts about this progress bar:
 - Whether to alert only the first time the progress bar reaches the goal line, or every time
 - How often you want Metabase to check to see if the goal has been reached.
 
-## Editing and deleting alerts
+## Editing, deleting, and unsubscribing from alerts
 
-To edit or delete alerts on a question, click on the Sharing icon in the top right corner. What you can edit depends on whether you're an admin.
+To edit or delete alerts on a question, click on the **three dots** icon in the top right corner and select **Edit alerts**. What you can edit depends on whether you're an admin.
 
 Everyone:
 
@@ -124,6 +116,14 @@ Admins:
 - Admins can add or remove recipients on any alert, even ones that they didn't create themselves.
 - Admins can bulk manage alerts per person from the [People menu in Admin settings](../people-and-groups/managing.md#unsubscribe-from-all-subscriptions-and-alerts).
 
+Metabase will email you when:
+
+- You set up an alert
+- You've been unsubscribed from an alert
+- One of your alerts has stopped working
+- You unsubscribed from an alert
+- An admin added you to an alert
+
 ## Avoid changing the name of the alerted channel in Slack
 
 Once you set up an alert to a Slack channel, avoid changing the name of that channel in Slack. If you rename the channel in Slack, but you want Metabase to continue to send alerts to that renamed channel, you'll need to update the alert in Metabase to point to the new channel name.

e2e/support/commands.js

@@ -1,3 +1,5 @@
+import "cypress-iframe";
+
 // this is the only place we allow direct helper import
 // eslint-disable-next-line no-direct-helper-import
 import { H } from "e2e/support";

e2e/support/component-webpack.config.js

@@ -3,6 +3,9 @@ const path = require("path");
 
 const webpack = require("webpack");
 
+const {
+  SVGO_CONFIG,
+} = require("../../frontend/build/shared/rspack/svgo-config");
 const mainConfig = require("../../rspack.main.config");
 
 const SDK_PACKAGE_NAME = "@metabase/embedding-sdk-react";
@@ -56,6 +59,7 @@ module.exports = {
             loader: "@svgr/webpack",
             options: {
               ref: true,
+              svgoConfig: SVGO_CONFIG,
             },
           },
         ],

e2e/support/helpers/e2e-document-helpers.ts

@@ -37,18 +37,12 @@ export const documentMetabotSuggestionItem = (name: RegExp | string) =>
 export const commandSuggestionItem = (name: RegExp | string) =>
   commandSuggestionDialog().findByRole("option", { name });
 
-export const getDocumentCard = (name: string, expectCardExists = true) => {
-  const cardsResult = documentContent()
+export const getDocumentCard = (name: string) => {
+  return documentContent()
     .findAllByTestId("card-embed-title")
-    .filter((index, el) => {
-      // Filter elements based on custom logic, e.g., text content, class, etc.
-      return el.innerText === name;
-    })
-    .should("have.length", expectCardExists ? 1 : 0);
-
-  return expectCardExists
-    ? cardsResult.closest('[data-testid="document-card-embed"]')
-    : cardsResult;
+    .filter((_index, element) => element.innerText === name)
+    .should("have.length", 1)
+    .closest('[data-testid="document-card-embed"]');
 };
 
 export const getDocumentCardResizeContainer = (name: string) =>

e2e/support/helpers/e2e-embedding-helpers.js

@@ -178,13 +178,9 @@ function getIframeUrl() {
  * @param {string} [selector]
  */
 export function getIframeBody(selector = "iframe") {
-  return cy
-    .get(selector)
-    .its("0.contentDocument")
-    .should("exist")
-    .its("body")
-    .should("not.be.undefined")
-    .then(cy.wrap);
+  cy.frameLoaded(selector);
+  cy.wait(1); // unclear why, but the tests are flaky without this
+  return cy.iframe(selector);
 }
 
 export function getEmbedModalSharingPane() {

e2e/support/helpers/e2e-embedding-iframe-sdk-helpers.ts

@@ -93,9 +93,10 @@ export const getSimpleEmbedIframeContent = (iframeIndex = 0) => {
  */
 export function loadSdkIframeEmbedTestPage({
   origin = "",
+  selector,
   onVisitPage,
   ...options
-}: BaseEmbedTestPageOptions) {
+}: BaseEmbedTestPageOptions & { selector?: string }) {
   const testPageSource = getSdkIframeEmbedHtml(options);
 
   const testPageUrl = `${origin}/sdk-iframe-test-page`;
@@ -108,7 +109,7 @@ export function loadSdkIframeEmbedTestPage({
   cy.visit(testPageUrl, { onLoad: onVisitPage });
   cy.title().should("include", "Metabase Embed Test");
 
-  return getIframeBody();
+  return getIframeBody(selector);
 }
 
 /**

e2e/test/scenarios/admin/transforms.cy.spec.ts

@@ -392,6 +392,7 @@ LIMIT
 
       H.NativeEditor.value().should("eq", EXPECTED_QUERY);
       getQueryEditor().button("Save changes").click();
+      getTransformPage().should("be.visible");
 
       cy.log("run the transform and make sure its table can be queried");
       runTransformAndWaitForSuccess();
@@ -1618,6 +1619,19 @@ LIMIT
         "This run succeeded before it had a chance to cancel.",
       );
     });
+
+    it("should be possible to cancel a SQL transform from the preview (metabase#64474)", () => {
+      createSlowTransform(500);
+      getTransformPage().findByText("Edit query").click();
+
+      getQueryEditor().within(() => {
+        cy.findAllByTestId("run-button").eq(0).click();
+        cy.findByTestId("loading-indicator").should("be.visible");
+
+        cy.findAllByTestId("run-button").eq(0).click();
+        cy.findByTestId("loading-indicator").should("not.exist");
+      });
+    });
   });
 
   describe("dependencies", () => {
@@ -2686,7 +2700,7 @@ function getQueryEditor() {
 }
 
 function getRunButton(options: { timeout?: number } = {}) {
-  return cy.findByTestId("run-button", options);
+  return cy.findAllByTestId("run-button").eq(0, options);
 }
 
 function getCancelButton() {

e2e/test/scenarios/collections/trash.cy.spec.js

@@ -4,6 +4,7 @@ const { H } = cy;
 import {
   FIRST_COLLECTION_ID,
   ORDERS_COUNT_QUESTION_ID,
+  ORDERS_MODEL_ID,
   ORDERS_QUESTION_ID,
   READ_ONLY_PERSONAL_COLLECTION_ID,
 } from "e2e/support/cypress_sample_instance_data";
@@ -864,6 +865,47 @@ describe("scenarios > collections > trash", () => {
       .and("contain", "Restore")
       .and("contain", "Delete permanently");
   });
+
+  it("should not deselect items when aborting operations (metabase#44911)", () => {
+    cy.request("PUT", `/api/card/${ORDERS_QUESTION_ID}`, { archived: true });
+    cy.request("PUT", `/api/card/${ORDERS_COUNT_QUESTION_ID}`, {
+      archived: true,
+    });
+    cy.request("PUT", `/api/card/${ORDERS_MODEL_ID}`, { archived: true });
+    cy.visit("/trash");
+
+    selectItem("Orders");
+    selectItem("Orders Model");
+
+    cy.findByTestId("toast-card")
+      .should("be.visible")
+      .findByText("Delete permanently")
+      .click();
+
+    H.modal().findByText("Cancel").click();
+
+    assertChecked("Orders");
+    assertChecked("Orders Model");
+
+    cy.findByTestId("toast-card")
+      .should("be.visible")
+      .findByText("Move")
+      .click();
+
+    H.entityPickerModal().findByText("Cancel").click();
+
+    assertChecked("Orders");
+    assertChecked("Orders Model");
+
+    cy.log("Going through with action should reset selection");
+    cy.findByTestId("toast-card")
+      .should("be.visible")
+      .findByText("Delete permanently")
+      .click();
+
+    H.modal().findByText("Delete permanently").click();
+    assertChecked("Orders, Count", false);
+  });
 });
 
 function toggleEllipsisMenuFor(item) {
@@ -937,9 +979,14 @@ function ensureCanRestoreFromPage(name) {
 }
 
 function selectItem(name) {
+  cy.findByText(name).closest("tr").findByRole("checkbox").check();
+}
+
+function assertChecked(name, checked = true) {
   cy.findByText(name)
     .closest("tr")
-    .within(() => cy.findByRole("checkbox").click());
+    .findByRole("checkbox")
+    .should(checked ? "have.attr" : "not.have.attr", "checked");
 }
 
 function assertTrashSelectedInNavigationSidebar() {

e2e/test/scenarios/documents/card-embed-node.cy.spec.ts

@@ -22,7 +22,7 @@ describe("documents card embed node custom logic", () => {
         idAlias: "documentId",
       });
 
-      cy.get("@documentId").then((id) => cy.visit(`/document/${id}`));
+      H.visitDocument("@documentId");
     });
 
     it("should create a flexContainer when dropping one cardEmbed onto another standalone cardEmbed", () => {
@@ -129,7 +129,7 @@ describe("documents card embed node custom logic", () => {
         idAlias: "documentId",
       });
 
-      cy.get("@documentId").then((id) => cy.visit(`/document/${id}`));
+      H.visitDocument("@documentId");
     });
 
     it("should allow you to resize the cards inside a flex container", () => {
@@ -439,7 +439,7 @@ describe("documents card embed node custom logic", () => {
         idAlias: "documentId",
       });
 
-      cy.get("@documentId").then((id) => cy.visit(`/document/${id}`));
+      H.visitDocument("@documentId");
 
       // Wait for cards to load
       H.getDocumentCard("Orders")
@@ -463,7 +463,10 @@ describe("documents card embed node custom logic", () => {
       cy.realPress("Backspace");
 
       // Verify the Orders card has been deleted
-      H.getDocumentCard("Orders", false).should("not.exist");
+      H.documentContent()
+        .findAllByTestId("card-embed-title")
+        .filter((_index, element) => element.innerText === "Orders")
+        .should("not.exist");
 
       // Verify only one card remains
       H.documentContent()
@@ -483,7 +486,7 @@ describe("documents card embed node custom logic", () => {
         idAlias: "documentId",
       });
 
-      cy.get("@documentId").then((id) => cy.visit(`/document/${id}`));
+      H.visitDocument("@documentId");
 
       // First create a flexContainer by dropping one card onto another
       H.getDocumentCard("Orders")
@@ -517,7 +520,10 @@ describe("documents card embed node custom logic", () => {
       cy.realPress("Backspace");
 
       // Verify the Orders card has been deleted from the flexContainer
-      H.getDocumentCard("Orders", false).should("not.exist");
+      H.documentContent()
+        .findAllByTestId("card-embed-title")
+        .filter((_index, element) => element.innerText === "Orders")
+        .should("not.exist");
 
       H.documentContent()
         .findAllByTestId("document-card-embed")

e2e/test/scenarios/documents/documents.cy.spec.ts

@@ -230,7 +230,7 @@ H.describeWithSnowplowEE("documents", () => {
           const cardEmbed = resizeNode?.content?.[0];
           const clonedCardId = cardEmbed?.attrs?.id;
           cy.request("DELETE", `/api/card/${clonedCardId}`);
-          cy.visit(`/document/${id}`);
+          H.visitDocument(id);
         });
         cy.findByTestId("document-card-embed").should(
           "have.text",
@@ -241,7 +241,7 @@ H.describeWithSnowplowEE("documents", () => {
       it("read only access", () => {
         cy.signIn("readonly");
 
-        cy.get("@documentId").then((id) => cy.visit(`/document/${id}`));
+        H.visitDocument("@documentId");
 
         H.documentContent()
           .findByRole("textbox")
@@ -254,7 +254,7 @@ H.describeWithSnowplowEE("documents", () => {
       it("no access", () => {
         cy.signIn("nocollection");
 
-        cy.get("@documentId").then((id) => cy.visit(`/document/${id}`));
+        H.visitDocument("@documentId");
         cy.findByRole("status").should(
           "contain.text",
           "Sorry, you don’t have permission to see that.",
@@ -262,17 +262,17 @@ H.describeWithSnowplowEE("documents", () => {
       });
 
       it("not found", () => {
-        cy.get("@documentId").then((id) =>
-          cy.visit(`/document/${(id as unknown as number) + 1}`),
-        );
+        H.visitDocument(9999);
         H.main().within(() => {
-          cy.findByText("We're a little lost...").should("exist");
-          cy.findByText("The page you asked for couldn't be found.");
+          cy.findByText("We're a little lost...").should("be.visible");
+          cy.findByText("The page you asked for couldn't be found.").should(
+            "be.visible",
+          );
         });
       });
 
       it("should allow you to print", () => {
-        cy.get("@documentId").then((id) => cy.visit(`/document/${id}`));
+        H.visitDocument("@documentId");
         cy.findByRole("button", { name: "More options" }).click();
 
         // This needs to be *after* the page load to work
@@ -295,7 +295,7 @@ H.describeWithSnowplowEE("documents", () => {
       it("should handle undo/redo properly, resetting the history whenever a different document is viewed", () => {
         const isMac = Cypress.platform === "darwin";
         const metaKey = isMac ? "Meta" : "Control";
-        cy.get("@documentId").then((id) => cy.visit(`/document/${id}`));
+        H.visitDocument("@documentId");
         H.getDocumentCard("Orders").should("exist");
         H.documentContent().within(() => {
           const originalText = "Lorem Ipsum and some more words";
@@ -326,7 +326,7 @@ H.describeWithSnowplowEE("documents", () => {
 
         const originalText = "Lorem Ipsum and some more words";
         const originalExact = new RegExp(`^${originalText}$`);
-        cy.get("@documentId").then((id) => cy.visit(`/document/${id}`));
+        H.visitDocument("@documentId");
         cy.findByTestId("document-card-embed").should("contain", "37.65"); // wait for data loading
         H.documentContent().contains(originalExact).click();
 
@@ -364,7 +364,7 @@ H.describeWithSnowplowEE("documents", () => {
     });
 
     it("should support typing with a markdown syntax", () => {
-      cy.get("@documentId").then((id) => cy.visit(`/document/${id}`));
+      H.visitDocument("@documentId");
       H.documentContent().click();
 
       H.addToDocument("# This is a heading level 1");
@@ -518,7 +518,7 @@ H.describeWithSnowplowEE("documents", () => {
             dashboard_id: id,
           });
         });
-        cy.get("@documentId").then((id) => cy.visit(`/document/${id}`));
+        H.visitDocument("@documentId");
       });
 
       it("should support keyboard and mouse selection in suggestions without double highlight", () => {
@@ -827,11 +827,10 @@ H.describeWithSnowplowEE("documents", () => {
           cy.location("pathname").should("equal", `/document/${id}`),
         );
 
-        H.getDocumentCard("Orders, Count, Grouped by Created At (year)").within(
-          () => {
-            H.cartesianChartCircle().eq(1).click();
-          },
+        H.getDocumentCard("Orders, Count, Grouped by Created At (year)").should(
+          "be.visible",
         );
+        H.cartesianChartCircle().eq(1).click();
 
         H.popover().findByText("See these Orders").click();
 
@@ -874,7 +873,7 @@ H.describeWithSnowplowEE("documents", () => {
         document: { type: "doc", content: [] },
         idAlias: "documentId",
       });
-      cy.get("@documentId").then((id) => cy.visit(`/document/${id}`));
+      H.visitDocument("@documentId");
 
       // make changes and attempt to save
       H.documentContent().click();

e2e/test/scenarios/embedding/embedding-reproductions.cy.spec.js

@@ -957,24 +957,23 @@ describe("issue 40660", () => {
     });
   });
 
-  it(
-    "static dashboard content shouldn't overflow its container (metabase#40660)",
-    { tags: "@flaky" },
-    () => {
-      H.openStaticEmbeddingModal({
-        activeTab: "parameters",
-        previewMode: "preview",
-      });
+  it("static dashboard content shouldn't overflow its container (metabase#40660)", () => {
+    H.openStaticEmbeddingModal({
+      activeTab: "parameters",
+      previewMode: "preview",
+    });
 
-      H.getIframeBody().within(() => {
-        cy.findByTestId("embed-frame").scrollTo("bottom");
+    H.getIframeBody().within(() => {
+      cy.findByText(dashboardDetails.name).should("be.visible");
+      cy.findByTestId("loading-indicator").should("not.exist");
+      cy.findAllByText("1018947080336").should("have.length", 3);
+      cy.findByTestId("embed-frame").scrollTo("bottom");
 
-        cy.findByRole("link", { name: "Powered by Metabase" })
-          .scrollIntoView()
-          .should("be.visible");
-      });
-    },
-  );
+      cy.findByRole("link", { name: "Powered by Metabase" })
+        .scrollIntoView()
+        .should("be.visible");
+    });
+  });
 });
 
 // Skipped since it does not make sense when CSP is disabled

e2e/test/scenarios/embedding/sdk-iframe-embedding/sdk-iframe-embedding.cy.spec.ts

@@ -351,6 +351,7 @@ H.describeWithSnowplowEE(
             attributes: {},
           },
         ],
+        selector: `[dashboard-id="${ORDERS_DASHBOARD_ID}"] > iframe`, // get only the first iframe
       });
 
       frame.within(() => {

e2e/test/scenarios/filters/filter.cy.spec.js

@@ -713,7 +713,7 @@ describe("scenarios > question > filter", () => {
     // Esc closes the suggestion popover
     cy.realPress("Escape");
 
-    H.CustomExpressionEditor.completions().should("be.visible");
+    H.CustomExpressionEditor.completions().should("not.be.visible");
   });
 
   it("should work on twice summarized questions and preserve both summaries (metabase#15620)", () => {

e2e/test/scenarios/joins/joins-reproductions.cy.spec.js

@@ -1156,6 +1156,9 @@ describe("issue 27521", () => {
     H.getNotebookStep("data").button("Pick columns").click();
     H.popover().findByText("Select all").click();
 
+    cy.log("close popover");
+    H.getNotebookStep("data").click();
+
     H.join();
 
     H.entityPickerModal().within(() => {

e2e/test/scenarios/onboarding/setup/user_settings.cy.spec.js

@@ -276,7 +276,7 @@ const assertLightMode = () =>
   cy.get("body").should("have.css", "background-color", "rgb(249, 249, 250)");
 
 const assertDarkMode = () =>
-  cy.get("body").should("have.css", "background-color", "rgb(0, 0, 0)");
+  cy.get("body").should("have.css", "background-color", "rgb(5, 14, 21)");
 
 /**
  * Stub the current user authentication method

e2e/test/scenarios/organization/timelines-question.cy.spec.js

@@ -90,7 +90,7 @@ describe("scenarios > organization > timelines > question", () => {
       // eslint-disable-next-line no-unscoped-text-selectors -- deprecated usage
       cy.findByText("v3").should("be.visible");
 
-      cy.findByLabelText("table2 icon").click();
+      cy.findByLabelText("Switch to data").click();
       // eslint-disable-next-line no-unscoped-text-selectors -- deprecated usage
       cy.findByText("v1").should("be.visible");
       // eslint-disable-next-line no-unscoped-text-selectors -- deprecated usage

e2e/test/scenarios/question-reproductions/reproductions.cy.spec.ts

@@ -119,11 +119,9 @@ describe("issue 39487", () => {
     },
   );
 
-  // broken after migration away from filter modal
-  // see https://github.com/metabase/metabase/issues/55688
   it(
-    "calendar has constant size when using date range picker filter (metabase#39487)",
-    { viewportHeight: 1000, tags: "@skip" },
+    "calendar has constant size when using date range picker filter, and text inputs are in sync with the calendar inputs (metabase#39487, metabase#64602)",
+    { viewportHeight: 1000 },
     () => {
       createTimeSeriesQuestionWithFilter([
         "between",
@@ -141,11 +139,35 @@ describe("issue 39487", () => {
       cy.findByTestId("filter-pill").click();
       checkDateRangeFilter();
 
-      cy.log("filter modal");
+      cy.log("filter dropdown");
       cy.button(/Filter/).click();
-      H.modal().findByText("May 1 – Jun 1, 2024").click();
+      H.popover().findByText("Created At").click();
+      H.popover().findByText("Fixed date range…").click();
+      H.popover().within(() => {
+        cy.log(
+          "changing text input values should navigate the calendars (metabase#64602)",
+        );
+        cy.findAllByRole("textbox").first().clear().type("2024/05/01");
+        cy.findByText("May 2024").should("be.visible");
+        cy.findByLabelText("1 May 2024").should(
+          "have.attr",
+          "data-first-in-range",
+          "true",
+        );
+
+        cy.findAllByRole("textbox")
+          .should("have.length", 2)
+          .last()
+          .clear()
+          .type("2024/06/01");
+        cy.findAllByLabelText("1 June 2024")
+          .filter(":visible")
+          .should("have.length", 1)
+          .and("have.attr", "data-last-in-range", "true");
+      });
+      previousButton().click();
       checkDateRangeFilter();
-      H.modal().button("Close").click();
+      cy.realPress("Escape");
 
       cy.log("filter drill");
       cy.findByLabelText("Switch to data").click();
@@ -153,8 +175,12 @@ describe("issue 39487", () => {
       H.popover().findByText("Filter by this column").click();
       H.popover().findByText("Fixed date range…").click();
       H.popover().findAllByRole("textbox").first().clear().type("2024/05/01");
-      // eslint-disable-next-line no-unsafe-element-filtering
-      H.popover().findAllByRole("textbox").last().clear().type("2024/06/01");
+      H.popover()
+        .findAllByRole("textbox")
+        .should("have.length", 2)
+        .last()
+        .clear()
+        .type("2024/06/01");
       previousButton().click();
       checkDateRangeFilter();
 

e2e/test/scenarios/visualizations-tabular/object_detail.cy.spec.js

@@ -287,24 +287,6 @@ describe("scenarios > question > object details", { tags: "@slow" }, () => {
       .should("be.visible");
   });
 
-  it("should fetch linked entities data only once per entity type when reopening the modal (metabase#32720)", () => {
-    cy.intercept("POST", "/api/dataset", cy.spy().as("fetchDataset"));
-
-    H.openProductsTable();
-    cy.get("@fetchDataset").should("have.callCount", 1);
-
-    drillPK({ id: 5 });
-    cy.get("@fetchDataset").should("have.callCount", 3);
-
-    cy.findByLabelText("Close").click();
-
-    drillPK({ id: 5 });
-    cy.get("@fetchDataset").should("have.callCount", 3);
-
-    cy.wait(100);
-    cy.get("@fetchDataset").should("have.callCount", 3);
-  });
-
   it("should not offer drill-through on the object detail records (metabase#20560)", () => {
     H.openPeopleTable({ limit: 2 });
 

enterprise/backend/src/metabase_enterprise/transforms/api.clj

@@ -9,11 +9,13 @@
    [metabase-enterprise.transforms.models.transform-run :as transform-run]
    [metabase-enterprise.transforms.models.transform-run-cancelation :as transform-run-cancelation]
    [metabase-enterprise.transforms.ordering :as transforms.ordering]
+   [metabase-enterprise.transforms.schema :as transforms.schema]
    [metabase-enterprise.transforms.util :as transforms.util]
    [metabase.api.common :as api]
    [metabase.api.macros :as api.macros]
    [metabase.api.routes.common :refer [+auth]]
    [metabase.api.util.handlers :as handlers]
+   [metabase.driver :as driver]
    [metabase.driver.util :as driver.u]
    [metabase.events.core :as events]
    [metabase.request.core :as request]
@@ -31,26 +33,6 @@
 
 (set! *warn-on-reflection* true)
 
-(mr/def ::transform-source
-  [:multi {:dispatch (comp keyword :type)}
-   [:query
-    [:map
-     [:type [:= "query"]]
-     [:query [:map [:database :int]]]]]
-   [:python
-    [:map {:closed true}
-     [:source-database {:optional true} :int]
-     [:source-tables   [:map-of :string :int]]
-     [:type [:= "python"]]
-     [:body :string]]]])
-
-(mr/def ::transform-target
-  [:map
-   [:database {:optional true} :int]
-   [:type [:enum "table"]]
-   [:schema {:optional true} [:or ms/NonBlankString :nil]]
-   [:name :string]])
-
 (mr/def ::run-trigger
   [:enum "none" "global-schedule"])
 
@@ -103,6 +85,21 @@
                 (map #(update % :last_run transforms.util/localize-run-timestamps)))
           (t2/hydrate transforms :last_run :transform_tag_ids))))
 
+(defn- watermark-table-for-transform [{:keys [target]}]
+  (assoc target :name "mb__watermark_table"))
+
+(defn- ensure-mb-watermark-table-exists! [{:keys [target] :as transform}]
+  (when (= :table-incremental (-> target :type keyword))
+    (let [db-id (transforms.util/target-database-id transform)
+          {driver :engine :as database} (t2/select-one :model/Database db-id)
+          watermark-table (watermark-table-for-transform transform)
+          watermark-table-name (transforms.util/qualified-table-name driver watermark-table)]
+      (when-not (driver/table-exists? driver database watermark-table)
+        (driver/create-table! driver db-id watermark-table-name
+                              {:transform_id (driver/type->database-type driver :type/Integer),
+                               :watermark_id (driver/type->database-type driver :type/Integer)}
+                              :primary-key [:transform_id])))))
+
 (api.macros/defendpoint :post "/"
   "Create a new transform."
   [_route-params
@@ -110,13 +107,14 @@
    body :- [:map
             [:name :string]
             [:description {:optional true} [:maybe :string]]
-            [:source ::transform-source]
-            [:target ::transform-target]
+            [:source ::transforms.schema/transform-source]
+            [:target ::transforms.schema/transform-target]
             [:run_trigger {:optional true} ::run-trigger]
             [:tag_ids {:optional true} [:sequential ms/PositiveInt]]]]
   (api/check-superuser)
   (check-database-feature body)
   (check-feature-enabled! body)
+  (ensure-mb-watermark-table-exists! body)
   (api/check (not (transforms.util/target-table-exists? body))
              403
              (deferred-tru "A table with that name already exists."))
@@ -185,8 +183,8 @@
    body :- [:map
             [:name {:optional true} :string]
             [:description {:optional true} [:maybe :string]]
-            [:source {:optional true} ::transform-source]
-            [:target {:optional true} ::transform-target]
+            [:source {:optional true} ::transforms.schema/transform-source]
+            [:target {:optional true} ::transforms.schema/transform-target]
             [:run_trigger {:optional true} ::run-trigger]
             [:tag_ids {:optional true} [:sequential ms/PositiveInt]]]]
   (log/info "put transform" id)
@@ -197,6 +195,7 @@
                           new (merge old body)
                           target-fields #(-> % :target (select-keys [:schema :name]))]
                       ;; we must validate on a full transform object
+                      (ensure-mb-watermark-table-exists! new)
                       (check-feature-enabled! new)
                       (check-database-feature new)
                       (when (transforms.util/query-transform? old)

enterprise/backend/src/metabase_enterprise/transforms/execute.clj

@@ -14,7 +14,7 @@
 
 (mr/def ::transform-details
   [:map
-   [:transform-type [:enum {:decode/normalize schema.common/normalize-keyword} :table]]
+   [:transform-type [:enum {:decode/normalize schema.common/normalize-keyword} :table :table-incremental]]
    [:conn-spec :any]
    [:query :string]
    [:output-table [:keyword {:decode/normalize schema.common/normalize-keyword}]]])
@@ -23,6 +23,13 @@
   [:map
    [:overwrite? :boolean]])
 
+(defn- transform-opts [{:keys [transform-type]}]
+  (case transform-type
+    :table {:overwrite? true}
+
+    ;; TODO: append vs merge, for now we're just append
+    :table-incremental {}))
+
 (defn run-mbql-transform!
   "Run `transform` and sync its target table.
 
@@ -42,7 +49,7 @@
                               :query (transforms.util/compile-source source)
                               :output-schema (:schema target)
                               :output-table (transforms.util/qualified-table-name driver target)}
-           opts {:overwrite? true}]
+           opts (transform-opts transform-details)]
        (when (transforms.util/db-routing-enabled? database)
          (throw (ex-info "Transforms are not supported on databases with DB routing enabled."
                          {:driver driver, :database database})))
@@ -57,9 +64,11 @@
          (transforms.util/run-cancelable-transform!
           run-id driver transform-details
           (fn [_cancel-chan] (driver/run-transform! driver transform-details opts)))
+         ;; Update watermark tracking for incremental transforms
+         (transforms.util/upsert-watermark! transform driver database)
          (transforms.instrumentation/with-stage-timing [run-id :table-sync]
            (transforms.util/sync-target! target database run-id)
-         ;; This event must be published only after the sync is complete - the new table needs to be in AppDB.
+           ;; This event must be published only after the sync is complete - the new table needs to be in AppDB.
            (events/publish-event! :event/transform-run-complete {:object transform-details}))))
      (catch Throwable t
        (log/error t "Error executing transform")

enterprise/backend/src/metabase_enterprise/transforms/schema.clj

@@ -0,0 +1,55 @@
+(ns metabase-enterprise.transforms.schema
+  (:require
+   [metabase.util.malli.registry :as mr]
+   [metabase.util.malli.schema :as ms]))
+
+(mr/def ::keyset-strategy
+  [:map
+   [:type [:= "keyset"]]
+   [:keyset-column :string]
+   [:table-name {:optional true} :string]]) ; for python transforms with multiple tables
+
+(mr/def ::source-strategy
+  [:multi {:dispatch :type}
+   ["keyset" ::keyset-strategy]])
+
+(mr/def ::transform-source
+  [:multi {:dispatch (comp keyword :type)}
+   [:query
+    [:map
+     [:type [:= "query"]]
+     [:query [:map [:database :int]]]
+     [:source-strategy {:optional true} ::source-strategy]]]
+   [:python
+    [:map {:closed true}
+     [:source-database {:optional true} :int]
+     [:source-tables   [:map-of :string :int]]
+     [:type [:= "python"]]
+     [:body :string]
+     [:source-strategy {:optional true} ::source-strategy]]]])
+
+(mr/def ::append-config
+  [:map
+   [:type [:= "append"]]
+   #_[:tag-run-id? {:optional true} :boolean]]) ; whether to tag rows with run_id
+
+#_(mr/def ::merge-config
+    [:map
+     [:type [:= "merge"]]
+     [:primary-key [:sequential :string]]
+     [:update-columns {:optional true} [:sequential :string]]]) ; columns to update on conflict
+
+(mr/def ::target-incremental-strategy
+  [:multi {:dispatch :type}
+   ["append" ::append-config]
+   #_["merge" ::merge-config]])
+
+(mr/def ::transform-target
+  [:map
+   [:database {:optional true} :int]
+   [:type [:enum "table" "table-incremental"]]
+   [:schema {:optional true} [:or ms/NonBlankString :nil]]
+   [:name :string]
+
+   ;; only for table-incremental, TODO: specify with a multi spec
+   [:target-incremental-strategy {:optional true} ::target-incremental-strategy]])

enterprise/backend/src/metabase_enterprise/transforms/util.clj

@@ -105,6 +105,35 @@
    (log/info "Syncing target" (pr-str target) "for transform")
    (activate-table-and-mark-computed! database target)))
 
+(defn upsert-watermark!
+  "Update the watermark tracking table for an incremental transform.
+   Deletes any existing watermark entry for this transform and inserts a new one
+   with the MAX value of the watermark field from the transform output table."
+  [{:keys [id target] :as transform} driver database]
+  (when (= :table-incremental (keyword (:type target)))
+    (let [watermark-field (or (:watermarkField target) "id")
+          output-table (qualified-table-name driver target)
+          watermark-table (qualified-table-name driver {:schema (:schema target)
+                                                        :name "mb__watermark_table"})
+          conn-spec (driver/connection-spec driver database)
+          ;; lol
+          select-query (format "SELECT %d as transform_id, MAX(%s) as watermark_id FROM %s"
+                               id
+                               watermark-field
+                               (name output-table))
+          delete-sql (format "DELETE FROM %s WHERE transform_id = %d"
+                             (name watermark-table)
+                             id)]
+      (log/info "Upserting watermark for transform" id "with field" watermark-field)
+      (try
+        (let [insert-queries (driver/compile-insert driver {:query select-query
+                                                            :output-table watermark-table})]
+          (driver/execute-raw-queries! driver conn-spec [[delete-sql]
+                                                         insert-queries]))
+        (catch Exception e
+          (log/error e "Failed to upsert watermark for transform" id))))))
+
+;; TODO this and target-database-id can be transforms multimethods?
 (mu/defn target-database-id :- ::lib.schema.id/database
   "Return the target database id of a transform"
   [transform]
@@ -182,25 +211,27 @@
 (defn required-database-feature
   "Returns the database feature necessary to execute `transform`."
   [transform]
+  ;; TODO: not quite right
   (if (python-transform? transform)
     :transforms/python
-    (case (-> transform :target :type)
-      "table"             :transforms/table)))
+    (case (-> transform :target :type keyword)
+      :table :transforms/table
+      :table-incremental :transforms/table)))
 
 (defn ->instant
   "Convert a temporal value `t` to an Instant in the system timezone."
   ^Instant [t]
   (when t
     (condp instance? t
-      Instant        t
-      Date           (.toInstant ^Date t)
+      Instant t
+      Date (.toInstant ^Date t)
       OffsetDateTime (.toInstant ^OffsetDateTime t)
-      ZonedDateTime  (.toInstant ^ZonedDateTime t)
-      LocalDateTime  (recur (.atZone ^LocalDateTime t (t/zone-id)))
-      String         (recur (u.date/parse t))
-      LocalTime      (recur (.atDate ^LocalTime t (t/local-date)))
-      OffsetTime     (recur (.atDate ^OffsetTime t (t/local-date)))
-      LocalDate      (recur (.atStartOfDay ^LocalDate t))
+      ZonedDateTime (.toInstant ^ZonedDateTime t)
+      LocalDateTime (recur (.atZone ^LocalDateTime t (t/zone-id)))
+      String (recur (u.date/parse t))
+      LocalTime (recur (.atDate ^LocalTime t (t/local-date)))
+      OffsetTime (recur (.atDate ^OffsetTime t (t/local-date)))
+      LocalDate (recur (.atStartOfDay ^LocalDate t))
       (throw (ex-info (str "Cannot convert temporal " t " of type " (type t) " to an Instant")
                       {:temporal t})))))
 
@@ -214,7 +245,7 @@
   [run]
   (-> run
       (u/update-some :start_time utc-timestamp-string)
-      (u/update-some :end_time   utc-timestamp-string)))
+      (u/update-some :end_time utc-timestamp-string)))
 
 (mr/def ::column-definition
   [:map
@@ -258,14 +289,14 @@
   "Generate a temporary table name with current timestamp in milliseconds.
   If table name would exceed max table name length for the driver, fallback to using a shorter timestamp"
   [driver schema]
-  (let [max-len   (max 1 (or (driver/table-name-length-limit driver) Integer/MAX_VALUE))
+  (let [max-len (max 1 (or (driver/table-name-length-limit driver) Integer/MAX_VALUE))
         timestamp (str (System/currentTimeMillis))
-        prefix    (str transform-temp-table-prefix "_")
+        prefix (str transform-temp-table-prefix "_")
         available (- max-len (count prefix))
         ;; If we don't have enough space, take the later digits of the timestamp
-        suffix    (if (>= available (count timestamp))
-                    timestamp
-                    (subs timestamp (- (count timestamp) available)))
+        suffix (if (>= available (count timestamp))
+                 timestamp
+                 (subs timestamp (- (count timestamp) available)))
         table-name (str prefix suffix)]
     (keyword schema table-name)))
 

enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj

@@ -139,22 +139,28 @@
           ret)))
     (thunk)))
 
-(defn- create-table-and-insert-data!
-  "Create a table from metadata and insert data from source."
-  [driver db-id table-name metadata data-source]
-  (let [table-schema {:name (if (keyword? table-name) table-name (keyword table-name))
-                      :columns (mapv (fn [{:keys [name base_type #_database_type]}]
-                                       {:name name
-                                        :type (python-runner/restricted-insert-type base_type)
-                                        ;; :database-type database_type
-                                        :nullable? true})
-                                     (:fields metadata))}
-        data-source (assoc data-source :table-schema table-schema)]
-    (transforms.util/create-table-from-schema! driver db-id table-schema)
+(defn- table-schema [table-name metadata]
+  {:name (if (keyword? table-name) table-name (keyword table-name))
+   :columns (mapv (fn [{:keys [name base_type #_database_type]}]
+                    {:name name
+                     :type (python-runner/restricted-insert-type base_type)
+                     ;; :database-type database_type
+                     :nullable? true})
+                  (:fields metadata))})
 
+(defn- insert-data!
+  "Insert data from source into an existing table."
+  [driver db-id table-schema data-source]
+  (let [data-source (assoc data-source :table-schema table-schema)]
     (->> #(driver/insert-from-source! driver db-id table-schema data-source)
          (maybe-retry-with-backoff driver))))
 
+(defn- create-table-and-insert-data!
+  "Create a table from metadata and insert data from source."
+  [driver db-id table-schema data-source]
+  (transforms.util/create-table-from-schema! driver db-id table-schema)
+  (insert-data! driver db-id table-schema data-source))
+
 (defn- transfer-with-rename-tables-strategy!
   "Transfer data using the rename-tables*! multimethod with atomicity guarantees.
    Creates new table, then atomically renames target->old and new->target, then drops old."
@@ -167,7 +173,7 @@
     (log/info "Using rename-tables strategy with atomicity guarantees")
     (try
 
-      (create-table-and-insert-data! driver db-id source-table-name metadata data-source)
+      (create-table-and-insert-data! driver db-id (table-schema source-table-name metadata) data-source)
       (transforms.util/rename-tables! driver db-id {table-name temp-table-name
                                                     source-table-name table-name})
       (transforms.util/drop-table! driver db-id temp-table-name)
@@ -187,7 +193,7 @@
     (log/info "Using create-drop-rename strategy to minimize downtime")
     (try
 
-      (create-table-and-insert-data! driver db-id source-table-name metadata data-source)
+      (create-table-and-insert-data! driver db-id (table-schema source-table-name metadata) data-source)
       (transforms.util/drop-table! driver db-id table-name)
       (driver/rename-table! driver db-id source-table-name table-name)
 
@@ -206,13 +212,35 @@
   (try
 
     (transforms.util/drop-table! driver db-id table-name)
-    (create-table-and-insert-data! driver db-id table-name metadata data-source)
+    (create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source)
 
     (catch Exception e
       (log/error e "Failed to transfer data using drop-create fallback strategy")
       (throw e))))
 
-(defn- transfer-file-to-db [driver {db-id :id :as db} {:keys [target] :as transform} metadata temp-file]
+(defmulti ^:private transfer-file-to-db
+  {:arglists '([driver db transform metadata temp-file])}
+  (fn [_ _ transform _ _] (-> transform :target :type keyword)))
+
+(defmethod transfer-file-to-db :table-incremental
+  [driver {db-id :id}
+   {:keys [target] :as transform}
+   metadata temp-file]
+  (let [table-name (transforms.util/qualified-table-name driver target)
+        table-exists? (transforms.util/target-table-exists? transform)
+        data-source {:type :jsonl-file
+                     :file temp-file}]
+    (if (not table-exists?)
+      (do
+        (log/info "New table")
+        (create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))
+
+      (insert-data! driver db-id (table-schema table-name metadata) data-source))))
+
+(defmethod transfer-file-to-db :table
+  [driver {db-id :id :as db}
+   {:keys [target] :as transform}
+   metadata temp-file]
   (let [table-name (transforms.util/qualified-table-name driver target)
         table-exists? (transforms.util/target-table-exists? transform)
         data-source {:type :jsonl-file
@@ -221,7 +249,7 @@
       (not table-exists?)
       (do
         (log/info "New table")
-        (create-table-and-insert-data! driver db-id table-name metadata data-source))
+        (create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))
 
       (driver.u/supports? driver :atomic-renames db)
       (transfer-with-rename-tables-strategy! driver db-id table-name metadata data-source)
@@ -333,6 +361,8 @@
                                 (save-log-to-transform-run-message! run-id message-log))
             ex-message-fn     #(exceptional-run-message message-log %)
             result            (transforms.util/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn)]
+        ;; Update watermark tracking for incremental transforms
+        (transforms.util/upsert-watermark! transform driver db)
         (transforms.instrumentation/with-stage-timing [run-id :table-sync]
           (transforms.util/sync-target! target db run-id))
         {:run_id run-id

enterprise/backend/test/metabase_enterprise/notification/payload/execute_test.clj

@@ -8,42 +8,69 @@
 (deftest viz-settings-is-correctly-returned
   (testing (str "2 questions with the same query but different viz-settings on a dashboard "
                 "with caching enabled should returns the correct viz-settings #57793")
-    (let [query (mt/mbql-query orders {:limit 1})
-          viz-settings-for-dashcard (fn [dashboard-id dashcard-id card-id]
-                                      (-> (mt/as-admin
-                                            (notification.payload.execute/execute-dashboard-subscription-card
-                                             {:dashboard_id dashboard-id
-                                              :card_id      card-id
-                                              :id           dashcard-id}
-                                             []))
-                                          :result
-                                          :data
-                                          :viz-settings :ver))
-          viz-settings-for-card (fn [card-id]
-                                  (-> (notification.payload.execute/execute-card
-                                       (mt/user->id :crowberto)
-                                       card-id)
-                                      :result
-                                      :data
-                                      :viz-settings :ver))]
+    (let [query (mt/mbql-query orders {:aggregation [[:count]
+                                                     [:sum $total]]})
+          result-for-dashcard (fn [dashboard-id dashcard-id card-id]
+                                (-> (mt/as-admin
+                                      (notification.payload.execute/execute-dashboard-subscription-card
+                                       {:dashboard_id dashboard-id
+                                        :card_id      card-id
+                                        :id           dashcard-id}
+                                       []))
+                                    :result))
+          result-for-card (fn [card-id]
+                            (-> (notification.payload.execute/execute-card
+                                 (mt/user->id :crowberto)
+                                 card-id)
+                                :result))]
       (t2/delete! :model/QueryCache)
-      (mt/with-temp
-        [:model/Card {card-1 :id}              {:dataset_query query
-                                                :visualization_settings {:ver 1}}
-         :model/Card {card-2 :id}              {:dataset_query query
-                                                :visualization_settings {:ver 2}}
-         :model/Dashboard {dashboard-id :id}   {}
-         :model/DashboardCard {dashcard-1 :id} {:dashboard_id dashboard-id
-                                                :card_id     card-1}
-         :model/DashboardCard {dashcard-2 :id} {:dashboard_id dashboard-id
-                                                :card_id     card-2}
-         :model/CacheConfig _                  {:model    "dashboard"
-                                                :model_id dashboard-id
-                                                :strategy :duration
-                                                :config   {:unit    :hours
-                                                           :duration 1}}]
-        (is (= 1 (viz-settings-for-dashcard dashboard-id dashcard-1 card-1)))
-        (is (= 1 (viz-settings-for-card card-1)))
-        (testing "card 2 should have viz-settings is 2"
-          (is (= 2 (viz-settings-for-card card-2)))
-          (is (= 2 (viz-settings-for-dashcard dashboard-id dashcard-2 card-2))))))))
+      (mt/with-premium-features #{:cache-granular-controls}
+        (mt/with-temp
+          ;; Card 1 has count hidden, sum visible
+          [:model/Card {card-1 :id}              {:dataset_query query
+                                                  :visualization_settings
+                                                  {:table.cell_column "count", :table.columns [{:name "count", :enabled false} {:name "sum", :enabled true}]}}
+           ;; Card 2 has count visible, sum hidden
+           :model/Card {card-2 :id}              {:dataset_query query
+                                                  :visualization_settings
+                                                  {:table.cell_column "count", :table.columns [{:name "count", :enabled true} {:name "sum", :enabled false}]}}
+           :model/Dashboard {dashboard-id :id}   {}
+           :model/DashboardCard {dashcard-1 :id} {:dashboard_id dashboard-id
+                                                  :card_id     card-1}
+           :model/DashboardCard {dashcard-2 :id} {:dashboard_id dashboard-id
+                                                  :card_id     card-2}
+           :model/CacheConfig _                  {:model    "database"
+                                                  :model_id (mt/id)
+                                                  :strategy :duration
+                                                  :config   {:unit    :hours
+                                                             :duration 1}}]
+          (testing "Card 1 has count hidden and sum visible"
+            (is (=? {:cached nil?
+                     :data {:viz-settings {:metabase.models.visualization-settings/table-columns
+                                           [{:metabase.models.visualization-settings/table-column-enabled false
+                                             :metabase.models.visualization-settings/table-column-name "count"}
+                                            {:metabase.models.visualization-settings/table-column-enabled true
+                                             :metabase.models.visualization-settings/table-column-name "sum"}]}}}
+                    (result-for-dashcard dashboard-id dashcard-1 card-1)))
+            (is (=? {:cached some?
+                     :data {:viz-settings {:metabase.models.visualization-settings/table-columns
+                                           [{:metabase.models.visualization-settings/table-column-enabled true
+                                             :metabase.models.visualization-settings/table-column-name "count"}
+                                            {:metabase.models.visualization-settings/table-column-enabled false
+                                             :metabase.models.visualization-settings/table-column-name "sum"}]}}}
+                    (result-for-dashcard dashboard-id dashcard-2 card-2))))
+          (testing "Card 2 has count visible and sum hidden"
+            (is (=? {:cached some?
+                     :data {:viz-settings {:metabase.models.visualization-settings/table-columns
+                                           [{:metabase.models.visualization-settings/table-column-enabled false
+                                             :metabase.models.visualization-settings/table-column-name "count"}
+                                            {:metabase.models.visualization-settings/table-column-enabled true
+                                             :metabase.models.visualization-settings/table-column-name "sum"}]}}}
+                    (result-for-card card-1)))
+            (is (=? {:cached some?
+                     :data {:viz-settings {:metabase.models.visualization-settings/table-columns
+                                           [{:metabase.models.visualization-settings/table-column-enabled true
+                                             :metabase.models.visualization-settings/table-column-name "count"}
+                                            {:metabase.models.visualization-settings/table-column-enabled false
+                                             :metabase.models.visualization-settings/table-column-name "sum"}]}}}
+                    (result-for-card card-2)))))))))

enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj

@@ -1672,14 +1672,15 @@
                    :model/DashboardCard       dc3 {:dashboard_id (:id d2) :card_id (:id c3)}
                    :model/DashboardCardSeries s   {:dashboardcard_id (:id dc1) :card_id (:id c2)}]
       (t2/with-call-count [qc]
-        (is (=? [(assoc d1
-                        :dashcards [(assoc dc1 :series [s])]
-                        :tabs nil)
-                 (assoc d2
-                        :dashcards [(assoc dc2 :series nil)
-                                    (assoc dc3 :series nil)]
-                        :tabs nil)]
-                (into [] (serdes/extract-query "Dashboard" {:where [:in :id [(:id d1) (:id d2)]]}))))
+        ;; Use sets to ensure order independence of `extract-query` result.
+        (is (=? #{(assoc d1
+                         :dashcards [(assoc dc1 :series [s])]
+                         :tabs nil)
+                  (assoc d2
+                         :dashcards [(assoc dc2 :series nil)
+                                     (assoc dc3 :series nil)]
+                         :tabs nil)}
+                (set (serdes/extract-query "Dashboard" {:where [:in :id [(:id d1) (:id d2)]]}))))
         ;; 1 per dashboard/dashcard/series/tabs
         (is (= 4 (qc)))))))
 

enterprise/frontend/src/metabase-enterprise/content_translation/utils.ts

@@ -1,9 +1,10 @@
 import * as I from "icepick";
 import { useCallback, useMemo } from "react";
-import { match } from "ts-pattern";
+import { P, match } from "ts-pattern";
 import _ from "underscore";
 
 import type { ContentTranslationFunction } from "metabase/i18n/types";
+import { isCartesianChart } from "metabase/visualizations";
 import type { HoveredObject } from "metabase/visualizations/types";
 import type {
   DictionaryArray,
@@ -135,6 +136,10 @@ export const translateFieldValuesInSeries = (
     }
     const untranslatedRows = singleSeries.data.rows.concat();
 
+    const defaultFn = () => {
+      return singleSeries.data.rows.map((row) => row.map((value) => tc(value)));
+    };
+
     const translatedRows: RowValue[][] = match(singleSeries.card?.display)
       .with("pie", () => {
         const pieRows =
@@ -164,11 +169,25 @@ export const translateFieldValuesInSeries = (
           }),
         );
       })
-      .otherwise(() => {
+      .with(P.when(isCartesianChart), () => {
+        // cartesian charts have series settings that can provide display names
+        // for fields, which we should translate if available
+        const seriesSettings =
+          singleSeries.card.visualization_settings?.series_settings ?? {};
+
         return singleSeries.data.rows.map((row) =>
-          row.map((value) => tc(value)),
+          row.map((value) => {
+            if (
+              typeof value === "string" &&
+              seriesSettings[value]?.title !== undefined
+            ) {
+              return tc(seriesSettings[value].title);
+            }
+            return tc(value);
+          }),
         );
-      });
+      })
+      .otherwise(defaultFn);
 
     return {
       ...singleSeries,

enterprise/frontend/src/metabase-enterprise/content_translation/utils.unit.spec.ts

@@ -1,14 +1,19 @@
 import type { ContentTranslationFunction } from "metabase/i18n/types";
+import registerVisualizations from "metabase/visualizations/register";
 import type {
   MaybeTranslatedSeries,
   Series,
+  SeriesSettings,
   SingleSeries,
+  VisualizationDisplay,
 } from "metabase-types/api";
 import { createMockDatasetData } from "metabase-types/api/mocks";
 
 import { leaveUntranslated } from "./use-translate-content";
 import { translateFieldValuesInSeries } from "./utils";
 
+registerVisualizations();
+
 const mockTranslateWithTranslations: ContentTranslationFunction = (value) => {
   if (typeof value === "string") {
     return `translated_${value}`;
@@ -112,68 +117,144 @@ describe("translateFieldValuesInSeries", () => {
     ]);
   });
 
-  it("should handle pie chart display with pie.rows visualization settings", () => {
-    const series: Series = [
-      {
-        card: {
-          display: "pie",
-          visualization_settings: {
-            "pie.rows": [
-              { key: "A", name: "Apple" },
-              { key: "B", name: "Banana" },
-              { key: "C", name: "Cherry" },
+  describe("for pies", () => {
+    it("should handle pie.rows visualization settings", () => {
+      const series: Series = [
+        {
+          card: {
+            display: "pie",
+            visualization_settings: {
+              "pie.rows": [
+                { key: "A", name: "Apple" },
+                { key: "B", name: "Banana" },
+                { key: "C", name: "Cherry" },
+              ],
+            },
+          },
+          data: {
+            rows: [
+              ["A", 10],
+              ["B", 20],
+              ["C", 30],
+              ["D", 40], // key not in pie.rows
             ],
           },
-        },
-        data: {
-          rows: [
-            ["A", 10],
-            ["B", 20],
-            ["C", 30],
-            ["D", 40], // key not in pie.rows
-          ],
-        },
-      } as SingleSeries,
-    ];
+        } as SingleSeries,
+      ];
 
-    const result = translateFieldValuesInSeries(
-      series,
-      mockTranslateWithTranslations,
-    ) as MaybeTranslatedSeries;
+      const result = translateFieldValuesInSeries(
+        series,
+        mockTranslateWithTranslations,
+      ) as MaybeTranslatedSeries;
 
-    expect(result[0].data?.rows).toEqual([
-      ["translated_Apple", 10],
-      ["translated_Banana", 20],
-      ["translated_Cherry", 30],
-      ["translated_D", 40], // falls back to translating the key itself
-    ]);
+      expect(result[0].data?.rows).toEqual([
+        ["translated_Apple", 10],
+        ["translated_Banana", 20],
+        ["translated_Cherry", 30],
+        ["translated_D", 40], // falls back to translating the key itself
+      ]);
+    });
+
+    it("should handle NO pie.rows visualization settings", () => {
+      const series: Series = [
+        {
+          card: {
+            display: "pie",
+            visualization_settings: {},
+          },
+          data: {
+            rows: [
+              ["apple", 10],
+              ["banana", 20],
+            ],
+          },
+        } as SingleSeries,
+      ];
+
+      const result = translateFieldValuesInSeries(
+        series,
+        mockTranslateWithTranslations,
+      ) as MaybeTranslatedSeries;
+
+      expect(result[0].data?.rows).toEqual([
+        ["translated_apple", 10],
+        ["translated_banana", 20],
+      ]);
+    });
   });
 
-  it("should handle pie chart display without pie.rows visualization settings", () => {
-    const series: Series = [
-      {
-        card: {
-          display: "pie",
-          visualization_settings: {},
-        },
-        data: {
-          rows: [
-            ["apple", 10],
-            ["banana", 20],
-          ],
-        },
-      } as SingleSeries,
-    ];
+  const cartesianDisplayTypes: VisualizationDisplay[] = [
+    "line",
+    "bar",
+    "area",
+    "scatter",
+    "combo",
+  ];
 
-    const result = translateFieldValuesInSeries(
-      series,
-      mockTranslateWithTranslations,
-    ) as MaybeTranslatedSeries;
+  describe.each(cartesianDisplayTypes)("for %s charts", (displayType) => {
+    it("should handle series_settings visualization settings", () => {
+      const seriesSettings: Record<string, SeriesSettings> = {
+        Banana: { title: "Coconut" },
+      };
 
-    expect(result[0].data?.rows).toEqual([
-      ["translated_apple", 10],
-      ["translated_banana", 20],
-    ]);
+      const series: Series = [
+        {
+          card: {
+            display: displayType,
+            visualization_settings: {
+              series_settings: seriesSettings,
+            },
+          },
+          data: {
+            rows: [
+              ["Apple", 100, 30],
+              ["Banana", 200, 70],
+              ["Cherry", 300, 40],
+            ],
+          },
+        } as SingleSeries,
+      ];
+
+      const result = translateFieldValuesInSeries(
+        series,
+        mockTranslateWithTranslations,
+      ) as MaybeTranslatedSeries;
+
+      expect(result[0].data?.rows).toEqual([
+        ["translated_Apple", 100, 30],
+        ["translated_Coconut", 200, 70],
+        ["translated_Cherry", 300, 40],
+      ]);
+    });
+
+    it("should handle NO series_settings visualization settings", () => {
+      const series: Series = [
+        {
+          card: {
+            display: displayType,
+            visualization_settings: {},
+          },
+          data: {
+            rows: [
+              ["Apple", 100, 30],
+              ["Banana", 200, 70],
+              ["Cherry", 300, 40],
+            ],
+          },
+        } as SingleSeries,
+      ];
+
+      const result = translateFieldValuesInSeries(
+        series,
+        mockTranslateWithTranslations,
+      ) as MaybeTranslatedSeries;
+
+      expect(result[0].data?.rows).toEqual([
+        ["translated_Apple", 100, 30],
+        ["translated_Banana", 200, 70],
+        ["translated_Cherry", 300, 40],
+      ]);
+    });
   });
 
   it("should handle multiple series", () => {

enterprise/frontend/src/metabase-enterprise/database_routing/DestinationDatabaseConnectionModal/DestinationDatabaseConnectionModal.tsx

@@ -115,7 +115,8 @@ export const DestinationDatabaseConnectionModalInner = ({
         <Flex
           py="sm"
           px="md"
-          mb="md"
+          mx="xl"
+          my="md"
           bg="accent-gray-light"
           align="center"
           justify="space-between"

enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-card-dependencies/use-check-card-dependencies.ts

@@ -14,7 +14,6 @@ import { useCheckDependencies } from "../use-check-dependencies";
 
 export function useCheckCardDependencies({
   onSave,
-  onError,
 }: UseCheckDependenciesProps<Question>): UseCheckDependenciesResult<Question> {
   const store = useStore();
 
@@ -35,6 +34,5 @@ export function useCheckCardDependencies({
     getCheckDependenciesRequest,
     useLazyCheckDependenciesQuery: useLazyCheckCardDependenciesQuery,
     onSave,
-    onError,
   });
 }

enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-card-dependencies/use-check-card-dependencies.unit.spec.ts

@@ -1,4 +1,7 @@
-import { setupCheckCardDependenciesEndpoint } from "__support__/server-mocks/dependencies";
+import {
+  setupCheckCardDependenciesEndpoint,
+  setupCheckCardDependenciesEndpointError,
+} from "__support__/server-mocks/dependencies";
 import { act, renderHookWithProviders } from "__support__/ui";
 import { SAMPLE_METADATA } from "metabase-lib/test-helpers";
 import Question from "metabase-lib/v1/Question";
@@ -9,21 +12,26 @@ import { useCheckCardDependencies } from "./use-check-card-dependencies";
 
 type SetupOpts = {
   response?: CheckDependenciesResponse;
+  hasError?: boolean;
 };
 
 function setup({
   response = createMockCheckDependenciesResponse(),
+  hasError = false,
 }: SetupOpts = {}) {
-  setupCheckCardDependenciesEndpoint(response);
+  if (hasError) {
+    setupCheckCardDependenciesEndpointError();
+  } else {
+    setupCheckCardDependenciesEndpoint(response);
+  }
 
   const onSave = jest.fn();
-  const onError = jest.fn();
   const { result } = renderHookWithProviders(
-    () => useCheckCardDependencies({ onSave, onError }),
+    () => useCheckCardDependencies({ onSave }),
     {},
   );
 
-  return { result, onSave, onError };
+  return { result, onSave };
 }
 
 describe("useCheckCardDependencies", () => {
@@ -74,4 +82,13 @@ describe("useCheckCardDependencies", () => {
     expect(result.current.isConfirmationShown).toBe(false);
     expect(onSave).not.toHaveBeenCalled();
   });
+
+  it("should save if there is a backend error when checking dependencies", async () => {
+    const question = Question.create({ metadata: SAMPLE_METADATA });
+    const { result, onSave } = setup({ hasError: true });
+
+    await act(() => result.current.handleInitialSave(question));
+
+    expect(onSave).toHaveBeenCalledWith(question);
+  });
 });

enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-dependencies/use-check-dependencies.ts

@@ -23,14 +23,12 @@ type UseCheckDependenciesProps<TChange, TRequest> = {
   getCheckDependenciesRequest: (change: TChange) => TRequest;
   useLazyCheckDependenciesQuery: () => UseCheckDependenciesQueryResult<TRequest>;
   onSave: (change: TChange) => Promise<void>;
-  onError: (error: unknown) => void;
 };
 
 export function useCheckDependencies<TChange, TRequest>({
   getCheckDependenciesRequest,
   useLazyCheckDependenciesQuery,
   onSave,
-  onError,
 }: UseCheckDependenciesProps<
   TChange,
   TRequest
@@ -46,7 +44,8 @@ export function useCheckDependencies<TChange, TRequest>({
         getCheckDependenciesRequest(change),
       );
       if (error != null) {
-        onError(error);
+        console.error("Error when checking dependencies.", error);
+        await onSave(change);
       } else if (data != null && !data.success) {
         setChange(change);
         setIsConfirmationShown(true);
@@ -54,7 +53,7 @@ export function useCheckDependencies<TChange, TRequest>({
         await onSave(change);
       }
     },
-    [getCheckDependenciesRequest, checkDependencies, onSave, onError],
+    [getCheckDependenciesRequest, checkDependencies, onSave],
   );
 
   const handleCloseConfirmation = useCallback(() => {

enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-snippet-dependencies/use-check-snippet-dependencies.ts

@@ -12,13 +12,11 @@ import { useCheckDependencies } from "../use-check-dependencies";
 
 export function useCheckSnippetDependencies({
   onSave,
-  onError,
 }: UseCheckDependenciesProps<UpdateSnippetRequest>): UseCheckDependenciesResult<UpdateSnippetRequest> {
   return useCheckDependencies({
     getCheckDependenciesRequest,
     useLazyCheckDependenciesQuery: useLazyCheckSnippetDependenciesQuery,
     onSave,
-    onError,
   });
 }
 

enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-snippet-dependencies/use-check-snippet-dependencies.unit.spec.ts

@@ -18,13 +18,12 @@ function setup({
   setupCheckSnippetDependenciesEndpoint(response);
 
   const onSave = jest.fn();
-  const onError = jest.fn();
   const { result } = renderHookWithProviders(
-    () => useCheckSnippetDependencies({ onSave, onError }),
+    () => useCheckSnippetDependencies({ onSave }),
     {},
   );
 
-  return { result, onSave, onError };
+  return { result, onSave };
 }
 
 describe("useCheckSnippetDependencies", () => {

enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-transform-dependencies/use-check-transform-dependencies.ts

@@ -12,13 +12,11 @@ import { useCheckDependencies } from "../use-check-dependencies";
 
 export function useCheckTransformDependencies({
   onSave,
-  onError,
 }: UseCheckDependenciesProps<UpdateTransformRequest>): UseCheckDependenciesResult<UpdateTransformRequest> {
   return useCheckDependencies({
     getCheckDependenciesRequest,
     useLazyCheckDependenciesQuery: useLazyCheckTransformDependenciesQuery,
     onSave,
-    onError,
   });
 }
 

enterprise/frontend/src/metabase-enterprise/dependencies/hooks/use-check-transform-dependencies/use-check-transform-dependencies.unit.spec.ts

@@ -18,13 +18,12 @@ function setup({
   setupCheckTransformDependenciesEndpoint(response);
 
   const onSave = jest.fn();
-  const onError = jest.fn();
   const { result } = renderHookWithProviders(
-    () => useCheckTransformDependencies({ onSave, onError }),
+    () => useCheckTransformDependencies({ onSave }),
     {},
   );
 
-  return { result, onSave, onError };
+  return { result, onSave };
 }
 
 describe("useCheckTransformDependencies", () => {

enterprise/frontend/src/metabase-enterprise/table-editing/common/utils.ts

@@ -1,41 +1,8 @@
-import { b64hash_to_utf8, utf8_to_b64url } from "metabase/lib/encoding";
 import { isPK } from "metabase-lib/v1/types/utils/isa";
-import type {
-  DatasetColumn,
-  Filter,
-  OrderBy,
-  RowValue,
-  RowValues,
-} from "metabase-types/api";
+import type { DatasetColumn, RowValue, RowValues } from "metabase-types/api";
 
 import type { RowCellsWithPkValue } from "../api/types";
 
-export const serializeMbqlParam = (filterMbql: Array<any>): string => {
-  return utf8_to_b64url(JSON.stringify(filterMbql));
-};
-
-export const deserializeTableFilter = (filterParam: string): Filter | null => {
-  const maybeFilter = JSON.parse(b64hash_to_utf8(filterParam));
-
-  // simple and hacky way to test if param is a valid filter
-  return Array.isArray(maybeFilter) && typeof maybeFilter[0] === "string"
-    ? (maybeFilter as Filter)
-    : null;
-};
-
-export const deserializeTableSorting = (
-  sortingParam: string,
-): Array<OrderBy> | null => {
-  const maybeParam = JSON.parse(b64hash_to_utf8(sortingParam));
-
-  // simple and hacky way to test if param is valid
-  return Array.isArray(maybeParam) &&
-    Array.isArray(maybeParam[0]) &&
-    typeof maybeParam[0][0] === "string"
-    ? (maybeParam as Array<OrderBy>)
-    : null;
-};
-
 export const getPkColumns = (
   columns: DatasetColumn[],
 ): { indexes: number[]; names: string[] } => {

enterprise/frontend/src/metabase-enterprise/table-editing/table-edit/EditTableDataContainer.tsx

@@ -44,7 +44,7 @@ type EditTableDataContainerProps = {
     tableId: string;
     objectId?: string;
   };
-  location: Location<{ filter?: string }>;
+  location: Location<{ query?: string }>;
 };
 
 export const EditTableDataContainer = ({

enterprise/frontend/src/metabase-enterprise/table-editing/table-edit/data-grid/getHeaderTemplate.tsx

@@ -1,12 +1,12 @@
 import { Ellipsified } from "metabase/common/components/Ellipsified";
 import { Box, Icon } from "metabase/ui";
-import type { OrderByDirection } from "metabase-lib";
+import type * as Lib from "metabase-lib";
 import type { DatasetColumn } from "metabase-types/api";
 
 import S from "./EditTableDataGrid.module.css";
 
 type Props = {
-  sortDirection?: OrderByDirection;
+  sortDirection?: Lib.OrderByDirection;
   column: DatasetColumn;
   onColumnSort?: (column: DatasetColumn) => void;
 };

enterprise/frontend/src/metabase-enterprise/table-editing/table-edit/use-adhoc-table-query.ts

@@ -2,23 +2,18 @@ import type { Location } from "history";
 import { useCallback, useEffect, useMemo } from "react";
 import { push } from "react-router-redux";
 
+import { b64url_to_utf8, utf8_to_b64url } from "metabase/lib/encoding";
 import { useDispatch, useSelector } from "metabase/lib/redux";
 import { loadMetadataForTable } from "metabase/questions/actions";
 import { getMetadata } from "metabase/selectors/metadata";
 import * as Lib from "metabase-lib";
 import Question from "metabase-lib/v1/Question";
-import type { StructuredDatasetQuery } from "metabase-types/api";
-
-import {
-  deserializeTableFilter,
-  deserializeTableSorting,
-  serializeMbqlParam,
-} from "../common/utils";
+import type { OpaqueDatasetQuery } from "metabase-types/api";
 
 type UseAdHocTableQueryProps = {
   tableId: number;
   databaseId: number;
-  location: Location<{ filter?: string; sorting?: string }>;
+  location: Location<{ query?: string }>;
 };
 
 export const useAdHocTableQuery = ({
@@ -26,80 +21,57 @@ export const useAdHocTableQuery = ({
   databaseId,
   location,
 }: UseAdHocTableQueryProps) => {
-  const filterQueryParam = useMemo(
-    () =>
-      location.query?.filter
-        ? deserializeTableFilter(location.query.filter)
-        : null,
-    [location.query.filter],
-  );
-
-  const sortingQueryParam = useMemo(
-    () =>
-      location.query?.sorting
-        ? deserializeTableSorting(location.query.sorting)
-        : null,
-    [location.query.sorting],
-  );
-
   const metadata = useSelector(getMetadata);
   const dispatch = useDispatch();
 
-  const table = metadata.table(tableId);
+  const queryParam = useMemo(
+    () =>
+      location.query?.query
+        ? deserializeQueryFromUrl(location.query.query)
+        : null,
+    [location.query.query],
+  );
+
+  const metadataProvider = useMemo(
+    () => Lib.metadataProvider(databaseId, metadata),
+    [databaseId, metadata],
+  );
+  const table = useMemo(
+    () => Lib.tableOrCardMetadata(metadataProvider, tableId),
+    [metadataProvider, tableId],
+  );
 
   useEffect(() => {
     dispatch(loadMetadataForTable(tableId));
   }, [dispatch, tableId]);
 
   const tableQuestion = useMemo(() => {
-    if (table) {
-      let question = Question.create({ databaseId, tableId, metadata });
-
-      if (filterQueryParam || sortingQueryParam) {
-        const legacyQuery = Lib.toLegacyQuery(
-          question.query(),
-        ) as StructuredDatasetQuery;
-
-        if (filterQueryParam) {
-          legacyQuery.query.filter = filterQueryParam;
-        }
-
-        if (sortingQueryParam) {
-          legacyQuery.query["order-by"] = sortingQueryParam;
-        }
-
-        question = question.setDatasetQuery(legacyQuery);
+    if (queryParam != null) {
+      const query = Lib.fromJsQuery(metadataProvider, queryParam);
+      if (Lib.sourceTableOrCardId(query) === tableId) {
+        return Question.create({
+          dataset_query: Lib.toJsQuery(query),
+          metadata,
+        });
       }
-
-      return question;
     }
-  }, [
-    table,
-    databaseId,
-    tableId,
-    metadata,
-    filterQueryParam,
-    sortingQueryParam,
-  ]);
+
+    if (table != null) {
+      const query = Lib.queryFromTableOrCardMetadata(metadataProvider, table);
+      return Question.create({ dataset_query: Lib.toJsQuery(query), metadata });
+    }
+  }, [tableId, table, queryParam, metadata, metadataProvider]);
 
   const handleTableQuestionChange = useCallback(
     (newQuestion: Question) => {
-      const legacyQuery = Lib.toLegacyQuery(
-        newQuestion.query(),
-      ) as StructuredDatasetQuery;
-      const newFilterMbql = legacyQuery.query.filter;
-      const newSortingMbql = legacyQuery.query["order-by"];
+      const newQuery = newQuestion.query();
+      const newFilters = Lib.filters(newQuery, 0);
+      const newOrderBys = Lib.orderBys(newQuery, 0);
 
-      if (newFilterMbql || newSortingMbql) {
+      // don't set the query string param if there are no filters or sorting
+      if (newFilters.length > 0 || newOrderBys.length > 0) {
         const searchParams = new URLSearchParams();
-        if (newFilterMbql) {
-          searchParams.append("filter", serializeMbqlParam(newFilterMbql));
-        }
-        if (newSortingMbql) {
-          searchParams.append("sorting", serializeMbqlParam(newSortingMbql));
-        }
-
-        // don't set filter param if it is empty
+        searchParams.set("query", serializeQueryToUrl(Lib.toJsQuery(newQuery)));
         dispatch(
           push(`${window.location.pathname}?${searchParams.toString()}`),
         );
@@ -112,7 +84,7 @@ export const useAdHocTableQuery = ({
 
   const tableQuery = useMemo(() => {
     if (tableQuestion) {
-      return Lib.toLegacyQuery(tableQuestion.query());
+      return Lib.toJsQuery(tableQuestion.query());
     }
   }, [tableQuestion]);
 
@@ -122,3 +94,11 @@ export const useAdHocTableQuery = ({
     handleTableQuestionChange,
   };
 };
+
+function serializeQueryToUrl(query: OpaqueDatasetQuery) {
+  return utf8_to_b64url(JSON.stringify(query));
+}
+
+function deserializeQueryFromUrl(query: string): OpaqueDatasetQuery {
+  return JSON.parse(b64url_to_utf8(query));
+}

enterprise/frontend/src/metabase-enterprise/table-editing/table-edit/use-edit-table-data.ts

@@ -2,17 +2,7 @@ import type { Location } from "history";
 import { useCallback } from "react";
 
 import { skipToken, useGetAdhocQueryQuery } from "metabase/api";
-import type { OrderByDirection, Query } from "metabase-lib";
-import {
-  changeDirection,
-  displayInfo,
-  findMatchingColumn,
-  fromLegacyColumn,
-  orderBy,
-  orderBys,
-  orderableColumns,
-  removeOrderBys,
-} from "metabase-lib";
+import * as Lib from "metabase-lib";
 import type Question from "metabase-lib/v1/Question";
 import type { DatasetColumn } from "metabase-types/api";
 
@@ -21,12 +11,12 @@ import { useAdHocTableQuery } from "./use-adhoc-table-query";
 type Props = {
   tableId: number;
   databaseId: number;
-  location: Location<{ filter?: string }>;
+  location: Location<{ query?: string }>;
 };
 
 export type TableDataGetColumnSortDirection = (
   column: DatasetColumn,
-) => OrderByDirection | undefined;
+) => Lib.OrderByDirection | undefined;
 
 export function useEditTableData({ tableId, databaseId, location }: Props) {
   const { tableQuestion, tableQuery, handleTableQuestionChange } =
@@ -44,11 +34,11 @@ export function useEditTableData({ tableId, databaseId, location }: Props) {
       );
 
       if (column != null) {
-        const columnInfo = displayInfo(query, stageIndex, column);
+        const columnInfo = Lib.displayInfo(query, stageIndex, column);
         if (columnInfo.orderByPosition != null) {
-          const orderBysMap = orderBys(query, stageIndex);
+          const orderBysMap = Lib.orderBys(query, stageIndex);
           const orderBy = orderBysMap[columnInfo.orderByPosition];
-          const orderByInfo = displayInfo(query, stageIndex, orderBy);
+          const orderByInfo = Lib.displayInfo(query, stageIndex, orderBy);
           return orderByInfo.direction;
         }
       }
@@ -70,18 +60,18 @@ export function useEditTableData({ tableId, databaseId, location }: Props) {
       if (column) {
         const sortDirection = getColumnSortDirection(columnOrField);
 
-        let newQuery: Query;
+        let newQuery: Lib.Query;
         if (sortDirection === "asc") {
           // have existing sorting by asc, then make it desc
-          const clauses = orderBys(query, stageIndex);
-          newQuery = changeDirection(query, clauses[0]);
+          const clauses = Lib.orderBys(query, stageIndex);
+          newQuery = Lib.changeDirection(query, clauses[0]);
         } else if (sortDirection === "desc") {
           // have existing sorting by desc, then remove sorting - WRK-446
-          newQuery = removeOrderBys(query, stageIndex);
+          newQuery = Lib.removeOrderBys(query, stageIndex);
         } else {
           // no existing sorting on this column - apply new sorting (default - by asc)
-          newQuery = orderBy(
-            removeOrderBys(query, stageIndex),
+          newQuery = Lib.orderBy(
+            Lib.removeOrderBys(query, stageIndex),
             stageIndex,
             column,
           );
@@ -113,11 +103,11 @@ export function useEditTableData({ tableId, databaseId, location }: Props) {
 const getQueryColumn = (question: Question, columnOrField: DatasetColumn) => {
   const query = question.query();
   const stageIndex = -1;
-  const column = findMatchingColumn(
+  const column = Lib.findMatchingColumn(
     query,
     stageIndex,
-    fromLegacyColumn(query, stageIndex, columnOrField),
-    orderableColumns(query, stageIndex),
+    Lib.fromLegacyColumn(query, stageIndex, columnOrField),
+    Lib.orderableColumns(query, stageIndex),
   );
 
   return {

enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonEditor/utils.ts

@@ -661,7 +661,6 @@ function genericPandasCompletions(context: CompletionContext) {
       }));
 
     return {
-      filter: false,
       from: word.from + text.length - partial.length,
       options,
     };

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/EditorHeader/EditorHeader.tsx

@@ -1,8 +1,9 @@
+import { useState } from "react";
 import { t } from "ttag";
 
 import Button from "metabase/common/components/Button";
 import EditBar from "metabase/common/components/EditBar";
-import { Tooltip } from "metabase/ui";
+import { Checkbox, Flex, Modal, Text, TextInput, Tooltip } from "metabase/ui";
 
 import type { QueryValidationResult } from "../types";
 
@@ -13,6 +14,10 @@ type EditorHeaderProps = {
   isSaving: boolean;
   onSave: () => void;
   onCancel: () => void;
+  transformId?: number;
+  isIncremental?: boolean;
+  onIncrementalChange?: (incremental: boolean, watermarkField?: string) => void;
+  watermarkField?: string | null;
 };
 
 export function EditorHeader({
@@ -22,20 +27,102 @@ export function EditorHeader({
   isSaving,
   onSave,
   onCancel,
+  transformId,
+  isIncremental = false,
+  onIncrementalChange,
+  watermarkField,
 }: EditorHeaderProps) {
+  const [localWatermarkField, setLocalWatermarkField] = useState(watermarkField || "id");
+
+  // Block saving if incremental is enabled but no watermark field is set
+  const hasWatermarkFieldIfNeeded = !isIncremental || localWatermarkField.trim() !== "";
   const canSave =
-    (isNew || isQueryDirty) && validationResult.isValid && !isSaving;
+    (isNew || isQueryDirty) &&
+    validationResult.isValid &&
+    !isSaving &&
+    hasWatermarkFieldIfNeeded;
+
+  const handleIncrementalToggle = (event: React.ChangeEvent<HTMLInputElement>) => {
+    const checked = event.currentTarget.checked;
+
+    if (isNew) {
+      // For new transforms, update the state and trigger save modal if checking
+      if (onIncrementalChange) {
+        onIncrementalChange(checked);
+      }
+      if (checked) {
+        // Show alert about needing to save first
+        alert(
+          t`You need to save the transform first before enabling incremental mode. The save dialog will open now.`,
+        );
+        // Trigger the save modal
+        onSave();
+      }
+    } else if (onIncrementalChange) {
+      // For existing transforms, toggle the incremental mode
+      if (checked && !localWatermarkField.trim()) {
+        // If enabling incremental but no watermark field, don't call the handler yet
+        // Just show an alert and don't actually toggle
+        alert(
+          t`Please enter a watermark field before enabling incremental mode.`,
+        );
+        // Prevent the checkbox from being checked
+        event.preventDefault();
+        return;
+      }
+      // Only call the handler if we have a watermark field (or if unchecking)
+      onIncrementalChange(checked, localWatermarkField);
+    }
+  };
+
+  const handleWatermarkFieldChange = (event: React.ChangeEvent<HTMLInputElement>) => {
+    const value = event.currentTarget.value;
+    setLocalWatermarkField(value);
+    if (onIncrementalChange && isIncremental) {
+      onIncrementalChange(true, value);
+    }
+  };
 
   return (
     <EditBar
-      title={getTitle(isNew)}
+      title={
+        <Flex align="center" gap="md">
+          {getTitle(isNew)}
+          <Checkbox
+            label={t`Incremental?`}
+            checked={isIncremental}
+            onChange={handleIncrementalToggle}
+          />
+          {isIncremental && !isNew && (
+            <TextInput
+              placeholder={t`Watermark field (e.g., id)`}
+              value={localWatermarkField}
+              onChange={handleWatermarkFieldChange}
+              size="xs"
+              w={200}
+            />
+          )}
+          {isIncremental && transformId && (
+            <Text size="sm" c="text-medium">
+              {t`Current Transform ID: ${transformId}`}
+            </Text>
+          )}
+        </Flex>
+      }
       admin
       buttons={[
         <Button key="cancel" small onClick={onCancel}>{t`Cancel`}</Button>,
         <Tooltip
           key="save"
-          label={validationResult.errorMessage}
-          disabled={validationResult.errorMessage == null}
+          label={
+            validationResult.errorMessage ||
+            (!hasWatermarkFieldIfNeeded
+              ? t`Watermark field is required for incremental transforms`
+              : undefined)
+          }
+          disabled={
+            validationResult.errorMessage == null && hasWatermarkFieldIfNeeded
+          }
         >
           <Button onClick={onSave} primary small disabled={!canSave}>
             {getSaveButtonLabel(isNew, isSaving)}
@@ -48,9 +135,9 @@ export function EditorHeader({
 
 function getTitle(isNew: boolean) {
   if (isNew) {
-    return t`You’re creating a new transform`;
+    return t`You're creating a new transform`;
   } else {
-    return t`You’re editing a transform`;
+    return t`You're editing a transform`;
   }
 }
 

enterprise/frontend/src/metabase-enterprise/transforms/components/QueryEditor/QueryEditor.tsx

@@ -40,6 +40,10 @@ type QueryEditorProps = {
   isSaving?: boolean;
   onSave: (source: QueryTransformSource) => void;
   onCancel: () => void;
+  transformId?: number;
+  isIncremental?: boolean;
+  onIncrementalChange?: (incremental: boolean, watermarkField?: string) => void;
+  watermarkField?: string | null;
 };
 
 export function QueryEditor({
@@ -48,6 +52,10 @@ export function QueryEditor({
   isSaving = false,
   onSave,
   onCancel,
+  transformId,
+  isIncremental,
+  onIncrementalChange,
+  watermarkField,
 }: QueryEditorProps) {
   const { question, isQueryDirty, setQuestion } = useQueryState(
     initialSource.query,
@@ -152,6 +160,10 @@ export function QueryEditor({
           isQueryDirty={isQueryDirty}
           onSave={handleSave}
           onCancel={onCancel}
+          transformId={transformId}
+          isIncremental={isIncremental}
+          onIncrementalChange={onIncrementalChange}
+          watermarkField={watermarkField}
         />
         <Flex h="100%" w="100%" mih="0">
           <Stack flex="2 1 100%" pos="relative">

enterprise/frontend/src/metabase-enterprise/transforms/hooks/use-query-results/use-query-results.ts

@@ -1,6 +1,7 @@
-import { useCallback, useMemo, useState } from "react";
+import { useCallback, useMemo, useRef, useState } from "react";
 import { useAsyncFn } from "react-use";
 
+import { type Deferred, defer } from "metabase/lib/promise";
 import { useSelector } from "metabase/lib/redux";
 import { getMetadata } from "metabase/selectors/metadata";
 import { runQuestionQuery } from "metabase/services";
@@ -11,9 +12,21 @@ import type { DatasetQuery } from "metabase-types/api";
 export function useQueryResults(question: Question) {
   const metadata = useSelector(getMetadata);
   const [lastRunQuery, setLastRunQuery] = useState<DatasetQuery | null>(null);
+  const deferredRef = useRef<Deferred>();
+
+  function deferred() {
+    // Cancel the previous query when a new one is started.
+    deferredRef.current?.resolve();
+    deferredRef.current = defer();
+
+    return deferredRef.current;
+  }
 
   const [{ value: results = null, loading: isRunning }, runQuery] = useAsyncFn(
-    () => runQuestionQuery(question),
+    () =>
+      runQuestionQuery(question, {
+        cancelDeferred: deferred(),
+      }),
     [question],
   );
 
@@ -48,7 +61,7 @@ export function useQueryResults(question: Question) {
   }, [question, runQuery]);
 
   const handleCancelQuery = useCallback(() => {
-    return null;
+    return deferredRef.current?.resolve();
   }, []);
 
   return {

enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx

@@ -1,3 +1,4 @@
+import { useFormikContext } from "formik";
 import { useMemo } from "react";
 import { t } from "ttag";
 import * as Yup from "yup";
@@ -11,6 +12,7 @@ import {
 import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
 import {
   Form,
+  FormCheckbox,
   FormErrorMessage,
   FormProvider,
   FormSubmitButton,
@@ -32,12 +34,14 @@ type CreateTransformModalProps = {
   source: TransformSource;
   onCreate: (transform: Transform) => void;
   onClose: () => void;
+  initialIncremental?: boolean;
 };
 
 export function CreateTransformModal({
   source,
   onCreate,
   onClose,
+  initialIncremental = false,
 }: CreateTransformModalProps) {
   return (
     <Modal title={t`Save your transform`} opened padding="xl" onClose={onClose}>
@@ -46,6 +50,7 @@ export function CreateTransformModal({
         source={source}
         onCreate={onCreate}
         onClose={onClose}
+        initialIncremental={initialIncremental}
       />
     </Modal>
   );
@@ -55,6 +60,7 @@ type CreateTransformFormProps = {
   source: TransformSource;
   onCreate: (transform: Transform) => void;
   onClose: () => void;
+  initialIncremental: boolean;
 };
 
 type NewTransformValues = {
@@ -62,6 +68,8 @@ type NewTransformValues = {
   description: string | null;
   targetName: string;
   targetSchema: string | null;
+  incremental: boolean;
+  watermarkField: string | null;
 };
 
 const NEW_TRANSFORM_SCHEMA = Yup.object({
@@ -69,12 +77,36 @@ const NEW_TRANSFORM_SCHEMA = Yup.object({
   description: Yup.string().nullable(),
   targetName: Yup.string().required(Errors.required),
   targetSchema: Yup.string().nullable(),
+  incremental: Yup.boolean().required(),
+  watermarkField: Yup.string().nullable().when("incremental", {
+    is: true,
+    then: (schema) => schema.required(Errors.required),
+    otherwise: (schema) => schema.nullable(),
+  }),
 });
 
+function WatermarkFieldInput() {
+  const { values } = useFormikContext<NewTransformValues>();
+
+  if (!values.incremental) {
+    return null;
+  }
+
+  return (
+    <FormTextInput
+      name="watermarkField"
+      label={t`Watermark field`}
+      placeholder={t`e.g., id, row_num`}
+      description={t`An integer field used to track incremental updates`}
+    />
+  );
+}
+
 function CreateTransformForm({
   source,
   onCreate,
   onClose,
+  initialIncremental,
 }: CreateTransformFormProps) {
   const databaseId =
     source.type === "query" ? source.query.database : source["source-database"];
@@ -100,8 +132,8 @@ function CreateTransformForm({
   const supportsSchemas = database && hasFeature(database, "schemas");
 
   const initialValues: NewTransformValues = useMemo(
-    () => getInitialValues(schemas),
-    [schemas],
+    () => getInitialValues(schemas, initialIncremental),
+    [schemas, initialIncremental],
   );
 
   if (isLoading || error != null) {
@@ -152,6 +184,11 @@ function CreateTransformForm({
             label={t`Table name`}
             placeholder={t`descriptive_name`}
           />
+          <FormCheckbox
+            name="incremental"
+            label={t`Incremental?`}
+          />
+          <WatermarkFieldInput />
           <Group>
             <Box flex={1}>
               <FormErrorMessage />
@@ -165,18 +202,23 @@ function CreateTransformForm({
   );
 }
 
-function getInitialValues(schemas: string[]): NewTransformValues {
+function getInitialValues(
+  schemas: string[],
+  initialIncremental: boolean,
+): NewTransformValues {
   return {
     name: "",
     description: null,
     targetName: "",
     targetSchema: schemas?.[0] || null,
+    incremental: initialIncremental,
+    watermarkField: initialIncremental ? "id" : null,
   };
 }
 
 function getCreateRequest(
   source: TransformSource,
-  { name, description, targetName, targetSchema }: NewTransformValues,
+  { name, description, targetName, targetSchema, incremental, watermarkField }: NewTransformValues,
   databaseId: number,
 ): CreateTransformRequest {
   return {
@@ -184,10 +226,11 @@ function getCreateRequest(
     description,
     source,
     target: {
-      type: "table",
+      type: incremental ? "table-incremental" : "table",
       name: targetName,
       schema: targetSchema,
       database: databaseId,
+      watermarkField: incremental ? watermarkField : null,
     },
   };
 }

enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/NewTransformPage.tsx

@@ -15,7 +15,11 @@ import type {
 } from "metabase-types/api";
 
 import { QueryEditor } from "../../components/QueryEditor";
-import { getTransformListUrl, getTransformUrl } from "../../urls";
+import {
+  getTransformListUrl,
+  getTransformQueryUrl,
+  getTransformUrl,
+} from "../../urls";
 
 import { CreateTransformModal } from "./CreateTransformModal";
 import {
@@ -41,12 +45,18 @@ export function NewTransformPage({ params }: NewTransformPageProps) {
   const { type, cardId } = getParsedParams(params);
 
   const [source, setSource] = useState<TransformSource | null>(null);
+  const [wantsIncremental, setWantsIncremental] = useState(false);
   const [isModalOpened, { open: openModal, close: closeModal }] =
     useDisclosure();
   const dispatch = useDispatch();
 
   const handleCreate = (transform: Transform) => {
-    dispatch(push(getTransformUrl(transform.id)));
+    // If user wanted incremental, redirect to query edit page instead of transform detail page
+    if (wantsIncremental) {
+      dispatch(push(getTransformQueryUrl(transform.id)));
+    } else {
+      dispatch(push(getTransformUrl(transform.id)));
+    }
   };
 
   const handleSave = (newSource: TransformSource) => {
@@ -58,6 +68,12 @@ export function NewTransformPage({ params }: NewTransformPageProps) {
     dispatch(push(getTransformListUrl()));
   };
 
+  const handleModalClose = () => {
+    // If modal is closed without saving, reset the incremental checkbox
+    setWantsIncremental(false);
+    closeModal();
+  };
+
   return (
     <>
       <NewTransformEditorBody
@@ -65,12 +81,15 @@ export function NewTransformPage({ params }: NewTransformPageProps) {
         cardId={cardId}
         onSave={handleSave}
         onCancel={handleCancel}
+        wantsIncremental={wantsIncremental}
+        onWantsIncrementalChange={setWantsIncremental}
       />
       {isModalOpened && source !== null && (
         <CreateTransformModal
           source={source}
           onCreate={handleCreate}
-          onClose={closeModal}
+          onClose={handleModalClose}
+          initialIncremental={wantsIncremental}
         />
       )}
     </>
@@ -82,6 +101,8 @@ function NewTransformEditorBody(props: {
   cardId?: CardId;
   onSave: (source: TransformSource) => void;
   onCancel: () => void;
+  wantsIncremental: boolean;
+  onWantsIncrementalChange: (wants: boolean) => void;
 }) {
   const { type, ...rest } = props;
   if (type === "python") {
@@ -103,11 +124,15 @@ function NewQueryTransformEditorBody({
   cardId,
   onSave,
   onCancel,
+  wantsIncremental,
+  onWantsIncrementalChange,
 }: {
   type: LegacyDatasetQuery["type"];
   cardId?: CardId;
   onSave: (source: TransformSource) => void;
   onCancel: () => void;
+  wantsIncremental: boolean;
+  onWantsIncrementalChange: (wants: boolean) => void;
 }) {
   const {
     data: card,
@@ -119,12 +144,18 @@ function NewQueryTransformEditorBody({
     return <LoadingAndErrorWrapper loading={isLoading} error={error} />;
   }
 
+  const handleIncrementalChange = (incremental: boolean) => {
+    onWantsIncrementalChange(incremental);
+  };
+
   return (
     <QueryEditor
       initialSource={getInitialQueryTransformSource(card, type)}
       isNew
       onSave={onSave}
       onCancel={onCancel}
+      isIncremental={wantsIncremental}
+      onIncrementalChange={handleIncrementalChange}
     />
   );
 }

enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/utils.ts

@@ -8,7 +8,7 @@ export function getInitialQueryTransformSource(
   const query =
     card != null
       ? card.dataset_query
-      : Question.create({ type }).datasetQuery();
+      : Question.create({ DEPRECATED_RAW_MBQL_type: type }).datasetQuery();
 
   return { type: "query" as const, query };
 }

enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformQueryPage/TransformQueryPage.tsx

@@ -75,9 +75,6 @@ export function TransformQueryPageBody({
         dispatch(push(getTransformUrl(transform.id)));
       }
     },
-    onError: () => {
-      sendErrorToast(t`Failed to update transform query`);
-    },
   });
 
   const handleSaveSource = async (source: TransformSource) => {
@@ -91,6 +88,30 @@ export function TransformQueryPageBody({
     dispatch(push(getTransformUrl(transform.id)));
   };
 
+  const handleIncrementalChange = async (
+    incremental: boolean,
+    watermarkField?: string,
+  ) => {
+    const newType = incremental ? "table-incremental" : "table";
+    const { error } = await updateTransform({
+      id: transform.id,
+      target: {
+        ...transform.target,
+        type: newType,
+        watermarkField: incremental ? watermarkField : null,
+      },
+    });
+    if (error) {
+      sendErrorToast(t`Failed to update transform type`);
+    } else {
+      sendSuccessToast(
+        incremental
+          ? t`Transform set to incremental mode`
+          : t`Transform set to full refresh mode`,
+      );
+    }
+  };
+
   if (transform.source.type === "python") {
     return (
       <PLUGIN_TRANSFORMS_PYTHON.TransformEditor
@@ -110,6 +131,10 @@ export function TransformQueryPageBody({
         isSaving={isSaving || isCheckingDependencies}
         onSave={handleSaveSource}
         onCancel={handleCancel}
+        transformId={transform.id}
+        isIncremental={transform.target.type === "table-incremental"}
+        onIncrementalChange={handleIncrementalChange}
+        watermarkField={transform.target.watermarkField}
       />
       {isConfirmationShown && checkData != null && (
         <PLUGIN_DEPENDENCIES.CheckDependenciesModal

enterprise/frontend/src/metabase-enterprise/user_provisioning/components/ScimInputs.tsx

@@ -17,9 +17,8 @@ export const getTextInputStyles = (params: {
   },
   input: {
     color: match(params)
-      .with({ masked: true }, () => `var(--mb-color-text-light) !important`)
       .with({ disabled: false }, () => `var(--mb-color-text-dark) !important`)
-      .otherwise(() => `black !important`),
+      .otherwise(() => `var(--mb-color-text-primary) !important`),
     fontFamily: fontFamilyMonospace,
   },
 });

frontend/build/shared/rspack/svgo-config.js

@@ -0,0 +1,22 @@
+/**
+ * Shared SVGO configuration for @svgr/webpack loader
+ *
+ * This configuration preserves the standalone="no" attribute in SVG files
+ * by disabling the defaultMarkupDeclarations option in the removeUnknownsAndDefaults plugin.
+ *
+ * In SVGO 3.3.2+, the removeUnknownsAndDefaults plugin removes XML declaration attributes
+ * with default values (like standalone="no"). Setting defaultMarkupDeclarations to false
+ * prevents this behavior.
+ */
+const SVGO_CONFIG = {
+  plugins: [
+    {
+      name: "removeUnknownsAndDefaults",
+      params: {
+        defaultMarkupDeclarations: false,
+      },
+    },
+  ],
+};
+
+module.exports = { SVGO_CONFIG };

frontend/src/metabase-lib/metadata.ts

@@ -5,6 +5,7 @@ import type {
   CardType,
   DatabaseId,
   DatasetColumn,
+  FieldId,
   TableId,
 } from "metabase-types/api";
 
@@ -170,6 +171,13 @@ export function tableOrCardMetadata(
   return ML.table_or_card_metadata(queryOrMetadataProvider, tableID);
 }
 
+export function fieldMetadata(
+  queryOrMetadataProvider: Query | MetadataProvider,
+  fieldID: FieldId,
+): ColumnMetadata | null {
+  return ML.field_metadata(queryOrMetadataProvider, fieldID);
+}
+
 export function visibleColumns(
   query: Query,
   stageIndex: number,

frontend/src/metabase-lib/v1/Question.ts

@@ -52,18 +52,18 @@ import { isDimensionTarget } from "metabase-types/guards";
 import type { Query } from "../types";
 
 export type QuestionCreatorOpts = {
-  databaseId?: DatabaseId;
   cardType?: CardType;
-  tableId?: TableId;
   collectionId?: CollectionId;
   dashboardId?: DashboardId;
   metadata?: Metadata;
   parameterValues?: ParameterValuesMap;
-  type?: "query" | "native";
   name?: string;
   display?: CardDisplayType;
   visualization_settings?: VisualizationSettings;
   dataset_query?: DatasetQuery;
+  DEPRECATED_RAW_MBQL_type?: "query" | "native";
+  DEPRECATED_RAW_MBQL_databaseId?: DatabaseId;
+  DEPRECATED_RAW_MBQL_tableId?: TableId;
 };
 
 export type QuestionDashboardProps = {
@@ -460,9 +460,11 @@ class Question {
       return this;
     }
     const query = this.composeQuestion().query();
-    return Question.create({ metadata: this.metadata(), ...options }).setQuery(
-      query,
-    );
+    return Question.create({
+      metadata: this.metadata(),
+      dataset_query: Lib.toJsQuery(query),
+      ...options,
+    });
   }
 
   /**
@@ -911,13 +913,13 @@ class Question {
    * but it would require changing the constructor signature so that `card` is an optional parameter and has a default value
    */
   static create({
-    databaseId,
-    tableId,
+    DEPRECATED_RAW_MBQL_type: type = "query",
+    DEPRECATED_RAW_MBQL_databaseId: databaseId,
+    DEPRECATED_RAW_MBQL_tableId: tableId,
     collectionId,
     dashboardId,
     metadata,
     parameterValues,
-    type = "query",
     name,
     display = "table",
     visualization_settings = {},

frontend/src/metabase-lib/v1/metadata/Table.ts

@@ -64,8 +64,8 @@ class Table {
 
   question() {
     return Question.create({
-      databaseId: this.db && this.db.id,
-      tableId: this.id,
+      DEPRECATED_RAW_MBQL_databaseId: this.db && this.db.id,
+      DEPRECATED_RAW_MBQL_tableId: this.id,
       metadata: this.metadata,
     });
   }

frontend/src/metabase-types/api/transform.ts

@@ -39,13 +39,14 @@ export type QueryTransformSource = {
 
 export type TransformSource = QueryTransformSource | PythonTransformSource;
 
-export type TransformTargetType = "table";
+export type TransformTargetType = "table" | "table-incremental";
 
 export type TransformTarget = {
   type: TransformTargetType;
   name: string;
   schema: string | null;
   database: number;
+  watermarkField?: string | null;
 };
 
 export type TransformRun = {

frontend/src/metabase/actions/components/ActionFormFieldWidget/ActionFormFieldWidget.tsx

@@ -4,16 +4,43 @@ import { forwardRef } from "react";
 import type { ActionFormFieldProps } from "metabase/actions/types";
 import FormInputWidget from "metabase/common/components/FormInput";
 import FormNumericInputWidget from "metabase/common/components/FormNumericInput";
-import type { FormRadioProps } from "metabase/common/components/FormRadio";
-import FormRadioWidget from "metabase/common/components/FormRadio";
 import FormSelectWidget from "metabase/common/components/FormSelect";
 import FormTextAreaWidget from "metabase/common/components/FormTextArea";
 import FormBooleanWidget from "metabase/common/components/FormToggle";
+import { FormRadioGroup, type FormRadioGroupProps } from "metabase/forms";
+import { Radio, Stack } from "metabase/ui";
 import type { InputComponentType } from "metabase-types/api";
 
-const VerticalRadio = (props: FormRadioProps) => (
-  <FormRadioWidget {...props} vertical />
-);
+type RadioOption = {
+  name: string;
+  // Radio.Group in our forms provides string values; stringify numeric values when rendering
+  value: string | number;
+};
+
+interface FormRadioProps extends FormRadioGroupProps {
+  title?: string;
+  options: RadioOption[];
+  actions?: ReactNode;
+  description?: ReactNode;
+  optional?: boolean;
+}
+
+const VerticalRadio = (props: FormRadioProps) => {
+  return (
+    <FormRadioGroup name={props.name} label={props.title}>
+      <Stack gap="sm" mt="xs">
+        {props.options.map((r) => (
+          // The value that comes from the `FormRadioGroup` wrapper is a string.
+          // It is crucial to match it properly or otherwise a radio button will
+          // not render as checked/selected.
+          // This is important because this component is used to render "Inline select"
+          // for both the string and the numeric variables!
+          <Radio label={r.name} key={r.value} value={r.value.toString()} />
+        ))}
+      </Stack>
+    </FormRadioGroup>
+  );
+};
 
 const WIDGETS: Record<InputComponentType, FunctionComponent<any>> = {
   text: FormInputWidget,

frontend/src/metabase/admin/permissions/components/FilterableTree/FilterableTree.styled.tsx

@@ -2,7 +2,6 @@
 import styled from "@emotion/styled";
 
 import { Tree } from "metabase/common/components/tree";
-import { lighten } from "metabase/lib/colors";
 import { color } from "metabase/ui/utils/colors";
 
 export const FilterableTreeRoot = styled.div`
@@ -31,12 +30,14 @@ export const EmptyStateContainer = styled.div`
 
 export const AdminTreeNode = styled(Tree.Node)`
   color: ${(props) =>
-    props.isSelected ? color("text-white") : color("text-medium")};
+    props.isSelected ? color("text-selected") : color("text-secondary")};
   background-color: ${(props) =>
-    props.isSelected ? color("filter") : "unset"};
+    props.isSelected ? color("admin-navbar") : "unset"};
 
   &:hover {
     background-color: ${(props) =>
-      props.isSelected ? color("filter") : lighten("filter", 0.6)};
+      props.isSelected
+        ? color("admin-navbar")
+        : color("admin-navbar-secondary")};
   }
 `;

frontend/src/metabase/admin/permissions/components/PermissionsTable/PermissionsTable.styled.tsx

@@ -44,7 +44,7 @@ export const PermissionsTableCell = styled.td`
     left: 0;
     top: 0;
     position: sticky;
-    padding-left: 0;
+    padding-left: 1.5rem;
     padding-right: 1.5rem;
 
     &:after {
@@ -89,7 +89,7 @@ export const EntityNameLink = styled(Link)`
   display: inline;
   font-weight: 700;
   text-decoration: underline;
-  color: ${() => color("admin-navbar")};
+  color: ${() => color("admin-navbar-inverse")};
 `;
 
 export const HintIcon = styled(

frontend/src/metabase/admin/settings/components/ApiKeys/SecretKeyModal.tsx

@@ -4,6 +4,7 @@ import { CopyTextInput } from "metabase/common/components/CopyTextInput";
 import CS from "metabase/css/core/index.css";
 import { Button, Flex, Group, Icon, Modal, Stack, Text } from "metabase/ui";
 import { getThemeOverrides } from "metabase/ui/theme";
+import { color } from "metabase/ui/utils/colors";
 const { fontFamilyMonospace } = getThemeOverrides();
 
 export const SecretKeyModal = ({
@@ -29,7 +30,8 @@ export const SecretKeyModal = ({
         disabled
         styles={{
           input: {
-            color: `black`,
+            // This color prop is needed to override disabled input color styles
+            color: color("text-primary"),
             fontFamily: fontFamilyMonospace as string,
           },
         }}

frontend/src/metabase/admin/settings/components/widgets/AnonymousTrackingInput.tsx

@@ -6,10 +6,13 @@ import { Stack } from "metabase/ui";
 import { trackTrackingPermissionChanged } from "../../analytics";
 import { SettingHeader } from "../SettingHeader";
 
-import { BasicAdminSettingInput } from "./AdminSettingInput";
+import {
+  BasicAdminSettingInput,
+  SetByEnvVarWrapper,
+} from "./AdminSettingInput";
 
 export function AnonymousTrackingInput() {
-  const { value, updateSetting, description } = useAdminSetting(
+  const { value, updateSetting, description, settingDetails } = useAdminSetting(
     "anon-tracking-enabled",
   );
 
@@ -34,12 +37,17 @@ export function AnonymousTrackingInput() {
         title={t`Anonymous tracking`}
         description={description}
       />
-      <BasicAdminSettingInput
-        name="anon-tracking-enabled"
-        inputType="boolean"
-        value={value}
-        onChange={(newValue) => handleChange(Boolean(newValue))}
-      />
+      <SetByEnvVarWrapper
+        settingKey="anon-tracking-enabled"
+        settingDetails={settingDetails}
+      >
+        <BasicAdminSettingInput
+          name="anon-tracking-enabled"
+          inputType="boolean"
+          value={value}
+          onChange={(newValue) => handleChange(Boolean(newValue))}
+        />
+      </SetByEnvVarWrapper>
     </Stack>
   );
 }

frontend/src/metabase/admin/settings/components/widgets/AnonymousTrackingInput.unit.spec.tsx

@@ -19,7 +19,13 @@ import { AnonymousTrackingInput } from "./AnonymousTrackingInput";
 
 const trackingFN = jest.spyOn(analytics, "trackTrackingPermissionChanged");
 
-const setup = ({ value }: { value: boolean }) => {
+const setup = ({
+  value,
+  isEnvSetting,
+}: {
+  value: boolean;
+  isEnvSetting?: boolean;
+}) => {
   const settings = createMockSettings({
     "anon-tracking-enabled": value,
   });
@@ -31,6 +37,8 @@ const setup = ({ value }: { value: boolean }) => {
       key: "anon-tracking-enabled",
       description: "Enable the collection of anonymous usage data",
       value: false,
+      is_env_setting: isEnvSetting,
+      env_name: isEnvSetting ? "MB_ANON_TRACKING_ENABLED" : undefined,
     }),
   ]);
 
@@ -75,4 +83,27 @@ describe("AnonymousTrackingInput", () => {
 
     expect(await screen.findByRole("switch")).toBeChecked();
   });
+
+  it("should show environment variable message when anonymous tracking is set via env var", async () => {
+    setup({ value: true, isEnvSetting: true });
+    expect(
+      await screen.findByText(/This has been set by the/),
+    ).toBeInTheDocument();
+    expect(
+      await screen.findByText("MB_ANON_TRACKING_ENABLED"),
+    ).toBeInTheDocument();
+  });
+
+  it("should not show the switch when anonymous tracking is set via env var", async () => {
+    setup({ value: true, isEnvSetting: true });
+    await screen.findByText(/This has been set by the/);
+
+    expect(screen.queryByRole("switch")).not.toBeInTheDocument();
+  });
+
+  it("should show the switch when anonymous tracking is not set via env var", async () => {
+    setup({ value: true, isEnvSetting: false });
+
+    expect(await screen.findByRole("switch")).toBeInTheDocument();
+  });
 });

frontend/src/metabase/admin/settings/components/widgets/SiteUrlWidget.tsx

@@ -9,8 +9,10 @@ import { Box, Text } from "metabase/ui";
 
 import { SettingHeader } from "../SettingHeader";
 
+import { SetByEnvVarWrapper } from "./AdminSettingInput";
+
 export function SiteUrlWidget() {
-  const { value, updateSetting, description, isLoading } =
+  const { value, updateSetting, description, isLoading, settingDetails } =
     useAdminSetting("site-url");
   const isHosted = useHasTokenFeature("hosting");
   const [errorMessage, setErrorMessage] = useState("");
@@ -46,18 +48,20 @@ export function SiteUrlWidget() {
           </>
         }
       />
-      <InputWithSelectPrefix
-        value={value || ""}
-        onChange={(newValue: string) => handleChange(newValue)}
-        prefixes={["https://", "http://"]}
-        defaultPrefix="http://"
-        placeholder={"http://example.com"}
-      />
-      {errorMessage && (
-        <Text size="sm" color="danger" mt="sm">
-          {errorMessage}
-        </Text>
-      )}
+      <SetByEnvVarWrapper settingKey="site-url" settingDetails={settingDetails}>
+        <InputWithSelectPrefix
+          value={value || ""}
+          onChange={(newValue: string) => handleChange(newValue)}
+          prefixes={["https://", "http://"]}
+          defaultPrefix="http://"
+          placeholder={"http://example.com"}
+        />
+        {errorMessage && (
+          <Text size="sm" color="danger" mt="sm">
+            {errorMessage}
+          </Text>
+        )}
+      </SetByEnvVarWrapper>
     </Box>
   );
 }

frontend/src/metabase/admin/settings/components/widgets/SiteUrlWidget.unit.spec.tsx

@@ -18,7 +18,7 @@ import { createMockSettingsState } from "metabase-types/store/mocks";
 
 import { SiteUrlWidget } from "./SiteUrlWidget";
 
-const setup = (props: { isHosted?: boolean }) => {
+const setup = (props: { isHosted?: boolean; isEnvSetting?: boolean }) => {
   const siteUrlWidgetSettings = {
     "site-url": "http://mysite.biz",
     "token-features": createMockTokenFeatures({
@@ -32,7 +32,13 @@ const setup = (props: { isHosted?: boolean }) => {
   setupUpdateSettingEndpoint();
   setupSettingsEndpoints(
     Object.entries(settings).map(([key, value]) =>
-      createMockSettingDefinition({ key: key as SettingKey, value }),
+      createMockSettingDefinition({
+        key: key as SettingKey,
+        value,
+        is_env_setting: props.isEnvSetting && key === "site-url",
+        env_name:
+          props.isEnvSetting && key === "site-url" ? "MB_SITE_URL" : undefined,
+      }),
     ),
   );
 
@@ -124,4 +130,33 @@ describe("siteUrlWidget", () => {
     setup({ isHosted: true });
     expect(screen.queryByText("Site url")).not.toBeInTheDocument();
   });
+
+  it("should show environment variable message when site URL is set via env var", async () => {
+    setup({ isEnvSetting: true });
+    expect(
+      await screen.findByText(/This has been set by the/),
+    ).toBeInTheDocument();
+    expect(await screen.findByText("MB_SITE_URL")).toBeInTheDocument();
+  });
+
+  it("should not show the input when site URL is set via env var", async () => {
+    setup({ isEnvSetting: true });
+    await screen.findByText(/This has been set by the/);
+
+    expect(
+      screen.queryByRole("textbox", { name: "input-prefix" }),
+    ).not.toBeInTheDocument();
+    expect(
+      screen.queryByPlaceholderText("http://example.com"),
+    ).not.toBeInTheDocument();
+  });
+
+  it("should show the input when site URL is not set via env var", async () => {
+    setup({ isEnvSetting: false });
+
+    expect(
+      await screen.findByRole("textbox", { name: "input-prefix" }),
+    ).toBeInTheDocument();
+    expect(await screen.findByDisplayValue("mysite.biz")).toBeInTheDocument();
+  });
 });

frontend/src/metabase/api/table.ts

@@ -1,5 +1,5 @@
 import { updateMetadata } from "metabase/lib/redux/metadata";
-import { TableSchema } from "metabase/schema";
+import { ForeignKeySchema, TableSchema } from "metabase/schema";
 import type {
   ForeignKey,
   GetTableDataRequest,
@@ -74,6 +74,10 @@ export const tableApi = Api.injectEndpoints({
         url: `/api/table/${id}/fks`,
       }),
       providesTags: [listTag("field")],
+      onQueryStarted: (_, { queryFulfilled, dispatch }) =>
+        handleQueryFulfilled(queryFulfilled, (data) =>
+          dispatch(updateMetadata(data, [ForeignKeySchema])),
+        ),
     }),
     updateTable: builder.mutation<Table, UpdateTableRequest>({
       query: ({ id, ...body }) => ({

frontend/src/metabase/collections/components/CollectionBulkActions/ArchivedBulkActions.tsx

@@ -43,6 +43,11 @@ export const ArchivedBulkActions = ({
   const handleCloseModal = () => {
     setSelectedItems(null);
     setSelectedAction(null);
+  };
+
+  const unselect = () => {
+    setSelectedItems(null);
+    setSelectedAction(null);
     clearSelected();
   };
 
@@ -55,7 +60,7 @@ export const ArchivedBulkActions = ({
 
   const handleBulkRestore = () => {
     const actions = selected.map((item) => item.setArchived(false));
-    Promise.all(actions).finally(() => clearSelected());
+    Promise.all(actions).finally(unselect);
   };
 
   // delete
@@ -70,7 +75,7 @@ export const ArchivedBulkActions = ({
 
   const handleBulkDeletePermanently = async () => {
     const actions = selected.map((item) => item.delete());
-    Promise.all(actions).finally(() => clearSelected());
+    Promise.all(actions).finally(unselect);
     dispatch(
       addUndo({
         message: ngettext(

frontend/src/metabase/collections/components/CollectionBulkActions/CollectionBulkActions.tsx

@@ -54,6 +54,12 @@ export const CollectionBulkActions = memo(
       clearSelected();
     };
 
+    const handleCancelModal = () => {
+      setSelectedItems(null);
+      setSelectedAction(null);
+      setRememberedDestination(null);
+    };
+
     const tryOrClear = (promise: Promise<any>) =>
       promise.finally(() => clearSelected());
 
@@ -157,7 +163,7 @@ export const CollectionBulkActions = memo(
         {selectedItems && hasSelectedItems && selectedAction === "move" && (
           <BulkMoveModal
             selectedItems={selectedItems}
-            onClose={handleCloseModal}
+            onClose={handleCancelModal}
             onMove={handleBulkMove}
             initialCollectionId={
               isTrashedCollection(collection) ? "root" : collection.id

frontend/src/metabase/common/components/CodeMirror/CodeMirror.module.css

@@ -59,6 +59,7 @@
 
     .cm-cursor {
       border-left-width: 1.6px;
+      border-left-color: var(--mb-color-text-secondary);
     }
 
     .cm-tooltip:not(.cm-tooltip-autocomplete) {
@@ -76,5 +77,87 @@
       font-style: italic;
       background: none;
     }
+
+    .cm-completionMatchedText {
+      text-decoration: none;
+      font-weight: bold;
+      color: var(--mb-color-focus);
+    }
+
+    .cm-completionDetail {
+      color: var(--mb-color-text-light);
+      font-style: normal;
+      text-align: right;
+      overflow: hidden;
+      text-overflow: ellipsis;
+    }
+
+    .cm-completionIcon {
+      display: none;
+    }
+
+    .cm-completionLabel {
+      display: block;
+      max-width: 18rem;
+      overflow: hidden;
+      text-overflow: ellipsis;
+      font-weight: bold;
+    }
+
+    .cm-tooltip-autocomplete {
+      box-shadow: 0 5px 14px rgba(0, 0, 0, 0.08);
+      border-radius: 4px;
+      background-color: var(--mb-color-bg-white);
+      color: var(--mb-color-text-dark);
+      border: 1px solid var(--mb-color-border);
+      max-width: 520px;
+
+      ul li {
+        color: inherit;
+        padding: 0 8px !important;
+        display: flex;
+        justify-content: space-between;
+
+        &:first-child {
+          margin-top: 4px;
+        }
+
+        &:last-child {
+          margin-bottom: 4px;
+        }
+
+        &:hover,
+        &[aria-selected="true"] {
+          background-color: var(--mb-color-bg-hover);
+        }
+      }
+
+      @supports (grid-template-columns: subgrid) {
+        ul:has(.cm-completionLabel) {
+          display: grid;
+          gap: 0 1rem;
+          grid-template-columns: 1fr;
+          grid-auto-rows: max-content;
+
+          &:has(.cm-completionDetail) {
+            grid-template-columns: max-content 1fr;
+          }
+
+          li {
+            display: grid;
+            grid-column: 1 / span 2;
+            grid-template-columns: subgrid;
+          }
+
+          .cm-completionLabel {
+            grid-column: 1;
+          }
+
+          .cm-completionDetail {
+            grid-column: 2;
+          }
+        }
+      }
+    }
   }
 }

frontend/src/metabase/common/components/FormRadio/FormRadio.stories.tsx

@@ -1,47 +0,0 @@
-import type { StoryFn } from "@storybook/react";
-
-import { Form, FormProvider } from "metabase/forms";
-
-import FormRadio from "./FormRadio";
-
-const TEST_OPTIONS = [
-  { name: "Line", value: "line" },
-  { name: "Area", value: "area" },
-  { name: "Bar", value: "bar" },
-];
-
-export default {
-  title: "Components/Ask Before Using/FormRadio",
-  component: FormRadio,
-};
-
-// TODO: mark this story as deprecated
-const Template: StoryFn<typeof FormRadio> = (args) => {
-  const initialValues = { value: undefined };
-  const handleSubmit = () => undefined;
-
-  return (
-    <FormProvider initialValues={initialValues} onSubmit={handleSubmit}>
-      <Form>
-        <FormRadio {...args} name="value" options={TEST_OPTIONS} />
-      </Form>
-    </FormProvider>
-  );
-};
-
-export const Default = {
-  render: Template,
-
-  args: {
-    title: "Title",
-  },
-};
-
-export const WithDescription = {
-  render: Template,
-
-  args: {
-    title: "Title",
-    description: "Description",
-  },
-};

frontend/src/metabase/common/components/FormRadio/FormRadio.tsx

@@ -1,67 +0,0 @@
-import { useField } from "formik";
-import type { Key, ReactNode, Ref } from "react";
-import { forwardRef } from "react";
-
-import FormField from "metabase/common/components/FormField";
-import type { RadioOption, RadioProps } from "metabase/common/components/Radio";
-import Radio from "metabase/common/components/Radio";
-
-export interface FormRadioProps<
-  TValue extends Key = string,
-  TOption = RadioOption<TValue>,
-> extends Omit<
-    RadioProps<TValue, TOption>,
-    "value" | "error" | "onChange" | "onBlur"
-  > {
-  name: string;
-  title?: string;
-  actions?: ReactNode;
-  description?: ReactNode;
-  optional?: boolean;
-}
-
-/**
- * @deprecated: use FormRadioGroup from "metabase/forms"
- */
-const FormRadio = forwardRef(function FormRadio<
-  TValue extends Key,
-  TOption = RadioOption<TValue>,
->(
-  {
-    name,
-    className,
-    style,
-    title,
-    actions,
-    description,
-    optional,
-    ...props
-  }: FormRadioProps<TValue, TOption>,
-  ref: Ref<HTMLDivElement>,
-) {
-  const [{ value, onBlur }, { error, touched }, { setValue }] = useField(name);
-
-  return (
-    <FormField
-      ref={ref}
-      className={className}
-      style={style}
-      title={title}
-      actions={actions}
-      description={description}
-      error={touched ? error : undefined}
-      optional={optional}
-    >
-      <Radio
-        {...props}
-        name={name}
-        value={value}
-        onChange={setValue}
-        onBlur={onBlur}
-      />
-    </FormField>
-  );
-});
-
-// eslint-disable-next-line import/no-default-export -- deprecated usage
-export default FormRadio;

frontend/src/metabase/common/components/FormRadio/FormRadio.unit.spec.tsx

@@ -1,70 +0,0 @@
-import userEvent from "@testing-library/user-event";
-import { Form, Formik } from "formik";
-import * as Yup from "yup";
-
-import { render, screen, waitFor } from "__support__/ui";
-
-import FormRadio from "./FormRadio";
-
-const TEST_SCHEMA = Yup.object({
-  value: Yup.string().notOneOf(["bar"], "error"),
-});
-
-const TEST_OPTIONS = [
-  { name: "Line", value: "line" },
-  { name: "Area", value: "area" },
-  { name: "Bar", value: "bar" },
-];
-
-interface TestFormRadioProps {
-  initialValue?: string;
-  onSubmit: () => void;
-}
-
-const TestFormRadio = ({ initialValue, onSubmit }: TestFormRadioProps) => {
-  return (
-    <Formik
-      initialValues={{ value: initialValue }}
-      validationSchema={TEST_SCHEMA}
-      onSubmit={onSubmit}
-    >
-      <Form>
-        <FormRadio name="value" options={TEST_OPTIONS} title="Label" />
-        <button type="submit">Submit</button>
-      </Form>
-    </Formik>
-  );
-};
-
-describe("FormRadio", () => {
-  it("should use the initial value from the form", () => {
-    const onSubmit = jest.fn();
-
-    render(<TestFormRadio initialValue="line" onSubmit={onSubmit} />);
-
-    expect(screen.getByRole("radio", { name: "Line" })).toBeChecked();
-  });
-
-  it("should propagate the changed value to the form", async () => {
-    const onSubmit = jest.fn();
-
-    render(<TestFormRadio onSubmit={onSubmit} />);
-    await userEvent.click(screen.getByRole("radio", { name: "Line" }));
-    await userEvent.click(screen.getByText("Submit"));
-
-    await waitFor(() => {
-      const values = { value: "line" };
-      expect(onSubmit).toHaveBeenCalledWith(values, expect.anything());
-    });
-  });
-
-  it("should be validated on blur", async () => {
-    const onSubmit = jest.fn();
-
-    render(<TestFormRadio initialValue="line" onSubmit={onSubmit} />);
-    await userEvent.click(screen.getByRole("radio", { name: "Bar" }));
-    await userEvent.tab();
-
-    expect(await screen.findByText(": error")).toBeInTheDocument();
-  });
-});

frontend/src/metabase/common/components/FormRadio/index.ts

@@ -1,3 +0,0 @@
-// eslint-disable-next-line import/no-default-export -- deprecated usage
-export { default } from "./FormRadio";
-export type { FormRadioProps } from "./FormRadio";

frontend/src/metabase/common/components/NewItemMenu/NewItemMenuView.tsx

@@ -65,11 +65,11 @@ const NewItemMenuView = ({
           key="native"
           component={ForwardRefLink}
           to={Urls.newQuestion({
-            type: "native",
+            DEPRECATED_RAW_MBQL_type: "native",
             creationType: "native_question",
             collectionId,
             cardType: "question",
-            databaseId: lastUsedDatabaseId || undefined,
+            DEPRECATED_RAW_MBQL_databaseId: lastUsedDatabaseId || undefined,
           })}
           leftSection={<Icon name="sql" />}
         >

frontend/src/metabase/common/components/SaveQuestionModal/SaveQuestionModal.tsx

@@ -30,9 +30,6 @@ export const SaveQuestionModal = ({
     handleSaveAfterConfirmation,
   } = PLUGIN_DEPENDENCIES.useCheckCardDependencies({
     onSave,
-    onError: (error) => {
-      throw error;
-    },
   });
   useEscapeToCloseModal(onClose);
 

frontend/src/metabase/dashboard/components/QuestionPicker/actions.ts

@@ -22,7 +22,7 @@ export const getNewQuestionUrl = ({
         } as const)
       : ({
           mode: "query",
-          type: "native",
+          DEPRECATED_RAW_MBQL_type: "native",
           creationType: "native_question",
         } as const);
 

frontend/src/metabase/detail-view/components/DetailViewPage/DetailViewPage.tsx

@@ -116,11 +116,8 @@ export function DetailViewPage({
             w={isMediumBreakpoint ? "100%" : rem(440)}
           >
             <Relationships
-              columns={columns}
-              row={row}
               rowId={rowId}
               rowName={rowName}
-              table={table}
               tableForeignKeys={tableForeignKeys}
             />
           </Box>

frontend/src/metabase/detail-view/components/DetailViewSidesheet/DetailViewSidesheet.tsx

@@ -372,11 +372,8 @@ export function DetailViewSidesheet({
                 py={rem(48)}
               >
                 <Relationships
-                  columns={columns}
-                  row={row}
                   rowId={rowId}
                   rowName={rowName}
-                  table={table}
                   tableForeignKeys={tableForeignKeys}
                   onClick={onClose}
                 />

frontend/src/metabase/detail-view/components/Relationships/Relationship/Relationship.tsx

@@ -2,50 +2,51 @@ import cx from "classnames";
 import { inflect } from "inflection";
 import { useMemo } from "react";
 import { Link } from "react-router";
+import { useLatest } from "react-use";
 import { t } from "ttag";
 
 import { skipToken, useGetAdhocQueryQuery } from "metabase/api";
+import { useSelector } from "metabase/lib/redux";
+import { getMetadata } from "metabase/selectors/metadata";
 import { Loader, Stack, Text, rem } from "metabase/ui";
-import { isNumeric, isPK } from "metabase-lib/v1/types/utils/isa";
-import type { ForeignKey, Table } from "metabase-types/api";
+import * as Lib from "metabase-lib";
+import type { ForeignKey } from "metabase-types/api";
 
 import S from "./Relationship.module.css";
+import {
+  getForeignKeyCountQuery,
+  getForeignKeyQuery,
+  getForeignKeyQuestionUrl,
+} from "./utils";
 
 interface Props {
   fk: ForeignKey;
-  href: string | undefined;
   rowId: string | number;
-  table: Table;
   onClick?: () => void;
 }
 
-export const Relationship = ({ fk, href, rowId, table, onClick }: Props) => {
-  const pk = (table.fields ?? []).find(isPK);
-  const fkOriginId =
-    fk.origin && typeof fk.origin.id == "number" ? fk.origin.id : undefined;
+export const Relationship = ({ fk, rowId, onClick }: Props) => {
+  const metadata = useSelector(getMetadata);
+  const metadataRef = useLatest(metadata);
+  const fkQuery = useMemo(
+    () => getForeignKeyQuery(fk, rowId, metadataRef.current),
+    [fk, rowId, metadataRef],
+  );
+  const fkCountQuery = useMemo(
+    () => (fkQuery != null ? getForeignKeyCountQuery(fkQuery) : undefined),
+    [fkQuery],
+  );
+  const fkQuestionUrl = useMemo(
+    () => (fkQuery != null ? getForeignKeyQuestionUrl(fkQuery) : undefined),
+    [fkQuery],
+  );
 
   const {
     data: dataset,
     error,
     isFetching,
   } = useGetAdhocQueryQuery(
-    fk.origin != null && fkOriginId != null
-      ? {
-          type: "query",
-          query: {
-            "source-table": fk.origin?.table_id,
-            filter: [
-              "=",
-              ["field", fkOriginId, null],
-              isNumeric(pk) && typeof rowId === "string"
-                ? parseFloat(rowId)
-                : rowId,
-            ],
-            aggregation: [["count"]],
-          },
-          database: fk.origin.table?.db_id ?? table.db_id,
-        }
-      : skipToken,
+    fkCountQuery != null ? Lib.toJsQuery(fkCountQuery) : skipToken,
   );
 
   const count = useMemo(() => {
@@ -68,7 +69,9 @@ export const Relationship = ({ fk, href, rowId, table, onClick }: Props) => {
         [S.clickable]: clickable,
       })}
       gap={rem(12)}
-      {...(clickable ? { component: Link, to: href, onClick } : undefined)}
+      {...(clickable
+        ? { component: Link, to: fkQuestionUrl, onClick }
+        : undefined)}
     >
       {isFetching && <Loader data-testid="loading-indicator" size="md" />}
 

frontend/src/metabase/detail-view/components/Relationships/Relationship/Relationship.unit.spec.tsx

@@ -4,6 +4,7 @@ import fetchMock from "fetch-mock";
 import { Route } from "react-router";
 
 import { setupCardDataset } from "__support__/server-mocks";
+import { createMockEntitiesState } from "__support__/store";
 import { renderWithProviders, waitForLoaderToBeRemoved } from "__support__/ui";
 import type { Table } from "metabase-types/api";
 import { createMockForeignKey } from "metabase-types/api/mocks";
@@ -16,7 +17,9 @@ import {
   PRODUCTS,
   createOrdersProductIdField,
   createOrdersTable,
+  createSampleDatabase,
 } from "metabase-types/api/mocks/presets";
+import { createMockState } from "metabase-types/store/mocks";
 
 import { Relationship } from "./Relationship";
 
@@ -51,23 +54,22 @@ interface SetupOpts {
   table?: Table;
 }
 
-function setup({ dataset = DATASET, table = ORDERS_TABLE }: SetupOpts = {}) {
+function setup({ dataset = DATASET }: SetupOpts = {}) {
   const onClick = jest.fn();
 
   const TestComponent = () => (
-    <Relationship
-      fk={FK}
-      href="/test"
-      rowId={ROW_ID}
-      table={table}
-      onClick={onClick}
-    />
+    <Relationship fk={FK} rowId={ROW_ID} onClick={onClick} />
   );
 
   setupCardDataset(dataset);
 
   renderWithProviders(<Route path="/" component={TestComponent} />, {
     withRouter: true,
+    storeInitialState: createMockState({
+      entities: createMockEntitiesState({
+        databases: [createSampleDatabase()],
+      }),
+    }),
   });
 
   return { onClick };

frontend/src/metabase/detail-view/components/Relationships/Relationship/utils.ts

@@ -0,0 +1,89 @@
+import { parseNumber } from "metabase/lib/number";
+import * as Urls from "metabase/lib/urls";
+import * as Lib from "metabase-lib";
+import Question from "metabase-lib/v1/Question";
+import type Metadata from "metabase-lib/v1/metadata/Metadata";
+import type { ForeignKey, RowValue } from "metabase-types/api";
+
+const STAGE_INDEX = 0;
+
+function getForeignKeyFilterClause(field: Lib.ColumnMetadata, rowId: RowValue) {
+  if (Lib.isStringOrStringLike(field) && typeof rowId === "string") {
+    return Lib.stringFilterClause({
+      operator: "=",
+      column: field,
+      values: [rowId],
+      options: {},
+    });
+  }
+
+  if (Lib.isNumeric(field) && typeof rowId === "string") {
+    const number = parseNumber(rowId);
+    if (number != null) {
+      return Lib.numberFilterClause({
+        operator: "=",
+        column: field,
+        values: [number],
+      });
+    }
+  }
+
+  if (Lib.isNumeric(field) && typeof rowId === "number") {
+    return Lib.numberFilterClause({
+      operator: "=",
+      column: field,
+      values: [rowId],
+    });
+  }
+
+  if (Lib.isBoolean(field) && typeof rowId === "boolean") {
+    return Lib.booleanFilterClause({
+      operator: "=",
+      column: field,
+      values: [rowId],
+    });
+  }
+}
+
+export function getForeignKeyQuery(
+  fk: ForeignKey,
+  rowId: RowValue,
+  metadata: Metadata,
+) {
+  if (fk.origin == null || fk.origin.table == null) {
+    return;
+  }
+
+  const metadataProvider = Lib.metadataProvider(
+    fk.origin.table.db_id,
+    metadata,
+  );
+  const table = Lib.tableOrCardMetadata(metadataProvider, fk.origin.table_id);
+  const field = Lib.fieldMetadata(metadataProvider, fk.origin_id);
+  if (table == null || field == null) {
+    return;
+  }
+
+  const filter = getForeignKeyFilterClause(field, rowId);
+  if (filter == null) {
+    return;
+  }
+
+  return Lib.filter(
+    Lib.queryFromTableOrCardMetadata(metadataProvider, table),
+    STAGE_INDEX,
+    filter,
+  );
+}
+
+export function getForeignKeyCountQuery(fkQuery: Lib.Query) {
+  return Lib.aggregateByCount(fkQuery, STAGE_INDEX);
+}
+
+export const getForeignKeyQuestionUrl = (query: Lib.Query): string => {
+  const question = Question.create({
+    dataset_query: Lib.toJsQuery(query),
+  });
+
+  return Urls.question(question.card(), { hash: question.card() });
+};

frontend/src/metabase/detail-view/components/Relationships/Relationships.tsx

@@ -2,36 +2,24 @@ import { type ReactNode, useMemo } from "react";
 import { jt, t } from "ttag";
 
 import { type BoxProps, Stack, Text } from "metabase/ui";
-import type {
-  DatasetColumn,
-  ForeignKey,
-  RowValues,
-  Table,
-} from "metabase-types/api";
+import type { ForeignKey } from "metabase-types/api";
 
 import { Relationship } from "./Relationship";
-import { getUrl } from "./utils";
 
 interface Props {
-  columns: DatasetColumn[];
-  row: RowValues;
   rowId: string | number;
   rowName: ReactNode;
-  table: Table;
   tableForeignKeys: ForeignKey[];
   onClick?: () => void;
 }
 
 export function Relationships({
-  columns,
-  row,
   rowId,
   rowName,
-  table,
   tableForeignKeys,
   onClick,
 }: Props & BoxProps): JSX.Element | null {
-  const sortedForeignTables = useMemo(
+  const sortedForeignKeys = useMemo(
     () =>
       tableForeignKeys.toSorted((a, b) => {
         const aDisplayName = a.origin?.table?.display_name ?? "";
@@ -58,14 +46,12 @@ export function Relationships({
       </Text>
 
       <Stack gap="md">
-        {sortedForeignTables.map((fk) => {
+        {sortedForeignKeys.map((fk) => {
           return (
             <Relationship
               key={`${fk.origin_id}-${fk.destination_id}`}
               fk={fk}
-              href={getUrl({ columns, row, table, fk })}
               rowId={rowId}
-              table={table}
               onClick={onClick}
             />
           );

frontend/src/metabase/detail-view/components/Relationships/utils.ts

@@ -1,51 +0,0 @@
-import { question } from "metabase/lib/urls";
-import { isPK } from "metabase-lib/v1/types/utils/isa";
-import type {
-  DatasetColumn,
-  ForeignKey,
-  RowValues,
-  Table,
-} from "metabase-types/api";
-
-export const getUrl = ({
-  columns,
-  row,
-  table,
-  fk,
-}: {
-  columns: DatasetColumn[];
-  fk: ForeignKey;
-  row: RowValues;
-  table: Table;
-}): string | undefined => {
-  const pkIndex = columns.findIndex(isPK);
-
-  if (pkIndex === -1) {
-    return undefined;
-  }
-
-  const objectId = row[pkIndex];
-
-  if (objectId == null) {
-    return undefined;
-  }
-
-  if (fk.origin?.table_id) {
-    const card = {
-      type: "question" as const,
-      dataset_query: {
-        type: "query" as const,
-        query: {
-          "source-table": fk.origin.table_id,
-          filter: ["=", ["field", fk.origin.id, null], objectId],
-        },
-        database: fk.origin.table?.db_id || table.db_id,
-      },
-    };
-
-    const questionUrl = question(card, { hash: card as any });
-    return questionUrl;
-  }
-
-  return undefined;
-};

frontend/src/metabase/home/components/Onboarding/Onboarding.tsx

@@ -98,11 +98,11 @@ export const Onboarding = () => {
   );
 
   const newNativeQuestionUrl = Urls.newQuestion({
-    type: "native",
+    DEPRECATED_RAW_MBQL_type: "native",
     creationType: "native_question",
     collectionId: "root",
     cardType: "question",
-    databaseId: lastUsedDatabaseId || undefined,
+    DEPRECATED_RAW_MBQL_databaseId: lastUsedDatabaseId || undefined,
   });
 
   const [lastItemOpened, setLastItemOpened] = useTempStorage(

frontend/src/metabase/lib/colors/colors.ts

@@ -79,6 +79,7 @@ const baseColors = {
 
   // Orion
   orion: {
+    110: "hsla(205, 63%, 5%, 1)",
     100: "hsla(204, 66%, 8%, 1)",
     90: "hsla(204, 34%, 14%, 1)",
     80: "hsla(205, 19%, 23%, 1)",
@@ -251,6 +252,14 @@ export const colorConfig = {
     light: baseColors.octopus[60],
     dark: baseColors.octopus[80],
   },
+  "admin-navbar-secondary": {
+    light: baseColors.octopus[40],
+    dark: baseColors.octopus[60],
+  },
+  "admin-navbar-inverse": {
+    light: baseColors.octopus[80],
+    dark: baseColors.octopus[60],
+  },
   "background-brand": {
     light: baseColors.brand[20],
     dark: baseColors.brand[70],
@@ -272,7 +281,11 @@ export const colorConfig = {
     light: baseColors.orion[80],
     dark: baseColors.orion[20],
   },
-  "background-light": { light: baseColors.orion[5], dark: baseColors.black }, //should be background-secondary
+  "background-light": {
+    //should be background-secondary
+    light: baseColors.orion[5],
+    dark: baseColors.orion[110],
+  },
   "background-selected": {
     light: baseColors.brand[50],
     dark: baseColors.brand[40],
@@ -291,7 +304,7 @@ export const colorConfig = {
     light: baseColors.lobster[10],
     dark: baseColors.lobster[90],
   },
-  "bg-light": { light: baseColors.orion[5], dark: baseColors.black }, //should be background-secondary
+  "bg-light": { light: baseColors.orion[5], dark: baseColors.orion[110] }, //should be background-secondary
   "bg-medium": { light: baseColors.orion[10], dark: baseColors.orion[80] }, //should be background-tertiary
   "bg-night": { light: baseColors.orion[70], dark: baseColors.orion[30] }, //merge with background-secondary-inverse?
   "bg-white-alpha-15": {
@@ -446,7 +459,7 @@ export const colorConfig = {
     dark: baseColors.brand[40],
   },
   "text-dark": {
-    /* @deprecated, use text-primary */ light: baseColors.orionAlpha[80],
+    light: baseColors.orionAlpha[80] /* @deprecated, use text-primary */,
     dark: baseColors.orionAlphaInverse[80],
   },
   "text-disabled": {
@@ -471,14 +484,18 @@ export const colorConfig = {
     light: baseColors.orionAlpha[80],
     dark: baseColors.orionAlphaInverse[80],
   },
-  "text-secondary-inverse": {
-    light: baseColors.orionAlphaInverse[60],
-    dark: baseColors.orionAlpha[60],
+  "text-primary-inverse": {
+    light: baseColors.orionAlphaInverse[80],
+    dark: baseColors.orionAlpha[80],
   },
   "text-secondary": {
     light: baseColors.orionAlpha[60],
     dark: baseColors.orionAlphaInverse[60],
   },
+  "text-secondary-inverse": {
+    light: baseColors.orionAlphaInverse[60],
+    dark: baseColors.orionAlpha[60],
+  },
   "text-selected": {
     light: baseColors.white,
     dark: baseColors.white,

frontend/src/metabase/metadata/.eslintrc

@@ -13,8 +13,6 @@
           "/embedding-sdk-package",
           "/embedding-sdk-bundle",
           "/embedding-sdk-shared",
-          "metabase-lib/v1/metadata",
-          "metabase-lib/v1/metadata/*",
           "metabase/entities",
           "metabase/entities/*",
         ],

frontend/src/metabase/metadata/pages/DataModel/components/FieldSection/BehaviorSection/RemappingPicker/CustomMappingModal/CustomMappingModal.tsx

@@ -131,7 +131,7 @@ export const CustomMappingModal = ({
             </Box>
 
             <Group
-              bg="white"
+              bg="background"
               bottom={0}
               className={S.footer}
               gap="md"

frontend/src/metabase/metadata/pages/DataModel/components/FieldSection/BehaviorSection/RemappingPicker/DisplayValuesPicker/DisplayValuesPicker.module.css

@@ -5,8 +5,9 @@
      * When it's disabled is uses pointer-events: none to prevent the colors being applied.
      * Putting an info icon with a tooltip requires it to be pointer-evens-enabled, which causes
      * its parent (i.e. SelectItem) to apply the hover styles. We override this here.
+     * This will effectively cover the element with a background
      */
-    background-color: white !important;
+    background-color: var(--mb-color-background) !important;
     color: var(--mb-color-text-primary) !important;
     /**
       * There's also margin-top: 1px involved, which causes 1px of the background color to be shown.

frontend/src/metabase/metadata/pages/DataModel/components/PreviewSection/ObjectDetailPreview.tsx

@@ -1,21 +1,22 @@
-import { memo, useMemo } from "react";
+import { memo, useMemo, useRef } from "react";
 import { t } from "ttag";
 
-import { useGetAdhocQueryQuery } from "metabase/api";
+import { skipToken, useGetAdhocQueryQuery } from "metabase/api";
 import { getErrorMessage } from "metabase/api/utils";
 import EmptyState from "metabase/common/components/EmptyState";
 import { DetailsGroup, Header } from "metabase/detail-view/components";
 import { getEntityIcon, getHeaderColumns } from "metabase/detail-view/utils";
+import { useSelector } from "metabase/lib/redux";
+import { getMetadataUnfiltered } from "metabase/selectors/metadata";
 import { Box, Repeat, Skeleton, Stack, rem } from "metabase/ui";
 import { extractRemappedColumns } from "metabase/visualizations";
+import * as Lib from "metabase-lib";
+import type Metadata from "metabase-lib/v1/metadata/Metadata";
 import type {
   DatabaseId,
   DatasetColumn,
-  DatasetQuery,
   Field,
-  FieldFilter,
   FieldId,
-  FieldReference,
   RawSeries,
   RowValues,
   TableId,
@@ -101,25 +102,50 @@ const ObjectDetailPreviewBase = ({
   );
 };
 
+function getDataSampleQuery(
+  metadata: Metadata,
+  databaseId: DatabaseId,
+  tableId: TableId,
+  fieldId: FieldId,
+) {
+  const metadataProvider = Lib.metadataProvider(databaseId, metadata);
+  const table = Lib.tableOrCardMetadata(metadataProvider, tableId);
+  const field = Lib.fieldMetadata(metadataProvider, fieldId);
+  if (table == null || field == null) {
+    return;
+  }
+
+  const tableQuery = Lib.queryFromTableOrCardMetadata(metadataProvider, table);
+  const stageIndex = 0;
+  return Lib.filter(
+    tableQuery,
+    stageIndex,
+    Lib.defaultFilterClause({
+      operator: "not-null",
+      column: field,
+    }),
+  );
+}
+
 function useDataSample({ databaseId, field, fieldId, tableId }: Props) {
-  const reference: FieldReference = ["field", fieldId, null];
-  const filter: FieldFilter = ["not-null", reference];
+  // do not generate a new query when metadata changes
+  const metadata = useSelector(getMetadataUnfiltered);
+  const metadataRef = useRef(metadata);
+  metadataRef.current = metadata;
+  const query = useMemo(
+    () => getDataSampleQuery(metadataRef.current, databaseId, tableId, fieldId),
+    [databaseId, tableId, fieldId],
+  );
 
-  const datasetQuery: DatasetQuery = {
-    type: "query" as const,
-    database: databaseId,
-    query: {
-      "source-table": tableId,
-      filter,
-      limit: 1,
-    },
-  };
-
-  const { data, ...rest } = useGetAdhocQueryQuery({
-    ...datasetQuery,
-    ignore_error: true,
-    _refetchDeps: field,
-  });
+  const { data, ...rest } = useGetAdhocQueryQuery(
+    query
+      ? {
+          ...Lib.toJsQuery(query),
+          ignore_error: true,
+          _refetchDeps: field,
+        }
+      : skipToken,
+  );
 
   const base = {
     ...rest,
@@ -144,14 +170,14 @@ function useDataSample({ databaseId, field, fieldId, tableId }: Props) {
     };
   }
 
-  if (!data?.data) {
+  if (data?.data == null || query == null) {
     return base;
   }
 
   const rawSeries: RawSeries = [
     {
       card: createMockCard({
-        dataset_query: datasetQuery,
+        dataset_query: Lib.toJsQuery(query),
         display: "object",
         visualization_settings: {},
       }),

frontend/src/metabase/metadata/pages/DataModel/components/PreviewSection/PreviewSection.tsx

@@ -51,11 +51,13 @@ const PreviewSectionBase = ({
   onPreviewTypeChange,
 }: Props) => {
   const data = useMemo(() => getPreviewTypeData(), []);
-  const pkFields = table.fields?.filter((field) => isPK(field)) ?? [];
+  const pkFields = useMemo(
+    () => table.fields?.filter((field) => isPK(field)) ?? [],
+    [table.fields],
+  );
 
   return (
     <Card
-      bg="white"
       className={className}
       data-testid="preview-section"
       h="100%"

frontend/src/metabase/metadata/pages/DataModel/components/PreviewSection/TablePreview.tsx

@@ -1,20 +1,20 @@
-import { memo } from "react";
+import { memo, useMemo, useRef } from "react";
 import { t } from "ttag";
 import _ from "underscore";
 
 import { skipToken, useGetAdhocQueryQuery } from "metabase/api";
 import { getErrorMessage } from "metabase/api/utils";
 import EmptyState from "metabase/common/components/EmptyState";
-import { getRawTableFieldId } from "metabase/metadata/utils/field";
+import { useSelector } from "metabase/lib/redux";
+import { getMetadataUnfiltered } from "metabase/selectors/metadata";
 import { Repeat, Skeleton, Stack } from "metabase/ui";
 import Visualization from "metabase/visualizations/components/Visualization";
+import * as Lib from "metabase-lib";
+import type Metadata from "metabase-lib/v1/metadata/Metadata";
 import type {
   DatabaseId,
-  DatasetQuery,
   Field,
-  FieldFilter,
   FieldId,
-  FieldReference,
   RawSeries,
   TableId,
 } from "metabase-types/api";
@@ -84,13 +84,27 @@ function useDataSample({
   pkFields,
   tableId,
 }: Props) {
-  const datasetQuery = getPreviewQuery(databaseId, tableId, fieldId, pkFields);
+  // do not generate a new query when metadata changes
+  const metadata = useSelector(getMetadataUnfiltered);
+  const metadataRef = useRef(metadata);
+  metadataRef.current = metadata;
+  const query = useMemo(
+    () =>
+      getPreviewQuery(
+        metadataRef.current,
+        databaseId,
+        tableId,
+        fieldId,
+        pkFields,
+      ),
+    [databaseId, tableId, fieldId, pkFields],
+  );
 
   const { data, refetch, ...rest } = useGetAdhocQueryQuery(
-    isFieldHidden(field)
+    query == null || isFieldHidden(field)
       ? skipToken
       : {
-          ...datasetQuery,
+          ...Lib.toJsQuery(query),
           ignore_error: true,
           _refetchDeps: field,
         },
@@ -113,14 +127,14 @@ function useDataSample({
     return { ...base, isError: true, error: getDataErrorMessage(data) };
   }
 
-  if (!data?.data || data.data.cols.length === 0) {
+  if (query == null || !data?.data || data.data.cols.length === 0) {
     return base;
   }
 
   const rawSeries: RawSeries = [
     {
       card: createMockCard({
-        dataset_query: datasetQuery,
+        dataset_query: Lib.toJsQuery(query),
         display: "table",
         visualization_settings: {},
       }),
@@ -135,30 +149,34 @@ function useDataSample({
 }
 
 function getPreviewQuery(
+  metadata: Metadata,
   databaseId: DatabaseId,
   tableId: TableId,
   fieldId: FieldId,
   pkFields: Field[],
-): DatasetQuery {
-  const fieldRef: FieldReference = ["field", fieldId, null];
-  const pkFieldRefs: FieldReference[] = pkFields.map((pkField) => {
-    return ["field", getRawTableFieldId(pkField), null];
-  });
-  const filter: FieldFilter = ["not-null", fieldRef];
+): Lib.Query | undefined {
+  const metadataProvider = Lib.metadataProvider(databaseId, metadata);
+  const table = Lib.tableOrCardMetadata(metadataProvider, tableId);
+  const field = Lib.fieldMetadata(metadataProvider, fieldId);
+  if (table == null || field == null) {
+    return;
+  }
 
-  return {
-    type: "query",
-    database: databaseId,
-    query: {
-      "source-table": tableId,
-      filter,
-      fields: [fieldRef],
-      // fetch more rows to increase probability of getting at least 5 unique values
-      limit: 50,
-      // order by PKs when possible to prevent SQL returning non-deterministically ordered values
-      "order-by": pkFieldRefs.map((pkFieldRef) => ["asc", pkFieldRef]),
-    },
-  };
+  const tableQuery = Lib.queryFromTableOrCardMetadata(metadataProvider, table);
+  const stageIndex = 0;
+  const fieldQuery = Lib.withFields(tableQuery, stageIndex, [field]);
+  const filterQuery = Lib.filter(
+    fieldQuery,
+    stageIndex,
+    Lib.defaultFilterClause({ operator: "not-null", column: field }),
+  );
+  // order by PKs when possible to prevent SQL returning non-deterministically ordered values
+  const orderByQuery = pkFields.reduce((query: Lib.Query, { id }: Field) => {
+    const pkField = Lib.fieldMetadata(metadataProvider, Number(id));
+    return pkField ? Lib.orderBy(query, stageIndex, pkField, "asc") : query;
+  }, filterQuery);
+  // fetch more rows to increase probability of getting at least 5 unique values
+  return Lib.limit(orderByQuery, stageIndex, 50);
 }
 
 export const TablePreview = memo(TablePreviewBase);

frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/TablePicker.tsx

@@ -1,197 +0,0 @@
-import { useDeferredValue, useEffect, useState } from "react";
-import { useKeyPressEvent } from "react-use";
-import { t } from "ttag";
-
-import { useDebouncedValue } from "metabase/common/hooks/use-debounced-value";
-import { Box, Flex, Icon, Input, Stack, rem } from "metabase/ui";
-
-import { Results } from "./Results";
-import S from "./TablePicker.module.css";
-import type { ChangeOptions, TreePath } from "./types";
-import { flatten, useExpandedState, useSearch, useTableLoader } from "./utils";
-
-export function TablePicker({
-  path,
-  onChange,
-}: {
-  path: TreePath;
-  onChange: (path: TreePath, options?: ChangeOptions) => void;
-}) {
-  const [query, setQuery] = useState("");
-  const deferredQuery = useDeferredValue(query);
-
-  return (
-    <Stack data-testid="table-picker" mih={rem(200)}>
-      <Box p="xl" pb={0}>
-        <Input
-          leftSection={<Icon name="search" />}
-          placeholder={t`Search tables`}
-          value={query}
-          onChange={(event) => setQuery(event.target.value)}
-        />
-      </Box>
-
-      {deferredQuery === "" ? (
-        <Tree path={path} onChange={onChange} />
-      ) : (
-        <Search query={deferredQuery} path={path} onChange={onChange} />
-      )}
-    </Stack>
-  );
-}
-
-function Tree({
-  path,
-  onChange,
-}: {
-  path: TreePath;
-  onChange: (path: TreePath, options?: ChangeOptions) => void;
-}) {
-  const { databaseId, schemaName } = path;
-  const { isExpanded, toggle } = useExpandedState(path);
-  const { tree, reload } = useTableLoader(path);
-
-  const items = flatten(tree, {
-    isExpanded,
-    addLoadingNodes: true,
-    canFlattenSingleSchema: true,
-  });
-  const isEmpty = items.length === 0;
-
-  useEffect(() => {
-    // When we detect only one database, we automatically select and expand it.
-    const databases = tree.children.filter((node) => node.type === "database");
-
-    if (databases.length !== 1) {
-      return;
-    }
-
-    const [database] = databases;
-
-    if (
-      !isExpanded({ databaseId: database.value.databaseId }) &&
-      databaseId == null
-    ) {
-      toggle(database.key, true);
-      onChange(database.value, { isAutomatic: true });
-    }
-  }, [databaseId, schemaName, tree, toggle, isExpanded, onChange]);
-
-  useEffect(() => {
-    // When we detect a database with just one schema, we automatically
-    // select and expand that schema.
-    const database = tree.children.find(
-      (node) =>
-        node.type === "database" && node.value.databaseId === databaseId,
-    );
-    if (
-      databaseId &&
-      isExpanded({ databaseId }) &&
-      database?.children.length === 1 &&
-      schemaName == null
-    ) {
-      const schema = database.children[0];
-      if (schema.type === "schema") {
-        toggle(schema.key, true);
-        onChange(schema.value, { isAutomatic: true });
-      }
-    }
-  }, [databaseId, schemaName, tree, toggle, isExpanded, onChange]);
-
-  if (isEmpty) {
-    return <EmptyState title={t`No data to show`} />;
-  }
-
-  return (
-    <Results
-      items={items}
-      path={path}
-      reload={reload}
-      toggle={toggle}
-      withMassToggle
-      onItemClick={onChange}
-    />
-  );
-}
-
-function Search({
-  query,
-  path,
-  onChange,
-}: {
-  query: string;
-  path: TreePath;
-  onChange: (path: TreePath) => void;
-}) {
-  const debouncedQuery = useDebouncedValue(query, 300);
-  const { tree, isLoading } = useSearch(debouncedQuery);
-  const [selectedIndex, setSelectedIndex] = useState(-1);
-
-  const items = flatten(tree);
-  const isEmpty = !isLoading && items.length === 0;
-
-  // search results need their own keypress handling logic
-  // because we want to keep the focus on the search input
-  useKeyPressEvent("ArrowDown", (event) => {
-    setSelectedIndex((selectedIndex) => {
-      const nextTableIndex = items.findIndex(
-        (item, index) => item.type === "table" && index > selectedIndex,
-      );
-      if (nextTableIndex >= 0) {
-        return nextTableIndex;
-      }
-      const firstTableIndex = items.findIndex((item) => item.type === "table");
-      return firstTableIndex;
-    });
-    event.preventDefault();
-  });
-
-  useKeyPressEvent("ArrowUp", (event) => {
-    setSelectedIndex((selectedIndex) => {
-      const previousTableIndex = items.findLastIndex(
-        (item, index) => item.type === "table" && index < selectedIndex,
-      );
-      if (previousTableIndex >= 0) {
-        return previousTableIndex;
-      }
-      const lastTableIndex = items.findLastIndex(
-        (item) => item.type === "table",
-      );
-      return lastTableIndex;
-    });
-    event.preventDefault();
-  });
-
-  useKeyPressEvent("Enter", (event) => {
-    const item = items[selectedIndex];
-    if (item?.value) {
-      onChange?.(item.value);
-      event.preventDefault();
-    }
-  });
-
-  if (isEmpty) {
-    return <EmptyState title={t`No results.`} />;
-  }
-
-  return (
-    <Results
-      items={items}
-      path={path}
-      onItemClick={onChange}
-      selectedIndex={selectedIndex}
-      onSelectedIndexChange={setSelectedIndex}
-    />
-  );
-}
-
-function EmptyState({ title }: { title: string }) {
-  return (
-    <Stack py="xl" align="center" className={S.emptyState} gap="md">
-      <Flex className={S.empyIcon} p="lg" align="center" justify="center">
-        <Icon name="table2" />
-      </Flex>
-      <Box>{title}</Box>
-    </Stack>
-  );
-}

frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/components/EmptyState.tsx

@@ -0,0 +1,18 @@
+import { Box, Flex, Icon, Stack } from "metabase/ui";
+
+import S from "./EmptyState.module.css";
+
+interface Props {
+  title: string;
+}
+
+export function EmptyState({ title }: Props) {
+  return (
+    <Stack py="xl" align="center" className={S.emptyState} gap="md">
+      <Flex className={S.empyIcon} p="lg" align="center" justify="center">
+        <Icon name="table2" />
+      </Flex>
+      <Box>{title}</Box>
+    </Stack>
+  );
+}

frontend/src/metabase/metadata/pages/DataModel/components/TablePicker/components/Results.tsx

@@ -5,13 +5,14 @@ import { Link } from "react-router";
 
 import { Box, Flex, Icon, Skeleton, rem } from "metabase/ui";
 
-import { getUrl } from "../../utils";
+import { getUrl } from "../../../utils";
+import { TYPE_ICONS } from "../constants";
+import type { FlatItem, TreePath } from "../types";
+import { hasChildren } from "../utils";
 
 import { BulkTableVisibilityToggle } from "./BulkTableVisibilityToggle";
 import S from "./Results.module.css";
 import { TableVisibilityToggle } from "./TableVisibilityToggle";
-import type { FlatItem, TreePath } from "./types";
-import { TYPE_ICONS, hasChildren } from "./utils";
 
 const VIRTUAL_OVERSCAN = 5;
 const ITEM_MIN_HEIGHT = 32; // items can vary in size because of text wrapping