Compare commits
206 Commits
python-sec
...
workspaces
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e481c02476 | ||
|
|
06dd052d57 | ||
|
|
a85d113127 | ||
|
|
3c5de95c17 | ||
|
|
98716ba649 | ||
|
|
b457d5ebaa | ||
|
|
32b4d0ae3c | ||
|
|
bd66d4c246 | ||
|
|
9ce4259dff | ||
|
|
6e8eadd2a0 | ||
|
|
ba279bfc29 | ||
|
|
49a00dea09 | ||
|
|
ce44815e8e | ||
|
|
a4c37283fc | ||
|
|
d191b6c0a1 | ||
|
|
0cfd5936c5 | ||
|
|
573ba81c1e | ||
|
|
de0616bd90 | ||
|
|
8f7fa76f87 | ||
|
|
7f9886044a | ||
|
|
becf8db5df | ||
|
|
c78784afef | ||
|
|
ac2c41b83f | ||
|
|
c38c69059e | ||
|
|
f83890077a | ||
|
|
a5922d9323 | ||
|
|
866dbe4be9 | ||
|
|
bd3c618303 | ||
|
|
ffbe648b7b | ||
|
|
e080d9e1ff | ||
|
|
af88617937 | ||
|
|
7bff4573e3 | ||
|
|
098370174a | ||
|
|
14facfdbb7 | ||
|
|
ddc9c7c6c3 | ||
|
|
63f4dcbf7c | ||
|
|
7f35c17f3e | ||
|
|
4f98f13971 | ||
|
|
651918f3e8 | ||
|
|
59c652d537 | ||
|
|
bc3687c4a5 | ||
|
|
05bb1daa42 | ||
|
|
6b6d55cd22 | ||
|
|
1361b9bda0 | ||
|
|
6644f846e8 | ||
|
|
98b5096eda | ||
|
|
0cc951e1a4 | ||
|
|
24910610c4 | ||
|
|
12fd210a41 | ||
|
|
7484249328 | ||
|
|
d034244735 | ||
|
|
3b837f7593 | ||
|
|
36419de6e7 | ||
|
|
b488c1b082 | ||
|
|
869cfcc024 | ||
|
|
81a7572e0a | ||
|
|
2ff73cd864 | ||
|
|
3ebdecc2d2 | ||
|
|
f0bfd6898f | ||
|
|
1b6528c9f1 | ||
|
|
f8f2e235f6 | ||
|
|
9086aa10e9 | ||
|
|
cfa8c389b2 | ||
|
|
c670809704 | ||
|
|
1f78793830 | ||
|
|
4e342e8481 | ||
|
|
b61455bb53 | ||
|
|
18c743403e | ||
|
|
b1fd062915 | ||
|
|
15b446ab04 | ||
|
|
0f61a79a6b | ||
|
|
9ebbf353f6 | ||
|
|
be25ff92a1 | ||
|
|
11ef81fdee | ||
|
|
ce08daf26f | ||
|
|
e2b5e9dd9d | ||
|
|
49bbcf7467 | ||
|
|
757cca20db | ||
|
|
b196d10ad3 | ||
|
|
3e92c2d2ad | ||
|
|
f1d495dfde | ||
|
|
a14b9c1b3b | ||
|
|
6cffef70f5 | ||
|
|
a5a8fa1f1e | ||
|
|
6c7102aa37 | ||
|
|
809e5b6724 | ||
|
|
5f75951717 | ||
|
|
8ff78faef0 | ||
|
|
2bd68fc928 | ||
|
|
288f07c709 | ||
|
|
1f199c1dfb | ||
|
|
51efac3d49 | ||
|
|
a9ccb8def6 | ||
|
|
b2dacc7a92 | ||
|
|
d769903e13 | ||
|
|
167af6a806 | ||
|
|
e2ad85b627 | ||
|
|
c1c1926751 | ||
|
|
40dbb2dd5e | ||
|
|
f206ac8d9b | ||
|
|
c93f52c810 | ||
|
|
fac1eda9fa | ||
|
|
5baa9d4709 | ||
|
|
78272586f8 | ||
|
|
86926f69d3 | ||
|
|
0ee7e2fa48 | ||
|
|
c3f7c0e3f8 | ||
|
|
1774564beb | ||
|
|
557ce7c3c6 | ||
|
|
a60fa0c2cb | ||
|
|
39e4f68e31 | ||
|
|
072c06e2bd | ||
|
|
e781690132 | ||
|
|
68b19966fc | ||
|
|
8b5db401e8 | ||
|
|
946fc08c43 | ||
|
|
f5cbb8cc48 | ||
|
|
44d6075e8c | ||
|
|
fa2ae8e884 | ||
|
|
36e3b6ec3a | ||
|
|
ba9a841466 | ||
|
|
5ebc738f29 | ||
|
|
045e9a224f | ||
|
|
a6e22daac1 | ||
|
|
6bdd48ffe3 | ||
|
|
5bb0f1f560 | ||
|
|
2d263d19e3 | ||
|
|
5e98f44b80 | ||
|
|
da6555f5cf | ||
|
|
937f3f7b03 | ||
|
|
71d065f0a9 | ||
|
|
b30071a69a | ||
|
|
407fc2a5ab | ||
|
|
de113f72ce | ||
|
|
6199e0056f | ||
|
|
fd6b176446 | ||
|
|
42d1202c18 | ||
|
|
d1a4656cb8 | ||
|
|
cc934ea766 | ||
|
|
8bf117b047 | ||
|
|
caea0b31cf | ||
|
|
1d6dd0d16d | ||
|
|
dca47362f8 | ||
|
|
2f8b666797 | ||
|
|
f3aef4f4ad | ||
|
|
5ac6249eb3 | ||
|
|
8acbee8000 | ||
|
|
bfd2ad8f0f | ||
|
|
a982db6bf5 | ||
|
|
3e6d266a33 | ||
|
|
e197cf97af | ||
|
|
97d89cc855 | ||
|
|
955cc0b4be | ||
|
|
1b4424393d | ||
|
|
3337769411 | ||
|
|
0b07779984 | ||
|
|
2e54d68fb2 | ||
|
|
e67c3e625d | ||
|
|
325fe07ab1 | ||
|
|
dc53fec17e | ||
|
|
17c3ed6635 | ||
|
|
f661f0c21e | ||
|
|
89d1394fb7 | ||
|
|
0cac006357 | ||
|
|
c9a5ea4dd1 | ||
|
|
79f7d83c4d | ||
|
|
56a1ac2862 | ||
|
|
8d0acd1436 | ||
|
|
b6628af72f | ||
|
|
0169970a8b | ||
|
|
9cc27dfa59 | ||
|
|
9221d59b7e | ||
|
|
22047bb890 | ||
|
|
772216505c | ||
|
|
3d7541f3c7 | ||
|
|
5a59eef925 | ||
|
|
dc05f59514 | ||
|
|
f64ab4ebab | ||
|
|
19497a7470 | ||
|
|
823690bc76 | ||
|
|
82930d7f02 | ||
|
|
fa9b90beb1 | ||
|
|
d4d0c64400 | ||
|
|
cb70117c05 | ||
|
|
3d9c45e126 | ||
|
|
516df0c70e | ||
|
|
4cd2aa7c3e | ||
|
|
e6943a7389 | ||
|
|
a43c90f357 | ||
|
|
91a42b4fac | ||
|
|
34689d9a05 | ||
|
|
528b264f87 | ||
|
|
0580e2dc34 | ||
|
|
91c985eca9 | ||
|
|
3bb34fa1c9 | ||
|
|
f539b334dd | ||
|
|
9cfe59b1a2 | ||
|
|
fb59bf9325 | ||
|
|
2b2ff2dae5 | ||
|
|
0dd7d78c07 | ||
|
|
33daf1d822 | ||
|
|
9faad636cb | ||
|
|
b201dca6cf | ||
|
|
f1982e56e3 | ||
|
|
439c3f3be5 | ||
|
|
37449925fa |
@@ -1325,6 +1325,7 @@
|
||||
metabase.sync.core
|
||||
metabase.sync.init
|
||||
metabase.sync.schedules
|
||||
metabase.sync.sync-metadata.tables
|
||||
metabase.sync.task.sync-databases
|
||||
metabase.sync.util}
|
||||
:uses #{analyze
|
||||
@@ -2277,6 +2278,7 @@
|
||||
app-db
|
||||
config
|
||||
driver
|
||||
events
|
||||
lib
|
||||
lib-be
|
||||
query-processor
|
||||
@@ -2291,6 +2293,23 @@
|
||||
models
|
||||
premium-features
|
||||
upload
|
||||
util}}
|
||||
|
||||
enterprise/workspaces
|
||||
{:team "Metabot"
|
||||
:api #{metabase-enterprise.workspaces.api
|
||||
metabase-enterprise.workspaces.core
|
||||
metabase-enterprise.workspaces.init}
|
||||
:uses #{api
|
||||
api-keys
|
||||
driver
|
||||
enterprise/transforms
|
||||
events
|
||||
models
|
||||
queries
|
||||
request
|
||||
sync
|
||||
system
|
||||
util}}}
|
||||
|
||||
;; namespaces matching these patterns (with `re-find`) are excluded from the module linter. Since regex literals
|
||||
|
||||
@@ -93,6 +93,7 @@
|
||||
metabase-enterprise.test/with-gtaps! [[:inner 0]]
|
||||
metabase-enterprise.test/with-gtaps-for-user! [[:inner 0]]
|
||||
metabase-enterprise.test/with-user-attributes [[:block 2]]
|
||||
metabase-enterprise.workspaces.models.workspace-log/track! [[:block 2]]
|
||||
metabase.actions.actions-test/with-actions-test-data-and-actions-permissively-enabled [[:block 0]]
|
||||
metabase.actions.test-util/with-actions [[:block 1]]
|
||||
metabase.actions.test-util/with-actions-disabled [[:block 0]]
|
||||
|
||||
@@ -61,6 +61,7 @@
|
||||
(eval . (put-clojure-indent 'u/prog1 1))
|
||||
(eval . (put-clojure-indent 'u/select-keys-when '(:form)))
|
||||
(eval . (put-clojure-indent 'with-meta '(:form)))
|
||||
(eval . (put-clojure-indent 'ws.log/track! 2))
|
||||
;; these ones have to be done with `define-clojure-indent' for now because of upstream bug
|
||||
;; https://github.com/clojure-emacs/clojure-mode/issues/600 once that's resolved we should use `put-clojure-indent'
|
||||
;; instead. Please don't add new entries unless they don't work with `put-clojure-indent'
|
||||
|
||||
@@ -144,6 +144,18 @@
|
||||
change-log
|
||||
#(pos? (compare % "v49.00-000"))))
|
||||
|
||||
(defn require-primary-key-exists-has-table-name
|
||||
"Ensures that all primaryKeyExists preconditions specify a tableName."
|
||||
[change-log]
|
||||
(doseq [{:keys [changeSet]} change-log
|
||||
:when changeSet
|
||||
:let [s (pr-str (:preConditions changeSet))]
|
||||
:when (and (str/includes? s ":primaryKeyExists")
|
||||
(not (str/includes? s ":tableName")))]
|
||||
(throw (validation-error
|
||||
(format "Migration '%s' has primaryKeyExists precondition without tableName" (:id changeSet))
|
||||
{:id (:id changeSet)}))))
|
||||
|
||||
(s/def ::changeSet
|
||||
(s/spec :change-set.strict/change-set))
|
||||
|
||||
@@ -163,6 +175,7 @@
|
||||
(require-no-bare-blob-or-text-types change-log)
|
||||
(require-no-bare-boolean-types change-log)
|
||||
(require-no-datetime-type change-log)
|
||||
(require-primary-key-exists-has-table-name change-log)
|
||||
(let [{:keys [changeSet]
|
||||
:as all} (group-by only-key change-log)]
|
||||
(when-not (set/subset? (set (keys all)) #{:property :objectQuotingStrategy :changeSet})
|
||||
|
||||
@@ -75,7 +75,11 @@ export function entityPickerModalItem(level, name) {
|
||||
}
|
||||
|
||||
export function entityPickerModalTab(name) {
|
||||
return cy.findAllByRole("tab").filter(`:contains(${name})`);
|
||||
if (typeof name === "string") {
|
||||
return cy.findAllByRole("tab").filter(`:contains(${name})`);
|
||||
} else {
|
||||
return cy.findAllByRole("tab").first().parent().findByText(name);
|
||||
}
|
||||
}
|
||||
|
||||
// displays at least these tabs:
|
||||
|
||||
83
e2e/support/helpers/e2e-workspaces-helpers.ts
Normal file
83
e2e/support/helpers/e2e-workspaces-helpers.ts
Normal file
@@ -0,0 +1,83 @@
|
||||
import type { WorkspaceId } from "metabase-types/api";
|
||||
|
||||
export const Workspaces = {
|
||||
visit: (id: WorkspaceId) => {
|
||||
return cy.visit(`/data-studio/workspace/${id}`);
|
||||
},
|
||||
|
||||
visitDataStudio() {
|
||||
return cy.visit("/data-studio");
|
||||
},
|
||||
|
||||
visitTransformListPage() {
|
||||
return cy.visit("/data-studio/transforms");
|
||||
},
|
||||
|
||||
getWorkspacesPage() {
|
||||
return cy.findByTestId("workspaces-page");
|
||||
},
|
||||
|
||||
getWorkspacePage() {
|
||||
return cy.findByTestId("workspace-page");
|
||||
},
|
||||
|
||||
getWorkspaceContent() {
|
||||
return cy.findByTestId("workspace-content");
|
||||
},
|
||||
|
||||
getWorkspaceSidebar() {
|
||||
return cy.findByTestId("workspace-sidebar");
|
||||
},
|
||||
|
||||
getWorkspacesSection() {
|
||||
return cy.findByTestId("workspaces-section");
|
||||
},
|
||||
|
||||
getWorkspaceItem(name: string | RegExp) {
|
||||
return Workspaces.getWorkspacesSection().findByRole("button", { name });
|
||||
},
|
||||
|
||||
getWorkspaceItemActions(name: string | RegExp) {
|
||||
return Workspaces.getWorkspaceItem(name).findByLabelText("More actions");
|
||||
},
|
||||
|
||||
getWorkspaceNameInput() {
|
||||
return cy.findByPlaceholderText("Workspace name");
|
||||
},
|
||||
|
||||
getMergeWorkspaceButton() {
|
||||
return cy.findByRole("button", { name: /Merge/ });
|
||||
},
|
||||
|
||||
getTransformTargetButton() {
|
||||
return cy.findByRole("button", { name: /Change target/ });
|
||||
},
|
||||
|
||||
getTransformRunButton() {
|
||||
return cy.findByRole("button", { name: /Run/ });
|
||||
},
|
||||
|
||||
getTransformSaveButton() {
|
||||
return cy.findByRole("button", { name: /Save/ });
|
||||
},
|
||||
|
||||
getWorkspaceTransforms() {
|
||||
return cy.findByTestId("workspace-transforms");
|
||||
},
|
||||
|
||||
getMainlandTransforms() {
|
||||
return cy.findByTestId("mainland-transforms");
|
||||
},
|
||||
|
||||
getNewWorkspaceButton() {
|
||||
return cy.findByRole("button", { name: /New workspace/ });
|
||||
},
|
||||
|
||||
getNewWorkspaceNameInput() {
|
||||
return cy.findByPlaceholderText("Enter workspace name");
|
||||
},
|
||||
|
||||
getNewWorkspaceDatabaseInput() {
|
||||
return cy.findByPlaceholderText("Select a database");
|
||||
},
|
||||
};
|
||||
@@ -13,9 +13,9 @@ export * from "./e2e-comment-helpers";
|
||||
export * from "./e2e-custom-column-helpers";
|
||||
export * from "./e2e-dashboard-helpers";
|
||||
export * from "./e2e-dashboard-visualizer-helpers";
|
||||
export * from "./e2e-data-studio-helpers";
|
||||
export * from "./e2e-database-metadata-helpers";
|
||||
export * from "./e2e-datamodel-helpers";
|
||||
export * from "./e2e-data-studio-helpers";
|
||||
export * from "./e2e-dependency-graph-helpers";
|
||||
export * from "./e2e-detail-view-helpers";
|
||||
export * from "./e2e-dimension-list-helpers";
|
||||
@@ -59,4 +59,5 @@ export * from "./e2e-upload-helpers";
|
||||
export * from "./e2e-users-helpers";
|
||||
export * from "./e2e-visual-tests-helpers";
|
||||
export * from "./e2e-viz-settings-helpers";
|
||||
export * from "./e2e-workspaces-helpers";
|
||||
export * from "./embedding-sdk-helpers/constants";
|
||||
|
||||
349
e2e/test/scenarios/data-studio/workspaces.cy.spec.ts
Normal file
349
e2e/test/scenarios/data-studio/workspaces.cy.spec.ts
Normal file
@@ -0,0 +1,349 @@
|
||||
import dedent from "ts-dedent";
|
||||
|
||||
import { SAMPLE_DB_ID, WRITABLE_DB_ID } from "e2e/support/cypress_data";
|
||||
import { SAMPLE_DATABASE } from "e2e/support/cypress_sample_database";
|
||||
import type {
|
||||
CardType,
|
||||
PythonTransformTableAliases,
|
||||
TransformTagId,
|
||||
} from "metabase-types/api";
|
||||
|
||||
const { H } = cy;
|
||||
const { DataStudio, Workspaces } = H;
|
||||
const { Transforms } = DataStudio;
|
||||
|
||||
const { ORDERS_ID } = SAMPLE_DATABASE;
|
||||
|
||||
const DB_NAME = "Writable Postgres12";
|
||||
const SOURCE_TABLE = "Animals";
|
||||
const TARGET_TABLE = "transform_table";
|
||||
const TARGET_TABLE_2 = "transform_table_2";
|
||||
const TARGET_SCHEMA = "Schema A";
|
||||
const TARGET_SCHEMA_2 = "Schema B";
|
||||
const CUSTOM_SCHEMA = "custom_schema";
|
||||
|
||||
describe("scenarios > data studio > workspaces", () => {
|
||||
beforeEach(() => {
|
||||
H.restore("postgres-writable");
|
||||
H.resetTestTable({ type: "postgres", table: "many_schemas" });
|
||||
H.resetSnowplow();
|
||||
cy.signInAsAdmin();
|
||||
H.activateToken("bleeding-edge");
|
||||
H.resyncDatabase({ dbId: WRITABLE_DB_ID, tableName: SOURCE_TABLE });
|
||||
|
||||
// cy.intercept("PUT", "/api/field/*").as("updateField");
|
||||
// cy.intercept("POST", "/api/ee/transform").as("createTransform");
|
||||
// cy.intercept("PUT", "/api/ee/transform/*").as("updateTransform");
|
||||
// cy.intercept("DELETE", "/api/ee/transform/*").as("deleteTransform");
|
||||
// cy.intercept("DELETE", "/api/ee/transform/*/table").as(
|
||||
// "deleteTransformTable",
|
||||
// );
|
||||
// cy.intercept("POST", "/api/ee/transform-tag").as("createTag");
|
||||
// cy.intercept("PUT", "/api/ee/transform-tag/*").as("updateTag");
|
||||
// cy.intercept("DELETE", "/api/ee/transform-tag/*").as("deleteTag");
|
||||
});
|
||||
|
||||
// afterEach(() => {
|
||||
// H.expectNoBadSnowplowEvents();
|
||||
// });
|
||||
|
||||
it("should be able to create, navigate, archive, and rename workspaces", () => {
|
||||
Workspaces.visitDataStudio();
|
||||
|
||||
Workspaces.getWorkspacesSection()
|
||||
.findByText("No workspaces yet")
|
||||
.should("be.visible");
|
||||
|
||||
createWorkspace("Workspace A");
|
||||
|
||||
cy.location("pathname").should("match", /data-studio\/workspaces\/\d+/);
|
||||
|
||||
cy.log("shows workspace name");
|
||||
Workspaces.getWorkspaceNameInput().should("have.value", "Workspace A");
|
||||
Workspaces.getMergeWorkspaceButton().should("be.disabled");
|
||||
|
||||
Workspaces.getWorkspaceContent().within(() => {
|
||||
cy.log("starts on setup tab, and has only 2 tabs");
|
||||
H.tabsShouldBe("Setup", ["Setup", "Agent Chat"]);
|
||||
|
||||
cy.log("shows workspace db");
|
||||
cy.findByText("Writable Postgres12").should("be.visible");
|
||||
|
||||
cy.log("shows workspace setup logs");
|
||||
cy.findByText("Provisioning database isolation").should("be.visible");
|
||||
cy.findByText("Setting up the workspace").should("be.visible");
|
||||
cy.findByText("Workspace ready!").should("be.visible");
|
||||
});
|
||||
|
||||
Workspaces.getWorkspaceSidebar().within(() => {
|
||||
cy.log("starts on Code tab, and has only 2 tabs");
|
||||
H.tabsShouldBe("Code", ["Code", "Data"]);
|
||||
|
||||
cy.log("shows transforms list");
|
||||
cy.findByText("Workspace is empty").should("be.visible");
|
||||
});
|
||||
|
||||
createWorkspace("Workspace B");
|
||||
Workspaces.getWorkspaceNameInput().should("have.value", "Workspace B");
|
||||
|
||||
Workspaces.getWorkspacesSection().within(() => {
|
||||
cy.findByText("Workspace B").should("be.visible");
|
||||
cy.findByText("Workspace A").should("be.visible").click();
|
||||
});
|
||||
|
||||
Workspaces.getWorkspaceNameInput().should("have.value", "Workspace A");
|
||||
|
||||
cy.log("can archive a workspace");
|
||||
Workspaces.getWorkspaceItemActions(/Workspace A/).click();
|
||||
H.popover().findByText("Archive").click();
|
||||
H.undoToast().should("have.text", "Workspace archived successfully");
|
||||
Workspaces.getWorkspaceItem(/Workspace A/).should("not.exist");
|
||||
cy.location("pathname").should("eq", "/data-studio/workspaces");
|
||||
|
||||
Workspaces.getWorkspacesPage().within(() => {
|
||||
cy.findByText("Workspaces").should("be.visible");
|
||||
cy.findByText("Workspace A").should("not.exist");
|
||||
cy.log("can navigate from workspaces list to a workspace");
|
||||
cy.findByText("Workspace B").should("be.visible").click();
|
||||
});
|
||||
|
||||
cy.location("pathname").should("match", /data-studio\/workspaces\/\d+/);
|
||||
Workspaces.getWorkspaceNameInput().should("have.value", "Workspace B");
|
||||
|
||||
Workspaces.getWorkspaceNameInput().clear().type("Renamed workspace").blur();
|
||||
// H.undoToast().should("have.text", "Workspace renamed"); // TODO: uncomment when implemented
|
||||
Workspaces.getWorkspaceItem(/Renamed workspace/).should("be.visible");
|
||||
|
||||
// TODO: workspace deletion?
|
||||
});
|
||||
|
||||
it("should be able to check out exisitng transforms into a new workspace from the transform page", () => {
|
||||
const sourceTable = `${TARGET_SCHEMA}.${SOURCE_TABLE}`;
|
||||
|
||||
createTransforms({ visit: true });
|
||||
|
||||
cy.findByRole("button", { name: /Edit transform/ }).click();
|
||||
H.popover().within(() => {
|
||||
cy.findByText("No workspaces yet").should("be.visible");
|
||||
cy.findByText("New workspace").should("be.visible").click();
|
||||
});
|
||||
|
||||
Workspaces.getNewWorkspaceNameInput().should("have.value", "New workspace");
|
||||
Workspaces.getNewWorkspaceDatabaseInput().should(
|
||||
"have.value",
|
||||
"Writable Postgres12",
|
||||
);
|
||||
H.modal().findByText("Create").click();
|
||||
|
||||
cy.location("pathname").should("match", /data-studio\/workspaces\/\d+/);
|
||||
Workspaces.getWorkspaceNameInput().should("have.value", "New workspace");
|
||||
|
||||
Workspaces.getWorkspacePage()
|
||||
.findByText("Workspace is empty")
|
||||
.should("not.exist");
|
||||
Workspaces.getMainlandTransforms()
|
||||
.findByText("Python transform")
|
||||
.should("be.visible");
|
||||
Workspaces.getMainlandTransforms()
|
||||
.findByText("MBQL transform")
|
||||
.should("not.exist");
|
||||
Workspaces.getWorkspaceTransforms()
|
||||
.findByText("SQL transform")
|
||||
.should("be.visible")
|
||||
.click();
|
||||
|
||||
Workspaces.getWorkspaceContent().within(() => {
|
||||
H.tabsShouldBe("SQL transform", ["Setup", "Agent Chat", "SQL transform"]);
|
||||
});
|
||||
Workspaces.getMergeWorkspaceButton().should("be.enabled");
|
||||
Workspaces.getTransformTargetButton().should("be.enabled");
|
||||
Workspaces.getTransformRunButton().should("be.enabled");
|
||||
Workspaces.getTransformSaveButton().should("be.disabled");
|
||||
|
||||
H.NativeEditor.type(" LIMIT 2");
|
||||
Workspaces.getMergeWorkspaceButton().should("be.disabled");
|
||||
Workspaces.getTransformTargetButton().should("be.disabled");
|
||||
Workspaces.getTransformRunButton().should("be.disabled");
|
||||
Workspaces.getTransformSaveButton().should("be.enabled").click();
|
||||
|
||||
Workspaces.getMergeWorkspaceButton().should("be.enabled");
|
||||
Workspaces.getTransformTargetButton().should("be.enabled");
|
||||
Workspaces.getTransformSaveButton().should("be.disabled");
|
||||
Workspaces.getTransformRunButton().should("be.enabled").click();
|
||||
|
||||
Workspaces.getWorkspaceSidebar().within(() => {
|
||||
cy.findByRole("tab", { name: "Data" }).click();
|
||||
cy.findByLabelText(sourceTable).should("be.visible").click();
|
||||
});
|
||||
|
||||
Workspaces.getWorkspaceContent().within(() => {
|
||||
H.tabsShouldBe(sourceTable, [
|
||||
"Setup",
|
||||
"Agent Chat",
|
||||
"SQL transform",
|
||||
sourceTable,
|
||||
]);
|
||||
H.assertTableData({
|
||||
columns: ["Name", "Score"],
|
||||
firstRows: [
|
||||
["Duck", "10"],
|
||||
["Horse", "20"],
|
||||
["Cow", "30"],
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
Workspaces.getWorkspaceSidebar().within(() => {
|
||||
cy.findByRole("tab", { name: "Data" }).click();
|
||||
cy.findByLabelText(/mb__isolation_/)
|
||||
.should("be.visible")
|
||||
.click();
|
||||
});
|
||||
|
||||
Workspaces.getWorkspaceContent().within(() => {
|
||||
H.tabsShouldBe(/mb__isolation_/, [
|
||||
"Setup",
|
||||
"Agent Chat",
|
||||
"SQL transform",
|
||||
sourceTable,
|
||||
/mb__isolation_/,
|
||||
]);
|
||||
H.assertTableData({
|
||||
columns: ["Name", "Score"],
|
||||
firstRows: [
|
||||
["Duck", "10"],
|
||||
["Horse", "20"],
|
||||
],
|
||||
});
|
||||
cy.findByText("Cow").should("not.exist");
|
||||
cy.findByText("30").should("not.exist");
|
||||
});
|
||||
|
||||
Workspaces.getMergeWorkspaceButton().click();
|
||||
|
||||
cy.findByRole("link", { name: /SQL transform/ }).click();
|
||||
H.NativeEditor.value().should(
|
||||
"eq",
|
||||
'SELECT * FROM "Schema A"."Animals" LIMIT 2',
|
||||
);
|
||||
Transforms.runTab().click();
|
||||
runTransformAndWaitForSuccess();
|
||||
Transforms.targetTab().click();
|
||||
getTableLink().should("contain.text", "transform_table").click();
|
||||
|
||||
H.assertTableData({
|
||||
columns: ["Name", "Score"],
|
||||
firstRows: [
|
||||
["Duck", "10"],
|
||||
["Horse", "20"],
|
||||
],
|
||||
});
|
||||
H.tableInteractiveBody().findByText("Cow").should("not.exist");
|
||||
H.tableInteractiveBody().findByText("30").should("not.exist");
|
||||
});
|
||||
|
||||
// it("should be able to check out existing transform into a new workspace from the workspace page", () => {
|
||||
// createTransforms({ visit: true });
|
||||
// });
|
||||
});
|
||||
|
||||
function createWorkspace(name: string) {
|
||||
Workspaces.getNewWorkspaceButton().click();
|
||||
H.modal().findByText("Create new workspace").should("be.visible");
|
||||
Workspaces.getNewWorkspaceNameInput().clear().type(name);
|
||||
Workspaces.getNewWorkspaceDatabaseInput().click();
|
||||
H.popover().within(() => {
|
||||
// cy.findByText("Internal Metabase Database").should("not.exist"); // TODO: uncomment once it works
|
||||
// cy.findByText("Sample Database").should("not.exist"); // TODO: uncomment once it works
|
||||
cy.findByText("Writable Postgres12").should("be.visible").click();
|
||||
});
|
||||
H.modal().findByText("Create").click();
|
||||
}
|
||||
|
||||
function createTransforms({ visit }: { visit?: boolean } = {}) {
|
||||
createMbqlTransform({
|
||||
targetTable: TARGET_TABLE,
|
||||
visitTransform: true,
|
||||
});
|
||||
|
||||
H.getTableId({ name: "Animals", databaseId: WRITABLE_DB_ID }).then((id) => {
|
||||
createPythonTransform({
|
||||
body: dedent`
|
||||
import pandas as pd
|
||||
|
||||
def transform(foo):
|
||||
return pd.DataFrame([{"foo": 42 }])
|
||||
`,
|
||||
sourceTables: { foo: id },
|
||||
});
|
||||
});
|
||||
|
||||
createSqlTransform({
|
||||
sourceQuery: `SELECT * FROM "${TARGET_SCHEMA}"."${SOURCE_TABLE}"`,
|
||||
visitTransform: visit,
|
||||
});
|
||||
}
|
||||
|
||||
function createMbqlTransform(
|
||||
opts: {
|
||||
sourceTable?: string;
|
||||
targetTable?: string;
|
||||
targetSchema?: string | null;
|
||||
tagIds?: TransformTagId[];
|
||||
name?: string;
|
||||
databaseId?: number;
|
||||
visitTransform?: boolean;
|
||||
} = {},
|
||||
) {
|
||||
return H.createMbqlTransform({
|
||||
sourceTable: SOURCE_TABLE,
|
||||
targetTable: TARGET_TABLE,
|
||||
targetSchema: TARGET_SCHEMA,
|
||||
name: "MBQL transform",
|
||||
...opts,
|
||||
});
|
||||
}
|
||||
|
||||
function createSqlTransform(opts: {
|
||||
sourceQuery: string;
|
||||
targetTable?: string;
|
||||
targetSchema?: string;
|
||||
tagIds?: TransformTagId[];
|
||||
visitTransform?: boolean;
|
||||
}) {
|
||||
return H.createSqlTransform({
|
||||
targetTable: TARGET_TABLE,
|
||||
targetSchema: TARGET_SCHEMA,
|
||||
...opts,
|
||||
});
|
||||
}
|
||||
|
||||
function createPythonTransform(opts: {
|
||||
body: string;
|
||||
sourceTables: PythonTransformTableAliases;
|
||||
targetTable?: string;
|
||||
targetSchema?: string;
|
||||
tagIds?: TransformTagId[];
|
||||
visitTransform?: boolean;
|
||||
}) {
|
||||
return H.createPythonTransform({
|
||||
targetTable: TARGET_TABLE,
|
||||
targetSchema: TARGET_SCHEMA,
|
||||
...opts,
|
||||
});
|
||||
}
|
||||
|
||||
function getTableLink({ isActive = true }: { isActive?: boolean } = {}) {
|
||||
return cy
|
||||
.findByTestId("table-link")
|
||||
.should("have.attr", "aria-disabled", String(!isActive));
|
||||
}
|
||||
|
||||
function runTransformAndWaitForSuccess() {
|
||||
getRunButton().click();
|
||||
getRunButton().should("have.text", "Ran successfully");
|
||||
}
|
||||
|
||||
function getRunButton(options: { timeout?: number } = {}) {
|
||||
return cy.findAllByTestId("run-button").eq(0, options);
|
||||
}
|
||||
@@ -42,6 +42,7 @@
|
||||
[metabase-enterprise.transforms-python.api]
|
||||
[metabase-enterprise.transforms.api]
|
||||
[metabase-enterprise.upload-management.api]
|
||||
[metabase-enterprise.workspaces.api]
|
||||
[metabase.api.macros :as api.macros]
|
||||
[metabase.api.routes.common :as api.routes.common]
|
||||
[metabase.api.util.handlers :as handlers]
|
||||
@@ -139,7 +140,8 @@
|
||||
"/transform" (premium-handler metabase-enterprise.transforms.api/routes :transforms)
|
||||
"/transform-job" (premium-handler metabase-enterprise.transforms.api/transform-job-routes :transforms)
|
||||
"/transform-tag" (premium-handler metabase-enterprise.transforms.api/transform-tag-routes :transforms)
|
||||
"/upload-management" (premium-handler metabase-enterprise.upload-management.api/routes :upload-management)})
|
||||
"/upload-management" (premium-handler metabase-enterprise.upload-management.api/routes :upload-management)
|
||||
"/workspace" metabase-enterprise.workspaces.api/routes})
|
||||
;;; ↑↑↑ KEEP THIS SORTED OR ELSE ↑↑↑
|
||||
|
||||
(def ^:private routes-map
|
||||
|
||||
@@ -24,4 +24,5 @@
|
||||
[metabase-enterprise.stale.init]
|
||||
[metabase-enterprise.support-access-grants.init]
|
||||
[metabase-enterprise.transforms-python.init]
|
||||
[metabase-enterprise.transforms.init]))
|
||||
[metabase-enterprise.transforms.init]
|
||||
[metabase-enterprise.workspaces.init]))
|
||||
|
||||
@@ -41,6 +41,12 @@
|
||||
query-deps
|
||||
param-card-ids)))
|
||||
|
||||
(mu/defn upstream-deps:python-transform :- ::deps.schema/upstream-deps
|
||||
"Given a Toucan `:model/Transform`, return its upstream dependencies as a map from the kind to a set of IDs."
|
||||
[{{tables :source-tables} :source :as _py-transform}
|
||||
:- [:map [:source-tables {:optional true} [:map-of :string :int]]]]
|
||||
{:table (set (vals tables))})
|
||||
|
||||
(mu/defn upstream-deps:transform :- ::deps.schema/upstream-deps
|
||||
"Given a Transform (in Toucan form), return its upstream dependencies."
|
||||
[{{:keys [query], source-type :type} :source :as transform} :-
|
||||
@@ -49,9 +55,15 @@
|
||||
[:query
|
||||
[:map [:query ::lib.schema/query]]]
|
||||
[:python
|
||||
[:map]]]]]]
|
||||
(if (= (keyword source-type) :query)
|
||||
[:map [:source-tables {:optional true} [:map-of :string :int]]]]]]]]
|
||||
(cond
|
||||
(= (keyword source-type) :query)
|
||||
(upstream-deps:query query)
|
||||
|
||||
(= (keyword source-type) :python)
|
||||
(upstream-deps:python-transform transform)
|
||||
|
||||
:else
|
||||
(do (log/warnf "Don't know how to analyze the deps of Transform %d with source type '%s'" (:id transform) source-type)
|
||||
{})))
|
||||
|
||||
|
||||
@@ -115,4 +115,7 @@
|
||||
"User"
|
||||
"UserKeyValue"
|
||||
"UserParameterValue"
|
||||
"ViewLog"])
|
||||
"ViewLog"
|
||||
"Workspace"
|
||||
"WorkspaceMappingTable"
|
||||
"WorkspaceMappingTransform"])
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
[metabase-enterprise.transforms.api.transform-job]
|
||||
[metabase-enterprise.transforms.api.transform-tag]
|
||||
[metabase-enterprise.transforms.canceling :as transforms.canceling]
|
||||
[metabase-enterprise.transforms.execute :as transforms.execute]
|
||||
[metabase-enterprise.transforms.interface :as transforms.i]
|
||||
[metabase-enterprise.transforms.models.transform :as transform.model]
|
||||
[metabase-enterprise.transforms.models.transform-run :as transform-run]
|
||||
@@ -79,14 +80,32 @@
|
||||
[402 (deferred-tru "Premium features required for this transform type are not enabled.")]))
|
||||
|
||||
(defn get-transforms
|
||||
"Get a list of transforms."
|
||||
[& {:keys [last_run_start_time last_run_statuses tag_ids]}]
|
||||
"Get a list of transforms.
|
||||
By default, only returns global transforms (workspace_id=null).
|
||||
When `exclude_workspace_id` is provided, additionally excludes global transforms
|
||||
that have been mirrored into the specified workspace (i.e., they have a
|
||||
workspace_mapping_transform entry pointing to a transform in that workspace)."
|
||||
[& {:keys [last_run_start_time last_run_statuses tag_ids exclude_workspace_id database_id type]}]
|
||||
(api/check-superuser)
|
||||
(let [transforms (t2/select :model/Transform {:order-by [[:id :asc]]})]
|
||||
(let [transforms (if exclude_workspace_id
|
||||
(t2/select :model/Transform
|
||||
{:left-join [[:workspace_mapping_transform :wmt]
|
||||
[:and
|
||||
[:= :transform.id :wmt.upstream_id]
|
||||
[:= :wmt.workspace_id exclude_workspace_id]]]
|
||||
:where [:and
|
||||
[:= :transform.workspace_id nil]
|
||||
[:= :wmt.upstream_id nil]]
|
||||
:order-by [[:id :asc]]})
|
||||
(t2/select :model/Transform
|
||||
{:where [:= :workspace_id nil]
|
||||
:order-by [[:id :asc]]}))]
|
||||
(into []
|
||||
(comp (transforms.util/->date-field-filter-xf [:last_run :start_time] last_run_start_time)
|
||||
(transforms.util/->status-filter-xf [:last_run :status] last_run_statuses)
|
||||
(transforms.util/->tag-filter-xf [:tag_ids] tag_ids)
|
||||
(transforms.util/->database-id-filter-xf database_id)
|
||||
(transforms.util/->type-filter-xf type)
|
||||
(map #(update % :last_run transforms.util/localize-run-timestamps)))
|
||||
(t2/hydrate transforms :last_run :transform_tag_ids :creator))))
|
||||
|
||||
@@ -94,13 +113,18 @@
|
||||
;; of the REST API
|
||||
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-query-params-use-kebab-case]}
|
||||
(api.macros/defendpoint :get "/"
|
||||
"Get a list of transforms."
|
||||
"Get a list of global transforms (workspace_id=null).
|
||||
When `exclude_workspace_id` is provided, additionally excludes global transforms
|
||||
that have been mirrored into the specified workspace."
|
||||
[_route-params
|
||||
query-params :-
|
||||
[:map
|
||||
[:last_run_start_time {:optional true} [:maybe ms/NonBlankString]]
|
||||
[:last_run_statuses {:optional true} [:maybe (ms/QueryVectorOf [:enum "started" "succeeded" "failed" "timeout"])]]
|
||||
[:tag_ids {:optional true} [:maybe (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]]]]
|
||||
[:tag_ids {:optional true} [:maybe (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]]
|
||||
[:exclude_workspace_id {:optional true} [:maybe ms/PositiveInt]]
|
||||
[:database_id {:optional true} [:maybe ms/PositiveInt]]
|
||||
[:type {:optional true} [:maybe (ms/QueryVectorOf [:enum "query" "native" "python"])]]]]
|
||||
(get-transforms query-params))
|
||||
|
||||
(api.macros/defendpoint :post "/"
|
||||
@@ -186,19 +210,10 @@
|
||||
:limit (request/limit)))
|
||||
(update :data #(map transforms.util/localize-run-timestamps %))))
|
||||
|
||||
(api.macros/defendpoint :put "/:id"
|
||||
"Update a transform."
|
||||
[{:keys [id]} :- [:map
|
||||
[:id ms/PositiveInt]]
|
||||
_query-params
|
||||
body :- [:map
|
||||
[:name {:optional true} :string]
|
||||
[:description {:optional true} [:maybe :string]]
|
||||
[:source {:optional true} ::transforms.schema/transform-source]
|
||||
[:target {:optional true} ::transforms.schema/transform-target]
|
||||
[:run_trigger {:optional true} ::run-trigger]
|
||||
[:tag_ids {:optional true} [:sequential ms/PositiveInt]]]]
|
||||
(api/check-superuser)
|
||||
(defn update-transform!
|
||||
"Update a transform. Validates features, database support, cycles, and target conflicts.
|
||||
Returns the updated transform with hydrated associations."
|
||||
[id body]
|
||||
(let [transform (t2/with-transaction [_]
|
||||
;; Cycle detection should occur within the transaction to avoid race
|
||||
(let [old (t2/select-one :model/Transform id)
|
||||
@@ -223,17 +238,36 @@
|
||||
(events/publish-event! :event/transform-update {:object transform :user-id api/*current-user-id*})
|
||||
transform))
|
||||
|
||||
(api.macros/defendpoint :put "/:id"
|
||||
"Update a transform."
|
||||
[{:keys [id]} :- [:map
|
||||
[:id ms/PositiveInt]]
|
||||
_query-params
|
||||
body :- [:map
|
||||
[:name {:optional true} :string]
|
||||
[:description {:optional true} [:maybe :string]]
|
||||
[:source {:optional true} ::transforms.schema/transform-source]
|
||||
[:target {:optional true} ::transforms.schema/transform-target]
|
||||
[:run_trigger {:optional true} ::run-trigger]
|
||||
[:tag_ids {:optional true} [:sequential ms/PositiveInt]]]]
|
||||
(api/check-superuser)
|
||||
(update-transform! id body))
|
||||
|
||||
(defn delete-transform!
|
||||
"Delete a transform and publish the delete event."
|
||||
[transform]
|
||||
(t2/delete! :model/Transform (:id transform))
|
||||
(events/publish-event! :event/transform-delete
|
||||
{:object transform
|
||||
:user-id api/*current-user-id*})
|
||||
nil)
|
||||
|
||||
(api.macros/defendpoint :delete "/:id"
|
||||
"Delete a transform."
|
||||
[{:keys [id]} :- [:map
|
||||
[:id ms/PositiveInt]]]
|
||||
(api/check-superuser)
|
||||
(let [transform (api/check-404 (t2/select-one :model/Transform id))]
|
||||
(t2/delete! :model/Transform id)
|
||||
(events/publish-event! :event/transform-delete
|
||||
{:object transform
|
||||
:user-id api/*current-user-id*}))
|
||||
nil)
|
||||
(delete-transform! (api/check-404 (t2/select-one :model/Transform id))))
|
||||
|
||||
(api.macros/defendpoint :delete "/:id/table"
|
||||
"Delete a transform's output table."
|
||||
@@ -255,17 +289,15 @@
|
||||
(transforms.canceling/cancel-run! (:id run))))
|
||||
nil)
|
||||
|
||||
(api.macros/defendpoint :post "/:id/run"
|
||||
"Run a transform."
|
||||
[{:keys [id]} :- [:map
|
||||
[:id ms/PositiveInt]]]
|
||||
(api/check-superuser)
|
||||
(let [transform (api/check-404 (t2/select-one :model/Transform id))
|
||||
_ (check-feature-enabled! transform)
|
||||
start-promise (promise)]
|
||||
(defn run-transform!
|
||||
"Run a transform. Returns a 202 response with run_id.
|
||||
The transform must already be fetched and validated."
|
||||
[transform]
|
||||
(check-feature-enabled! transform)
|
||||
(let [start-promise (promise)]
|
||||
(u.jvm/in-virtual-thread*
|
||||
(transforms.i/execute! transform {:start-promise start-promise
|
||||
:run-method :manual}))
|
||||
(transforms.execute/execute! transform {:start-promise start-promise
|
||||
:run-method :manual}))
|
||||
(when (instance? Throwable @start-promise)
|
||||
(throw @start-promise))
|
||||
(let [result @start-promise
|
||||
@@ -275,6 +307,13 @@
|
||||
:run_id run-id})
|
||||
(assoc :status 202)))))
|
||||
|
||||
(api.macros/defendpoint :post "/:id/run"
|
||||
"Run a transform."
|
||||
[{:keys [id]} :- [:map
|
||||
[:id ms/PositiveInt]]]
|
||||
(api/check-superuser)
|
||||
(run-transform! (api/check-404 (t2/select-one :model/Transform id))))
|
||||
|
||||
(defn- extract-columns-from-query
|
||||
"Attempts to extract column names from an MBQL query without executing it.
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
(ns metabase-enterprise.transforms.core
|
||||
"API namespace for the `metabase-enterprise.transform` module."
|
||||
(:require
|
||||
[metabase-enterprise.transforms.models.transform]
|
||||
[metabase-enterprise.transforms.models.transform-run]
|
||||
[metabase-enterprise.transforms.settings]
|
||||
[metabase-enterprise.transforms.util]
|
||||
@@ -14,4 +15,6 @@
|
||||
python-transform?
|
||||
query-transform?]
|
||||
[metabase-enterprise.transforms.models.transform-run
|
||||
timeout-run!])
|
||||
timeout-run!]
|
||||
[metabase-enterprise.transforms.models.transform
|
||||
update-transform-tags!])
|
||||
|
||||
@@ -1,81 +1,22 @@
|
||||
(ns metabase-enterprise.transforms.execute
|
||||
(:require
|
||||
[metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
|
||||
[metabase-enterprise.transforms.util :as transforms.util]
|
||||
[metabase.driver :as driver]
|
||||
[metabase.driver.util :as driver.u]
|
||||
[metabase.events.core :as events]
|
||||
[metabase.lib.schema.common :as schema.common]
|
||||
[metabase.query-processor.compile :as qp.compile]
|
||||
[metabase.util.log :as log]
|
||||
[metabase.util.malli.registry :as mr]
|
||||
[toucan2.core :as t2]))
|
||||
[metabase-enterprise.transforms.interface :as transforms.i]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(mr/def ::transform-details
|
||||
[:map
|
||||
[:transform-type [:enum {:decode/normalize schema.common/normalize-keyword} :table :table-incremental]]
|
||||
[:conn-spec :any]
|
||||
[:query ::qp.compile/compiled]
|
||||
[:output-table [:keyword {:decode/normalize schema.common/normalize-keyword}]]])
|
||||
(defn execute!
|
||||
"Execute a transform.
|
||||
|
||||
(mr/def ::transform-opts
|
||||
[:map
|
||||
[:overwrite? :boolean]])
|
||||
|
||||
(defn- transform-opts [{:keys [transform-type]}]
|
||||
(case transform-type
|
||||
:table {:overwrite? true}
|
||||
|
||||
;; once we have more than just append, dispatch on :target-incremental-strategy
|
||||
:table-incremental {}))
|
||||
|
||||
(defn run-mbql-transform!
|
||||
"Run `transform` and sync its target table.
|
||||
|
||||
This is executing synchronously, but supports being kicked off in the background
|
||||
by delivering the `start-promise` just before the start when the beginning of the execution has been booked
|
||||
in the database."
|
||||
([transform] (run-mbql-transform! transform nil))
|
||||
([{:keys [id source target] :as transform} {:keys [run-method start-promise]}]
|
||||
(try
|
||||
(let [db (get-in source [:query :database])
|
||||
{driver :engine :as database} (t2/select-one :model/Database db)
|
||||
transform-details {:db-id db
|
||||
:database database
|
||||
:transform-id id
|
||||
:transform-type (keyword (:type target))
|
||||
:conn-spec (driver/connection-spec driver database)
|
||||
:query (transforms.util/compile-source transform)
|
||||
:output-schema (:schema target)
|
||||
:output-table (transforms.util/qualified-table-name driver target)}
|
||||
opts (transform-opts transform-details)
|
||||
features (transforms.util/required-database-features transform)]
|
||||
|
||||
(when (transforms.util/db-routing-enabled? database)
|
||||
(throw (ex-info "Transforms are not supported on databases with DB routing enabled."
|
||||
{:driver driver, :database database})))
|
||||
(when-not (every? (fn [feature] (driver.u/supports? (:engine database) feature database)) features)
|
||||
(throw (ex-info "The database does not support the requested transform target type."
|
||||
{:driver driver, :database database, :features features})))
|
||||
;; mark the execution as started and notify any observers
|
||||
(let [{run-id :id} (transforms.util/try-start-unless-already-running id run-method)]
|
||||
(when start-promise
|
||||
(deliver start-promise [:started run-id]))
|
||||
(log/info "Executing transform" id "with target" (pr-str target))
|
||||
(transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]]
|
||||
(transforms.util/run-cancelable-transform!
|
||||
run-id driver transform-details
|
||||
(fn [_cancel-chan] (driver/run-transform! driver transform-details opts))))
|
||||
(transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]
|
||||
(transforms.util/sync-target! target database)
|
||||
;; This event must be published only after the sync is complete - the new table needs to be in AppDB.
|
||||
(events/publish-event! :event/transform-run-complete {:object transform-details}))))
|
||||
(catch Throwable t
|
||||
(log/error t "Error executing transform")
|
||||
(when start-promise
|
||||
;; if the start-promise has been delivered, this is a no-op,
|
||||
;; but we assume nobody would catch the exception anyway
|
||||
(deliver start-promise t))
|
||||
(throw t)))))
|
||||
For workspace transforms, execution must go through the workspace-specific endpoint.
|
||||
Calling this function directly with a workspace transform will throw a 404 error."
|
||||
[transform {:keys [start-promise] :as opts}]
|
||||
;; This is used on FE for workspace transform execution until proper API is provided in workspaces module!
|
||||
;; Hence temp enabling that.
|
||||
;; TODO: Cleanup when moving to woskapce transform execution api.
|
||||
(if false #_(:workspace_id transform)
|
||||
(deliver start-promise
|
||||
(ex-info "Workspace transforms must be executed through the workspace API"
|
||||
{:status-code 404
|
||||
:transform-id (:id transform)
|
||||
:workspace-id (:workspace_id transform)}))
|
||||
(transforms.i/execute! transform opts)))
|
||||
|
||||
@@ -223,12 +223,14 @@
|
||||
[_model-name opts]
|
||||
{:copy [:name :description :entity_id]
|
||||
:skip [:dependency_analysis_version :source_type]
|
||||
:transform {:created_at (serdes/date)
|
||||
:creator_id (serdes/fk :model/User)
|
||||
:source {:export #(update % :query serdes/export-mbql)
|
||||
:import #(update % :query serdes/import-mbql)}
|
||||
:target {:export serdes/export-mbql :import serdes/import-mbql}
|
||||
:tags (serdes/nested :model/TransformTransformTag :transform_id opts)}})
|
||||
:transform {:created_at (serdes/date)
|
||||
:creator_id (serdes/fk :model/User)
|
||||
:workspace_id (serdes/fk :model/Workspace)
|
||||
:collection_id (serdes/fk :model/Collection)
|
||||
:source {:export #(update % :query serdes/export-mbql)
|
||||
:import #(update % :query serdes/import-mbql)}
|
||||
:target {:export serdes/export-mbql :import serdes/import-mbql}
|
||||
:tags (serdes/nested :model/TransformTransformTag :transform_id opts)}})
|
||||
|
||||
(defmethod serdes/dependencies "Transform"
|
||||
[{:keys [source tags]}]
|
||||
|
||||
@@ -1,7 +1,15 @@
|
||||
(ns metabase-enterprise.transforms.query-impl
|
||||
(:require
|
||||
[metabase-enterprise.transforms.execute :as transforms.execute]
|
||||
[metabase-enterprise.transforms.interface :as transforms.i]))
|
||||
[metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
|
||||
[metabase-enterprise.transforms.interface :as transforms.i]
|
||||
[metabase-enterprise.transforms.util :as transforms.util]
|
||||
[metabase.driver :as driver]
|
||||
[metabase.driver.util :as driver.u]
|
||||
[metabase.events.core :as events]
|
||||
[metabase.lib.schema.common :as schema.common]
|
||||
[metabase.util.log :as log]
|
||||
[metabase.util.malli.registry :as mr]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
@@ -13,5 +21,71 @@
|
||||
[transform]
|
||||
(-> transform :source :query :database))
|
||||
|
||||
(mr/def ::transform-details
|
||||
[:map
|
||||
[:transform-type [:enum {:decode/normalize schema.common/normalize-keyword} :table]]
|
||||
[:conn-spec :any]
|
||||
[:query :string]
|
||||
[:output-table [:keyword {:decode/normalize schema.common/normalize-keyword}]]])
|
||||
|
||||
(mr/def ::transform-opts
|
||||
[:map
|
||||
[:overwrite? :boolean]])
|
||||
|
||||
(defn- transform-opts [{:keys [transform-type]}]
|
||||
(case transform-type
|
||||
:table {:overwrite? true}
|
||||
|
||||
;; once we have more than just append, dispatch on :target-incremental-strategy
|
||||
:table-incremental {}))
|
||||
|
||||
(defn run-mbql-transform!
|
||||
"Run `transform` and sync its target table.
|
||||
|
||||
This is executing synchronously, but supports being kicked off in the background
|
||||
by delivering the `start-promise` just before the start when the beginning of the execution has been booked
|
||||
in the database."
|
||||
([transform] (run-mbql-transform! transform nil))
|
||||
([{:keys [id source target] :as transform} {:keys [run-method start-promise]}]
|
||||
(try
|
||||
(let [db (get-in source [:query :database])
|
||||
{driver :engine :as database} (t2/select-one :model/Database db)
|
||||
transform-details {:db-id db
|
||||
:database database
|
||||
:transform-id id
|
||||
:transform-type (keyword (:type target))
|
||||
:conn-spec (driver/connection-spec driver database)
|
||||
:query (transforms.util/compile-source transform)
|
||||
:output-schema (:schema target)
|
||||
:output-table (transforms.util/qualified-table-name driver target)}
|
||||
opts (transform-opts transform-details)
|
||||
features (transforms.util/required-database-features transform)]
|
||||
(when (transforms.util/db-routing-enabled? database)
|
||||
(throw (ex-info "Transforms are not supported on databases with DB routing enabled."
|
||||
{:driver driver, :database database})))
|
||||
(when-not (every? (fn [feature] (driver.u/supports? (:engine database) feature database)) features)
|
||||
(throw (ex-info "The database does not support the requested transform target type."
|
||||
{:driver driver, :database database, :features features})))
|
||||
;; mark the execution as started and notify any observers
|
||||
(let [{run-id :id} (transforms.util/try-start-unless-already-running id run-method)]
|
||||
(when start-promise
|
||||
(deliver start-promise [:started run-id]))
|
||||
(log/info "Executing transform" id "with target" (pr-str target))
|
||||
(transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]]
|
||||
(transforms.util/run-cancelable-transform!
|
||||
run-id driver transform-details
|
||||
(fn [_cancel-chan] (driver/run-transform! driver transform-details opts))))
|
||||
(transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]
|
||||
(transforms.util/sync-target! target database)
|
||||
;; This event must be published only after the sync is complete - the new table needs to be in AppDB.
|
||||
(events/publish-event! :event/transform-run-complete {:object transform-details}))))
|
||||
(catch Throwable t
|
||||
(log/error t "Error executing transform")
|
||||
(when start-promise
|
||||
;; if the start-promise has been delivered, this is a no-op,
|
||||
;; but we assume nobody would catch the exception anyway
|
||||
(deliver start-promise t))
|
||||
(throw t)))))
|
||||
|
||||
(defmethod transforms.i/execute! :query [transform opts]
|
||||
(transforms.execute/run-mbql-transform! transform opts))
|
||||
(run-mbql-transform! transform opts))
|
||||
|
||||
@@ -45,14 +45,14 @@
|
||||
[:map
|
||||
[:database {:optional true} :int]
|
||||
[:type [:= "table"]]
|
||||
[:schema {:optional true} [:or ms/NonBlankString :nil]]
|
||||
[:schema {:optional true} [:maybe ms/NonBlankString]]
|
||||
[:name :string]])
|
||||
|
||||
(mr/def ::table-incremental-target
|
||||
[:map
|
||||
[:database {:optional true} :int]
|
||||
[:type [:= "table-incremental"]]
|
||||
[:schema {:optional true} [:or ms/NonBlankString :nil]]
|
||||
[:schema {:optional true} [:maybe ms/NonBlankString]]
|
||||
[:name :string]
|
||||
[:target-incremental-strategy ::target-incremental-strategy]])
|
||||
|
||||
|
||||
@@ -436,33 +436,49 @@
|
||||
(defn- matching-timestamp?
|
||||
[job field-path {:keys [start end]}]
|
||||
(when-let [field-instant (->instant (get-in job field-path))]
|
||||
(let [start-instant (some-> start u.date/parse ->instant)
|
||||
end-instant (some-> end u.date/parse ->instant)]
|
||||
(and (or (nil? start)
|
||||
(not (.isBefore field-instant start-instant)))
|
||||
(or (nil? end)
|
||||
(.isAfter end-instant field-instant))))))
|
||||
(let [parse #(-> % u.date/parse ->instant)]
|
||||
;; logic here is to find when it's not matching and invert this
|
||||
(not (or (and start (.isBefore field-instant (parse start)))
|
||||
(and end (.isAfter field-instant (parse end))))))))
|
||||
|
||||
(defn ->date-field-filter-xf
|
||||
"Returns an xform for a date filter."
|
||||
[field-path filter-value]
|
||||
(let [range (some-> filter-value (params.dates/date-string->range {:inclusive-end? false}))]
|
||||
(if range
|
||||
(filter #(matching-timestamp? % field-path range))
|
||||
identity)))
|
||||
(if-let [range (some-> filter-value (params.dates/date-string->range {:inclusive-end? false}))]
|
||||
(filter #(matching-timestamp? % field-path range))
|
||||
identity))
|
||||
|
||||
(defn ->status-filter-xf
|
||||
"Returns an xform for a transform run status filter."
|
||||
[field-path statuses]
|
||||
(let [statuses (->> statuses (map keyword) set not-empty)]
|
||||
(if statuses
|
||||
(filter #(statuses (get-in % field-path)))
|
||||
identity)))
|
||||
(if-let [statuses (->> statuses (map keyword) set not-empty)]
|
||||
(filter #(statuses (get-in % field-path)))
|
||||
identity))
|
||||
|
||||
(defn ->tag-filter-xf
|
||||
"Returns an xform for a transform tag filter."
|
||||
[field-path tag-ids]
|
||||
(let [tag-ids (-> tag-ids set not-empty)]
|
||||
(if tag-ids
|
||||
(filter #(some tag-ids (get-in % field-path)))
|
||||
identity)))
|
||||
(if-let [tag-ids (-> tag-ids set not-empty)]
|
||||
(filter #(some tag-ids (get-in % field-path)))
|
||||
identity))
|
||||
|
||||
(defn ->type-filter-xf
|
||||
"Returns an xform for a transform source type filter."
|
||||
;; TODO: Ngoc - 2025-12-05 We have transform.source_type column so we can just use it to query
|
||||
[types]
|
||||
(if-let [types (->> types (map keyword) set not-empty)]
|
||||
(filter (fn [transform]
|
||||
(contains? types (:source_type transform))))
|
||||
identity))
|
||||
|
||||
(defn ->database-id-filter-xf
|
||||
"Returns an xform for a transform database ID filter.
|
||||
Matches transforms where the source database OR target database matches the given database-id."
|
||||
[database-id]
|
||||
(if database-id
|
||||
(filter (fn [transform]
|
||||
(let [python-source-db-id (transforms.i/source-db-id transform)
|
||||
target-db-id (transforms.i/target-db-id transform)]
|
||||
(or (= database-id python-source-db-id)
|
||||
(= database-id target-db-id)))))
|
||||
identity))
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
[metabase.app-db.core :as app-db]
|
||||
[metabase.driver :as driver]
|
||||
[metabase.driver.util :as driver.u]
|
||||
[metabase.events.core :as events]
|
||||
[metabase.util :as u]
|
||||
[metabase.util.format :as u.format]
|
||||
[metabase.util.i18n :as i18n]
|
||||
@@ -360,6 +361,7 @@
|
||||
(log/info "Executing Python transform" transform-id "with target" (pr-str target))
|
||||
(let [start-ms (u/start-timer)
|
||||
transform-details {:db-id (:id db)
|
||||
:transform-id transform-id
|
||||
:transform-type (keyword (:type target))
|
||||
:conn-spec (driver/connection-spec driver db)
|
||||
:output-schema (:schema target)
|
||||
@@ -372,7 +374,8 @@
|
||||
result (transforms.instrumentation/with-stage-timing [run-id [:computation :python-execution]]
|
||||
(transforms.util/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn))]
|
||||
(transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]
|
||||
(transforms.util/sync-target! target db))
|
||||
(transforms.util/sync-target! target db)
|
||||
(events/publish-event! :event/transform-run-complete {:object transform-details}))
|
||||
{:run_id run-id
|
||||
:result result}))
|
||||
(catch Throwable t
|
||||
|
||||
552
enterprise/backend/src/metabase_enterprise/workspaces/api.clj
Normal file
552
enterprise/backend/src/metabase_enterprise/workspaces/api.clj
Normal file
@@ -0,0 +1,552 @@
|
||||
(ns metabase-enterprise.workspaces.api
|
||||
"`/api/ee/workspace/` routes"
|
||||
(:require
|
||||
[clojure.string :as str]
|
||||
[honey.sql.helpers :as sql.helpers]
|
||||
[metabase-enterprise.transforms.api :as transforms.api]
|
||||
[metabase-enterprise.transforms.util :as transforms.util]
|
||||
[metabase-enterprise.workspaces.common :as ws.common]
|
||||
[metabase-enterprise.workspaces.models.workspace-log]
|
||||
[metabase-enterprise.workspaces.types :as ws.t]
|
||||
[metabase.api.common :as api]
|
||||
[metabase.api.macros :as api.macros]
|
||||
[metabase.api.routes.common :refer [+auth]]
|
||||
[metabase.driver.util :as driver.u]
|
||||
[metabase.queries.schema :as queries.schema]
|
||||
[metabase.request.core :as request]
|
||||
[metabase.util :as u]
|
||||
[metabase.util.i18n :refer [deferred-tru]]
|
||||
[metabase.util.malli.registry :as mr]
|
||||
[metabase.util.malli.schema :as ms]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(def ^:private log-limit "Maximum number of recent workspace log items to show" 20)
|
||||
|
||||
(mr/def ::appdb-or-ref-id [:or ::ws.t/appdb-id ::ws.t/ref-id])
|
||||
|
||||
;; Workspace status
|
||||
|
||||
(mr/def ::graph-status [:enum :stale :calculating :ready])
|
||||
|
||||
(mr/def ::db-status [:enum :uninitialized :provisioning :granting :cleaning :ready])
|
||||
|
||||
;; Proposed simplified FE statuses
|
||||
|
||||
(mr/def ::display-status [:enum :pending :ready] #_[:enum :uninitialized :database-not-read :graph-not-ready :ready])
|
||||
|
||||
(defn- display-status [graph-status db-status]
|
||||
(cond
|
||||
(= :uninitialized db-status) :uninitialized
|
||||
(not (#{:ready :cleaning} db-status)) :database-not-ready
|
||||
(not= :ready graph-status) :graph-not-ready
|
||||
:else :ready))
|
||||
|
||||
(def ^:private Workspace
|
||||
[:map
|
||||
[:id ::ws.t/appdb-id]
|
||||
[:name :string]
|
||||
[:collection_id ::ws.t/appdb-id]
|
||||
[:database_id ::ws.t/appdb-id]
|
||||
[:status ::display-status]
|
||||
[:created_at ms/TemporalInstant]
|
||||
[:updated_at ms/TemporalInstant]
|
||||
[:archived_at [:maybe :any]]])
|
||||
|
||||
;; Transform-related schemas (adapted from transforms/api.clj)
|
||||
|
||||
(mr/def ::transform-source
|
||||
[:multi {:dispatch (comp keyword :type)}
|
||||
[:query
|
||||
[:map
|
||||
[:type [:= "query"]]
|
||||
[:query ::queries.schema/query]]]
|
||||
[:python
|
||||
[:map {:closed true}
|
||||
[:source-database {:optional true} :int]
|
||||
[:source-tables [:map-of [:string {:min 1}] :int]]
|
||||
[:type [:= "python"]]
|
||||
[:body :string]]]])
|
||||
|
||||
(mr/def ::transform-target
|
||||
[:map {:closed true}
|
||||
[:database {:optional true} ::ws.t/appdb-id]
|
||||
[:type [:enum "table"]]
|
||||
[:schema {:optional true} [:maybe [:string {:min 1}]]]
|
||||
[:name [:string {:min 1}]]])
|
||||
|
||||
(mr/def ::run-trigger
|
||||
[:enum "none" "global-schedule"])
|
||||
|
||||
(defn- check-transforms-enabled!
|
||||
[db-id]
|
||||
(let [database (api/check-400 (t2/select-one :model/Database db-id)
|
||||
(deferred-tru "The target database cannot be found."))]
|
||||
(api/check (transforms.util/check-feature-enabled nil)
|
||||
[402 (deferred-tru "Premium features required for transforms are not enabled.")])
|
||||
(api/check-400 (not (:is_sample database))
|
||||
(deferred-tru "Cannot run transforms on the sample database."))
|
||||
(api/check-400 (not (:is_audit database))
|
||||
(deferred-tru "Cannot run transforms on audit databases."))
|
||||
(api/check-400 (driver.u/supports? (:engine database) :transforms/table database)
|
||||
(deferred-tru "The database does not support the requested transform target type."))
|
||||
(api/check-400 (not (transforms.util/db-routing-enabled? database))
|
||||
(deferred-tru "Transforms are not supported on databases with DB routing enabled."))))
|
||||
|
||||
(defn- ws->response [ws]
|
||||
(select-keys ws [:id :name :collection_id :database_id :status :created_at :updated_at :archived_at]))
|
||||
|
||||
;;; routes
|
||||
|
||||
(def ^:private WorkspaceListing
|
||||
[:map {:closed true}
|
||||
[:id ::ws.t/appdb-id]
|
||||
[:name :string]
|
||||
[:archived :boolean]])
|
||||
|
||||
(api.macros/defendpoint :get "/" :- [:map {:closed true}
|
||||
[:items [:sequential WorkspaceListing]]
|
||||
[:limit [:maybe :int]]
|
||||
[:offset [:maybe :int]]]
|
||||
"Get a list of all workspaces"
|
||||
[_route-params
|
||||
_query-params]
|
||||
{:items (t2/select [:model/Workspace :id :name [[:not= nil :archived_at] :archived]]
|
||||
(cond-> {:order-by [[:created_at :desc]]}
|
||||
(request/limit) (sql.helpers/limit (request/limit))
|
||||
(request/offset) (sql.helpers/offset (request/offset))))
|
||||
:limit (request/limit)
|
||||
:offset (request/offset)})
|
||||
|
||||
(mr/def ::input-table
|
||||
[:map
|
||||
;; Future-proof with cross-db Python transforms
|
||||
[:db_id ::ws.t/appdb-id]
|
||||
[:schema :string]
|
||||
[:table :string]
|
||||
[:table_id [:maybe ::ws.t/appdb-id]]])
|
||||
|
||||
(mr/def ::output-table
|
||||
[:map
|
||||
;; Future-proof with multi-db Workspaces, plus necessary to resolve table references when id is null.
|
||||
[:db_id ::ws.t/appdb-id]
|
||||
[:global [:map
|
||||
[:transform_id [:maybe ::ws.t/appdb-id]]
|
||||
[:schema :string]
|
||||
[:table :string]
|
||||
[:table_id [:maybe ::ws.t/appdb-id]]]]
|
||||
[:isolated [:map
|
||||
[:transform_id ::ws.t/ref-id]
|
||||
[:schema :string]
|
||||
[:table :string]
|
||||
[:table_id [:maybe ::ws.t/appdb-id]]]]])
|
||||
|
||||
(api.macros/defendpoint :get "/:id/table"
|
||||
:- [:map {:closed true}
|
||||
[:inputs [:sequential ::input-table]]
|
||||
[:outputs [:sequential ::output-table]]]
|
||||
"Get workspace tables"
|
||||
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
|
||||
_query-params]
|
||||
;; This typically needs the big 'ol graph, to know about the enclosed transforms (for their outputs)
|
||||
;; Easy optimization for the <= 1 entities case and just return the stuff directly though :lightbulb
|
||||
(let [{:keys [database_id schema]} (api/check-404 (t2/select-one [:model/Workspace :database_id :schema] id))]
|
||||
;; TODO fix N+1
|
||||
{:inputs []
|
||||
:outputs (for [{:keys [ref_id target]} (t2/select [:model/WorkspaceTransform :ref_id :target] :workspace_id id)]
|
||||
;; transform_id is not guaranteed to be the global_id, topology may have changed
|
||||
{:db_id database_id
|
||||
:global {:transform_id nil
|
||||
:schema (:schema target)
|
||||
:table (:name target)
|
||||
:table_id (t2/select-one-pk :model/Table
|
||||
:db_id database_id
|
||||
:schema (:schema target)
|
||||
:name (:name target))}
|
||||
:isolated {:transform_id ref_id
|
||||
:schema schema
|
||||
:table (str (:schema target) "__" (:name target))
|
||||
:table_id (t2/select-one-pk :model/Table
|
||||
:db_id database_id
|
||||
:schema schema
|
||||
:name (str (:schema target) "__" (:name target)))}})}))
|
||||
|
||||
(api.macros/defendpoint :get "/:id" :- Workspace
|
||||
"Get a single workspace by ID"
|
||||
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
|
||||
_query-params]
|
||||
(-> (t2/select-one :model/Workspace :id id) api/check-404 ws->response))
|
||||
|
||||
(api.macros/defendpoint :get "/:id/log"
|
||||
:- [:map
|
||||
[:workspace_id ms/PositiveInt]
|
||||
[:status [:enum :pending :ready]]
|
||||
[:updated_at :any]
|
||||
[:last_completed_at [:maybe :any]]
|
||||
[:logs [:sequential [:map
|
||||
[:id ms/PositiveInt]
|
||||
[:task :keyword]
|
||||
[:started_at :any]
|
||||
[:updated_at :any]
|
||||
[:completed_at [:maybe :any]]
|
||||
[:status [:maybe :keyword]]
|
||||
[:message [:maybe :string]]]]]]
|
||||
"Get workspace creation status and recent log entries for polling during async setup"
|
||||
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
|
||||
_query-params]
|
||||
(let [workspace (api/check-404 (t2/select-one :model/Workspace :id id))
|
||||
logs (t2/select [:model/WorkspaceLog
|
||||
:id :task :started_at :completed_at :status :message
|
||||
:updated_at]
|
||||
:workspace_id id
|
||||
{:order-by [[:started_at :desc]]
|
||||
:limit log-limit})]
|
||||
{:workspace_id id
|
||||
:status (:status workspace)
|
||||
:logs logs
|
||||
:updated_at (->> (map :updated_at logs) sort reverse first)
|
||||
:last_completed_at (->> (seq (keep :completed_at logs)) sort reverse first)}))
|
||||
|
||||
(def ^:private CreateWorkspace
|
||||
[:map
|
||||
[:name [:string {:min 1}]]
|
||||
[:database_id ::ws.t/appdb-id]])
|
||||
|
||||
(api.macros/defendpoint :post "/" :- Workspace
|
||||
"Create a new workspace
|
||||
|
||||
Potential payload:
|
||||
{:name \"a\" :database_id 2}}"
|
||||
[_route-params
|
||||
_query-params
|
||||
{:keys [database_id] :as body} :- CreateWorkspace]
|
||||
|
||||
(when database_id
|
||||
(check-transforms-enabled! database_id))
|
||||
|
||||
(ws->response (ws.common/create-workspace! api/*current-user-id* body)))
|
||||
|
||||
(api.macros/defendpoint :put "/:id" :- Workspace
|
||||
"Update simple workspace properties, like name."
|
||||
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
|
||||
_query-params
|
||||
{:keys [name]} :- [:map {:closed true} [:name [:string {:min 1}]]]]
|
||||
(u/prog1 (api/check-404 (t2/select-one :model/Workspace :id id))
|
||||
(api/check-400 (nil? (:archived_at <>)) "Cannot update an archived workspace"))
|
||||
(t2/update! :model/Workspace id {:name name})
|
||||
(-> (t2/select-one :model/Workspace :id id)
|
||||
ws->response))
|
||||
|
||||
(api.macros/defendpoint :post "/:id/archive" :- Workspace
|
||||
"Archive a workspace. Deletes the isolated schema and tables, but preserves mirrored entities."
|
||||
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
|
||||
_query-params
|
||||
_body-params]
|
||||
(let [ws (api/check-404 (t2/select-one :model/Workspace :id id))]
|
||||
(api/check-400 (nil? (:archived_at ws)) "You cannot archive an archived workspace")
|
||||
;; TODO tear down the isolated database resources, and delete the graph
|
||||
(t2/update! :model/Workspace id {:archived_at [:now]})
|
||||
(-> (t2/select-one :model/Workspace :id id)
|
||||
ws->response)))
|
||||
|
||||
(api.macros/defendpoint :post "/:id/unarchive" :- Workspace
|
||||
"Restore an archived workspace. Recreates the isolated schema and tables."
|
||||
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
|
||||
_query-params
|
||||
_body-params]
|
||||
(let [ws (api/check-404 (t2/select-one :model/Workspace :id id))]
|
||||
(api/check-400 (some? (:archived_at ws)) "You cannot unarchive a workspace that is not archived")
|
||||
;; TODO re-provision the isolated database resources, and recompute the graph
|
||||
(t2/update! :model/Workspace id {:archived_at nil})
|
||||
(-> (t2/select-one :model/Workspace :id id)
|
||||
ws->response)))
|
||||
|
||||
(api.macros/defendpoint :delete "/:id" :- [:map [:ok [:= true]]]
|
||||
"Delete a workspace and all its contents, including mirrored entities."
|
||||
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
|
||||
_query-params]
|
||||
(api/check-404 (t2/select-one :model/Workspace :id id))
|
||||
;; TODO implement it https://linear.app/metabase/issue/BOT-574/delete-workspaceid
|
||||
{:ok true})
|
||||
|
||||
(api.macros/defendpoint :post "/:id/run"
|
||||
:- [:map
|
||||
[:succeeded [:sequential ::ws.t/ref-id]]
|
||||
[:failed [:sequential ::ws.t/ref-id]]
|
||||
[:not_run [:sequential ::ws.t/ref-id]]]
|
||||
"Execute all transforms in the workspace in dependency order.
|
||||
Returns which transforms succeeded, failed, and were not run."
|
||||
[{:keys [id]} :- [:map [:id ::ws.t/appdb-id]]
|
||||
_query-params
|
||||
_body-params]
|
||||
(u/prog1 (t2/select-one :model/Workspace :id id)
|
||||
(api/check-404 <>)
|
||||
(api/check-400 (nil? (:archived_at <>)) "Cannot execute archived workspace"))
|
||||
;; get topo-sorted enclosed transforms, run them in order
|
||||
;; to keep things simple, stop execution as soon as there is any failure
|
||||
;; in future we can continue running anything as long as its dependencies succeeded
|
||||
;; TODO not only is the order dodgy, we're not actually running them.
|
||||
{:succeeded (or (t2/select-fn-vec :ref_id [:model/WorkspaceTransform :ref_id] :workspace_id id) [])
|
||||
:failed []
|
||||
:not_run []})
|
||||
|
||||
(mr/def ::graph-node-type [:enum :table :output-table :transform :workspace-transform])
|
||||
|
||||
(mr/def ::graph-node
|
||||
[:map
|
||||
[:id ::appdb-or-ref-id]
|
||||
[:type [:enum :input-table :output-table :workspace-transform]]
|
||||
[:dependents_count [:map-of ::graph-node-type ms/PositiveInt]]
|
||||
[:data :map]])
|
||||
|
||||
(mr/def ::graph-edge
|
||||
[:map
|
||||
[:from_entity_id ::appdb-or-ref-id]
|
||||
[:from_entity_type :string]
|
||||
[:to_entity_id ::appdb-or-ref-id]
|
||||
[:to_entity_type :string]])
|
||||
|
||||
(def ^:private GraphResult
|
||||
[:map
|
||||
[:nodes [:sequential ::graph-node]]
|
||||
[:edges [:sequential ::graph-edge]]])
|
||||
|
||||
(api.macros/defendpoint :get "/:id/graph" :- GraphResult
|
||||
"Display the dependency graph between the Changeset and the (potentially external) entities that they depend on."
|
||||
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
|
||||
_query-params]
|
||||
(api/check-404 (t2/select-one :model/Workspace :id id))
|
||||
;; TODO decide on whether to show output tables, or to rather show dependencies directly between transforms.
|
||||
{:nodes [{:id 1, :type "table", :data {:name "Bob"}, :dependants_count {:workspace-transform 1}}
|
||||
{:id "2", :type "workspace-transform", :data {:name "MyTrans"}, :dependants_count {:output-table 1}}
|
||||
{:id 3, :type "output-table", :data {:external {:table_id 2, :name "Clarence"}, :internal {:name "_-sdre4rcc@"}}, :dependants_count {}}]
|
||||
;; I can't remember which way this is supposed to point - it might be meant to point *backwards* rather.
|
||||
:edges [{:from_entity_type "table"
|
||||
:from_entity_id 1
|
||||
:to_entity_type "workspace-transform"
|
||||
:to_entity_id "3"}
|
||||
{:from_entity_type "workspace-transform"
|
||||
:from_entity_id "3"
|
||||
:to_entity_type "output-table"
|
||||
:to_entity_id 3}]})
|
||||
|
||||
(def ^:private db+schema+table (juxt :database :schema :name))
|
||||
|
||||
(defn- internal-target-conflict?
|
||||
"Check whether the given table is the target of another transform within the workspace. Ignores global transforms."
|
||||
[ws-id target & [tx-id]]
|
||||
(contains?
|
||||
(t2/select-fn-set (comp db+schema+table :target)
|
||||
[:model/WorkspaceTransform :target]
|
||||
:ref_id [:not= tx-id]
|
||||
:workspace_id ws-id)
|
||||
(db+schema+table target)))
|
||||
|
||||
(api.macros/defendpoint :post "/:id/transform/validate/target"
|
||||
"Validate the target table for a workspace transform"
|
||||
[{:keys [id]} :- [:map [:id ::ws.t/appdb-id]]
|
||||
{:keys [transform-id]} :- [:map [:transform-id {:optional true} ::ws.t/ref-id]]
|
||||
{:keys [db_id target]} :- [:map
|
||||
[:db_id {:optional true} ::ws.t/appdb-id]
|
||||
[:target [:map
|
||||
[:database {:optional true} ::ws.t/appdb-id]
|
||||
[:type :string]
|
||||
[:schema :string]
|
||||
[:name :string]]]]]
|
||||
(let [workspace (api/check-404 (t2/select-one [:model/Workspace :database_id] id))
|
||||
target (update target :database #(or % db_id))
|
||||
tx-id (when transform-id (parse-long transform-id))
|
||||
ws-db-id (:database_id workspace)]
|
||||
(cond
|
||||
(not= "table" (:type target))
|
||||
{:status 403 :body (deferred-tru "Unsupported target type")}
|
||||
|
||||
(and db_id ws-db-id (not= db_id ws-db-id))
|
||||
{:status 403 :body (deferred-tru "Must target the workspace database")}
|
||||
|
||||
(not (or db_id ws-db-id))
|
||||
{:status 403 :body (deferred-tru "Must target a database")}
|
||||
|
||||
(str/starts-with? (:schema target) "mb__isolation_")
|
||||
{:status 403 :body (deferred-tru "Must not target an isolated workspace schema")}
|
||||
|
||||
;; Within a workspace, we defer blocking on conflicts outside the workspace
|
||||
#_{:status 403 :body (deferred-tru "A table with that name already exists.")}
|
||||
|
||||
;; TODO consider deferring this validation until merge also.
|
||||
(internal-target-conflict? id target tx-id)
|
||||
{:status 403 :body (deferred-tru "Another transform in this workspace already targets that table.")}
|
||||
|
||||
:else
|
||||
{:status 200 :body "OK"})))
|
||||
|
||||
(api.macros/defendpoint :post "/:id/transform"
|
||||
"Add another transform to the Changeset. This could be a fork of an existing global transform, or something new."
|
||||
[{:keys [id]} :- [:map [:id ::ws.t/appdb-id]]
|
||||
_query-params
|
||||
body :- [:map #_{:closed true}
|
||||
[:name :string]
|
||||
[:description {:optional true} [:maybe :string]]
|
||||
[:source ::transform-source]
|
||||
;; Not sure why this schema is giving trouble
|
||||
#_[:target ::transform-target]]]
|
||||
(api/check (transforms.util/check-feature-enabled body)
|
||||
[402 (deferred-tru "Premium features required for this transform type are not enabled.")])
|
||||
(let [workspace (u/prog1 (api/check-404 (t2/select-one :model/Workspace :id id))
|
||||
(api/check-400 (nil? (:archived_at <>)) "Cannot create transforms in an archived workspace"))
|
||||
;; TODO why 400 here and 403 in the validation route? T_T
|
||||
_ (api/check-400 (not (internal-target-conflict? id (:target body)))
|
||||
(deferred-tru "Another transform in this workspace already targets that table."))
|
||||
global-id (:global_id body (:id body))
|
||||
body (-> body (dissoc :global_id) (update :target assoc :database_id (:database_id workspace)))]
|
||||
(ws.common/add-to-changeset! api/*current-user-id* workspace :transform global-id body)))
|
||||
|
||||
(defn- malli-map-keys [schema]
|
||||
(map first (rest schema)))
|
||||
|
||||
;; TODO Confirm precisely which fields are needed by the FE
|
||||
(def ^:private WorkspaceTransformListing
|
||||
"Schema for a transform in a workspace"
|
||||
[:map {:closed true}
|
||||
[:ref_id ::ws.t/ref-id]
|
||||
[:name :string]
|
||||
[:source_type [:maybe :keyword]]
|
||||
;[:creator_id ::ws.t/appdb-id]
|
||||
;[:last_run :map]
|
||||
; See https://metaboat.slack.com/archives/C099RKNLP6U/p1765205882655869?thread_ts=1765205222.888209&cid=C099RKNLP6U
|
||||
#_[:target_stale :boolean]])
|
||||
|
||||
(defn- map-source-type [ws-tx]
|
||||
(-> ws-tx
|
||||
(assoc :source_type (keyword (:type (:source ws-tx))))
|
||||
(dissoc :source)))
|
||||
|
||||
(api.macros/defendpoint :get "/:id/transform" :- [:map [:transforms [:sequential WorkspaceTransformListing]]]
|
||||
"Get all transforms in a workspace."
|
||||
[{:keys [id]} :- [:map [:id ::ws.t/appdb-id]]]
|
||||
(api/check-404 (t2/select-one :model/Workspace :id id))
|
||||
{:transforms (map map-source-type (t2/select [:model/WorkspaceTransform :ref_id :name :source] :workspace_id id))})
|
||||
|
||||
(def ^:private WorkspaceTransform
|
||||
[:map
|
||||
[:ref_id ::ws.t/ref-id]
|
||||
[:name :string]
|
||||
[:description [:maybe :string]]
|
||||
[:source :map]
|
||||
[:target :map]
|
||||
[:workspace_id ::ws.t/appdb-id]
|
||||
;[:creator_id ::ws.t/appdb-id]
|
||||
[:created_at :any]
|
||||
[:updated_at :any]])
|
||||
|
||||
(defn- fetch-ws-transform [ws-id tx-id]
|
||||
;; TODO We still need to do some hydration, e.g. of the target table (both internal and external)
|
||||
(-> (into [:model/WorkspaceTransform] (malli-map-keys WorkspaceTransform))
|
||||
(t2/select-one :ref_id tx-id :workspace_id ws-id)
|
||||
(api/check-404)))
|
||||
|
||||
(api.macros/defendpoint :get "/:id/transform/:tx-id" :- WorkspaceTransform
|
||||
"Get a specific transform in a workspace."
|
||||
[{:keys [id tx-id]} :- [:map [:id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]
|
||||
(fetch-ws-transform id tx-id))
|
||||
|
||||
(api.macros/defendpoint :put "/:id/transform/:tx-id" :- WorkspaceTransform
|
||||
"Update a transform in a workspace."
|
||||
[{:keys [id tx-id]} :- [:map [:id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]
|
||||
_query-params
|
||||
body :- [:map
|
||||
[:name {:optional true} :string]
|
||||
[:description {:optional true} [:maybe :string]]
|
||||
[:source {:optional true} ::transform-source]
|
||||
[:target {:optional true} ::transform-target]]]
|
||||
(api/check-404 (t2/select-one :model/WorkspaceTransform :ref_id tx-id :workspace_id id))
|
||||
(t2/update! :model/WorkspaceTransform tx-id body)
|
||||
(fetch-ws-transform id tx-id))
|
||||
|
||||
(api.macros/defendpoint :post "/:id/transform/:tx-id/archive" :- :nil
|
||||
"Mark the given transform to be archived when the workspace is merged.
|
||||
For provisional transforms we will skip even creating it in the first place."
|
||||
[{:keys [id tx-id]} :- [:map [:id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]
|
||||
(api/check-404 (pos? (t2/update! :model/WorkspaceTransform {:ref_id tx-id :workspace id} {:archived_at [:now]})))
|
||||
nil)
|
||||
|
||||
(api.macros/defendpoint :post "/:id/transform/:tx-id/unarchive" :- :nil
|
||||
"Unmark the given transform for archival. This will recall the last definition it had within the workspace."
|
||||
[{:keys [id tx-id]} :- [:map [:id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]
|
||||
(api/check-404 (pos? (t2/update! :model/WorkspaceTransform {:ref_id tx-id :workspace_id id} {:archived_at nil})))
|
||||
nil)
|
||||
|
||||
(api.macros/defendpoint :delete "/:id/transform/:tx-id" :- :nil
|
||||
"Discard a transform from the changeset.
|
||||
Equivalent to resetting a checked-out transform to its global definition, or deleting a provisional transform."
|
||||
[{:keys [id tx-id]} :- [:map [:id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]
|
||||
(api/check-404 (pos? (t2/delete! :model/WorkspaceTransform :ref_id tx-id :workspace_id id)))
|
||||
nil)
|
||||
|
||||
(api.macros/defendpoint :post "/:id/transform/:tx-id/run"
|
||||
:- [:map [:message :string] [:run_id {:optional true} [:maybe :int]]]
|
||||
"Run a transform in a workspace."
|
||||
[{:keys [id tx-id]} :- [:map [:id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]
|
||||
(let [transform (api/check-404 (t2/select-one :model/WorkspaceTransform :ref_id tx-id :workspace_id id))]
|
||||
;; We want to run *this* transform, not its global (which might not exist)
|
||||
;; ... in order to get this working quickly we'll need to create a temporary global one (scream)
|
||||
(when-let [global-id (:global_id transform)]
|
||||
(transforms.api/run-transform! (t2/select :model/Transform global-id)))))
|
||||
|
||||
(api.macros/defendpoint :get "/checkout"
|
||||
:- [:map
|
||||
[:transforms [:sequential
|
||||
[:map
|
||||
[:id ::ws.t/ref-id]
|
||||
[:name :string]
|
||||
[:workspace [:map
|
||||
[:id ms/PositiveInt]
|
||||
[:name :string]]]]]]]
|
||||
"Get all downstream transforms for a transform that is not in a workspace.
|
||||
Returns the transforms that were mirrored from this upstream transform, with workspace info."
|
||||
[_route-params
|
||||
{:keys [transform-id]} :- [:map {:closed true} [:id ::ws.t/appdb-id]]]
|
||||
(let [transforms (t2/select [:model/WorkspaceTransform :ref_id :name :workspace_id] :global_id transform-id)
|
||||
workspace-ids (map :workspace_id transforms)
|
||||
workspaces-by-id (when (seq transforms)
|
||||
(t2/select-fn->fn :id identity [:model/Workspace :id :name] :id [:in workspace-ids]))]
|
||||
{:transforms (for [transform transforms]
|
||||
(assoc transform :workspace (get workspaces-by-id (:workspace_id transform))))}))
|
||||
|
||||
(api.macros/defendpoint :post "/:id/merge"
|
||||
:- [:or
|
||||
[:map
|
||||
[:errors [:maybe [:sequential [:map [:id ::ws.t/ref-id] [:name :string] [:error :string]]]]]
|
||||
[:workspace [:map [:id ::ws.t/appdb-id] [:name :string]]]
|
||||
[:archived_at [:maybe :any]]]
|
||||
;; error message from check-404 or check-400
|
||||
:string]
|
||||
"This will:
|
||||
1. Update original transforms with workspace versions
|
||||
2. Archive the workspace and clean up isolated resources
|
||||
Returns a report of merged entities, and any errors."
|
||||
[{:keys [id]} :- [:map [:id ::ws.t/appdb-id]]
|
||||
_query-params
|
||||
_body-params]
|
||||
(let [ws (u/prog1 (t2/select-one :model/Workspace :id id)
|
||||
(api/check-404 <>)
|
||||
(api/check-400 (nil? (:archived_at <>)) "Cannot merge an archived workspace"))
|
||||
;; TODO implement individual entity merging
|
||||
;; return new global ids for provisional transforms
|
||||
{:keys [merged
|
||||
errors]} {:merged {:transforms (vec (t2/select-fn-vec :global_id [:model/WorkspaceTransform :global_id] :workspace_id id))}
|
||||
:errors []}]
|
||||
(u/prog1
|
||||
{:merged merged
|
||||
:errors errors
|
||||
:workspace {:id id, :name (:name ws)}
|
||||
:archived_at (when-not (seq errors)
|
||||
;; TODO call a ws.common method, which can handle the clean-up too
|
||||
(t2/update! :model/Workspace :id id {:archived_at [:now]})
|
||||
(t2/select-one-fn :archived_at [:model/Workspace :archived_at] :id id))}
|
||||
(when-not (seq errors)
|
||||
;; Most of the APIs and the FE are not respecting when a Workspace is archived yet.
|
||||
(t2/delete! :model/Workspace id)))))
|
||||
|
||||
(def ^{:arglists '([request respond raise])} routes
|
||||
"`/api/ee/workspace/` routes."
|
||||
(api.macros/ns-handler *ns* api/+check-superuser +auth))
|
||||
223
enterprise/backend/src/metabase_enterprise/workspaces/common.clj
Normal file
223
enterprise/backend/src/metabase_enterprise/workspaces/common.clj
Normal file
@@ -0,0 +1,223 @@
|
||||
(ns metabase-enterprise.workspaces.common
|
||||
(:require
|
||||
[clojure.string :as str]
|
||||
[metabase-enterprise.workspaces.dag :as ws.dag]
|
||||
[metabase-enterprise.workspaces.driver.common :as driver.common]
|
||||
[metabase-enterprise.workspaces.isolation :as ws.isolation]
|
||||
[metabase-enterprise.workspaces.models.workspace-log :as ws.log]
|
||||
[metabase-enterprise.workspaces.util :as ws.u]
|
||||
[metabase.api-keys.core :as api-key]
|
||||
[metabase.api.common :as api]
|
||||
[metabase.util :as u]
|
||||
[metabase.util.quick-task :as quick-task]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
;; should be encapsulated in our dag namespace, or dependency module
|
||||
(defn check-no-card-dependencies!
|
||||
"Check that transforms don't depend on cards. Throws 400 if they do."
|
||||
[transform-ids]
|
||||
(when-let [card-ids (seq (ws.dag/unsupported-dependency? transform-ids))]
|
||||
(api/check-400 false
|
||||
(format "Cannot add transforms that depend on saved questions (cards). Found dependencies on card IDs: %s"
|
||||
(pr-str (vec card-ids))))))
|
||||
|
||||
(defn- extract-suffix-number
|
||||
"Extract the numeric suffix from a workspace name like 'Foo (3)', or nil if no valid suffix."
|
||||
[ws-name base-name]
|
||||
(when (= (str/replace ws-name #"\s*\(\d+\)$" "") base-name)
|
||||
(when-let [[_ n] (re-find #"\((\d+)\)$" ws-name)]
|
||||
(parse-long n))))
|
||||
|
||||
(defn- generate-unique-workspace-name
|
||||
"Generate a unique workspace name by appending (N) if the name already exists.
|
||||
|
||||
Strategy:
|
||||
1. If the base name doesn't exist, use it as-is
|
||||
2. If it exists, strip any existing (N) suffix and find the highest N among similar names
|
||||
3. Return the base name with (N+1) appended
|
||||
|
||||
Note: This function has a potential race condition between checking existence and insertion.
|
||||
Callers should handle unique constraint violations by retrying."
|
||||
[base-name]
|
||||
(when (str/blank? base-name)
|
||||
(throw (ex-info "Workspace name cannot be empty" {:status-code 400})))
|
||||
(if-not (t2/exists? :model/Workspace :name base-name)
|
||||
base-name
|
||||
(let [stripped-name (str/replace base-name #"\s*\(\d+\)$" "")
|
||||
existing (t2/select-fn-set :name :model/Workspace
|
||||
:name [:like (str stripped-name " (%)")])
|
||||
numbers (keep #(extract-suffix-number % stripped-name) existing)
|
||||
next-num (inc (apply max 0 numbers))]
|
||||
(str stripped-name " (" next-num ")"))))
|
||||
|
||||
;; TODO: Generate new metabase user for the workspace
|
||||
;; TODO: Should we move this to model as per the diagram?
|
||||
(defn- create-workspace-container!
|
||||
"Create the workspace and its related collection, user, and api key."
|
||||
[creator-id db-id workspace-name status]
|
||||
;; TODO (Chris 2025-11-19) Unsure API key name is unique, and remove this (insecure) workaround.
|
||||
(let [api-key (let [key-name (format "API key for Workspace %s" workspace-name)]
|
||||
(or (t2/select-one :model/ApiKey :name key-name)
|
||||
(api-key/create-api-key-with-new-user! {:key-name key-name})))
|
||||
;; TODO (Chris 2025-11-19) Associate the api-key user with the workspace as well.
|
||||
ws (t2/insert-returning-instance! :model/Workspace
|
||||
{:name workspace-name
|
||||
:creator_id creator-id
|
||||
:database_id db-id
|
||||
:api_key_id (:id api-key)
|
||||
:execution_user (:user_id api-key)
|
||||
:status status})
|
||||
coll (t2/insert-returning-instance! :model/Collection
|
||||
{:name (format "Collection for Workspace %s" workspace-name)
|
||||
:namespace "workspace"
|
||||
:workspace_id (:id ws)})
|
||||
schema (driver.common/isolation-namespace-name ws)
|
||||
ws (assoc ws
|
||||
:collection_id (:id coll)
|
||||
:schema schema)]
|
||||
;; Set the backlink from the workspace to the collection inside it and set the schema.
|
||||
(t2/update! :model/Workspace (:id ws) {:collection_id (:id coll)
|
||||
:schema schema})
|
||||
;; TODO (Sanya 2025-11-18) - not sure how to transfer this api key to agent
|
||||
#_(log/infof "Generated API key for workspace: %s" (u.secret/expose (:unmasked_key api-key)))
|
||||
ws))
|
||||
|
||||
(defn- unique-constraint-violation?
|
||||
"Check if an exception is due to a unique constraint violation."
|
||||
[e]
|
||||
(let [msg (str (ex-message e) (some-> (ex-cause e) ex-message))]
|
||||
(or (str/includes? msg "unique")
|
||||
(str/includes? msg "duplicate")
|
||||
(str/includes? msg "UNIQUE"))))
|
||||
|
||||
(defn- run-workspace-setup!
|
||||
"Background job: runs isolation, mirroring, grants. Updates status to :ready when done."
|
||||
[{ws-id :id :as workspace} database]
|
||||
(ws.log/track! ws-id :workspace-setup
|
||||
(let [{:keys [database_details]} (ws.log/track! ws-id :database-isolation
|
||||
(-> (ws.isolation/ensure-database-isolation! workspace database)
|
||||
;; it actually returns just those, this is more like a doc than behavior
|
||||
(select-keys [:schema :database_details])
|
||||
(u/prog1 (t2/update! :model/Workspace ws-id <>))))
|
||||
;; TODO analyze graph. in the
|
||||
{:keys [inputs]} {}]
|
||||
(when-let [table-ids (seq (keep #(when (= :table (:type %)) (:id %)) inputs))]
|
||||
(ws.log/track! ws-id :grant-read-access
|
||||
(let [user (:user database_details)
|
||||
input-tables (t2/select :model/Table :id [:in table-ids])]
|
||||
(ws.isolation/grant-read-access-to-tables! database user input-tables)))))
|
||||
(t2/update! :model/Workspace ws-id {:status :ready})))
|
||||
|
||||
(defn- create-workspace-with-unique-name!
|
||||
"Create a workspace with status=updating, then kick off async setup."
|
||||
[creator-id database ws-name max-retries]
|
||||
(loop [attempt 1]
|
||||
(let [unique-name (generate-unique-workspace-name ws-name)
|
||||
{:keys [retry
|
||||
workspace]} (try
|
||||
{:workspace (create-workspace-container! creator-id (:id database) unique-name :pending)}
|
||||
(catch Exception e
|
||||
(if (and (< attempt max-retries) (unique-constraint-violation? e))
|
||||
{:retry true}
|
||||
(throw e))))]
|
||||
(if retry
|
||||
(recur (inc attempt))
|
||||
(do
|
||||
(quick-task/submit-task! #(run-workspace-setup! workspace database))
|
||||
workspace)))))
|
||||
|
||||
;; TODO internal: test!
|
||||
(defn create-workspace!
|
||||
"Create workspace"
|
||||
[creator-id {ws-name-maybe :name
|
||||
db-id :database_id}]
|
||||
(let [ws-name (or ws-name-maybe (str (random-uuid)))
|
||||
database (t2/select-one :model/Database :id db-id)]
|
||||
(create-workspace-with-unique-name! creator-id database ws-name 5)))
|
||||
|
||||
(defn add-to-changeset!
|
||||
"Add the given "
|
||||
[_creator-id workspace entity-type global-id body]
|
||||
(ws.u/assert-transform! entity-type)
|
||||
(u/prog1
|
||||
(t2/with-transaction [_]
|
||||
(let [workspace-id (:id workspace)
|
||||
workspace-db-id (:database_id workspace)
|
||||
body (assoc-in body [:target :database] workspace-db-id)]
|
||||
(t2/insert-returning-instance!
|
||||
:model/WorkspaceTransform
|
||||
(assoc (select-keys body [:name :description :source :target])
|
||||
;; TODO add this to workspace_transform, or implicitly use the id of the user that does the merge?
|
||||
;:creator_id creator-id
|
||||
:global_id global-id
|
||||
:workspace_id workspace-id))))))
|
||||
|
||||
(defn- mirror-table-to-delete-where
|
||||
[database-id targets]
|
||||
(let [s+t (mapv (fn [[schema name]]
|
||||
[:and
|
||||
[:= [:inline schema] :schema]
|
||||
[:= [:inline name] :name]])
|
||||
targets)]
|
||||
(into [:and
|
||||
[:= [:inline database-id] :db_id]
|
||||
(into [:or] s+t)])))
|
||||
|
||||
(defn remove-entities!
|
||||
"Remove multiple entities from the workspace."
|
||||
[workspace entities]
|
||||
;; count does not have to be 1
|
||||
(assert (= 1 (count entities))
|
||||
"Single kv")
|
||||
(assert (= :transforms (-> entities keys first))
|
||||
"For transforms")
|
||||
(assert (every? pos-int? (:transforms entities))
|
||||
"With seq of ids")
|
||||
|
||||
;; drop transform, drop its target table, no checking of dependencies
|
||||
(let [mirror-transforms-ids (set (:transforms entities))
|
||||
mirror-transforms-data (t2/select :model/Transform :id [:in mirror-transforms-ids])
|
||||
database (t2/select-one :model/Database :id (:database_id workspace))
|
||||
targets (mapv (fn [{:keys [target]}]
|
||||
(assert (= "table" (:type target)))
|
||||
[(:schema target) (:name target)])
|
||||
mirror-transforms-data)
|
||||
_ (assert (< 0 (count targets)))
|
||||
tables-where-clause (mirror-table-to-delete-where (:database_id workspace) targets)
|
||||
tables-data (t2/select :model/Table {:where tables-where-clause})
|
||||
tables-ids (into #{} (map :id) tables-data)]
|
||||
(assert (every? pos-int? tables-ids))
|
||||
(when (seq targets)
|
||||
(ws.isolation/drop-isolated-tables! database targets))
|
||||
(when (seq tables-ids)
|
||||
(t2/delete! :model/Table :id [:in tables-ids]))
|
||||
(when (seq mirror-transforms-ids)
|
||||
(t2/delete! :model/Transform :id [:in mirror-transforms-ids]))))
|
||||
|
||||
#_:clj-kondo/ignore
|
||||
(comment
|
||||
(defn- clean-up-ws!* [& [ws-id]]
|
||||
;; Pass nil to clean up all workspaces
|
||||
(let [ws-clause (or ws-id [:not= nil])]
|
||||
(t2/delete! :model/Collection :workspace_id ws-clause)
|
||||
(t2/delete! :model/Transform :workspace_id ws-clause)
|
||||
(doseq [[db_id schema] (t2/select-fn-set (juxt :db_id :schema) :model/Table :workspace_id [:not= nil])]
|
||||
(t2/delete! :model/Table :schema schema)
|
||||
(let [db (t2/select-one :model/Database db_id)
|
||||
driver (metabase.driver.util/database->driver db)
|
||||
make-spec (requiring-resolve 'metabase.driver.sql-jdbc.connection/connection-details->spec)
|
||||
jdbc-spec (make-spec driver (:details db))]
|
||||
(clojure.java.jdbc/execute! jdbc-spec [(str "DROP SCHEMA \"" schema "\" CASCADE")])))
|
||||
(t2/delete! :model/Workspace :id ws-clause)))
|
||||
|
||||
(def upstream-id (t2/select-one-pk :model/Transform :workspace_id nil {:order-by [:id]}))
|
||||
(def upstream-ids (t2/select-pks-vec :model/Transform :workspace_id nil {:order-by [:id]}))
|
||||
|
||||
;; Ensure output table exists
|
||||
(#'metabase-enterprise.transforms.execute/run-mbql-transform! (t2/select-one :model/Transform upstream-id))
|
||||
|
||||
(let [admin-id (t2/select-one-pk :model/User :is_superuser true {:order-by [:id]})]
|
||||
(binding [api/*current-user-id* admin-id]
|
||||
(create-workspace! admin-id {:name "Workplace Workspace", :upstream {:transforms upstream-ids #_[upstream-id]}})))
|
||||
|
||||
(clean-up-ws!*))
|
||||
@@ -0,0 +1,8 @@
|
||||
(ns metabase-enterprise.workspaces.core
|
||||
(:require
|
||||
[metabase-enterprise.workspaces.isolation]
|
||||
[potemkin :as p]))
|
||||
|
||||
(p/import-vars
|
||||
[metabase-enterprise.workspaces.isolation
|
||||
with-workspace-isolation])
|
||||
103
enterprise/backend/src/metabase_enterprise/workspaces/dag.clj
Normal file
103
enterprise/backend/src/metabase_enterprise/workspaces/dag.clj
Normal file
@@ -0,0 +1,103 @@
|
||||
;; TODO consider renaming this to dataflow
|
||||
(ns metabase-enterprise.workspaces.dag
|
||||
"Used to do graph computations with respect to dataflow within a workspace.
|
||||
|
||||
This workspace should encapsulate all coupling to the [[metabase-enterprise.dependencies]] module, including through
|
||||
database artifacts.
|
||||
|
||||
It is purely concerned with calculation, any side effects wanted as a result of the analysis should happen outside.")
|
||||
|
||||
#_(def ^:private empty-subgraph
|
||||
{:working-set []
|
||||
:inputs []
|
||||
:outputs []
|
||||
:entities []
|
||||
:dependencies {}})
|
||||
|
||||
;; Is this working with appdb-ids or ref-ids?
|
||||
(defn unsupported-dependency?
|
||||
"Return a map of the entities which have unsupported dependencies, or nil if there are none."
|
||||
[{transform-ids :transforms :as _entity-map}]
|
||||
;; TODO filter out those that don't have a card dependency
|
||||
(when (seq transform-ids)
|
||||
{:transforms transform-ids}))
|
||||
|
||||
;;;; Public API
|
||||
|
||||
(defn- path-induced-subgraph
|
||||
"Given a map of entity types to IDs, compute the path-induced subgraph.
|
||||
`entities-by-type` is a map like {:transform [1 2 3]}.
|
||||
|
||||
Returns a map with:
|
||||
- :changeset - the entities being edited within the workspace, in topological order
|
||||
{type, ref-id, parent-id}
|
||||
- :inputs - tables that the subgraph directly depends on, that are not themselves produced by the subgraph.
|
||||
ordered lexically by db+schema+table, to be stable across metabase instances, and visibly sorted.
|
||||
- :outputs - outputs of both the changeset transforms, and the external transforms enclosed by the changeset.
|
||||
both the external names, and the internal (isolated) names. sorted by the external names.
|
||||
point back to their relevant transform.
|
||||
|
||||
- :entities - the full list of entities, both in the changeset, and those that are enclosed. topo-sorted.
|
||||
- :dependencies - association list for the subgraph, with keys and values both in topological order"
|
||||
[working-set]
|
||||
{:working-set (or working-set [{:type "transform", :ref_id "2", :global_id 1}])
|
||||
:inputs [{:db_id 1, :schema "public", :table "orders", :table_id 1}]
|
||||
;; or global + isolated?
|
||||
:outputs [{:global {:transform_id nil
|
||||
:schema "public"
|
||||
:table "customers"
|
||||
:table_id nil}
|
||||
:isolated {:transform_id "1"
|
||||
:schema "isolated__blah"
|
||||
:table "public__customers"
|
||||
:table_id 2}}
|
||||
|
||||
{:db_id 1
|
||||
:global {:transform_id 1
|
||||
:schema "public"
|
||||
:table "mega_orders"
|
||||
:table_id 1}
|
||||
;; integer id - enclosed
|
||||
:isolated {:transform_id nil
|
||||
:schema "isolated__blah"
|
||||
:table "public__mega_orders"
|
||||
:table_id 3}}
|
||||
|
||||
{:id 1
|
||||
:db_id 1
|
||||
:global {:transform_id nil
|
||||
:schema "public"
|
||||
:table "augmented_orders"
|
||||
:table_id 4}
|
||||
;; string id - changeset
|
||||
:isolated {:transform_id "2"
|
||||
:schema "isolated__blah"
|
||||
:table "public__augmented_orders"
|
||||
:table-id nil}}]
|
||||
|
||||
;; should we include tables in the :dependencies graph? i kinda don't like it
|
||||
;; 1. bigger and uglier for test validation
|
||||
;; 2. easier to expand than to contract (and not sure which we want to show in ui)
|
||||
;; 3. don't need the tables for execute.
|
||||
;; 4. don't need the dependencies list for data tab.
|
||||
|
||||
:enclosed [{:type "transform", :id 2}]
|
||||
;; If writing to JSON, we need to serialize to an association list because of rich keys.
|
||||
:dependencies {{:type "external-transform", :id 1} [{:type "workspace-transform", :id "1"}]
|
||||
{:type "workspace-transform", :id "2"} [{:type "input-table", :id 1}
|
||||
{:type "global-transform" :id 1}]}})
|
||||
|
||||
;; source-table ---------> checked-out-transform
|
||||
;; \____external-transform__/
|
||||
;;
|
||||
;; should external-transform be enclosed? answer: no, because it won't become stale from changes in the workspace
|
||||
;;
|
||||
;; source-table ----> model ---> checked-out-transform
|
||||
;;
|
||||
;; this time we *need* the model to be serialized, because we need to read it in a fresh instance.
|
||||
;; it doesn't need to be remapped or "run" independently though, so does it make sense to call it enclosed?
|
||||
;;
|
||||
;; enclosed: between changeset only? or including the model in the example above?
|
||||
;; ancestors: as far back as you can go
|
||||
;;
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
(ns metabase-enterprise.workspaces.driver.common
|
||||
"Common utilities for workspace isolation drivers."
|
||||
(:require
|
||||
[clojure.string :as str]
|
||||
[metabase.system.core :as system]))
|
||||
|
||||
;;;; Naming
|
||||
|
||||
;; re-using https://github.com/metabase/metabase/pull/61887/commits/c92e4a9cc451c61a13fef19ed9d6107873b17f07
|
||||
;; (original ws isolation code)
|
||||
(defn- instance-uuid-slug
|
||||
"Create a slug from the site UUID, taking the first character of each section."
|
||||
[site-uuid-string]
|
||||
(apply str (map first (str/split site-uuid-string #"-"))))
|
||||
|
||||
(defn isolation-namespace-name
|
||||
"Generate namespace/database name for workspace isolation following mb__isolation_<slug>_<workspace-id> pattern.
|
||||
Uses 'namespace' as the generic term that maps to 'schema' in Postgres, 'database' in ClickHouse, etc."
|
||||
[workspace]
|
||||
(assert (some? (:id workspace)) "Workspace must have an :id")
|
||||
(let [instance-slug (instance-uuid-slug (str (system/site-uuid)))
|
||||
clean-workspace-id (str/replace (str (:id workspace)) #"[^a-zA-Z0-9]" "_")]
|
||||
(format "mb__isolation_%s_%s" instance-slug clean-workspace-id)))
|
||||
|
||||
(defn isolated-table-name
|
||||
"Generate name for a table mirroring transform target table in the isolated database namespace."
|
||||
[{:keys [schema name] :as _source-table}]
|
||||
;; TODO: This naming scheme is not guaranteed to give a unique name, even if it's likely
|
||||
;; to always be unique in practice. Consider adding a hash suffix or conflict detection
|
||||
;; before merging to master.
|
||||
(format "%s__%s" schema name))
|
||||
|
||||
(defn isolation-user-name
|
||||
"Generate username for workspace isolation."
|
||||
[workspace]
|
||||
(let [instance-slug (instance-uuid-slug (str (system/site-uuid)))]
|
||||
(format "mb_isolation_%s_%s" instance-slug (:id workspace))))
|
||||
@@ -0,0 +1,56 @@
|
||||
(ns metabase-enterprise.workspaces.driver.h2
|
||||
"H2-specific implementations for workspace isolation.
|
||||
|
||||
H2 differences from PostgreSQL:
|
||||
- H2 doesn't support CREATE USER with PASSWORD syntax like Postgres
|
||||
- H2 uses CREATE USER userName PASSWORD 'password'
|
||||
- H2 GRANT syntax: GRANT { SELECT | ALL } ON { SCHEMA schemaName | tableName } TO userName
|
||||
- H2 supports CREATE TABLE ... AS SELECT syntax for table duplication"
|
||||
(:require
|
||||
[clojure.java.jdbc :as jdbc]
|
||||
[metabase-enterprise.workspaces.driver.common :as driver.common]
|
||||
[metabase-enterprise.workspaces.isolation :as isolation]
|
||||
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
|
||||
[metabase.driver.util :as driver.u]))
|
||||
|
||||
(defmethod isolation/grant-read-access-to-tables! :h2
|
||||
[database username tables]
|
||||
(let [driver (driver.u/database->driver database)
|
||||
jdbc-spec (sql-jdbc.conn/connection-details->spec driver (:details database))
|
||||
schemas (distinct (map :schema tables))]
|
||||
;; H2 uses GRANT SELECT ON SCHEMA schemaName TO userName
|
||||
(doseq [schema schemas]
|
||||
(jdbc/execute! jdbc-spec [(format "GRANT SELECT ON SCHEMA \"%s\" TO \"%s\"" schema username)]))
|
||||
;; Also grant on individual tables for more fine-grained access
|
||||
(doseq [table tables]
|
||||
(jdbc/execute! jdbc-spec [(format "GRANT SELECT ON \"%s\".\"%s\" TO \"%s\""
|
||||
(:schema table) (:name table) username)]))))
|
||||
|
||||
(defmethod isolation/init-workspace-database-isolation! :h2
|
||||
[database workspace]
|
||||
(let [driver (driver.u/database->driver database)
|
||||
schema-name (driver.common/isolation-namespace-name workspace)
|
||||
;; H2 uses simpler user creation syntax
|
||||
read-user {:user (driver.common/isolation-user-name workspace)
|
||||
:password (str (random-uuid))}
|
||||
jdbc-spec (sql-jdbc.conn/connection-details->spec driver (:details database))]
|
||||
(doseq [sql [(format "CREATE SCHEMA IF NOT EXISTS \"%s\"" schema-name)
|
||||
;; H2 syntax: CREATE USER userName PASSWORD 'password'
|
||||
(format "CREATE USER IF NOT EXISTS \"%s\" PASSWORD '%s'"
|
||||
(:user read-user) (:password read-user))
|
||||
;; Grant access on the isolation schema
|
||||
(format "GRANT ALL ON SCHEMA \"%s\" TO \"%s\"" schema-name (:user read-user))]]
|
||||
(jdbc/execute! jdbc-spec [sql]))
|
||||
{:schema schema-name
|
||||
:database_details read-user}))
|
||||
|
||||
(defmethod isolation/drop-isolated-tables! :h2
|
||||
[database s+t-tuples]
|
||||
(let [driver (driver.u/database->driver database)
|
||||
jdbc-spec (sql-jdbc.conn/connection-details->spec driver (:details database))]
|
||||
(run!
|
||||
(fn [[schema-name table-name]]
|
||||
(jdbc/execute! jdbc-spec
|
||||
[(format "DROP TABLE IF EXISTS \"%s\".\"%s\""
|
||||
schema-name table-name)]))
|
||||
s+t-tuples)))
|
||||
@@ -0,0 +1,6 @@
|
||||
(ns metabase-enterprise.workspaces.driver.init
|
||||
"Load all workspace driver implementations.
|
||||
Require this namespace to ensure all driver methods are registered."
|
||||
(:require
|
||||
[metabase-enterprise.workspaces.driver.h2]
|
||||
[metabase-enterprise.workspaces.driver.postgres]))
|
||||
@@ -0,0 +1,46 @@
|
||||
(ns metabase-enterprise.workspaces.driver.postgres
|
||||
"Postgres-specific implementations for workspace isolation."
|
||||
(:require
|
||||
[clojure.java.jdbc :as jdbc]
|
||||
[metabase-enterprise.workspaces.driver.common :as driver.common]
|
||||
[metabase-enterprise.workspaces.isolation :as isolation]
|
||||
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
|
||||
[metabase.driver.util :as driver.u]))
|
||||
|
||||
(defmethod isolation/grant-read-access-to-tables! :postgres
|
||||
[database username tables]
|
||||
(let [driver (driver.u/database->driver database)
|
||||
jdbc-spec (sql-jdbc.conn/connection-details->spec driver (:details database))
|
||||
schemas (distinct (map :schema tables))]
|
||||
(doseq [schema schemas]
|
||||
(jdbc/execute! jdbc-spec [(format "GRANT USAGE ON SCHEMA %s TO %s" schema username)]))
|
||||
(doseq [table tables]
|
||||
(jdbc/execute! jdbc-spec [(format "GRANT SELECT ON TABLE %s.%s TO %s" (:schema table) (:name table) username)]))))
|
||||
|
||||
(defmethod isolation/init-workspace-database-isolation! :postgres
|
||||
[database workspace]
|
||||
(let [driver (driver.u/database->driver database)
|
||||
schema-name (driver.common/isolation-namespace-name workspace)
|
||||
read-user {:user (driver.common/isolation-user-name workspace)
|
||||
:password (str (random-uuid))}
|
||||
jdbc-spec (sql-jdbc.conn/connection-details->spec driver (:details database))]
|
||||
(doseq [sql [(format "CREATE SCHEMA %s" schema-name)
|
||||
(format "CREATE USER %s WITH PASSWORD '%s'" (:user read-user) (:password read-user))
|
||||
;; grant all access on the destination schema
|
||||
(format "GRANT USAGE ON SCHEMA %s TO %s" schema-name (:user read-user))
|
||||
;; need to be able insert and dropping, rename tables from this chema
|
||||
(format "GRANT ALL PRIVILEGES ON SCHEMA %s TO %s" schema-name (:user read-user))]]
|
||||
(jdbc/execute! jdbc-spec [sql]))
|
||||
{:schema schema-name
|
||||
:database_details read-user}))
|
||||
|
||||
(defmethod isolation/drop-isolated-tables! :postgres
|
||||
[database s+t-tuples]
|
||||
(let [driver (driver.u/database->driver database)
|
||||
jdbc-spec (sql-jdbc.conn/connection-details->spec driver (:details database))]
|
||||
(run!
|
||||
(fn [[schema-name table-name]]
|
||||
(jdbc/execute! jdbc-spec
|
||||
[(format "DROP TABLE IF EXISTS \"%s\".\"%s\""
|
||||
schema-name table-name)]))
|
||||
s+t-tuples)))
|
||||
@@ -0,0 +1,7 @@
|
||||
(ns metabase-enterprise.workspaces.init
|
||||
(:require
|
||||
[metabase-enterprise.workspaces.driver.h2]
|
||||
[metabase-enterprise.workspaces.driver.postgres]
|
||||
[metabase-enterprise.workspaces.models.workspace]
|
||||
[metabase-enterprise.workspaces.models.workspace-mapping]
|
||||
[metabase-enterprise.workspaces.models.workspace-transform]))
|
||||
@@ -0,0 +1,60 @@
|
||||
(ns metabase-enterprise.workspaces.isolation
|
||||
(:require
|
||||
[metabase-enterprise.workspaces.driver.common :as driver.common]
|
||||
[metabase.driver :as driver]
|
||||
[metabase.driver.util :as driver.u]
|
||||
[metabase.util :as u]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
;;;; Driver multimethods
|
||||
;; Implementations are in metabase-enterprise.workspaces.driver.{postgres,h2}
|
||||
|
||||
(defn dispatch-on-engine
|
||||
"Take engine from database `db` and dispatch on that."
|
||||
[database & _args]
|
||||
(driver.u/database->driver database))
|
||||
|
||||
(defmulti grant-read-access-to-tables!
|
||||
"Grant read access to these tables."
|
||||
{:added "0.59.0" :arglists '([database username tables])}
|
||||
#'dispatch-on-engine
|
||||
:hierarchy #'driver/hierarchy)
|
||||
|
||||
(defmulti init-workspace-database-isolation!
|
||||
"Create database isolation for a workspace. Return the database details."
|
||||
{:added "0.59.0" :arglists '([database workspace])}
|
||||
#'dispatch-on-engine
|
||||
:hierarchy #'driver/hierarchy)
|
||||
|
||||
(defmulti duplicate-output-table!
|
||||
"Create an isolated copy of the given output tables, for a workspace transform to write to.
|
||||
|
||||
TODO: Consider removing this method once we have 'remap-on-execute' semantics, where
|
||||
transforms write directly to the isolated location without needing to duplicate existing tables."
|
||||
{:added "0.59.0" :arglists '([database workspace output])}
|
||||
#'dispatch-on-engine
|
||||
:hierarchy #'driver/hierarchy)
|
||||
|
||||
(defmulti drop-isolated-tables!
|
||||
"Drop isolated tables"
|
||||
{:added "0.59.0" :arglists '([database s+t-tuples])}
|
||||
#'dispatch-on-engine)
|
||||
|
||||
;;;; Public API
|
||||
|
||||
(defn ensure-database-isolation!
|
||||
"Wrapper around the driver method, to make migrations easier in future."
|
||||
[workspace database]
|
||||
(init-workspace-database-isolation! database workspace))
|
||||
|
||||
(defn do-with-workspace-isolation
|
||||
"Impl of* with-workspace-isolation*."
|
||||
[workspace thunk]
|
||||
(driver/with-swapped-connection-details (:database_id workspace)
|
||||
(:database_details workspace)
|
||||
(thunk)))
|
||||
|
||||
(defmacro with-workspace-isolation
|
||||
"Execute body with necessary isolation."
|
||||
[workspace & body]
|
||||
`(do-with-workspace-isolation ~workspace (fn [] ~@body)))
|
||||
@@ -0,0 +1,49 @@
|
||||
(ns metabase-enterprise.workspaces.models.workspace
|
||||
(:require
|
||||
[metabase.models.interface :as mi]
|
||||
[metabase.util.json :as json]
|
||||
[methodical.core :as methodical]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(def ^:private type->grouping "How models are grouped in lists"
|
||||
;; do not forget to update `entities` if you add stuff here
|
||||
{:transform :transforms})
|
||||
|
||||
(methodical/defmethod t2/table-name :model/Workspace [_model] :workspace)
|
||||
|
||||
(t2/deftransforms :model/Workspace
|
||||
{:graph mi/transform-json
|
||||
:database_details mi/transform-encrypted-json
|
||||
:status mi/transform-keyword})
|
||||
|
||||
(doto :model/Workspace
|
||||
(derive :metabase/model)
|
||||
(derive :hook/timestamped?))
|
||||
|
||||
(defn- entities [entity-type ids]
|
||||
(->> (case entity-type
|
||||
:transform
|
||||
(t2/select [:model/Transform
|
||||
:id :name :description :source :target :source_type :creator_id :collection_id
|
||||
:created_at :updated_at :transform.workspace_id :wmt.upstream_id
|
||||
[:upstream.target :upstream]]
|
||||
:transform.workspace_id [:in ids]
|
||||
{:left-join [[:workspace_mapping_transform :wmt] [:= :wmt.downstream_id :transform.id]
|
||||
[:transform :upstream] [:= :upstream.id :wmt.upstream_id]]
|
||||
:order-by [:id]}))
|
||||
(mapv #(-> %
|
||||
(update :upstream json/decode+kw)
|
||||
(assoc :type entity-type)))))
|
||||
|
||||
(defn- ws-contents [ids]
|
||||
(fn []
|
||||
(->> (mapcat #(entities % ids) (keys type->grouping))
|
||||
(reduce (fn [acc e]
|
||||
(update-in acc [(:workspace_id e) (type->grouping (:type e))] (fnil conj []) e))
|
||||
{}))))
|
||||
|
||||
(methodical/defmethod t2/batched-hydrate [:model/Workspace :contents]
|
||||
"Batch hydrate `Workspace` contents"
|
||||
[_model k wses]
|
||||
(let [ids (mapv :id wses)]
|
||||
(mi/instances-with-hydrated-data wses k (ws-contents ids) :id {:default {}})))
|
||||
@@ -0,0 +1,55 @@
|
||||
(ns metabase-enterprise.workspaces.models.workspace-log
|
||||
(:require
|
||||
[java-time.api :as t]
|
||||
[metabase.models.interface :as mi]
|
||||
[methodical.core :as methodical]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(methodical/defmethod t2/table-name :model/WorkspaceLog [_model] :workspace_log)
|
||||
|
||||
(doto :model/WorkspaceLog
|
||||
(derive :metabase/model)
|
||||
(derive :hook/timestamped?))
|
||||
|
||||
(t2/deftransforms :model/WorkspaceLog
|
||||
{:task mi/transform-keyword
|
||||
:status mi/transform-keyword})
|
||||
|
||||
(defn start!
|
||||
"Create a log entry for a workspace setup task. Returns the created log."
|
||||
[workspace-id task]
|
||||
(t2/insert-returning-instance! :model/WorkspaceLog
|
||||
{:workspace_id workspace-id
|
||||
:task task
|
||||
:started_at (t/offset-date-time)
|
||||
:status :started}))
|
||||
|
||||
(defn- complete!
|
||||
[status log-id & [message]]
|
||||
(assert (#{:success :failure} status) "It can only be either success or failure")
|
||||
(let [data (cond-> {:completed_at (t/offset-date-time)
|
||||
:status status}
|
||||
message (assoc :message message))]
|
||||
(t2/update! :model/WorkspaceLog log-id data)))
|
||||
|
||||
(defn success!
|
||||
"Task was completed successfully"
|
||||
[log-id & [message]]
|
||||
(complete! :success log-id message))
|
||||
|
||||
(defn failure!
|
||||
"Task has failed"
|
||||
[log-id & [message]]
|
||||
(complete! :failure log-id message))
|
||||
|
||||
(defmacro track!
|
||||
"Wrap body with automatic logging: starts a log entry, marks success on completion, failure on exception."
|
||||
[workspace-id task & body]
|
||||
`(let [log# (start! ~workspace-id ~task)]
|
||||
(try
|
||||
(let [result# (do ~@body)]
|
||||
(success! (:id log#))
|
||||
result#)
|
||||
(catch Exception e#
|
||||
(failure! (:id log#) (ex-message e#))
|
||||
(throw e#)))))
|
||||
@@ -0,0 +1,18 @@
|
||||
(ns metabase-enterprise.workspaces.models.workspace-mapping
|
||||
(:require
|
||||
[methodical.core :as methodical]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(methodical/defmethod t2/table-name :model/WorkspaceMappingTable [_model] :workspace_mapping_table)
|
||||
|
||||
(methodical/defmethod t2/primary-keys :model/WorkspaceMappingTable [_model] [:upstream_id :downstream_id])
|
||||
|
||||
(doto :model/WorkspaceMappingTable
|
||||
(derive :metabase/model))
|
||||
|
||||
(methodical/defmethod t2/table-name :model/WorkspaceMappingTransform [_model] :workspace_mapping_transform)
|
||||
|
||||
(methodical/defmethod t2/primary-keys :model/WorkspaceMappingTransform [_model] [:upstream_id :downstream_id])
|
||||
|
||||
(doto :model/WorkspaceMappingTransform
|
||||
(derive :metabase/model))
|
||||
@@ -0,0 +1,86 @@
|
||||
(ns metabase-enterprise.workspaces.models.workspace-transform
|
||||
"Model for WorkspaceTransform - holds the changeset of transforms being created
|
||||
and edited within a workspace."
|
||||
(:require
|
||||
[clojure.string :as str]
|
||||
[metabase.models.interface :as mi]
|
||||
[methodical.core :as methodical]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
;;; -------------------------------------------------- Cute Names --------------------------------------------------
|
||||
|
||||
(def ^:private adjectives
|
||||
"Adjectives for generating cute ref IDs, inspired by Docker's naming scheme."
|
||||
["admiring" "adoring" "affectionate" "agitated" "amazing" "angry" "awesome"
|
||||
"beautiful" "blissful" "bold" "boring" "brave" "busy" "charming" "clever"
|
||||
"cool" "compassionate" "competent" "condescending" "confident" "cranky"
|
||||
"crazy" "dazzling" "determined" "distracted" "dreamy" "eager" "ecstatic"
|
||||
"elastic" "elated" "elegant" "eloquent" "epic" "exciting" "fervent"
|
||||
"festive" "flamboyant" "focused" "friendly" "frosty" "funny" "gallant"
|
||||
"gifted" "goofy" "gracious" "great" "happy" "hardcore" "heuristic" "hopeful"
|
||||
"hungry" "infallible" "inspiring" "intelligent" "interesting" "jolly"
|
||||
"jovial" "keen" "kind" "laughing" "loving" "lucid" "magical" "modest"
|
||||
"musing" "mystifying" "naughty" "nervous" "nice" "nifty" "nostalgic"
|
||||
"objective" "optimistic" "peaceful" "pedantic" "pensive" "practical"
|
||||
"priceless" "quirky" "quizzical" "recursing" "relaxed" "reverent" "romantic"
|
||||
"sad" "serene" "sharp" "silly" "sleepy" "stoic" "strange" "stupefied"
|
||||
"suspicious" "sweet" "tender" "thirsty" "trusting" "unruffled" "upbeat"
|
||||
"vibrant" "vigilant" "vigorous" "wizardly" "wonderful" "xenodochial" "youthful"
|
||||
"zealous" "zen"])
|
||||
|
||||
(def ^:private animals
|
||||
"Animals for generating cute ref IDs."
|
||||
["albatross" "alligator" "alpaca" "ant" "anteater" "antelope" "armadillo"
|
||||
"baboon" "badger" "bat" "bear" "beaver" "bee" "bison" "boar" "buffalo"
|
||||
"butterfly" "camel" "capybara" "caribou" "cat" "caterpillar" "cheetah"
|
||||
"chicken" "chimpanzee" "chinchilla" "cobra" "condor" "cormorant" "coyote"
|
||||
"crab" "crane" "crocodile" "crow" "deer" "dingo" "dog" "dolphin" "donkey"
|
||||
"dove" "dragonfly" "duck" "eagle" "echidna" "eel" "elephant" "elk" "emu"
|
||||
"falcon" "ferret" "finch" "firefly" "flamingo" "fox" "frog" "gazelle"
|
||||
"gerbil" "giraffe" "goat" "goose" "gopher" "gorilla" "grasshopper" "grouse"
|
||||
"hamster" "hare" "hawk" "hedgehog" "heron" "hippopotamus" "hornet" "horse"
|
||||
"hummingbird" "hyena" "ibex" "iguana" "impala" "jackal" "jaguar" "jellyfish"
|
||||
"kangaroo" "kingfisher" "kiwi" "koala" "komodo" "kookaburra" "lemur"
|
||||
"leopard" "lion" "lizard" "llama" "lobster" "lynx" "magpie" "mallard"
|
||||
"manatee" "mandrill" "marmot" "meerkat" "mink" "mole" "mongoose" "monkey"
|
||||
"moose" "moth" "mouse" "mule" "narwhal" "newt" "nightingale" "ocelot"
|
||||
"octopus" "okapi" "opossum" "orangutan" "orca" "oryx" "osprey" "ostrich"
|
||||
"otter" "owl" "ox" "panda" "panther" "parrot" "partridge" "peacock"
|
||||
"pelican" "penguin" "pheasant" "pig" "pigeon" "platypus" "pony" "porcupine"
|
||||
"porpoise" "puma" "quail" "rabbit" "raccoon" "ram" "raven" "reindeer"
|
||||
"rhinoceros" "salamander" "salmon" "sardine" "scorpion" "seahorse" "seal"
|
||||
"shark" "sheep" "shrew" "skunk" "sloth" "snail" "snake" "sparrow" "spider"
|
||||
"squid" "squirrel" "starfish" "stingray" "stork" "swallow" "swan" "tapir"
|
||||
"tiger" "toad" "toucan" "turkey" "turtle" "viper" "vulture" "wallaby"
|
||||
"walrus" "warthog" "wasp" "weasel" "whale" "wolf" "wolverine" "wombat"
|
||||
"woodpecker" "yak" "zebra"])
|
||||
|
||||
(defn generate-ref-id
|
||||
"Generate a cute adjective-animal style ref ID with a random suffix.
|
||||
Format: adjective-animal-xxxx (where xxxx is a random 4-char hex suffix)"
|
||||
[]
|
||||
(let [adj (rand-nth adjectives)
|
||||
animal (rand-nth animals)
|
||||
suffix (format "%04x" (rand-int 65536))]
|
||||
(str adj "-" animal "-" suffix)))
|
||||
|
||||
;;; -------------------------------------------------- Model Setup --------------------------------------------------
|
||||
|
||||
(methodical/defmethod t2/table-name :model/WorkspaceTransform [_model] :workspace_transform)
|
||||
|
||||
(methodical/defmethod t2/primary-keys :model/WorkspaceTransform [_model] [:ref_id])
|
||||
|
||||
(doto :model/WorkspaceTransform
|
||||
(derive :metabase/model)
|
||||
(derive :hook/timestamped?)
|
||||
(derive :hook/entity-id))
|
||||
|
||||
(t2/deftransforms :model/WorkspaceTransform
|
||||
{:ref_id {:in identity :out str/trim}
|
||||
:source mi/transform-json
|
||||
:target mi/transform-json})
|
||||
|
||||
(t2/define-before-insert :model/WorkspaceTransform
|
||||
[instance]
|
||||
(cond-> instance
|
||||
(not (:ref_id instance)) (assoc :ref_id (generate-ref-id))))
|
||||
@@ -0,0 +1,23 @@
|
||||
(ns metabase-enterprise.workspaces.types
|
||||
(:require
|
||||
[metabase.util.malli.registry :as mr]
|
||||
[metabase.util.malli.schema :as ms]))
|
||||
|
||||
(mr/def ::appdb-id ms/PositiveInt)
|
||||
|
||||
(mr/def ::ref-id [:string {:min 1}])
|
||||
|
||||
(def entity-types
|
||||
"The kinds of entities we can store within a Workspace."
|
||||
[{:name "Transform"
|
||||
:key :transform
|
||||
:group :transforms
|
||||
:model :model/Transform}])
|
||||
|
||||
(mr/def ::entity-type (into [:enum] (map :key) entity-types))
|
||||
|
||||
(mr/def ::entity-grouping (into [:enum] (map :group) entity-types))
|
||||
|
||||
;; Map like {:transforms [1 2 3]}
|
||||
(mr/def ::entity-map
|
||||
[:map-of ::entity-grouping [:sequential ms/PositiveInt]])
|
||||
@@ -0,0 +1,36 @@
|
||||
(ns metabase-enterprise.workspaces.util)
|
||||
|
||||
(defn assert-transform!
|
||||
"Test whether we support the given entity type within workspaces yet.
|
||||
Named for the only case we support currently, to make call sites assumptions more obvious."
|
||||
[entity-type]
|
||||
(when (not= "transform" (name entity-type))
|
||||
(throw (ex-info "Entity type not supported"
|
||||
{:status-code 400
|
||||
:entity-type entity-type}))))
|
||||
|
||||
(defn- toposort-visit [node child->parents visited result]
|
||||
(cond
|
||||
(visited node) [visited result]
|
||||
:else (let [parents (child->parents node [])
|
||||
[visited' result'] (reduce (fn [[v r] p]
|
||||
(toposort-visit p child->parents v r))
|
||||
[(conj visited node) result]
|
||||
parents)]
|
||||
[visited' (conj result' node)])))
|
||||
|
||||
(defn toposort-dfs
|
||||
"Perform a topological sort using depth-first search.
|
||||
Takes a map from child nodes to their parent nodes (dependencies).
|
||||
Returns nodes in topological order (dependencies before dependents)."
|
||||
[child->parents]
|
||||
;; TODO (Chris 2025-11-20): Detect cycles and throw an error. (In practice inputs will never be cyclic, but still.)
|
||||
(let [all-nodes (set (keys child->parents))]
|
||||
(loop [visited #{}
|
||||
result []
|
||||
remaining all-nodes]
|
||||
(if (empty? remaining)
|
||||
result
|
||||
(let [node (first remaining)
|
||||
[visited' result'] (toposort-visit node child->parents visited result)]
|
||||
(recur visited' result' (disj remaining node)))))))
|
||||
@@ -169,6 +169,20 @@
|
||||
:table #{checkins-id venues-id}}
|
||||
(calculation/upstream-deps:transform transform)))))))
|
||||
|
||||
(deftest ^:parallel upstream-deps-python-transform-test
|
||||
(mt/with-premium-features #{:transforms}
|
||||
(let [products-id (mt/id :products)
|
||||
orders-id (mt/id :orders)]
|
||||
(mt/with-temp [:model/Transform transform {:name "Test Transform"
|
||||
:source {:type :python
|
||||
:source-tables {"PRODUCTS" products-id
|
||||
"ORDERS" orders-id}
|
||||
:body "..."}
|
||||
:target {:schema "PUBLIC"
|
||||
:name "test_output"}}]
|
||||
(is (= {:table #{products-id orders-id}}
|
||||
(calculation/upstream-deps:transform transform)))))))
|
||||
|
||||
(deftest ^:parallel upstream-deps-card-native-with-parameter-source-test
|
||||
(let [mp (mt/metadata-provider)
|
||||
products-id (mt/id :products)
|
||||
|
||||
@@ -96,6 +96,8 @@
|
||||
:model/TransformJobRun
|
||||
:model/TransformJobTransformTag
|
||||
:model/TransformTransformTag
|
||||
:model/WorkspaceMappingTable
|
||||
:model/WorkspaceMappingTransform
|
||||
:model/PythonLibrary
|
||||
:model/Undo
|
||||
:model/User
|
||||
@@ -106,7 +108,8 @@
|
||||
:model/ConnectionImpersonation
|
||||
:model/CloudMigration
|
||||
:model/Comment
|
||||
:model/CommentReaction})
|
||||
:model/CommentReaction
|
||||
:model/Workspace})
|
||||
|
||||
(deftest ^:parallel comprehensive-entity-id-test
|
||||
(let [entity-id-models (->> (v2.entity-ids/toucan-models)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
(ns ^:mb/driver-tests metabase-enterprise.transforms.api-test
|
||||
"Tests for /api/transform endpoints."
|
||||
(:require
|
||||
[clojure.string :as str]
|
||||
[clojure.test :refer :all]
|
||||
[medley.core :as m]
|
||||
[metabase-enterprise.transforms.query-test-util :as query-test-util]
|
||||
@@ -269,6 +270,229 @@
|
||||
(is (=? [{:id t2-id}]
|
||||
(mt/user-http-request :crowberto :get 200 "ee/transform" :tag_ids [tag2-id])))))))))
|
||||
|
||||
(deftest filter-transforms-by-workspace-test
|
||||
(testing "GET /api/ee/transform filters out workspace transforms by default"
|
||||
(mt/with-premium-features #{:transforms :workspaces}
|
||||
(mt/with-temp [:model/Transform {global-id :id} {}
|
||||
:model/Workspace {ws-id :id} {:name "Test Workspace"}
|
||||
:model/Transform {ws-transform-id :id} {:workspace_id ws-id}]
|
||||
(testing "by default, only global transforms (workspace_id=null) are returned"
|
||||
(let [transforms (mt/user-http-request :crowberto :get 200 "ee/transform")
|
||||
ids (set (map :id transforms))]
|
||||
(is (contains? ids global-id))
|
||||
(is (not (contains? ids ws-transform-id)))))))))
|
||||
|
||||
(deftest exclude-workspace-id-filter-test
|
||||
(testing "GET /api/ee/transform with exclude_workspace_id excludes mirrored transforms"
|
||||
(mt/with-premium-features #{:transforms :workspaces}
|
||||
(mt/with-temp [;; Global transforms (workspace_id = null)
|
||||
:model/Transform {global1-id :id} {:name "Global Transform 1"}
|
||||
:model/Transform {global2-id :id} {:name "Global Transform 2"}
|
||||
:model/Transform {global3-id :id} {:name "Global Transform 3"}
|
||||
;; Workspace
|
||||
:model/Workspace {ws-id :id} {:name "Test Workspace"}
|
||||
;; Workspace transforms (mirrored from global1 and global2)
|
||||
:model/Transform {ws-xf1-id :id} {:name "Workspace Transform 1"
|
||||
:workspace_id ws-id}
|
||||
:model/Transform {ws-xf2-id :id} {:name "Workspace Transform 2"
|
||||
:workspace_id ws-id}
|
||||
;; Mappings: global1 -> ws-xf1, global2 -> ws-xf2
|
||||
:model/WorkspaceMappingTransform _ {:upstream_id global1-id
|
||||
:downstream_id ws-xf1-id
|
||||
:workspace_id ws-id}
|
||||
:model/WorkspaceMappingTransform _ {:upstream_id global2-id
|
||||
:downstream_id ws-xf2-id
|
||||
:workspace_id ws-id}]
|
||||
(testing "without exclude_workspace_id, returns all global transforms"
|
||||
(let [transforms (mt/user-http-request :crowberto :get 200 "ee/transform")
|
||||
ids (set (map :id transforms))]
|
||||
(is (contains? ids global1-id))
|
||||
(is (contains? ids global2-id))
|
||||
(is (contains? ids global3-id))
|
||||
;; workspace transforms should NOT be returned (they have workspace_id set)
|
||||
(is (not (contains? ids ws-xf1-id)))
|
||||
(is (not (contains? ids ws-xf2-id)))))
|
||||
|
||||
(testing "with exclude_workspace_id, excludes global transforms mirrored into that workspace"
|
||||
(let [transforms (mt/user-http-request :crowberto :get 200 "ee/transform"
|
||||
:exclude_workspace_id ws-id)
|
||||
ids (set (map :id transforms))]
|
||||
;; global1 and global2 are mirrored into the workspace, so they should be excluded
|
||||
(is (not (contains? ids global1-id)))
|
||||
(is (not (contains? ids global2-id)))
|
||||
;; global3 is NOT mirrored, so it should be returned
|
||||
(is (contains? ids global3-id))
|
||||
;; workspace transforms should still NOT be returned
|
||||
(is (not (contains? ids ws-xf1-id)))
|
||||
(is (not (contains? ids ws-xf2-id))))))))
|
||||
|
||||
(testing "exclude_workspace_id only excludes transforms mirrored into that specific workspace"
|
||||
(mt/with-premium-features #{:transforms :workspaces}
|
||||
(mt/with-temp [;; Global transforms
|
||||
:model/Transform {global1-id :id} {:name "Global Transform 1"}
|
||||
:model/Transform {global2-id :id} {:name "Global Transform 2"}
|
||||
;; Two workspaces
|
||||
:model/Workspace {ws1-id :id} {:name "Workspace 1"}
|
||||
:model/Workspace {ws2-id :id} {:name "Workspace 2"}
|
||||
;; global1 mirrored only into ws1
|
||||
:model/Transform {ws1-xf-id :id} {:name "WS1 Transform"
|
||||
:workspace_id ws1-id}
|
||||
:model/WorkspaceMappingTransform _ {:upstream_id global1-id
|
||||
:downstream_id ws1-xf-id
|
||||
:workspace_id ws1-id}
|
||||
;; global2 mirrored only into ws2
|
||||
:model/Transform {ws2-xf-id :id} {:name "WS2 Transform"
|
||||
:workspace_id ws2-id}
|
||||
:model/WorkspaceMappingTransform _ {:upstream_id global2-id
|
||||
:downstream_id ws2-xf-id
|
||||
:workspace_id ws2-id}]
|
||||
(testing "exclude_workspace_id=ws1 excludes only global1"
|
||||
(let [transforms (mt/user-http-request :crowberto :get 200 "ee/transform"
|
||||
:exclude_workspace_id ws1-id)
|
||||
ids (set (map :id transforms))]
|
||||
(is (not (contains? ids global1-id)))
|
||||
(is (contains? ids global2-id))))
|
||||
|
||||
(testing "exclude_workspace_id=ws2 excludes only global2"
|
||||
(let [transforms (mt/user-http-request :crowberto :get 200 "ee/transform"
|
||||
:exclude_workspace_id ws2-id)
|
||||
ids (set (map :id transforms))]
|
||||
(is (contains? ids global1-id))
|
||||
(is (not (contains? ids global2-id)))))))))
|
||||
|
||||
(deftest filter-transforms-by-type-test
|
||||
(testing "should be able to filter transforms by source type"
|
||||
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
|
||||
(mt/with-premium-features #{:transforms :transforms-python}
|
||||
(mt/with-temp [:model/Transform {native-transform-id :id}
|
||||
{:name "Native Transform"
|
||||
:source {:type "query"
|
||||
:query {:database (mt/id)
|
||||
:type "native"
|
||||
:native {:query "SELECT 1"
|
||||
:template-tags {}}}}
|
||||
:target {:type "table"
|
||||
:name (str "test_native_" (u/generate-nano-id))}}
|
||||
:model/Transform {python-transform-id :id}
|
||||
{:name "Python Transform"
|
||||
:source {:type "python"
|
||||
:body "print('hello')"
|
||||
:source-tables {}}
|
||||
:target {:type "table"
|
||||
:name (str "test_python_" (u/generate-nano-id))
|
||||
:database (mt/id)}}]
|
||||
(testing "filter by native type"
|
||||
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :type ["native"])]
|
||||
(is (some #(= native-transform-id (:id %)) results))
|
||||
(is (not (some #(= python-transform-id (:id %)) results)))))
|
||||
(testing "filter by python type"
|
||||
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :type ["python"])]
|
||||
(is (some #(= python-transform-id (:id %)) results))
|
||||
(is (not (some #(= native-transform-id (:id %)) results)))))
|
||||
(testing "filter by both types returns all"
|
||||
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :type ["native" "python"])]
|
||||
(is (some #(= native-transform-id (:id %)) results))
|
||||
(is (some #(= python-transform-id (:id %)) results)))))))))
|
||||
|
||||
(deftest filter-transforms-by-database-id-test
|
||||
(testing "should be able to filter transforms by database_id"
|
||||
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
|
||||
(mt/with-premium-features #{:transforms}
|
||||
(mt/with-temp [:model/Database {other-db-id :id} {:engine :h2 :details {}}
|
||||
:model/Transform {transform1-id :id}
|
||||
{:name "Transform for main DB"
|
||||
:source {:type "query"
|
||||
:query {:database (mt/id)
|
||||
:type "native"
|
||||
:native {:query "SELECT 1"
|
||||
:template-tags {}}}}
|
||||
:target {:type "table"
|
||||
:name (str "test_main_" (u/generate-nano-id))}}
|
||||
:model/Transform {transform2-id :id}
|
||||
{:name "Transform for other DB"
|
||||
:source {:type "query"
|
||||
:query {:database other-db-id
|
||||
:type "native"
|
||||
:native {:query "SELECT 2"
|
||||
:template-tags {}}}}
|
||||
:target {:type "table"
|
||||
:name (str "test_other_" (u/generate-nano-id))}}]
|
||||
(testing "filter by main database id"
|
||||
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :database_id (mt/id))]
|
||||
(is (some #(= transform1-id (:id %)) results))
|
||||
(is (not (some #(= transform2-id (:id %)) results)))))
|
||||
(testing "filter by other database id"
|
||||
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :database_id other-db-id)]
|
||||
(is (some #(= transform2-id (:id %)) results))
|
||||
(is (not (some #(= transform1-id (:id %)) results)))))
|
||||
(testing "filter by non-existent database returns empty for our transforms"
|
||||
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :database_id 999999)]
|
||||
(is (not (some #(= transform1-id (:id %)) results)))
|
||||
(is (not (some #(= transform2-id (:id %)) results))))))))))
|
||||
|
||||
(deftest filter-transforms-by-database-id-target-test
|
||||
(testing "should match transforms where target database matches"
|
||||
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
|
||||
(mt/with-premium-features #{:transforms :transforms-python}
|
||||
(mt/with-temp [:model/Database {target-db-id :id} {:engine :h2 :details {}}
|
||||
:model/Transform {transform-id :id}
|
||||
{:name "Python Transform with target DB"
|
||||
:source {:type "python"
|
||||
:body "print('hello')"
|
||||
:source-tables {}}
|
||||
:target {:type "table"
|
||||
:name (str "test_target_" (u/generate-nano-id))
|
||||
:database target-db-id}}]
|
||||
(testing "filter by target database id"
|
||||
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :database_id target-db-id)]
|
||||
(is (some #(= transform-id (:id %)) results)))))))))
|
||||
|
||||
(deftest filter-transforms-combined-test
|
||||
(testing "should be able to combine database_id and type filters"
|
||||
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
|
||||
(mt/with-premium-features #{:transforms :transforms-python}
|
||||
(mt/with-temp [:model/Database {other-db-id :id} {:engine :h2 :details {}}
|
||||
:model/Transform {query-main-id :id}
|
||||
{:name "Query on main DB"
|
||||
:source {:type "query"
|
||||
:query {:database (mt/id)
|
||||
:type "native"
|
||||
:native {:query "SELECT 1"
|
||||
:template-tags {}}}}
|
||||
:target {:type "table"
|
||||
:name (str "test_qm_" (u/generate-nano-id))}}
|
||||
:model/Transform {python-main-id :id}
|
||||
{:name "Python on main DB"
|
||||
:source {:type "python"
|
||||
:body "print('hello')"
|
||||
:source-tables {}}
|
||||
:target {:type "table"
|
||||
:name (str "test_pm_" (u/generate-nano-id))
|
||||
:database (mt/id)}}
|
||||
:model/Transform {query-other-id :id}
|
||||
{:name "Query on other DB"
|
||||
:source {:type "query"
|
||||
:query {:database other-db-id
|
||||
:type "native"
|
||||
:native {:query "SELECT 2"
|
||||
:template-tags {}}}}
|
||||
:target {:type "table"
|
||||
:name (str "test_qo_" (u/generate-nano-id))}}]
|
||||
(testing "filter by main database and query type"
|
||||
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform"
|
||||
:database_id (mt/id)
|
||||
:type ["query"])]
|
||||
(is (some #(= query-main-id (:id %)) results))
|
||||
(is (not (some #(= python-main-id (:id %)) results)))
|
||||
(is (not (some #(= query-other-id (:id %)) results)))))
|
||||
(testing "filter by main database and python type"
|
||||
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform"
|
||||
:database_id (mt/id)
|
||||
:type ["python"])]
|
||||
(is (some #(= python-main-id (:id %)) results))
|
||||
(is (not (some #(= query-main-id (:id %)) results)))
|
||||
(is (not (some #(= query-other-id (:id %)) results))))))))))
|
||||
|
||||
(deftest get-transforms-test
|
||||
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
|
||||
(mt/with-premium-features #{:transforms}
|
||||
|
||||
@@ -108,3 +108,81 @@
|
||||
(catch Exception _e
|
||||
;; Ignore cleanup errors
|
||||
nil)))))))))
|
||||
|
||||
;;; ------------------------------------------------------------
|
||||
;;; Filter xf tests
|
||||
;;; ------------------------------------------------------------
|
||||
|
||||
(deftest ^:parallel type-filter-xf-test
|
||||
(testing "->type-filter-xf filters transforms by source type"
|
||||
;; Note: :query :type "query" = MBQL, :query :type "native" = native SQL
|
||||
(let [mbql-transform {:id 1 :source {:type "query" :query {:type "query"}}}
|
||||
native-transform {:id 2 :source {:type "query" :query {:type "native"}}}
|
||||
python-transform {:id 3 :source {:type "python"}}
|
||||
transforms [mbql-transform native-transform python-transform]]
|
||||
|
||||
(are [x y] (= x (into [] (transforms.util/->type-filter-xf y) transforms))
|
||||
transforms nil
|
||||
transforms []
|
||||
[mbql-transform] ["query"]
|
||||
[native-transform] ["native"]
|
||||
[python-transform] ["python"]
|
||||
[mbql-transform native-transform] ["query" "native"]
|
||||
transforms ["query" "native" "python"]
|
||||
[] ["whatever"]))))
|
||||
|
||||
(deftest ^:parallel database-id-filter-xf-test
|
||||
(testing "->database-id-filter-xf filters transforms by database ID"
|
||||
(let [db1-query-x {:id 1
|
||||
:name "Query on DB1"
|
||||
:source {:type "query" :query {:database 10}}
|
||||
:target {:type "table" :name "t1" :database 1}}
|
||||
db2-target-x {:id 2
|
||||
:name "Python targeting DB4"
|
||||
:source {:type "python" :body "" :source-tables {}}
|
||||
:target {:type "table" :name "t4" :database 2}}
|
||||
transforms [db1-query-x db2-target-x]]
|
||||
|
||||
(are [x y] (= x (into [] (transforms.util/->database-id-filter-xf y) transforms))
|
||||
transforms nil
|
||||
[db1-query-x] 1
|
||||
[] 10 ; source does not match
|
||||
[db2-target-x] 2
|
||||
[] 999))))
|
||||
|
||||
(deftest ^:parallel matching-timestamp?-test
|
||||
(testing "matching-timestamp? checks if a timestamp falls within a date range [start, end)"
|
||||
(let [matching-timestamp? #'transforms.util/matching-timestamp?
|
||||
field-path [:start_time]
|
||||
range-jan-feb {:start "2024-01-01T00:00:00Z" :end "2024-02-01T00:00:00Z"}
|
||||
range-start-only {:start "2024-01-01T00:00:00Z" :end nil}
|
||||
range-end-only {:start nil :end "2024-02-01T00:00:00Z"}]
|
||||
|
||||
(testing "with both start and end bounds"
|
||||
(are [expected timestamp]
|
||||
(= expected (matching-timestamp? {:start_time timestamp} field-path range-jan-feb))
|
||||
nil nil ; missing field returns nil
|
||||
true "2024-01-15T12:00:00Z" ; timestamp in middle of range
|
||||
false "2023-12-15T12:00:00Z" ; timestamp before range
|
||||
false "2024-02-15T12:00:00Z" ; timestamp after range
|
||||
true "2024-01-01T00:00:00Z" ; start boundary is inclusive
|
||||
true "2024-02-01T00:00:00Z")) ; end boundary is inclusive too 🤷
|
||||
|
||||
(testing "with only start bound"
|
||||
(are [expected timestamp]
|
||||
(= expected (matching-timestamp? {:start_time timestamp} field-path range-start-only))
|
||||
true "2024-01-15T12:00:00Z" ; timestamp after start
|
||||
true "2024-02-15T12:00:00Z" ; any timestamp after start
|
||||
false "2023-12-15T12:00:00Z")) ; timestamp before start
|
||||
|
||||
(testing "with only end bound"
|
||||
(are [expected timestamp]
|
||||
(= expected (matching-timestamp? {:start_time timestamp} field-path range-end-only))
|
||||
true "2024-01-15T12:00:00Z" ; timestamp before end
|
||||
true "2023-12-15T12:00:00Z" ; any timestamp before end
|
||||
false "2024-02-15T12:00:00Z")) ; timestamp after end
|
||||
|
||||
(testing "returns nil when field value is missing"
|
||||
(are [job] (nil? (matching-timestamp? job field-path range-jan-feb))
|
||||
{}
|
||||
{:other "value"})))))
|
||||
|
||||
@@ -0,0 +1,645 @@
|
||||
(ns ^:mb/driver-tests metabase-enterprise.workspaces.api-test
|
||||
(:require
|
||||
[clojure.string :as str]
|
||||
[clojure.test :refer :all]
|
||||
[metabase-enterprise.transforms.interface :as transforms.i]
|
||||
[metabase-enterprise.transforms.test-dataset :as transforms-dataset]
|
||||
[metabase-enterprise.transforms.test-util :refer [with-transform-cleanup!]]
|
||||
[metabase-enterprise.workspaces.isolation :as ws.isolation]
|
||||
[metabase.lib.core :as lib]
|
||||
[metabase.search.test-util :as search.tu]
|
||||
[metabase.test :as mt]
|
||||
[metabase.test.util :as tu]
|
||||
[toucan2.core :as t2])
|
||||
(:import (java.time OffsetDateTime)))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(use-fixtures :once (fn [tests]
|
||||
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
|
||||
(mt/with-premium-features [:workspaces :dependencies :transforms]
|
||||
(search.tu/with-index-disabled
|
||||
(tests))))))
|
||||
|
||||
(use-fixtures :each (fn [tests]
|
||||
(mt/with-model-cleanup [:model/Collection
|
||||
:model/Transform
|
||||
:model/Workspace
|
||||
:model/WorkspaceTransform
|
||||
:model/WorkspaceMappingTransform]
|
||||
(tests))))
|
||||
|
||||
(defn ws-url [id & [path]]
|
||||
(str "ee/workspace/" id path))
|
||||
|
||||
(defn- ws-ready
|
||||
"Poll until workspace status becomes :ready or timeout"
|
||||
[ws-or-id]
|
||||
(let [ws-id (cond-> ws-or-id
|
||||
(map? ws-or-id) :id)]
|
||||
(try
|
||||
(tu/poll-until 300 (or (t2/select-one :model/Workspace :id ws-id :status :ready)
|
||||
(Thread/sleep 10)))
|
||||
(catch Exception e
|
||||
(if (:timeout-ms (ex-data e))
|
||||
(throw (ex-info "Workspace is not ready yet" {:logs (t2/select [:model/WorkspaceLog :task :status :message]
|
||||
:workspace_id ws-id
|
||||
{:order-by [[:started_at :desc]]})}))
|
||||
(throw e))))))
|
||||
|
||||
(deftest workspace-endpoints-require-superuser-test
|
||||
(mt/with-temp [:model/Workspace workspace {:name "Private Workspace"}]
|
||||
(testing "GET /api/ee/workspace requires superuser"
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :get 403 "ee/workspace"))))
|
||||
|
||||
(testing "GET /api/ee/workspace/:id requires superuser"
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :get 403 (ws-url (:id workspace) "")))))
|
||||
|
||||
(testing "POST /api/ee/workspace requires superuser"
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :post 403 "ee/workspace"
|
||||
{:name "Unauthorized Workspace"})))))
|
||||
|
||||
(mt/with-temp [:model/Workspace workspace {:name "Put Workspace"}]
|
||||
(testing "PUT /api/ee/workspace/:id requires superuser"
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :put 403 (ws-url (:id workspace) "")
|
||||
{:name "Updated"})))))
|
||||
|
||||
(mt/with-temp [:model/Workspace workspace {:name "Delete Workspace"}]
|
||||
(testing "DELETE /api/ee/workspace/:id requires superuser"
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :delete 403 (ws-url (:id workspace) ""))))))
|
||||
|
||||
(mt/with-temp [:model/Workspace workspace {:name "Promote Workspace"}]
|
||||
(testing "POST /api/ee/workspace/:id/promote requires superuser"
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :post 403 (ws-url (:id workspace) "/merge")))))))
|
||||
|
||||
(deftest workspace-crud-flow-test
|
||||
(let [workspace-name (str "Workspace " (random-uuid))
|
||||
created (mt/user-http-request :crowberto :post 200 "ee/workspace"
|
||||
{:name workspace-name
|
||||
:database_id (mt/id)})
|
||||
workspace-id (:id created)
|
||||
collection-id (:collection_id created)]
|
||||
(is (=? {:id int?
|
||||
:collection_id int?
|
||||
:name workspace-name}
|
||||
created))
|
||||
(testing "the collection exists, and we have a back reference"
|
||||
(is (t2/exists? :model/Workspace :id workspace-id :collection_id collection-id))
|
||||
(is (t2/exists? :model/Collection :id collection-id :workspace_id workspace-id)))
|
||||
|
||||
(testing "workspace appears in list response"
|
||||
(let [{:keys [items]} (mt/user-http-request :crowberto :get 200 "ee/workspace")]
|
||||
(is (some #(= workspace-id (:id %)) items))))
|
||||
|
||||
(testing "workspace can be fetched individually"
|
||||
(let [response (mt/user-http-request :crowberto :get 200 (ws-url workspace-id ""))]
|
||||
(is (= workspace-id (:id response)))))
|
||||
|
||||
(testing "workspace can be archived"
|
||||
(let [updated (mt/user-http-request :crowberto :post 200 (str "ee/workspace/" workspace-id "/archive"))]
|
||||
(is (some? (:archived_at updated)))))
|
||||
|
||||
(testing "workspace can be unarchived"
|
||||
(let [updated (mt/user-http-request :crowberto :post 200 (str "ee/workspace/" workspace-id "/unarchive"))]
|
||||
(is (nil? (:archived_at updated)))))))
|
||||
|
||||
(deftest ^:parallel promote-workspace-test
|
||||
(testing "POST /api/ee/workspace/:id/promote requires superuser"
|
||||
(mt/with-temp [:model/Workspace workspace {:name "Promote Test"}]
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :post 403 (ws-url (:id workspace) "/merge"))))))
|
||||
|
||||
(testing "Cannot merge an already archived workspace"
|
||||
(mt/with-temp [:model/Workspace workspace {:name "Archived Workspace"
|
||||
:archived_at (OffsetDateTime/now)}]
|
||||
(is (= "Cannot merge an archived workspace"
|
||||
(mt/user-http-request :crowberto :post 400 (ws-url (:id workspace) "/merge")))))))
|
||||
|
||||
(deftest merge-workspace-with-transform-test
|
||||
(testing "POST /api/ee/workspace/:id/merge promotes transforms and archives workspace"
|
||||
(mt/with-temp [:model/Table _table {:schema "public" :name "merge_test_table"}
|
||||
:model/Transform x1 {:name "Upstream Transform"
|
||||
:description "Original description"
|
||||
:target {:type "table"
|
||||
:database 1
|
||||
:schema "public"
|
||||
:name "merge_test_table"}}]
|
||||
(let [{ws-id :id} (ws-ready (mt/user-http-request :crowberto :post 200 "ee/workspace"
|
||||
{:name "Merge test"
|
||||
:database_id (mt/id)}))]
|
||||
(mt/user-http-request :crowberto :post 200 (ws-url ws-id "/transform")
|
||||
(merge {:global_id (:id x1)}
|
||||
(select-keys x1 [:name :description :source :target])))
|
||||
(testing "We've got our workspace with transform to merge"
|
||||
(is (int? ws-id))
|
||||
;; (sanya) TODO: maybe switch to using transform APIs once we get our own
|
||||
(let [x2-id (t2/select-one-fn :downstream_id :model/WorkspaceMappingTransform :upstream_id (:id x1))]
|
||||
(t2/update! :model/Transform :id x2-id {:description "Modified in workspace"})))
|
||||
(testing "returns merged transforms"
|
||||
(is (=? {:merged {:transforms [(:id x1)]}
|
||||
:errors []
|
||||
:workspace {:id ws-id :name "Merge test"}}
|
||||
(mt/user-http-request :crowberto :post 200 (ws-url ws-id "/merge")))))
|
||||
;; TODO re-implement merge
|
||||
#_(testing "original transform was updated with workspace version"
|
||||
(is (= "Modified in workspace"
|
||||
(t2/select-one-fn :description :model/Transform :id (:id x1)))))
|
||||
(testing "workspace was deleted after successful merge"
|
||||
(is (nil? (t2/select-one :model/Workspace :id ws-id))))))))
|
||||
|
||||
(deftest create-workspace-transform-permissions-test
|
||||
(testing "POST /api/ee/workspace/:id/transform requires superuser"
|
||||
(mt/with-temp [:model/Workspace workspace {:name "Transform Test"}]
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :post 403 (ws-url (:id workspace) "/transform")
|
||||
{:name "Should Fail"
|
||||
:source {:type "query"
|
||||
:query {}}
|
||||
:target {:type "table"
|
||||
:name "should_fail"}}))))))
|
||||
|
||||
(deftest create-workspace-transform-archived-test
|
||||
(testing "Cannot create transform in archived workspace"
|
||||
(mt/with-temp [:model/Workspace workspace {:name "Archived"
|
||||
:archived_at (OffsetDateTime/now)}]
|
||||
(is (= "Cannot create transforms in an archived workspace"
|
||||
(mt/user-http-request :crowberto :post 400 (ws-url (:id workspace) "/transform")
|
||||
{:name "Should Fail"
|
||||
:source {:type "query"
|
||||
:query (mt/mbql-query venues)}
|
||||
:target {:type "table"
|
||||
:name "should_fail"}}))))))
|
||||
|
||||
(deftest add-transforms-to-workspace-test
|
||||
(testing "Add transforms to workspace via POST /transform"
|
||||
(with-transform-cleanup! [orig-name "ws_tables_test"]
|
||||
(mt/with-temp [:model/Transform {x1-id :id :as x1} {:target {:type "table"
|
||||
:database (mt/id)
|
||||
:schema "public"
|
||||
:name orig-name}}]
|
||||
(let [{ws-id :id} (ws-ready (mt/user-http-request :crowberto :post 200 "ee/workspace"
|
||||
{:name "Add Transforms Test"
|
||||
:database_id (mt/id)}))]
|
||||
(is (int? ws-id))
|
||||
(testing "Can check out a global transform into workspace"
|
||||
(is (=? {:ref_id string?
|
||||
:global_id x1-id}
|
||||
(mt/user-http-request :crowberto :post 200 (ws-url ws-id "/transform")
|
||||
(merge {:global_id x1-id}
|
||||
(select-keys x1 [:name :description :source :target]))))))
|
||||
|
||||
(testing "Can create a new provisional transform"
|
||||
(is (=? {:ref_id string?
|
||||
:global_id nil?
|
||||
:name "New Transform"}
|
||||
(mt/user-http-request :crowberto :post 200 (ws-url ws-id "/transform")
|
||||
{:name "New Transform"
|
||||
:source {:type "query"
|
||||
:query (mt/mbql-query venues)}
|
||||
:target {:type "table"
|
||||
:name "new_transform_output"}}))))
|
||||
|
||||
(testing "Cannot add transforms to archived workspace"
|
||||
(t2/update! :model/Workspace ws-id {:archived_at (OffsetDateTime/now)})
|
||||
(is (= "Cannot create transforms in an archived workspace"
|
||||
(mt/user-http-request :crowberto :post 400 (ws-url ws-id "/transform")
|
||||
{:name "Should Fail"
|
||||
:source {:type "query"
|
||||
:query (mt/mbql-query venues)}
|
||||
:target {:type "table"
|
||||
:name "should_fail"}})))))))))
|
||||
|
||||
(deftest add-entities-requires-superuser-test
|
||||
(testing "POST /api/ee/workspace/:id/add requires superuser"
|
||||
(mt/with-temp [:model/Workspace workspace {:name "Permission Test"}]
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :post 403 (ws-url (:id workspace) "/transform")
|
||||
{:name "blah", :source {}, :target {}}))))))
|
||||
|
||||
(deftest create-workspace-transform-test
|
||||
(mt/dataset transforms-dataset/transforms-test
|
||||
(let [{ws-id :id} (ws-ready (mt/user-http-request :crowberto :post 200 "ee/workspace"
|
||||
{:name "Test Workspace"
|
||||
:database_id (mt/id)}))]
|
||||
(with-transform-cleanup! [table-name "workspace_transform_test"]
|
||||
(is (=? {:ref_id string?
|
||||
:workspace_id ws-id
|
||||
;:creator_id (mt/user->id :crowberto)
|
||||
:target {:database (mt/id)}}
|
||||
(mt/user-http-request :crowberto :post 200 (ws-url ws-id "/transform")
|
||||
{:name "Workspace Transform"
|
||||
:source {:type "query"
|
||||
:query (mt/mbql-query transforms_products)}
|
||||
:target {:type "table"
|
||||
:name table-name}})))
|
||||
(is (=? {:id ws-id, :status "ready"}
|
||||
(mt/user-http-request :crowberto :get 200 (ws-url ws-id))))
|
||||
(is (=? {:transforms [{:ref_id string?, :name "Workspace Transform", :source_type "query"}]}
|
||||
(mt/user-http-request :crowberto :get 200 (ws-url ws-id "/transform"))))))))
|
||||
|
||||
(deftest tables-endpoint-empty-ws-test
|
||||
(let [user-id (mt/user->id :crowberto)
|
||||
ws (mt/user-http-request :crowberto :post 200 "ee/workspace/"
|
||||
{:name "My test ws"
|
||||
:creator_id user-id
|
||||
:database_id (mt/id)})]
|
||||
(is (= {:inputs []
|
||||
:outputs []}
|
||||
(mt/user-http-request :crowberto :get 200 (ws-url (:id ws) "/table"))))))
|
||||
|
||||
(deftest tables-endpoint-transform-not-run-test
|
||||
(let [mp (mt/metadata-provider)
|
||||
query (lib/native-query mp "select * from orders limit 10;")
|
||||
orig-schema "public"]
|
||||
(with-transform-cleanup! [orig-name "ws_tables_test"]
|
||||
(mt/with-temp [:model/Transform x1 {:source_type "native"
|
||||
:name "My X1"
|
||||
:source {:type "query"
|
||||
:query query}
|
||||
:target {:type "table"
|
||||
:database (mt/id)
|
||||
:schema "public"
|
||||
:name orig-name}}]
|
||||
(let [;; create the workspace
|
||||
{ws-id :id} (mt/user-http-request :crowberto :post 200 "ee/workspace"
|
||||
{:name "Test Workspace"
|
||||
:database_id (mt/id)})
|
||||
;; add the transform
|
||||
ref_id (:ref_id (mt/user-http-request :crowberto :post 200 (ws-url ws-id "/transform") x1))
|
||||
;; get the tables
|
||||
tables-result (mt/user-http-request :crowberto :get 200 (ws-url ws-id "/table"))]
|
||||
(testing "/tables returns expected results"
|
||||
(is (=? {:inputs [#_{:id (mt/id :orders) :schema orig-schema :table "orders"}]
|
||||
:outputs [{:db_id (mt/id)
|
||||
:global {:schema "public", :table orig-name}
|
||||
:isolated {:transform_id ref_id}}]}
|
||||
tables-result))))))))
|
||||
|
||||
(deftest tables-endpoint-test
|
||||
(let [mp (mt/metadata-provider)
|
||||
query (lib/native-query mp "select * from orders limit 10;")
|
||||
orig-schema "public"]
|
||||
(with-transform-cleanup! [orig-name "ws_tables_test"]
|
||||
(mt/with-temp [:model/Transform x1 {:source_type "native"
|
||||
:name "My X1"
|
||||
:source {:type "query"
|
||||
:query query}
|
||||
:target {:type "table"
|
||||
:database (mt/id)
|
||||
:schema "public"
|
||||
:name orig-name}}]
|
||||
;; TODO: add a provisional transform, and make sure one of the output tables is created
|
||||
;; create the global table
|
||||
(transforms.i/execute! x1 {:run-method :manual})
|
||||
(let [;; create the workspace
|
||||
workspace (ws-ready (mt/user-http-request :crowberto :post 200 "ee/workspace"
|
||||
{:name "Test Workspace"
|
||||
:database_id (mt/id)}))
|
||||
;; add the transform
|
||||
ref-id (:ref_id (mt/user-http-request :crowberto :post 200 (ws-url (:id workspace) "/transform") x1))
|
||||
;; get the tables
|
||||
tables-result (mt/user-http-request :crowberto :get 200 (ws-url (:id workspace) "/table"))
|
||||
mirror-transform (t2/select-one :model/WorkspaceTransform :workspace_id (:id workspace))
|
||||
mirror-table (t2/select-one :model/Table
|
||||
:schema (-> mirror-transform :target :schema)
|
||||
:name (-> mirror-transform :target :name))]
|
||||
(testing "/table returns expected results"
|
||||
;; TODO: implement inputs (requires query analysis)
|
||||
(is (=? {:inputs [#_{:id (mt/id :orders) :schema orig-schema :table "orders"}]
|
||||
:outputs
|
||||
[{:db_id (mt/id)
|
||||
:global {:schema orig-schema :table orig-name, :table_id (:id mirror-table)}
|
||||
:isolated {:transform_id ref-id}}]}
|
||||
tables-result))))))))
|
||||
|
||||
;;;; Card dependency rejection tests
|
||||
|
||||
(defn- query-with-source-card
|
||||
"Create a pMBQL query that uses a card as its source."
|
||||
[card-id]
|
||||
{:lib/type :mbql/query
|
||||
:database (mt/id)
|
||||
:stages [{:lib/type :mbql.stage/mbql
|
||||
:source-card card-id}]})
|
||||
|
||||
(defn- create-transform-with-card-source!
|
||||
"Create a transform whose source query depends on a card.
|
||||
The after-insert hook triggers dependency calculation automatically."
|
||||
[card]
|
||||
(t2/insert-returning-instance! :model/Transform
|
||||
{:name "Transform depending on card"
|
||||
:source {:type :query
|
||||
:query (query-with-source-card (:id card))}
|
||||
:target {:type "table"
|
||||
:database (mt/id)
|
||||
:schema "public"
|
||||
:name "card_dep_output"}}))
|
||||
|
||||
(deftest add-transform-rejects-card-dependencies-test
|
||||
(testing "Cannot add transforms that depend on cards"
|
||||
(mt/with-temp [:model/Card card {:name "Test Card"
|
||||
:database_id (mt/id)
|
||||
:dataset_query (mt/mbql-query venues)}]
|
||||
(let [tx (create-transform-with-card-source! card)
|
||||
ws (mt/user-http-request :crowberto :post 200 "ee/workspace"
|
||||
{:name "Card Dep Workspace"
|
||||
:database_id (mt/id)})
|
||||
;; TODO add validation
|
||||
_response (mt/user-http-request :crowberto :post 200 #_403 (ws-url (:id ws) "/transform") tx)]
|
||||
#_(is (re-find #"Cannot add transforms that depend on saved questions" ws))))))
|
||||
|
||||
(deftest add-transform-rejects-transitive-card-dependencies-test
|
||||
(testing "Cannot add transforms that transitively depend on cards"
|
||||
(mt/with-temp [:model/Card card {:name "Base Card"
|
||||
:database_id (mt/id)
|
||||
:dataset_query (mt/mbql-query venues)}]
|
||||
;; tx1 depends on card
|
||||
(let [tx1 (create-transform-with-card-source! card)
|
||||
;; tx2 depends on tx1 (via a manually created dependency - simulating transform chain)
|
||||
tx2 (t2/insert-returning-instance! :model/Transform
|
||||
{:name "Transform 2 - depends on tx1"
|
||||
:source {:type :query
|
||||
:query {:database (mt/id)
|
||||
:type :native
|
||||
:native {:query "SELECT 1"}}}
|
||||
:target {:type "table"
|
||||
:database (mt/id)
|
||||
:schema "public"
|
||||
:name "tx2_output"}})]
|
||||
;; Create dependency: tx2 depends on tx1
|
||||
(t2/insert! :model/Dependency
|
||||
{:from_entity_type "transform"
|
||||
:from_entity_id (:id tx2)
|
||||
:to_entity_type "transform"
|
||||
:to_entity_id (:id tx1)})
|
||||
;; Try to add tx2 (which transitively depends on card via tx1) to workspace
|
||||
(let [ws (mt/user-http-request :crowberto :post 200 "ee/workspace"
|
||||
{:name "Transitive Card Dep Workspace"
|
||||
:database_id (mt/id)})
|
||||
;; TODO add validation
|
||||
_response (mt/user-http-request :crowberto :post 200 #_403 (ws-url (:id ws) "/transform") tx2)]
|
||||
#_(is (re-find #"Cannot add transforms that depend on saved questions" response)))))))
|
||||
|
||||
(deftest rename-workspace-test
|
||||
(testing "POST /api/ee/workspace/:id/name updates the workspace name"
|
||||
(mt/with-temp [:model/Collection {coll-id :id} {}
|
||||
:model/Workspace workspace {:name "Original Name"
|
||||
:database_id (mt/id)
|
||||
:collection_id coll-id}]
|
||||
(let [response (mt/user-http-request :crowberto :put 200 (ws-url (:id workspace))
|
||||
{:name "Updated Name"})]
|
||||
(is (= "Updated Name"
|
||||
(:name response)
|
||||
(t2/select-one-fn :name :model/Workspace :id (:id workspace)))))))
|
||||
|
||||
(testing "Requires superuser"
|
||||
(mt/with-temp [:model/Workspace workspace {:name "Permission Test"}]
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :put 403 (ws-url (:id workspace))
|
||||
{:name "Should Fail"})))))
|
||||
|
||||
(testing "Cannot rename an archived workspace"
|
||||
(mt/with-temp [:model/Workspace workspace {:name "Archived"
|
||||
:archived_at (OffsetDateTime/now)}]
|
||||
(is (= "Cannot update an archived workspace"
|
||||
(mt/user-http-request :crowberto :put 400 (ws-url (:id workspace))
|
||||
{:name "Should Fail"}))))))
|
||||
|
||||
(deftest add-transform-rejects-card-dependencies-to-existing-workspace-test
|
||||
(testing "Cannot add transforms with card dependencies to existing workspace"
|
||||
(mt/with-temp [:model/Card card {:name "Test Card"
|
||||
:database_id (mt/id)
|
||||
:dataset_query (mt/mbql-query venues)}]
|
||||
(let [tx (create-transform-with-card-source! card)
|
||||
workspace-id (:id (mt/user-http-request :crowberto :post 200 "ee/workspace"
|
||||
{:name "Empty Workspace"
|
||||
:database_id (mt/id)}))
|
||||
;; TODO add validation
|
||||
_response (mt/user-http-request :crowberto :post 200 (ws-url workspace-id "/transform") tx)]
|
||||
#_(is (re-find #"Cannot add transforms that depend on saved questions" response))))))
|
||||
|
||||
(deftest validate-target-test
|
||||
(let [table (t2/select-one :model/Table :active true)]
|
||||
(mt/with-temp [:model/Workspace {ws-id :id} {:name "test"}
|
||||
:model/WorkspaceTransform _x1 {:workspace_id ws-id
|
||||
:target {:database (:db_id table)
|
||||
:type "table"
|
||||
:schema (:schema table)
|
||||
:name (str "q_" (:name table))}}]
|
||||
(testing "Unique"
|
||||
(is (= "OK"
|
||||
(mt/with-log-level [metabase.driver.sql-jdbc.sync.describe-table :fatal]
|
||||
(mt/user-http-request :crowberto :post 200 (ws-url ws-id "/transform/validate/target")
|
||||
{:db_id (mt/id)
|
||||
:target {:type "table"
|
||||
:schema "public"
|
||||
:name (str/replace (str (random-uuid)) "-" "_")}})))))
|
||||
;; We've decided to defer this error until merge.
|
||||
;; Also, this logic is going to become more relaxed, where we're allowed to take over a "dormant" table.
|
||||
#_(testing "Conflict outside of workspace"
|
||||
(is (= "A table with that name already exists."
|
||||
(mt/user-http-request :crowberto :post 403 (ws-url ws-id "/transform/validate/target")
|
||||
{:db_id (:db_id table)
|
||||
:target {:type "table"
|
||||
:schema (:schema table)
|
||||
:name (:name table)}}))))
|
||||
(testing "Conflict inside of workspace"
|
||||
(let [table (t2/select-one :model/Table :active true)]
|
||||
(is (= "Another transform in this workspace already targets that table."
|
||||
(mt/with-log-level [metabase.driver.sql-jdbc.sync.describe-table :fatal]
|
||||
(mt/user-http-request :crowberto :post 403 (ws-url ws-id "/transform/validate/target")
|
||||
{:db_id (:db_id table)
|
||||
:target {:type "table"
|
||||
:schema (:schema table)
|
||||
:name (str "q_" (:name table))}})))))))))
|
||||
|
||||
;;;; Async workspace creation tests
|
||||
|
||||
(deftest create-workspace-returns-updating-status-test
|
||||
(testing "Creating workspace returns status :pending immediately"
|
||||
(is (=? {:status "pending"}
|
||||
(mt/user-http-request :crowberto :post 200 "ee/workspace"
|
||||
{:name "async-test" :database_id (mt/id)})))))
|
||||
|
||||
(deftest workspace-status-becomes-ready-test
|
||||
(testing "Workspace status transitions to :ready after setup completes"
|
||||
(let [ws (ws-ready (mt/user-http-request :crowberto :post 200 "ee/workspace"
|
||||
{:name "async-ready-test" :database_id (mt/id)}))]
|
||||
(is (= :ready (:status ws))))))
|
||||
|
||||
(deftest workspace-log-endpoint-test
|
||||
(testing "GET /api/ee/workspace/:id/log returns status and log entries"
|
||||
(let [{ws-id :id} (ws-ready (mt/user-http-request :crowberto :post 200 "ee/workspace"
|
||||
{:name "log-test" :database_id (mt/id)}))]
|
||||
(is (=? {:workspace_id ws-id
|
||||
:status "ready"
|
||||
:updated_at some?
|
||||
:last_completed_at some?
|
||||
:logs [{:task "database-isolation"
|
||||
:status "success"}
|
||||
{:task "workspace-setup"}]}
|
||||
(mt/user-http-request :crowberto :get 200 (ws-url ws-id "/log")))))))
|
||||
|
||||
(deftest workspace-log-endpoint-404-test
|
||||
(testing "GET /api/ee/workspace/:id/log returns 404 for non-existent workspace"
|
||||
(is (= "Not found."
|
||||
(mt/user-http-request :crowberto :get 404 "ee/workspace/999999/log")))))
|
||||
|
||||
(deftest workspace-log-entries-created-test
|
||||
(testing "WorkspaceLog entries are created during setup"
|
||||
(let [{ws-id :id} (ws-ready (mt/user-http-request :crowberto :post 200 "ee/workspace"
|
||||
{:name "log-entries-test" :database_id (mt/id)}))]
|
||||
(is (=? [{:task :database-isolation
|
||||
:status :success}
|
||||
{:task :workspace-setup
|
||||
:status :success}]
|
||||
(t2/select :model/WorkspaceLog :workspace_id ws-id {:order-by [[:started_at :desc]]}))))))
|
||||
|
||||
(deftest workspace-setup-failure-logs-error-test
|
||||
(testing "Failed workspace setup logs error message"
|
||||
(with-redefs [ws.isolation/ensure-database-isolation!
|
||||
(fn [& _] (throw (ex-info "Test isolation error" {})))]
|
||||
(let [{ws-id :id} (mt/user-http-request :crowberto :post 200 "ee/workspace"
|
||||
{:name "fail-test" :database_id (mt/id)})
|
||||
_ (try (ws-ready ws-id) (catch Exception _))]
|
||||
(Thread/sleep 500)
|
||||
(is (=? [{:task :database-isolation
|
||||
:status :failure
|
||||
:message "Test isolation error"}
|
||||
{:task :workspace-setup
|
||||
:status :failure}]
|
||||
(t2/select :model/WorkspaceLog :workspace_id ws-id {:order-by [[:started_at :desc]]})))))))
|
||||
|
||||
;;; ---------------------------------------- Workspace Transform CRUD Tests ----------------------------------------
|
||||
|
||||
(deftest get-workspace-transforms-test
|
||||
(testing "GET /api/ee/workspace/:id/transform"
|
||||
(mt/with-temp [:model/Workspace workspace {:name "List Transforms Test"}
|
||||
:model/WorkspaceTransform tx1 {:name "Transform 1"
|
||||
:workspace_id (:id workspace)}
|
||||
:model/WorkspaceTransform tx2 {:name "Transform 2"
|
||||
:workspace_id (:id workspace)}
|
||||
:model/Transform _tx3 {:name "Global Transform"}]
|
||||
(testing "returns transforms in workspace"
|
||||
(is (=? {:transforms [{:ref_id (:ref_id tx1)}
|
||||
{:ref_id (:ref_id tx2)}]}
|
||||
(mt/user-http-request :crowberto :get 200 (ws-url (:id workspace) "/transform")))))
|
||||
(testing "requires superuser"
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :get 403 (ws-url (:id workspace) "/transform"))))))
|
||||
(testing "returns empty list when no transforms"
|
||||
(mt/with-temp [:model/Workspace workspace {:name "Empty Workspace"}]
|
||||
(is (= {:transforms []}
|
||||
(mt/user-http-request :crowberto :get 200 (ws-url (:id workspace) "/transform"))))))
|
||||
(testing "returns 404 for non-existent workspace"
|
||||
(is (= "Not found."
|
||||
(mt/user-http-request :crowberto :get 404 "ee/workspace/999999/transform"))))))
|
||||
|
||||
(deftest get-workspace-transform-by-id-test
|
||||
(testing "GET /api/ee/workspace/:id/transform/:txid"
|
||||
(mt/with-temp [:model/Workspace workspace1 {:name "Workspace 1"}
|
||||
:model/Workspace workspace2 {:name "Workspace 2"}
|
||||
:model/WorkspaceTransform transform {:name "My Transform"
|
||||
:description "Test description"
|
||||
:workspace_id (:id workspace1)}]
|
||||
(testing "returns specific transform"
|
||||
(is (=? {:ref_id (:ref_id transform)
|
||||
:name "My Transform"
|
||||
:description "Test description"}
|
||||
(mt/user-http-request :crowberto :get 200
|
||||
(ws-url (:id workspace1) (str "/transform/" (:ref_id transform)))))))
|
||||
(testing "returns 404 if transform not in workspace"
|
||||
(is (= "Not found."
|
||||
(mt/user-http-request :crowberto :get 404
|
||||
(ws-url (:id workspace2) (str "/transform/" (:ref_id transform)))))))
|
||||
(testing "requires superuser"
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :get 403
|
||||
(ws-url (:id workspace1) (str "/transform/" (:ref_id transform))))))))))
|
||||
|
||||
(deftest update-workspace-transform-test
|
||||
(testing "PUT /api/ee/workspace/:id/transform/:txid"
|
||||
(mt/with-temp [:model/Workspace workspace1 {:name "Workspace 1"}
|
||||
:model/Workspace workspace2 {:name "Workspace 2"}
|
||||
:model/WorkspaceTransform transform {:name "Original Name"
|
||||
:description "Original description"
|
||||
:workspace_id (:id workspace1)}]
|
||||
(testing "updates transform"
|
||||
(is (=? {:ref_id (:ref_id transform)
|
||||
:name "Updated Name"
|
||||
:description "Updated description"}
|
||||
(mt/user-http-request :crowberto :put 200
|
||||
(ws-url (:id workspace1) (str "/transform/" (:ref_id transform)))
|
||||
{:name "Updated Name"
|
||||
:description "Updated description"})))
|
||||
(is (= "Updated Name" (t2/select-one-fn :name :model/WorkspaceTransform :ref_id (:ref_id transform)))))
|
||||
(testing "returns 404 if transform not in workspace"
|
||||
(is (= "Not found."
|
||||
(mt/user-http-request :crowberto :put 404
|
||||
(ws-url (:id workspace2) (str "/transform/" (:ref_id transform)))
|
||||
{:name "Should Fail"}))))
|
||||
(testing "requires superuser"
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :put 403
|
||||
(ws-url (:id workspace1) (str "/transform/" (:ref_id transform)))
|
||||
{:name "Should Fail"})))))))
|
||||
|
||||
(deftest delete-workspace-transform-test
|
||||
(testing "DELETE /api/ee/workspace/:id/transform/:txid"
|
||||
(mt/with-temp [:model/Workspace workspace1 {:name "Workspace 1"}
|
||||
:model/Workspace workspace2 {:name "Workspace 2"}
|
||||
:model/WorkspaceTransform transform1 {:name "Transform in WS1"
|
||||
:workspace_id (:id workspace1)}
|
||||
:model/WorkspaceTransform transform2 {:name "To Delete"
|
||||
:workspace_id (:id workspace1)}]
|
||||
(testing "returns 404 if transform not in workspace"
|
||||
(is (= "Not found."
|
||||
(mt/user-http-request :crowberto :delete 404
|
||||
(ws-url (:id workspace2) (str "/transform/" (:ref_id transform1)))))))
|
||||
(testing "requires superuser"
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :delete 403
|
||||
(ws-url (:id workspace1) (str "/transform/" (:ref_id transform1)))))))
|
||||
(testing "deletes transform"
|
||||
(is (nil? (mt/user-http-request :crowberto :delete 204
|
||||
(ws-url (:id workspace1) (str "/transform/" (:ref_id transform2))))))
|
||||
(is (some? (t2/select-one :model/WorkspaceTransform :ref_id (:ref_id transform1))))
|
||||
(is (nil? (t2/select-one :model/WorkspaceTransform :ref_id (:ref_id transform2))))))))
|
||||
|
||||
(deftest run-workspace-transform-test
|
||||
(testing "POST /api/ee/workspace/:id/transform/:txid/run"
|
||||
(mt/with-temp [:model/Workspace workspace1 {:name "Workspace 1"}
|
||||
:model/Workspace workspace2 {:name "Workspace 2"}
|
||||
:model/WorkspaceTransform transform {:name "Transform in WS1"
|
||||
:workspace_id (:id workspace1)}]
|
||||
(testing "returns 404 if transform not in workspace"
|
||||
(is (= "Not found."
|
||||
(mt/user-http-request :crowberto :post 404
|
||||
(ws-url (:id workspace2) (str "/transform/" (:ref_id transform) "/run"))))))
|
||||
(testing "requires superuser"
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :post 403
|
||||
(ws-url (:id workspace1) (str "/transform/" (:ref_id transform) "/run")))))))))
|
||||
|
||||
(deftest run-workspace-test
|
||||
(testing "POST /api/ee/workspace/:id/execute"
|
||||
(mt/with-temp [:model/Workspace workspace1 {:name "Workspace 1"}
|
||||
:model/Workspace workspace2 {:name "Workspace 2"}
|
||||
:model/WorkspaceTransform transform {:name "Transform in WS1"
|
||||
:workspace_id (:id workspace1)}]
|
||||
(testing "returns empty when no transforms"
|
||||
(is (= {:succeeded []
|
||||
:failed []
|
||||
:not_run []}
|
||||
(mt/user-http-request :crowberto :post 200 (ws-url (:id workspace2) "/run")))))
|
||||
(testing "executes transforms in workspace"
|
||||
(is (= {:succeeded [(:ref_id transform)]
|
||||
:failed []
|
||||
:not_run []}
|
||||
(mt/user-http-request :crowberto :post 200 (ws-url (:id workspace1) "/run"))))))))
|
||||
@@ -0,0 +1,422 @@
|
||||
(ns ^:mb/driver-tests metabase-enterprise.workspaces.common-test
|
||||
(:require
|
||||
[clojure.java.jdbc :as jdbc]
|
||||
[clojure.test :refer [deftest testing is]]
|
||||
[medley.core :as m]
|
||||
[metabase-enterprise.transforms.interface :as transforms.i]
|
||||
[metabase-enterprise.transforms.test-util :as transforms.tu]
|
||||
[metabase-enterprise.workspaces.common :as ws.common]
|
||||
[metabase.driver :as driver]
|
||||
[metabase.driver.sql-jdbc.connection :as driver.conn]
|
||||
[metabase.lib.core :as lib]
|
||||
[metabase.test :as mt]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(defn- setup!
|
||||
[schema]
|
||||
(let [spec (driver.conn/connection-details->spec :postgres (:details (mt/db)))]
|
||||
(jdbc/execute! spec (format "DROP SCHEMA IF EXISTS %s CASCADE" schema))
|
||||
(jdbc/execute! spec (format "CREATE SCHEMA %s" schema))))
|
||||
|
||||
(defn teardown!
|
||||
[ws-id]
|
||||
(let [[ws-graph ws-schema] (t2/select-one-fn (juxt :graph :schema) :model/Workspace :id ws-id)
|
||||
spec (driver.conn/connection-details->spec :postgres (:details (mt/db)))
|
||||
output-ids (-> ws-graph :outputs (->> (map :id)))
|
||||
dup-tables-ids (into #{}
|
||||
(map #(-> % :mapping :id))
|
||||
(-> ws-graph :outputs))]
|
||||
(when (seq dup-tables-ids)
|
||||
(t2/delete! :model/Table :id [:in dup-tables-ids]))
|
||||
(when (seq output-ids)
|
||||
(t2/delete! :model/Table :id [:in output-ids]))
|
||||
(when (some? ws-id)
|
||||
(t2/delete! :model/Workspace :id ws-id))
|
||||
(when (string? (not-empty ws-schema))
|
||||
(jdbc/execute! spec (format "DROP SCHEMA IF EXISTS %s CASCADE" ws-schema)))))
|
||||
|
||||
(deftest create-workspace-independent-native-transforms-test
|
||||
(mt/test-driver
|
||||
:postgres
|
||||
(let [mp (mt/metadata-provider)
|
||||
native-base (lib/native-query mp "select * from ORDERS LIMIT 10")
|
||||
test-schema "test_schema"
|
||||
new-workspace (atom nil)]
|
||||
(try
|
||||
(setup! test-schema)
|
||||
(mt/with-temp
|
||||
[:model/Transform x1 {:name "x1"
|
||||
:source_type :native
|
||||
:source {:type :query
|
||||
:query native-base}
|
||||
:target {:type :table
|
||||
:database (mt/id)
|
||||
:schema test-schema
|
||||
:name "t1"}}
|
||||
:model/Transform x2 {:name "x2"
|
||||
:source_type :native
|
||||
:source {:type :query
|
||||
:query (assoc-in native-base [:stages 0 :native]
|
||||
(str "SELECT *\n"
|
||||
"FROM PRODUCTS\n"
|
||||
"LIMIT 10\n"))}
|
||||
:target {:type :table
|
||||
:database (mt/id)
|
||||
:schema test-schema
|
||||
:name "t2"}}
|
||||
:model/Transform x3 {:name "x3"
|
||||
:source_type :native
|
||||
:source {:type :query
|
||||
:query (assoc-in native-base [:stages 0 :native]
|
||||
(str "SELECT *\n"
|
||||
"FROM PEOPLE\n"
|
||||
"LIMIT 10\n"))}
|
||||
:target {:type :table
|
||||
:database (mt/id)
|
||||
:schema test-schema
|
||||
:name "t3"}}]
|
||||
(transforms.i/execute! x1 {:run-method :manual})
|
||||
(transforms.i/execute! x2 {:run-method :manual})
|
||||
(transforms.i/execute! x3 {:run-method :manual})
|
||||
(let [result (mt/with-current-user (mt/user->id :crowberto)
|
||||
(ws.common/create-workspace! (mt/user->id :crowberto)
|
||||
{:name "test_ws"
|
||||
:database_id (mt/id)
|
||||
:upstream {:transforms (mapv :id [x1 x2 x3])}}))
|
||||
checkouts (-> result :graph :check-outs)
|
||||
transforms (-> result :graph :transforms)]
|
||||
(reset! new-workspace result)
|
||||
(testing "Name is correct"
|
||||
(is (= "test_ws" (:name result))))
|
||||
(testing "Checkouts are as expected"
|
||||
(is (= [{:type :transform :id (:id x1)}
|
||||
{:type :transform :id (:id x2)}
|
||||
{:type :transform :id (:id x3)}]
|
||||
(sort-by :id checkouts))))
|
||||
(testing "Transforms look as expected"
|
||||
(is (= #{{:id (:id x1), :type :transform, :mapping {#_#_:id pos-int?, :name "x1"}}
|
||||
{:id (:id x2), :type :transform, :mapping {#_#_:id pos-int?, :name "x2"}}
|
||||
{:id (:id x3), :type :transform, :mapping {#_#_:id pos-int?, :name "x3"}}}
|
||||
(into #{} (map #(m/update-existing % :mapping dissoc :id)) transforms))))))
|
||||
;; For development, subject to change
|
||||
(catch Throwable t
|
||||
(def ttt t)
|
||||
(throw t))
|
||||
(finally
|
||||
(teardown! (:id @new-workspace)))))))
|
||||
|
||||
(deftest mirroring-multiple-interdependent-native-transforms-test
|
||||
(mt/test-driver
|
||||
:postgres
|
||||
(let [mp (mt/metadata-provider)
|
||||
native-base (lib/native-query mp "select * from ORDERS LIMIT 10")
|
||||
test-schema "test_schema"
|
||||
new-workspace (atom nil)]
|
||||
(try
|
||||
(setup! test-schema)
|
||||
(testing "Interdependent native transforms"
|
||||
(mt/with-temp
|
||||
[:model/Transform x1 {:name "x1"
|
||||
:source_type :native
|
||||
:source {:type :query
|
||||
:query native-base}
|
||||
:target {:type :table
|
||||
:database (mt/id)
|
||||
:schema test-schema
|
||||
:name "t1"}}
|
||||
:model/Transform x2 {:name "x2"
|
||||
:source_type :native
|
||||
:source {:type :query
|
||||
:query (assoc-in native-base [:stages 0 :native]
|
||||
(str "SELECT *\n"
|
||||
(format "FROM \"%s\".\"%s\"\n" test-schema "t1")
|
||||
"LIMIT 10\n"))}
|
||||
:target {:type :table
|
||||
:database (mt/id)
|
||||
:schema test-schema
|
||||
:name "t2"}}
|
||||
:model/Transform x3 {:name "x3"
|
||||
:source_type :native
|
||||
:source {:type :query
|
||||
:query (assoc-in native-base [:stages 0 :native]
|
||||
(str "SELECT *\n"
|
||||
(format "FROM \"%s\".\"%s\"\n" test-schema "t2")
|
||||
"LIMIT 10\n"))}
|
||||
:target {:type :table
|
||||
:database (mt/id)
|
||||
:schema test-schema
|
||||
:name "t3"}}]
|
||||
(transforms.i/execute! x1 {:run-method :manual})
|
||||
(transforms.i/execute! x2 {:run-method :manual})
|
||||
(transforms.i/execute! x3 {:run-method :manual})
|
||||
(let [result-ws (mt/with-current-user (mt/user->id :crowberto)
|
||||
(ws.common/create-workspace! (mt/user->id :crowberto)
|
||||
{:name "test_ws"
|
||||
:database_id (mt/id)
|
||||
:upstream {:transforms (mapv :id [x1 x3])}}))
|
||||
checkouts (-> result-ws :graph :check-outs)
|
||||
transforms (-> result-ws :graph :transforms)]
|
||||
(reset! new-workspace result-ws)
|
||||
(testing "Workspace has expected name"
|
||||
(is (= "test_ws" (:name result-ws))))
|
||||
(testing "Checkouts contain only the explicitly checked out transforms"
|
||||
(is (= #{{:type :transform :id (:id x1)}
|
||||
{:type :transform :id (:id x3)}}
|
||||
(set checkouts))))
|
||||
(testing "Workspace has correct mappings in graph"
|
||||
(is (= #{{:id (:id x1), :type :transform, :mapping {#_#_:id pos-int?, :name "x1"}}
|
||||
{:id (:id x2), :type :transform, :mapping {#_#_:id pos-int?, :name "x2"}}
|
||||
{:id (:id x3), :type :transform, :mapping {#_#_:id pos-int?, :name "x3"}}}
|
||||
(into #{} (map #(m/update-existing % :mapping dissoc :id)) transforms))))
|
||||
(let [orig-id->mirror-id (into {}
|
||||
(map (fn [{:keys [id mapping]}]
|
||||
[id (:id mapping)]))
|
||||
transforms)
|
||||
mirror-id->mirror (t2/select-fn->fn :id identity :model/Transform :id
|
||||
[:in (vals orig-id->mirror-id)])
|
||||
x1-mirror (-> (:id x1) orig-id->mirror-id mirror-id->mirror)
|
||||
x2-mirror (-> (:id x2) orig-id->mirror-id mirror-id->mirror)
|
||||
x3-mirror (-> (:id x3) orig-id->mirror-id mirror-id->mirror)]
|
||||
(testing "X1 was properly mirrored"
|
||||
(testing "name"
|
||||
(is (= "x1" (:name x1-mirror))))
|
||||
(testing "target adjusted"
|
||||
(is (=? {:type "table"
|
||||
:database (mt/id)
|
||||
:schema (:schema result-ws)
|
||||
;; table name = orig schema + __ + orig target name atm
|
||||
:name (str test-schema "__" (-> x1 :target :name))}
|
||||
(:target x1-mirror))))
|
||||
(testing "source not modifed"
|
||||
(is (=? (:source x1)
|
||||
(:source x1-mirror)))))
|
||||
(testing "X2 was properly mirrored"
|
||||
(testing "name"
|
||||
(is (= (:name x2) (:name x2-mirror))))
|
||||
(testing "target"
|
||||
(is (=? {:type "table"
|
||||
:database (mt/id)
|
||||
:schema (:schema result-ws)
|
||||
:name (str test-schema "__" (-> x2 :target :name))}
|
||||
(:target x2-mirror))))
|
||||
(testing "source query referencing target of x1 mirror"
|
||||
;; TODO (lbrdnk 2025-11-27): Quotes are removed, figure out whether what has
|
||||
;; some unwanted consequences.
|
||||
(is (=? {:query {:stages [{:native (format "SELECT *\nFROM %s.%s\nLIMIT 10\n"
|
||||
(-> x1-mirror :target :schema)
|
||||
(-> x1-mirror :target :name))}]}}
|
||||
(:source x2-mirror)))))
|
||||
(testing "X3 was properly mirrored"
|
||||
(testing "name"
|
||||
(is (= (:name x3) (:name x3-mirror))))
|
||||
(testing "target"
|
||||
(is (=? {:type "table"
|
||||
:database (mt/id)
|
||||
:schema (:schema result-ws)
|
||||
:name (str test-schema "__" (-> x3 :target :name))}
|
||||
(:target x3-mirror))))
|
||||
(testing "source query referencing target of x2 mirror"
|
||||
(is (=? {:query {:stages [{:native (format "SELECT *\nFROM %s.%s\nLIMIT 10\n"
|
||||
(-> x2-mirror :target :schema)
|
||||
(-> x2-mirror :target :name))}]}}
|
||||
(:source x3-mirror)))))))))
|
||||
;; For development, subject to change
|
||||
(catch Throwable t
|
||||
(def ttt t)
|
||||
(throw t))
|
||||
(finally
|
||||
(teardown! (:id @new-workspace)))))))
|
||||
|
||||
(deftest python-transforms-checkout-test
|
||||
(mt/test-driver
|
||||
:postgres
|
||||
(let [target-schema "public"]
|
||||
(try (transforms.tu/with-transform-cleanup! [t1 "t1"
|
||||
t2 "t2"]
|
||||
(mt/with-temp
|
||||
[:model/Transform x1 {:name "initial"
|
||||
:source {:type "python"
|
||||
:source-database (mt/id)
|
||||
:source-tables {"checkins" (mt/id :checkins)
|
||||
"venues" (mt/id :venues)}
|
||||
:body (str "import pandas as pd\n"
|
||||
"\n"
|
||||
"def transform(checkins, venues):\n"
|
||||
" return checkins.set_index('venue_id').join(venues.set_index('id'), lsuffix='_venues')\n")}
|
||||
:target {:type "table"
|
||||
:database (mt/id)
|
||||
:schema target-schema
|
||||
:name t1}}]
|
||||
(transforms.i/execute! x1 {:run-method :manual})
|
||||
(let [t1-table (t2/select-one :model/Table :name t1)]
|
||||
(mt/with-temp [:model/Transform x2 {:name "dependent"
|
||||
:source {:type "python"
|
||||
:source-database (mt/id)
|
||||
:source-tables {"t1" (:id t1-table)}
|
||||
:body (str "import pandas as pd\n"
|
||||
"\n"
|
||||
"def transform(t1):\n"
|
||||
" return t1")}
|
||||
:target {:type "table"
|
||||
:database (mt/id)
|
||||
:schema target-schema
|
||||
:name t2}}]
|
||||
(transforms.i/execute! x2 {:run-method :manual})
|
||||
(let [user-id (mt/user->id :crowberto)
|
||||
result-workspace (mt/with-test-user :crowberto
|
||||
(ws.common/create-workspace! user-id
|
||||
{:name "my test workspace x"
|
||||
:database_id (mt/id)
|
||||
:upstream {:transforms [(:id x1) (:id x2)]}}))
|
||||
mirror-initial (t2/select-one :model/Transform :name "initial" :workspace_id (:id result-workspace))
|
||||
mirror-dependent (t2/select-one :model/Transform :name "dependent" :workspace_id (:id result-workspace))
|
||||
t1-mirror (t2/select-one :model/Table
|
||||
:schema (-> mirror-initial :target :schema)
|
||||
:name (-> mirror-initial :target :name))]
|
||||
(testing "targets"
|
||||
(is (=? {:name (str "public__" (-> x1 :target :name))
|
||||
:schema (:schema result-workspace)}
|
||||
(:target mirror-initial)))
|
||||
(is (=? {:name (str "public__" (-> x2 :target :name))
|
||||
:schema (:schema result-workspace)}
|
||||
(:target mirror-dependent))))
|
||||
(testing "sources"
|
||||
(is (=? {:source-tables {"venues" (mt/id :venues)
|
||||
"checkins" (mt/id :checkins)}}
|
||||
(:source mirror-initial)))
|
||||
(is (=? {:source-tables {"t1" (:id t1-mirror)}}
|
||||
(:source mirror-dependent)))))))))
|
||||
(catch Throwable t
|
||||
(def ttt t)
|
||||
(throw t))))))
|
||||
|
||||
(deftest remove-transforms-test
|
||||
(mt/test-driver
|
||||
:postgres
|
||||
(let [mp (mt/metadata-provider)
|
||||
q1 (lib/native-query mp "select * from orders limit 10;")
|
||||
q2 (lib/native-query mp "select * from products limit 10;")
|
||||
test-schema "my_test_schema"
|
||||
ws-id (atom nil)]
|
||||
(try
|
||||
(setup! test-schema)
|
||||
(transforms.tu/with-transform-cleanup! [t1 "t1"
|
||||
t2 "t2"]
|
||||
(mt/with-temp
|
||||
[:model/Transform x1 {:source_type :native
|
||||
:source {:type :query
|
||||
:query q1}
|
||||
:target {:type :table
|
||||
:database (mt/id)
|
||||
:schema test-schema
|
||||
:name t1}}
|
||||
:model/Transform x2 {:source_type :native
|
||||
:source {:type :query
|
||||
:query q2}
|
||||
:target {:type :table
|
||||
:database (mt/id)
|
||||
:schema test-schema
|
||||
:name t2}}]
|
||||
;; Create the target tables
|
||||
(transforms.i/execute! x1 {:run-method :manual})
|
||||
(transforms.i/execute! x2 {:run-method :manual})
|
||||
;; Add into a workspace
|
||||
(let [creator-id (mt/user->id :crowberto)
|
||||
ws (mt/with-test-user :crowberto
|
||||
(ws.common/create-workspace! creator-id {:name "my test ws xxx"
|
||||
:database_id (mt/id)
|
||||
:upstream {:transforms [(:id x1) (:id x2)]}}))
|
||||
_ (reset! ws-id (:id ws))
|
||||
mirrored-transforms-ids (t2/select-fn-vec :downstream_id :model/WorkspaceMappingTransform :workspace_id (:id ws))
|
||||
ws-db (t2/select-one :model/Database :id (:database_id ws))
|
||||
mirrored-tables-ids (t2/select-fn-vec :downstream_id :model/WorkspaceMappingTable :workspace_id (:id ws))
|
||||
mirrored-tables (t2/select :model/Table :id [:in mirrored-tables-ids])]
|
||||
(testing "Sanity"
|
||||
(testing "Mirrored transforms created"
|
||||
(is (= 2 (count mirrored-transforms-ids))))
|
||||
(testing "Mirrored tables created in isolated schema"
|
||||
(run!
|
||||
(fn [{:keys [schema name]}]
|
||||
(is (true? (driver/table-exists? :postgres ws-db {:schema schema :name name}))))
|
||||
mirrored-tables))
|
||||
(testing "Mirrored tables created in appdb"
|
||||
(is (= 2 (count mirrored-tables-ids)))))
|
||||
(testing "Remove transforms from a workspace"
|
||||
(mt/with-test-user :crowberto
|
||||
;; ACT!
|
||||
(ws.common/remove-entities! ws {:transforms mirrored-transforms-ids}))
|
||||
(testing "Mirrored transforms deleted"
|
||||
(is (empty? (t2/select :model/Transform :id [:in mirrored-transforms-ids]))))
|
||||
(testing "Mirrored tables dropped from the isolated schema"
|
||||
(run!
|
||||
(fn [{:keys [schema name]}]
|
||||
;; driver/table-exists? -- _prints_ exception on non-existent relation, while having correct
|
||||
;; return value.
|
||||
;; one way or another -- this test will be rewritten completely. Then we will handle that.
|
||||
(is (false? (driver/table-exists? :postgres ws-db {:schema schema :name name}))))
|
||||
mirrored-tables))
|
||||
(testing "Mirrored tables deleted from the appdb"
|
||||
(is (empty? (t2/select :model/Table :id [:in mirrored-tables-ids]))))
|
||||
(testing "Mirrored entites are not present in the mapping tables"
|
||||
(is (empty? (t2/select :model/WorkspaceMappingTable :workspace_id (:id ws))))
|
||||
(is (empty? (t2/select :model/WorkspaceMappingTransform :workspace_id (:id ws)))))))))
|
||||
(catch Throwable t
|
||||
(def ttt t)
|
||||
(throw t))
|
||||
(finally
|
||||
(teardown! @ws-id))))))
|
||||
|
||||
(deftest add-one-by-one-and-remove-test
|
||||
(mt/test-driver
|
||||
:postgres
|
||||
(let [mp (mt/metadata-provider)
|
||||
q1 (lib/native-query mp "select * from orders limit 10;")
|
||||
test-schema "my_test_schema"
|
||||
ws-id (atom nil)]
|
||||
(try
|
||||
(setup! test-schema)
|
||||
(transforms.tu/with-transform-cleanup! [t1 "t1"]
|
||||
(mt/with-temp
|
||||
[:model/Transform x1 {:source_type :native
|
||||
:source {:type :query
|
||||
:query q1}
|
||||
:target {:type :table
|
||||
:database (mt/id)
|
||||
:schema test-schema
|
||||
:name t1}}]
|
||||
;; Create the target tables
|
||||
(transforms.i/execute! x1 {:run-method :manual})
|
||||
;; Add into a workspace
|
||||
(let [creator-id (mt/user->id :crowberto)
|
||||
;; create empty workspace
|
||||
ws (mt/user-http-request :crowberto :post 200
|
||||
"/ee/workspace"
|
||||
{:name "my test ws xxx"
|
||||
:creator_id creator-id
|
||||
:database_id (mt/id)})
|
||||
_ (reset! ws-id (:id ws))]
|
||||
(let [add-res (mt/user-http-request :crowberto :post 200
|
||||
(str "ee/workspace/" (:id ws) "/contents")
|
||||
{:add {:transforms [(:id x1)]}})]
|
||||
(testing "Successfully added"
|
||||
(is (some? (-> add-res :contents :transforms (nth 0)))))
|
||||
(let [rem-res (mt/user-http-request :crowberto :post 200
|
||||
(str "ee/workspace/" (:id ws) "/contents")
|
||||
{:remove {:transforms [(-> add-res :contents :transforms (nth 0) :id)]}})]
|
||||
(testing "successfully removed"
|
||||
(is (empty? (-> rem-res :contents :transforms))))))
|
||||
(let [add-res (mt/user-http-request :crowberto :post 200
|
||||
(str "ee/workspace/" (:id ws) "/contents")
|
||||
{:add {:transforms [(:id x1)]}})]
|
||||
(testing "Successfully added"
|
||||
(is (some? (-> add-res :contents :transforms (nth 0)))))
|
||||
(let [rem-res (mt/user-http-request :crowberto :post 200
|
||||
(str "ee/workspace/" (:id ws) "/contents")
|
||||
{:remove {:transforms [(-> add-res :contents :transforms (nth 0) :id)]}})]
|
||||
(testing "successfully removed"
|
||||
(is (empty? (-> rem-res :contents :transforms)))))))))
|
||||
(catch Throwable t
|
||||
(def ttt t)
|
||||
(throw t))
|
||||
(finally
|
||||
(teardown! @ws-id))))))
|
||||
@@ -0,0 +1,115 @@
|
||||
(ns metabase-enterprise.workspaces.dag-abstract
|
||||
"Abstract graph solver using shorthand notation for testing.
|
||||
|
||||
Shorthand uses keywords like :x1, :t2, :m3 where:
|
||||
- :x<n> is a transform
|
||||
- :t<n> is a table (source/generated)
|
||||
- :m<n> is a model (card)"
|
||||
(:require
|
||||
[clojure.set :as set]
|
||||
[flatland.ordered.map :as ordered-map]
|
||||
[metabase-enterprise.workspaces.dag :as ws.dag]
|
||||
[metabase-enterprise.workspaces.util :as ws.u]
|
||||
[metabase.util :as u]))
|
||||
|
||||
;;;; Shorthand notation helpers
|
||||
|
||||
(defn- related [t id]
|
||||
(keyword (str (name t) (subs (name id) 1))))
|
||||
|
||||
(def ^:private x->t (partial related :t))
|
||||
|
||||
(defn is?
|
||||
"Check if id is of type t (e.g., (is? :x :x1) => true)."
|
||||
[t id]
|
||||
(= (name t) (str (first (name id)))))
|
||||
|
||||
(defn- lexico [id]
|
||||
(let [n (name id)]
|
||||
[(parse-long (subs n 1))
|
||||
({\x 1 \t 2 \m 3} (first n))]))
|
||||
|
||||
(defn expand-shorthand
|
||||
"As shorthand, we can use transforms in the place of their induced tables.
|
||||
This function expands the graph to insert the induced tables, and replace the corresponding references."
|
||||
[{:keys [dependencies] :as graph}]
|
||||
(assoc graph :dependencies
|
||||
(let [xs (set (filter #(is? :x %) (concat (keys dependencies) (mapcat val dependencies))))]
|
||||
(into (sorted-map-by #(compare (lexico %1) (lexico %2)))
|
||||
(merge (zipmap (map x->t xs) (map vector xs))
|
||||
(into {} (filter (comp #(is? :t %) key)) dependencies)
|
||||
(update-vals
|
||||
(into {} (remove (comp #(is? :t %) key)) dependencies)
|
||||
(fn [deps] (->> deps
|
||||
(map (fn [id]
|
||||
(if (is? :x id)
|
||||
(x->t id)
|
||||
id)))
|
||||
(sort-by lexico)
|
||||
(vec)))))))))
|
||||
|
||||
;;;; Graph traversal helpers
|
||||
|
||||
(defn- children [dependencies]
|
||||
(reduce
|
||||
(fn [acc [child parents]]
|
||||
(reduce (fn [acc p] (update acc p u/conjv child)) acc parents))
|
||||
(sorted-map-by #(compare (lexico %1) (lexico %2)))
|
||||
dependencies))
|
||||
|
||||
(defn- ->ts [ids]
|
||||
(into #{}
|
||||
(keep (fn [id]
|
||||
(cond (is? :t id) id
|
||||
(is? :x id) (x->t id))))
|
||||
ids))
|
||||
|
||||
;;;; Toposort
|
||||
|
||||
(defn- toposort-map [child->parents]
|
||||
(let [child->parents (if (map? child->parents) child->parents (into {} child->parents))]
|
||||
(into (ordered-map/ordered-map)
|
||||
(keep (fn [n] (when-let [deps (get child->parents n)] [n deps])))
|
||||
(ws.u/toposort-dfs child->parents))))
|
||||
|
||||
;;;; Abstract path-induced subgraph solver
|
||||
|
||||
(def ^:private max-iterations 100)
|
||||
|
||||
(defn path-induced-subgraph
|
||||
"Abstract solver for path-induced subgraph using shorthand notation."
|
||||
[{:keys [check-outs dependencies] :as _graph}]
|
||||
(let [->parents dependencies
|
||||
->children (children dependencies)]
|
||||
(loop [iteration 1
|
||||
below check-outs
|
||||
above check-outs
|
||||
seen-down? check-outs
|
||||
seen-up? check-outs
|
||||
to-descend check-outs
|
||||
to-ascend check-outs]
|
||||
(if (and (empty? to-descend) (empty? to-ascend))
|
||||
(let [nodes (set/intersection above below)
|
||||
entities (remove #(is? :t %) nodes)
|
||||
tables (sort-by lexico (filter #(is? :t %) nodes))]
|
||||
{:check-outs (sort-by lexico check-outs)
|
||||
:inputs (sort-by lexico (remove nodes (->ts (mapcat ->parents check-outs))))
|
||||
:tables tables
|
||||
:transforms (sort-by lexico (filter #(is? :x %) nodes))
|
||||
:entities (sort-by lexico entities)
|
||||
:dependencies (apply dissoc
|
||||
(toposort-map
|
||||
(for [n nodes]
|
||||
[n (vec (filter nodes (get dependencies n)))]))
|
||||
tables)})
|
||||
(if (> iteration max-iterations)
|
||||
{:error "max iterations exceeded"}
|
||||
(let [next-down (into #{} (comp (mapcat ->children) (remove seen-down?)) to-descend)
|
||||
next-up (into #{} (comp (mapcat ->parents) (remove seen-up?)) to-ascend)]
|
||||
(recur (inc iteration)
|
||||
(into below next-down)
|
||||
(into above next-up)
|
||||
(into seen-down? next-down)
|
||||
(into seen-up? next-up)
|
||||
next-down
|
||||
next-up)))))))
|
||||
@@ -0,0 +1,202 @@
|
||||
(ns metabase-enterprise.workspaces.dag-test
|
||||
(:require
|
||||
[clojure.string :as str]
|
||||
[clojure.test :refer :all]
|
||||
[flatland.ordered.map :as ordered-map]
|
||||
[metabase-enterprise.workspaces.dag :as ws.dag]
|
||||
[metabase-enterprise.workspaces.dag-abstract :as dag-abstract]
|
||||
[metabase.test :as mt]
|
||||
[metabase.util :as u]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(use-fixtures
|
||||
:once
|
||||
(fn [f]
|
||||
(mt/with-premium-features [:dependencies :transforms :workspaces]
|
||||
(mt/with-model-cleanup [:model/Dependency :model/Transform :model/Table]
|
||||
(f)))))
|
||||
|
||||
;;;; Example graphs for testing
|
||||
|
||||
(def ^:private example
|
||||
{:check-outs #{:x3 :m6 :m10 :m13}
|
||||
:dependencies {:x3 [:x1 :t2]
|
||||
:x4 [:x3]
|
||||
:m6 [:x4 :t5]
|
||||
:m10 [:t9]
|
||||
:x11 [:m10]
|
||||
:m12 [:x11]
|
||||
:m13 [:x11 :m12]}})
|
||||
|
||||
;;;; Test data helpers
|
||||
|
||||
(defn- transform? [kw] (= \x (first (name kw))))
|
||||
(defn- table? [kw] (= \t (first (name kw))))
|
||||
(defn- kw->id [kw] (parse-long (subs (name kw) 1)))
|
||||
|
||||
(defn- create-test-graph!
|
||||
"Create test transforms and tables from shorthand notation, returning id mappings.
|
||||
|
||||
Shorthand uses keywords like :x1, :t2, :m3 where:
|
||||
- :x<n> is a transform
|
||||
- :t<n> is a table (source/generated)
|
||||
- :m<n> is a model (card)
|
||||
|
||||
Returns a map from shorthand id to real database id, e.g. {:x1 123, :t2 456}
|
||||
|
||||
NOTE: Caller should wrap in mt/with-model-cleanup for proper cleanup."
|
||||
[{:keys [check-outs dependencies]}]
|
||||
(let [schema (str/replace (str (random-uuid)) "-" "_")
|
||||
all-ids (set (concat (keys dependencies)
|
||||
(mapcat val dependencies)
|
||||
check-outs))
|
||||
transforms (filter transform? all-ids)
|
||||
tables (filter table? all-ids)
|
||||
table-ids (u/for-map [t tables]
|
||||
[t (t2/insert-returning-pk! :model/Table
|
||||
{:db_id (mt/id)
|
||||
:schema schema
|
||||
:name (str "test_table_" (kw->id t))
|
||||
:active true})])
|
||||
;; Create transforms that reference their input tables
|
||||
tx-ids (u/for-map [tx transforms]
|
||||
(let [parent-tables (->> (get dependencies tx []) (filter table?) (map table-ids))
|
||||
id (t2/insert-returning-pk! :model/Transform
|
||||
{:name (str "Test " (name tx))
|
||||
:source {:type :query
|
||||
:query (if (seq parent-tables)
|
||||
{:database (mt/id)
|
||||
:type :query
|
||||
:query {:source-table (first parent-tables)
|
||||
:joins (for [pt (rest parent-tables)]
|
||||
{:source-table pt
|
||||
:condition [:= 1 1]})}}
|
||||
{:database (mt/id)
|
||||
:type :native
|
||||
:native {:query "SELECT 1"}})}
|
||||
:target {:type "table"
|
||||
:database (mt/id)
|
||||
:schema schema
|
||||
:name (str "test_table_" (kw->id tx))}})]
|
||||
[tx id]))]
|
||||
|
||||
;; TODO This is a workaround for the dependency between transforms and their output only being inserted on run.
|
||||
;; We will need to do something about this when we mirror as well - ideally the deps module would "just work"
|
||||
(doseq [[tx-kw tx-id] tx-ids]
|
||||
(t2/insert! :model/Dependency
|
||||
{:from_entity_type "table"
|
||||
:from_entity_id (table-ids (keyword (str "t" (kw->id tx-kw))))
|
||||
:to_entity_type "transform"
|
||||
:to_entity_id tx-id}))
|
||||
|
||||
(merge tx-ids table-ids)))
|
||||
|
||||
(defn- translate-result
|
||||
"Translate result from real IDs back to shorthand notation for easier comparison."
|
||||
[result id-map]
|
||||
(let [reverse-map (u/for-map [[k v] id-map] [v k])
|
||||
translate (fn [{:keys [id]}] (get reverse-map id))]
|
||||
(-> (reduce (fn [m k] (update m k #(set (map translate %))))
|
||||
result
|
||||
[:check-outs :inputs :outputs :transforms])
|
||||
(update :dependencies
|
||||
(fn [deps]
|
||||
(into {}
|
||||
(map (fn [[k v]]
|
||||
[(translate k) (set (map translate v))]))
|
||||
deps))))))
|
||||
|
||||
;;;; Tests
|
||||
|
||||
(deftest path-induced-subgraph-shorthand-test
|
||||
(testing "graph built from shorthand matches abstract solver"
|
||||
(let [shorthand {:check-outs #{:x1}
|
||||
:dependencies {:x1 [:t1]}}
|
||||
id-map (create-test-graph! shorthand)
|
||||
result nil #_(ws.dag/path-induced-subgraph {:transforms [(id-map :x1)]})
|
||||
translated (translate-result result id-map)]
|
||||
#_(is (= {:check-outs #{:x1}
|
||||
:transforms #{:x1}
|
||||
:inputs #{:t1}}
|
||||
translated)))))
|
||||
|
||||
(deftest path-induced-subgraph-larger-test
|
||||
(testing "graph built from shorthand matches abstract solver"
|
||||
(let [shorthand {:check-outs #{:x2, :x4}
|
||||
:dependencies {:x1 [:t0]
|
||||
:x2 [:x1, :t10]
|
||||
:x3 [:x2, :t8]
|
||||
:x4 [:x3]
|
||||
:x5 [:x2, :x4, :t9]}}
|
||||
id-map (create-test-graph! (dag-abstract/expand-shorthand shorthand))
|
||||
result nil #_(ws.dag/path-induced-subgraph {:transforms (mapv id-map (:check-outs shorthand))})
|
||||
translated (translate-result result id-map)]
|
||||
#_(is (=? {:check-outs #{:x2, :x4}
|
||||
:transforms #{:x2, :x3, :x4}
|
||||
:inputs #{:t0, :t1, :t8, :t10}
|
||||
:outputs #{:t2, :t3, :t4}
|
||||
;; Dependencies outside the subgraph are not listed
|
||||
:dependencies {:x2 #{}
|
||||
:x3 #{:x2}
|
||||
:x4 #{:x3}}}
|
||||
translated)))))
|
||||
|
||||
(deftest expand-solver-test
|
||||
(testing "expand-shorthand inserts interstitial nodes for transform output tables"
|
||||
(is (= {:check-outs #{:x3 :m6 :m10 :m13}
|
||||
:dependencies {:t1 [:x1]
|
||||
:x3 [:t1 :t2]
|
||||
:t3 [:x3]
|
||||
:x4 [:t3]
|
||||
:t4 [:x4]
|
||||
:m6 [:t4 :t5]
|
||||
:m10 [:t9]
|
||||
:x11 [:m10]
|
||||
:t11 [:x11]
|
||||
:m12 [:t11]
|
||||
:m13 [:t11 :m12]}}
|
||||
(dag-abstract/expand-shorthand example)))))
|
||||
|
||||
(deftest abstract-path-induced-subgraph-test
|
||||
(testing "path-induced-subgraph computes correct result for example graph"
|
||||
(is (= {:check-outs [:x3 :m6 :m10 :m13]
|
||||
:inputs [:t1 :t2 :t5 :t9]
|
||||
:tables [:t3 :t4 :t11]
|
||||
:transforms [:x3 :x4 :x11]
|
||||
:entities [:x3 :x4 :m6 :m10 :x11 :m12 :m13]
|
||||
:dependencies (ordered-map/ordered-map
|
||||
:m10 []
|
||||
:x11 [:m10]
|
||||
:m12 [:t11]
|
||||
:m13 [:t11 :m12]
|
||||
:x3 []
|
||||
:x4 [:t3]
|
||||
:m6 [:t4])}
|
||||
(dag-abstract/path-induced-subgraph (dag-abstract/expand-shorthand example))))))
|
||||
|
||||
;;;; Card dependency detection tests
|
||||
|
||||
(deftest unsupported-dependency-no-transforms-test
|
||||
(testing "empty input returns nil"
|
||||
(is (nil? (ws.dag/unsupported-dependency? {})))
|
||||
(is (nil? (ws.dag/unsupported-dependency? {:transforms []})))))
|
||||
|
||||
(deftest unsupported-dependency-mbql-card-test
|
||||
#_(testing "transform with card dependencies are unsupported"
|
||||
(let [tx-with-no-dependencies 1
|
||||
;; These tests transforms must seem pretty redundant - but it's intentional: we may relax them one case at
|
||||
;; at time in the future.
|
||||
;; ----------------------
|
||||
;; We don't allow MBQL card dependencies, as we do not support re-mapping their references on execution.
|
||||
;; This is only a problem if the card depends on a table that is shadowed in the isolated schema, but we
|
||||
;; want to keep the semantics simple.
|
||||
tx-with-direct-mbql-card-dependency 2
|
||||
;; Even a transitive dependency would need to be remapped.
|
||||
tx-with-indirect-mbql-card-dependency 3
|
||||
;; In fact, we don't allow *any* card dependency for two reasons:
|
||||
;; 1. We don't yet support reference re-mapping across entity references.
|
||||
;; 2. It's an anti-pattern to build transforms on models, the relationship is intended to be the other way
|
||||
;; around.
|
||||
tx-with-sql-card-dependency 4]
|
||||
(is (= nil (ws.dag/unsupported-dependency? {:transforms [1]})))
|
||||
(is (= {:transforms [2 3 4]} (ws.dag/unsupported-dependency? {:transforms [1 2 3 4]}))))))
|
||||
@@ -0,0 +1,20 @@
|
||||
(ns ^:mb/driver-tests metabase-enterprise.workspaces.drivers-test
|
||||
(:require
|
||||
[clojure.test :refer :all]
|
||||
[metabase.driver :as driver]
|
||||
[metabase.test :as mt]))
|
||||
|
||||
(deftest drivers-support-workspaces-has-to-support-db-swapping-test
|
||||
(mt/test-drivers (mt/normal-drivers-with-feature :workspace)
|
||||
(let [db-swapped? (fn []
|
||||
(try
|
||||
(mt/run-mbql-query venues {:limit 1})
|
||||
false
|
||||
(catch Exception _e
|
||||
true)))]
|
||||
(testing "db is swapped if being executed inside a macros"
|
||||
(driver/with-swapped-connection-details (mt/id) {:user "unicorn"}
|
||||
(is (db-swapped?))))
|
||||
|
||||
(testing "sanity check that it's not swapped outside of the macros"
|
||||
(is (not (db-swapped?)))))))
|
||||
@@ -0,0 +1,56 @@
|
||||
(ns ^:mb/driver-tests metabase-enterprise.workspaces.e2e-test
|
||||
(:require
|
||||
[clojure.string :as str]
|
||||
[clojure.test :refer :all]
|
||||
[metabase-enterprise.transforms.interface :as transforms.i]
|
||||
[metabase-enterprise.transforms.test-util :as transforms.tu]
|
||||
[metabase-enterprise.workspaces.common :as ws.common]
|
||||
[metabase-enterprise.workspaces.core :as workspaces]
|
||||
[metabase.driver :as driver]
|
||||
[metabase.driver.sql.query-processor :as sql.qp]
|
||||
[metabase.query-processor.preprocess :as qp.preprocess]
|
||||
^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]
|
||||
[metabase.test :as mt]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(defn- execute-workspace-transform!
|
||||
"Execute a transform within workspace isolation context. For testing purposes."
|
||||
[workspace transform opts]
|
||||
(workspaces/with-workspace-isolation workspace
|
||||
(transforms.i/execute! transform opts)))
|
||||
|
||||
(defn- mbql->native [query]
|
||||
(qp.store/with-metadata-provider (mt/id)
|
||||
(sql.qp/mbql->native driver/*driver* (qp.preprocess/preprocess query))))
|
||||
|
||||
(deftest isolation-e2e-test
|
||||
(mt/test-drivers #_(mt/normal-drivers-with-feature :workspace) #{:postgres #_:h2}
|
||||
(transforms.tu/with-transform-cleanup! [output-table-name (str "test_table_1_" (mt/random-name))]
|
||||
(mt/with-model-cleanup [:model/Transform :model/Workspace]
|
||||
(mt/with-temp [:model/Transform {transform-id :id} {:name "Transform 1"
|
||||
:source {:type "query"
|
||||
:query (mt/native-query (mbql->native (mt/mbql-query orders {:limit 1})))}
|
||||
:target {:type "table"
|
||||
:name output-table-name}}]
|
||||
(let [workspace (mt/with-current-user (mt/user->id :crowberto)
|
||||
(ws.common/create-workspace! (mt/user->id :crowberto)
|
||||
{:name (mt/random-name)
|
||||
:database_id (mt/id)
|
||||
:upstream {:transforms [transform-id]}}))
|
||||
isolated-transform-id (t2/select-one-fn :downstream_id :model/WorkspaceMappingTransform :workspace_id (:id workspace))
|
||||
isolated-transform (t2/select-one :model/Transform isolated-transform-id)
|
||||
executed-transform (execute-workspace-transform! workspace isolated-transform {:run-method :manual})
|
||||
output-table (-> executed-transform :object :output-table)]
|
||||
(testing "execute the transform with the original query is fine and the output is in an isolated schema"
|
||||
(transforms.tu/wait-for-table (name output-table) 1000)
|
||||
(is (str/starts-with? (namespace output-table) "mb__isolation"))
|
||||
(is (= 1 (count (transforms.tu/table-rows (name output-table))))))
|
||||
|
||||
(testing "changing the query without granting access will fail"
|
||||
(t2/update! :model/Transform isolated-transform-id {:source {:type "query"
|
||||
;; TODO we should use native query here
|
||||
:query (mt/mbql-query venues {:limit 1})}})
|
||||
(is (thrown-with-msg?
|
||||
Exception
|
||||
#"ERROR: permission denied for table.*"
|
||||
(execute-workspace-transform! workspace (t2/select-one :model/Transform isolated-transform-id) {:run-method :manual}))))))))))
|
||||
@@ -5,12 +5,27 @@ import { useMetabotAgent } from "metabase-enterprise/metabot/hooks";
|
||||
|
||||
import { trackQueryFixClicked } from "../../analytics";
|
||||
|
||||
export function FixSqlQueryButton() {
|
||||
type FixSqlQueryButtonProps = {
|
||||
rawSql?: string | null;
|
||||
errorMessage?: string | null;
|
||||
};
|
||||
|
||||
export function FixSqlQueryButton({
|
||||
rawSql,
|
||||
errorMessage,
|
||||
}: FixSqlQueryButtonProps) {
|
||||
const { submitInput } = useMetabotAgent();
|
||||
|
||||
const handleClick = () => {
|
||||
trackQueryFixClicked();
|
||||
submitInput("Fix this SQL query");
|
||||
const promptParts = ["Fix this SQL query."];
|
||||
if (errorMessage) {
|
||||
promptParts.push(`The database returned this error: ${errorMessage}`);
|
||||
}
|
||||
if (rawSql) {
|
||||
promptParts.push(`SQL:\n${rawSql}`);
|
||||
}
|
||||
submitInput(promptParts.join("\n\n"));
|
||||
};
|
||||
|
||||
return <Button onClick={handleClick}>{t`Have Metabot fix it`}</Button>;
|
||||
|
||||
@@ -21,3 +21,4 @@ export * from "./transform";
|
||||
export * from "./transform-job";
|
||||
export * from "./transform-tag";
|
||||
export * from "./upload-management";
|
||||
export * from "./workspace";
|
||||
|
||||
@@ -25,6 +25,9 @@ import type {
|
||||
TransformJob,
|
||||
TransformRun,
|
||||
TransformTag,
|
||||
Workspace,
|
||||
WorkspaceContentItem,
|
||||
WorkspaceContents,
|
||||
} from "metabase-types/api";
|
||||
|
||||
export const ENTERPRISE_TAG_TYPES = [
|
||||
@@ -49,6 +52,7 @@ export const ENTERPRISE_TAG_TYPES = [
|
||||
"remote-sync-branches",
|
||||
"remote-sync-current-task",
|
||||
"python-transform-library",
|
||||
"workspace",
|
||||
"support-access-grant",
|
||||
"support-access-grant-current",
|
||||
"library-collection",
|
||||
@@ -82,6 +86,32 @@ export function invalidateTags(
|
||||
return !error ? tags : [];
|
||||
}
|
||||
|
||||
export function provideWorkspacesTags(
|
||||
workspaces: Workspace[],
|
||||
): TagDescription<EnterpriseTagType>[] {
|
||||
return [listTag("workspace"), ...workspaces.flatMap(provideWorkspaceTags)];
|
||||
}
|
||||
|
||||
export function provideWorkspaceTags(
|
||||
workspace: Workspace,
|
||||
): TagDescription<EnterpriseTagType>[] {
|
||||
return [idTag("workspace", workspace.id)];
|
||||
}
|
||||
|
||||
export function provideWorkspaceContentsTags(
|
||||
workspaceContents: WorkspaceContents,
|
||||
): TagDescription<EnterpriseTagType>[] {
|
||||
return provideWorkspaceContentItemsTags(
|
||||
workspaceContents.contents.transforms,
|
||||
);
|
||||
}
|
||||
|
||||
export function provideWorkspaceContentItemsTags(
|
||||
items: WorkspaceContentItem[],
|
||||
): TagDescription<EnterpriseTagType>[] {
|
||||
return items.flatMap((transform) => idTag("transform", transform.id));
|
||||
}
|
||||
|
||||
export function provideTransformTags(
|
||||
transform: Transform,
|
||||
): TagDescription<EnterpriseTagType>[] {
|
||||
|
||||
208
enterprise/frontend/src/metabase-enterprise/api/workspace.ts
Normal file
208
enterprise/frontend/src/metabase-enterprise/api/workspace.ts
Normal file
@@ -0,0 +1,208 @@
|
||||
import type {
|
||||
CreateWorkspaceRequest,
|
||||
CreateWorkspaceTransformRequest,
|
||||
CreateWorkspaceTransformResponse,
|
||||
TransformDownstreamMapping,
|
||||
TransformId,
|
||||
TransformUpstreamMapping,
|
||||
ValidateTableNameRequest,
|
||||
ValidateTableNameResponse,
|
||||
Workspace,
|
||||
WorkspaceContents,
|
||||
WorkspaceExecuteRequest,
|
||||
WorkspaceExecuteResponse,
|
||||
WorkspaceId,
|
||||
WorkspaceListResponse,
|
||||
WorkspaceLogResponse,
|
||||
WorkspaceMergeResponse,
|
||||
WorkspaceTablesResponse,
|
||||
WorkspaceUpdateContentsRequest,
|
||||
} from "metabase-types/api";
|
||||
|
||||
import { EnterpriseApi } from "./api";
|
||||
import {
|
||||
idTag,
|
||||
invalidateTags,
|
||||
listTag,
|
||||
provideWorkspaceContentItemsTags,
|
||||
provideWorkspaceContentsTags,
|
||||
provideWorkspaceTags,
|
||||
provideWorkspacesTags,
|
||||
tag,
|
||||
} from "./tags";
|
||||
|
||||
export const workspaceApi = EnterpriseApi.injectEndpoints({
|
||||
endpoints: (builder) => ({
|
||||
getWorkspaces: builder.query<WorkspaceListResponse, void>({
|
||||
query: () => ({
|
||||
method: "GET",
|
||||
url: "/api/ee/workspace",
|
||||
}),
|
||||
providesTags: (response) =>
|
||||
response ? provideWorkspacesTags(response.items) : [],
|
||||
}),
|
||||
getWorkspace: builder.query<Workspace, WorkspaceId>({
|
||||
query: (id) => ({
|
||||
method: "GET",
|
||||
url: `/api/ee/workspace/${id}`,
|
||||
}),
|
||||
providesTags: (workspace) =>
|
||||
workspace ? provideWorkspaceTags(workspace) : [],
|
||||
}),
|
||||
createWorkspace: builder.mutation<Workspace, CreateWorkspaceRequest>({
|
||||
query: (body) => ({
|
||||
method: "POST",
|
||||
url: "/api/ee/workspace",
|
||||
body,
|
||||
}),
|
||||
invalidatesTags: (_, error) =>
|
||||
invalidateTags(error, [listTag("workspace"), listTag("transform")]),
|
||||
}),
|
||||
getWorkspaceContents: builder.query<WorkspaceContents, WorkspaceId>({
|
||||
query: (id) => ({
|
||||
method: "GET",
|
||||
url: `/api/ee/workspace/${id}`,
|
||||
}),
|
||||
transformResponse: (workspace: any) => {
|
||||
return {
|
||||
contents: workspace?.contents ?? { transforms: [] },
|
||||
} as WorkspaceContents;
|
||||
},
|
||||
providesTags: (workspaceContents) =>
|
||||
workspaceContents
|
||||
? provideWorkspaceContentsTags(workspaceContents)
|
||||
: [],
|
||||
}),
|
||||
updateWorkspaceContents: builder.mutation<
|
||||
WorkspaceContents,
|
||||
WorkspaceUpdateContentsRequest
|
||||
>({
|
||||
query: ({ id, ...body }) => ({
|
||||
method: "POST",
|
||||
url: `/api/ee/workspace/${id}/contents`,
|
||||
body,
|
||||
}),
|
||||
invalidatesTags: (_, error, { id }) =>
|
||||
invalidateTags(error, [idTag("workspace", id), tag("transform")]),
|
||||
}),
|
||||
createWorkspaceTransform: builder.mutation<
|
||||
CreateWorkspaceTransformResponse,
|
||||
{ id: WorkspaceId } & CreateWorkspaceTransformRequest
|
||||
>({
|
||||
query: ({ id, ...body }) => ({
|
||||
method: "POST",
|
||||
url: `/api/ee/workspace/${id}/transform`,
|
||||
body,
|
||||
}),
|
||||
invalidatesTags: (_, error, { id }) =>
|
||||
invalidateTags(error, [idTag("workspace", id), tag("transform")]),
|
||||
}),
|
||||
getTransformUpstreamMapping: builder.query<
|
||||
TransformUpstreamMapping,
|
||||
TransformId
|
||||
>({
|
||||
query: (id) => ({
|
||||
method: "GET",
|
||||
url: `/api/ee/workspace/mapping/transform/${id}/upstream`,
|
||||
}),
|
||||
providesTags: (mapping) =>
|
||||
mapping?.transform
|
||||
? provideWorkspaceContentItemsTags([mapping.transform])
|
||||
: [],
|
||||
}),
|
||||
getTransformDownstreamMapping: builder.query<
|
||||
TransformDownstreamMapping,
|
||||
TransformId
|
||||
>({
|
||||
query: (id) => ({
|
||||
method: "GET",
|
||||
url: `/api/ee/workspace/mapping/transform/${id}/downstream`,
|
||||
}),
|
||||
providesTags: (mapping) =>
|
||||
mapping ? provideWorkspaceContentItemsTags(mapping.transforms) : [],
|
||||
}),
|
||||
mergeWorkspace: builder.mutation<WorkspaceMergeResponse, WorkspaceId>({
|
||||
query: (id) => ({
|
||||
method: "POST",
|
||||
url: `/api/ee/workspace/${id}/merge`,
|
||||
}),
|
||||
invalidatesTags: (_, error) =>
|
||||
invalidateTags(error, [tag("workspace"), tag("transform")]),
|
||||
}),
|
||||
archiveWorkspace: builder.mutation<Workspace, WorkspaceId>({
|
||||
query: (id) => ({
|
||||
method: "POST",
|
||||
url: `/api/ee/workspace/${id}/archive`,
|
||||
}),
|
||||
invalidatesTags: (_, error) =>
|
||||
invalidateTags(error, [tag("workspace"), tag("transform")]),
|
||||
}),
|
||||
validateTableName: builder.mutation<
|
||||
ValidateTableNameResponse,
|
||||
ValidateTableNameRequest
|
||||
>({
|
||||
query: ({ id, ...body }) => ({
|
||||
method: "POST",
|
||||
url: `/api/ee/workspace/${id}/validate-target`,
|
||||
body,
|
||||
}),
|
||||
}),
|
||||
updateWorkspaceName: builder.mutation<
|
||||
Workspace,
|
||||
{ id: WorkspaceId; name: string }
|
||||
>({
|
||||
query: ({ id, ...body }) => ({
|
||||
method: "POST",
|
||||
url: `/api/ee/workspace/${id}/name`,
|
||||
body,
|
||||
}),
|
||||
invalidatesTags: (_, error, { id }) =>
|
||||
invalidateTags(error, [idTag("workspace", id)]),
|
||||
}),
|
||||
getWorkspaceTables: builder.query<WorkspaceTablesResponse, WorkspaceId>({
|
||||
query: (id) => ({
|
||||
method: "GET",
|
||||
url: `/api/ee/workspace/${id}/tables`,
|
||||
}),
|
||||
providesTags: (_, __, id) => [idTag("workspace", id)],
|
||||
}),
|
||||
getWorkspaceLog: builder.query<WorkspaceLogResponse, WorkspaceId>({
|
||||
query: (id) => ({
|
||||
method: "GET",
|
||||
url: `/api/ee/workspace/${id}/log`,
|
||||
}),
|
||||
providesTags: (_, _error, id) => [idTag("workspace", id)],
|
||||
}),
|
||||
executeWorkspace: builder.mutation<
|
||||
WorkspaceExecuteResponse,
|
||||
WorkspaceExecuteRequest
|
||||
>({
|
||||
query: ({ id, stale_only }) => ({
|
||||
method: "POST",
|
||||
url: `/api/ee/workspace/${id}/execute`,
|
||||
params: stale_only ? { stale_only: 1 } : undefined,
|
||||
}),
|
||||
invalidatesTags: (_, error, { id }) =>
|
||||
invalidateTags(error, [idTag("workspace", id), tag("transform")]),
|
||||
}),
|
||||
}),
|
||||
});
|
||||
|
||||
export const {
|
||||
useGetWorkspacesQuery,
|
||||
useGetWorkspaceQuery,
|
||||
useCreateWorkspaceMutation,
|
||||
useGetWorkspaceContentsQuery,
|
||||
useUpdateWorkspaceContentsMutation,
|
||||
useLazyGetWorkspaceContentsQuery,
|
||||
useCreateWorkspaceTransformMutation,
|
||||
useGetTransformUpstreamMappingQuery,
|
||||
useGetTransformDownstreamMappingQuery,
|
||||
useMergeWorkspaceMutation,
|
||||
useArchiveWorkspaceMutation,
|
||||
useValidateTableNameMutation,
|
||||
useUpdateWorkspaceNameMutation,
|
||||
useGetWorkspaceTablesQuery,
|
||||
useGetWorkspaceLogQuery,
|
||||
useExecuteWorkspaceMutation,
|
||||
} = workspaceApi;
|
||||
@@ -1,5 +1,13 @@
|
||||
.nav {
|
||||
border-right: 1px solid var(--mb-color-border);
|
||||
transition: width 200ms ease;
|
||||
width: auto;
|
||||
min-width: 250px;
|
||||
}
|
||||
|
||||
.navCollapsed {
|
||||
width: 60px;
|
||||
min-width: 60px;
|
||||
}
|
||||
|
||||
.tab {
|
||||
@@ -14,9 +22,99 @@
|
||||
.tab,
|
||||
.toggle {
|
||||
color: var(--mb-color-text-primary);
|
||||
font-weight: 600;
|
||||
|
||||
&:hover,
|
||||
&.selected {
|
||||
color: var(--mb-color-brand);
|
||||
}
|
||||
}
|
||||
|
||||
.workspacesSection {
|
||||
border-top: 1px solid var(--mb-color-border);
|
||||
padding-top: 0.75rem;
|
||||
}
|
||||
|
||||
.workspacesSectionHeader {
|
||||
background-color: var(--mb-color-bg-secondary);
|
||||
color: var(--mb-color-text-primary);
|
||||
font-weight: 600;
|
||||
transition: background-color 150ms ease;
|
||||
|
||||
&:hover {
|
||||
background-color: var(--mb-color-brand-light);
|
||||
color: var(--mb-color-brand);
|
||||
}
|
||||
|
||||
&.selected {
|
||||
background-color: var(--mb-color-brand-light);
|
||||
color: var(--mb-color-brand);
|
||||
}
|
||||
}
|
||||
|
||||
.workspaceItem {
|
||||
background-color: var(--mb-color-bg-white);
|
||||
color: var(--mb-color-text-primary);
|
||||
cursor: pointer;
|
||||
border: 1px solid var(--mb-color-border);
|
||||
transition:
|
||||
background-color 150ms ease,
|
||||
border-color 150ms ease,
|
||||
color 150ms ease;
|
||||
|
||||
&:hover {
|
||||
background-color: var(--mb-color-bg-light);
|
||||
border-color: var(--mb-color-border-dark);
|
||||
}
|
||||
|
||||
&:hover .workspaceMenuButton {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
&.selected {
|
||||
background-color: var(--mb-color-brand-lighter);
|
||||
border-color: var(--mb-color-brand);
|
||||
color: var(--mb-color-brand);
|
||||
}
|
||||
|
||||
&.selected .workspaceMenuButton {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
&:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
}
|
||||
|
||||
.workspaceMenuButton {
|
||||
color: var(--mb-color-text-secondary);
|
||||
opacity: 0;
|
||||
transition:
|
||||
opacity 150ms ease,
|
||||
background-color 150ms ease,
|
||||
color 150ms ease;
|
||||
flex-shrink: 0;
|
||||
|
||||
&:hover {
|
||||
background-color: var(--mb-color-bg-medium);
|
||||
color: var(--mb-color-text-primary);
|
||||
}
|
||||
}
|
||||
|
||||
.newWorkspaceButton {
|
||||
background-color: transparent;
|
||||
color: var(--mb-color-text-primary);
|
||||
cursor: pointer;
|
||||
border: none;
|
||||
transition: color 150ms ease;
|
||||
|
||||
&:hover {
|
||||
color: var(--mb-color-brand);
|
||||
}
|
||||
|
||||
&:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
import cx from "classnames";
|
||||
import { type ReactNode, useMemo, useState } from "react";
|
||||
import dayjs from "dayjs";
|
||||
import { type ReactNode, useCallback, useMemo, useState } from "react";
|
||||
import { push } from "react-router-redux";
|
||||
import { t } from "ttag";
|
||||
|
||||
import { useListDatabasesQuery } from "metabase/api";
|
||||
import { ForwardRefLink } from "metabase/common/components/Link";
|
||||
import { useSelector } from "metabase/lib/redux";
|
||||
import { useDispatch, useSelector } from "metabase/lib/redux";
|
||||
import * as Urls from "metabase/lib/urls";
|
||||
import { useMetadataToasts } from "metabase/metadata/hooks";
|
||||
import {
|
||||
PLUGIN_DEPENDENCIES,
|
||||
PLUGIN_FEATURE_LEVEL_PERMISSIONS,
|
||||
@@ -12,15 +16,27 @@ import {
|
||||
} from "metabase/plugins";
|
||||
import { getLocation } from "metabase/selectors/routing";
|
||||
import {
|
||||
ActionIcon,
|
||||
Box,
|
||||
FixedSizeIcon,
|
||||
Flex,
|
||||
Icon,
|
||||
type IconName,
|
||||
Menu,
|
||||
Skeleton,
|
||||
Stack,
|
||||
Text,
|
||||
Tooltip,
|
||||
UnstyledButton,
|
||||
} from "metabase/ui";
|
||||
import {
|
||||
useArchiveWorkspaceMutation,
|
||||
useCreateWorkspaceMutation,
|
||||
useGetWorkspacesQuery,
|
||||
} from "metabase-enterprise/api";
|
||||
import { DataStudioContext } from "metabase-enterprise/data-studio/common/contexts/DataStudioContext";
|
||||
import { CreateWorkspaceModal } from "metabase-enterprise/data-studio/workspaces/components/CreateWorkspaceModal/CreateWorkspaceModal";
|
||||
import type { Database, WorkspaceId } from "metabase-types/api";
|
||||
|
||||
import S from "./DataStudioLayout.module.css";
|
||||
|
||||
@@ -31,6 +47,7 @@ type DataStudioLayoutProps = {
|
||||
export function DataStudioLayout({ children }: DataStudioLayoutProps) {
|
||||
const [isSidebarOpened, setIsSidebarOpened] = useState(false);
|
||||
const [isSidebarAvailable, setIsSidebarAvailable] = useState(false);
|
||||
const [isNavExpanded, setIsNavExpanded] = useState(true);
|
||||
const contextValue = useMemo(
|
||||
() => ({
|
||||
isSidebarOpened,
|
||||
@@ -48,6 +65,8 @@ export function DataStudioLayout({ children }: DataStudioLayoutProps) {
|
||||
isSidebarOpened={isSidebarOpened}
|
||||
isSidebarAvailable={isSidebarAvailable}
|
||||
onSidebarToggle={setIsSidebarOpened}
|
||||
isNavExpanded={isNavExpanded}
|
||||
onNavToggle={setIsNavExpanded}
|
||||
/>
|
||||
<Box h="100%" flex={1} miw={0}>
|
||||
{children}
|
||||
@@ -61,12 +80,16 @@ type DataStudioNavProps = {
|
||||
isSidebarOpened: boolean;
|
||||
isSidebarAvailable: boolean;
|
||||
onSidebarToggle: (isOpened: boolean) => void;
|
||||
isNavExpanded: boolean;
|
||||
onNavToggle: (isExpanded: boolean) => void;
|
||||
};
|
||||
|
||||
function DataStudioNav({
|
||||
isSidebarOpened,
|
||||
isSidebarAvailable,
|
||||
onSidebarToggle,
|
||||
isNavExpanded,
|
||||
onNavToggle,
|
||||
}: DataStudioNavProps) {
|
||||
const { pathname } = useSelector(getLocation);
|
||||
const canAccessDataModel = useSelector(
|
||||
@@ -76,42 +99,62 @@ function DataStudioNav({
|
||||
PLUGIN_TRANSFORMS.canAccessTransforms,
|
||||
);
|
||||
const isDataTab = pathname.startsWith(Urls.dataStudioData());
|
||||
const isTransformsTab = pathname.startsWith(Urls.transformList());
|
||||
const isWorkspacePage = pathname.startsWith(Urls.dataStudioWorkspaceList());
|
||||
const isTransformsTab =
|
||||
pathname.startsWith(Urls.transformList()) && !isWorkspacePage;
|
||||
const isModelingTab = pathname.startsWith(Urls.dataStudioModeling());
|
||||
const isDependenciesTab = pathname.startsWith(Urls.dependencyGraph());
|
||||
|
||||
return (
|
||||
<Stack className={S.nav} h="100%" p="0.75rem" justify="space-between">
|
||||
<Stack
|
||||
className={cx(S.nav, { [S.navCollapsed]: !isNavExpanded })}
|
||||
h="100%"
|
||||
p="0.75rem"
|
||||
justify="space-between"
|
||||
>
|
||||
<Stack gap="0.75rem">
|
||||
{canAccessDataModel && (
|
||||
<DataStudioTab
|
||||
label={t`Data`}
|
||||
icon="database"
|
||||
to={Urls.dataStudioData()}
|
||||
isSelected={isDataTab}
|
||||
/>
|
||||
)}
|
||||
{canAccessTransforms && (
|
||||
<DataStudioTab
|
||||
label={t`Transforms`}
|
||||
icon="transform"
|
||||
to={Urls.transformList()}
|
||||
isSelected={isTransformsTab}
|
||||
/>
|
||||
)}
|
||||
<DataStudioTab
|
||||
label={t`Modeling`}
|
||||
icon="model"
|
||||
to={Urls.dataStudioModeling()}
|
||||
isSelected={isModelingTab}
|
||||
<DataStudioNavToggle
|
||||
isNavExpanded={isNavExpanded}
|
||||
onNavToggle={onNavToggle}
|
||||
/>
|
||||
{PLUGIN_DEPENDENCIES.isEnabled && (
|
||||
<Stack gap="0.75rem">
|
||||
{canAccessDataModel && (
|
||||
<DataStudioTab
|
||||
label={t`Data`}
|
||||
icon="database"
|
||||
to={Urls.dataStudioData()}
|
||||
isSelected={isDataTab}
|
||||
isExpanded={isNavExpanded}
|
||||
/>
|
||||
)}
|
||||
{canAccessTransforms && (
|
||||
<DataStudioTab
|
||||
label={t`Transforms`}
|
||||
icon="transform"
|
||||
to={Urls.transformList()}
|
||||
isSelected={isTransformsTab}
|
||||
isExpanded={isNavExpanded}
|
||||
/>
|
||||
)}
|
||||
<DataStudioTab
|
||||
label={t`Dependency graph`}
|
||||
icon="schema"
|
||||
to={Urls.dependencyGraph()}
|
||||
isSelected={isDependenciesTab}
|
||||
label={t`Modeling`}
|
||||
icon="model"
|
||||
to={Urls.dataStudioModeling()}
|
||||
isSelected={isModelingTab}
|
||||
isExpanded={isNavExpanded}
|
||||
/>
|
||||
{PLUGIN_DEPENDENCIES.isEnabled && (
|
||||
<DataStudioTab
|
||||
label={t`Dependency graph`}
|
||||
icon="schema"
|
||||
to={Urls.dependencyGraph()}
|
||||
isSelected={isDependenciesTab}
|
||||
isExpanded={isNavExpanded}
|
||||
/>
|
||||
)}
|
||||
</Stack>
|
||||
{canAccessTransforms && (
|
||||
<WorkspacesSection isExpanded={isNavExpanded} />
|
||||
)}
|
||||
</Stack>
|
||||
{isSidebarAvailable && (
|
||||
@@ -129,23 +172,241 @@ type DataStudioTabProps = {
|
||||
icon: IconName;
|
||||
to: string;
|
||||
isSelected: boolean;
|
||||
isExpanded: boolean;
|
||||
};
|
||||
|
||||
const TOOLTIP_OPEN_DELAY = 1000;
|
||||
|
||||
function DataStudioTab({ label, icon, to, isSelected }: DataStudioTabProps) {
|
||||
function DataStudioTab({
|
||||
label,
|
||||
icon,
|
||||
to,
|
||||
isSelected,
|
||||
isExpanded,
|
||||
}: DataStudioTabProps) {
|
||||
const content = (
|
||||
<Box
|
||||
className={cx(S.tab, { [S.selected]: isSelected })}
|
||||
component={ForwardRefLink}
|
||||
to={to}
|
||||
p="0.75rem"
|
||||
bdrs="md"
|
||||
>
|
||||
<Flex
|
||||
align="center"
|
||||
gap="sm"
|
||||
justify={isExpanded ? "flex-start" : "center"}
|
||||
>
|
||||
<FixedSizeIcon name={icon} display="block" />
|
||||
{isExpanded && (
|
||||
<Text size="sm" fw={600}>
|
||||
{label}
|
||||
</Text>
|
||||
)}
|
||||
</Flex>
|
||||
</Box>
|
||||
);
|
||||
|
||||
if (!isExpanded) {
|
||||
return (
|
||||
<Tooltip label={label} position="right" openDelay={TOOLTIP_OPEN_DELAY}>
|
||||
{content}
|
||||
</Tooltip>
|
||||
);
|
||||
}
|
||||
|
||||
return content;
|
||||
}
|
||||
|
||||
type WorkspacesSectionProps = {
|
||||
isExpanded: boolean;
|
||||
};
|
||||
|
||||
function WorkspacesSection({
|
||||
isExpanded: isNavExpanded,
|
||||
}: WorkspacesSectionProps) {
|
||||
const dispatch = useDispatch();
|
||||
const [isWorkspacesExpanded, setIsWorkspacesExpanded] = useState(true);
|
||||
const [isCreateModalOpen, setIsCreateModalOpen] = useState(false);
|
||||
const { pathname } = useSelector(getLocation);
|
||||
const { data: workspacesData, isLoading } = useGetWorkspacesQuery();
|
||||
const { data: databaseData, isLoading: isLoadingDatabases } =
|
||||
useListDatabasesQuery({ include_analytics: true });
|
||||
const [createWorkspace, { isLoading: isCreating }] =
|
||||
useCreateWorkspaceMutation();
|
||||
|
||||
const workspaces = useMemo(
|
||||
() =>
|
||||
[...(workspacesData?.items ?? [])].sort(
|
||||
(a, b) =>
|
||||
new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime(),
|
||||
),
|
||||
[workspacesData],
|
||||
);
|
||||
|
||||
const databaseOptions = useMemo(
|
||||
() =>
|
||||
(databaseData?.data ?? []).map((db: Database) => ({
|
||||
value: String(db.id),
|
||||
label: db.name,
|
||||
})),
|
||||
[databaseData],
|
||||
);
|
||||
|
||||
const handleOpenWorkspace = useCallback(
|
||||
(workspaceId: number) => {
|
||||
dispatch(push(Urls.dataStudioWorkspace(workspaceId)));
|
||||
},
|
||||
[dispatch],
|
||||
);
|
||||
|
||||
const handleOpenCreateModal = useCallback(() => {
|
||||
setIsCreateModalOpen(true);
|
||||
}, []);
|
||||
|
||||
const handleCloseCreateModal = useCallback(() => {
|
||||
setIsCreateModalOpen(false);
|
||||
}, []);
|
||||
|
||||
const { sendErrorToast, sendSuccessToast } = useMetadataToasts();
|
||||
|
||||
const handleCreateWorkspace = useCallback(
|
||||
async ({ name, databaseId }: { name: string; databaseId: string }) => {
|
||||
try {
|
||||
const workspace = await createWorkspace({
|
||||
name,
|
||||
database_id: Number(databaseId),
|
||||
upstream: {},
|
||||
}).unwrap();
|
||||
handleCloseCreateModal();
|
||||
handleOpenWorkspace(workspace.id);
|
||||
} catch (error) {
|
||||
sendErrorToast(t`Failed to create workspace`);
|
||||
}
|
||||
},
|
||||
[
|
||||
createWorkspace,
|
||||
handleCloseCreateModal,
|
||||
handleOpenWorkspace,
|
||||
sendErrorToast,
|
||||
],
|
||||
);
|
||||
|
||||
const [archiveWorkspace] = useArchiveWorkspaceMutation();
|
||||
const handleWorkspaceArchive = async (id: WorkspaceId) => {
|
||||
try {
|
||||
await archiveWorkspace(id).unwrap();
|
||||
sendSuccessToast(t`Workspace archived successfully`);
|
||||
dispatch(push(Urls.dataStudioWorkspaceList()));
|
||||
} catch (error) {
|
||||
sendErrorToast(t`Failed to archive workspace`);
|
||||
}
|
||||
};
|
||||
|
||||
const isWorkspaceListPage = pathname === Urls.dataStudioWorkspaceList();
|
||||
|
||||
if (!isNavExpanded) {
|
||||
// In collapsed mode, show only an icon button with tooltip
|
||||
return (
|
||||
<Tooltip
|
||||
label={t`Workspaces`}
|
||||
position="right"
|
||||
openDelay={TOOLTIP_OPEN_DELAY}
|
||||
>
|
||||
<UnstyledButton
|
||||
className={cx(S.tab, { [S.selected]: isWorkspaceListPage })}
|
||||
component={ForwardRefLink}
|
||||
to={Urls.dataStudioWorkspaceList()}
|
||||
p="0.75rem"
|
||||
bdrs="md"
|
||||
>
|
||||
<Flex align="center" justify="center">
|
||||
<FixedSizeIcon name="git_branch" display="block" />
|
||||
</Flex>
|
||||
</UnstyledButton>
|
||||
</Tooltip>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<Tooltip label={label} position="right" openDelay={TOOLTIP_OPEN_DELAY}>
|
||||
<Box
|
||||
className={cx(S.tab, { [S.selected]: isSelected })}
|
||||
component={ForwardRefLink}
|
||||
to={to}
|
||||
<Stack
|
||||
data-testid="workspaces-section"
|
||||
gap="0.5rem"
|
||||
className={S.workspacesSection}
|
||||
>
|
||||
<UnstyledButton
|
||||
className={cx(S.workspacesSectionHeader, {
|
||||
[S.selected]: isWorkspaceListPage,
|
||||
})}
|
||||
onClick={() => setIsWorkspacesExpanded(!isWorkspacesExpanded)}
|
||||
p="0.75rem"
|
||||
bdrs="md"
|
||||
>
|
||||
<FixedSizeIcon name={icon} display="block" />
|
||||
</Box>
|
||||
</Tooltip>
|
||||
<Flex align="center" justify="space-between" w="100%">
|
||||
<Text size="sm" fw={600}>
|
||||
{t`Workspaces`}
|
||||
</Text>
|
||||
<Icon
|
||||
name={isWorkspacesExpanded ? "chevrondown" : "chevronright"}
|
||||
size={16}
|
||||
/>
|
||||
</Flex>
|
||||
</UnstyledButton>
|
||||
|
||||
{isWorkspacesExpanded && (
|
||||
<>
|
||||
<UnstyledButton
|
||||
className={S.newWorkspaceButton}
|
||||
onClick={handleOpenCreateModal}
|
||||
disabled={isLoadingDatabases}
|
||||
p="0.5rem"
|
||||
bdrs="md"
|
||||
>
|
||||
<Flex align="center" gap="xs">
|
||||
<Icon name="add" size={16} />
|
||||
<Text size="sm" fw={500}>
|
||||
{t`New workspace`}
|
||||
</Text>
|
||||
</Flex>
|
||||
</UnstyledButton>
|
||||
<Stack gap="0.75rem" style={{ overflowY: "auto", maxHeight: "50vh" }}>
|
||||
{isLoading ? (
|
||||
<>
|
||||
<Skeleton height={80} radius="md" />
|
||||
<Skeleton height={80} radius="md" />
|
||||
</>
|
||||
) : workspaces.length === 0 ? (
|
||||
<Text c="text-secondary" size="xs" px="0.5rem">
|
||||
{t`No workspaces yet`}
|
||||
</Text>
|
||||
) : (
|
||||
workspaces.map((workspace) => {
|
||||
const isSelected =
|
||||
pathname === Urls.dataStudioWorkspace(workspace.id);
|
||||
|
||||
return (
|
||||
<WorkspaceItem
|
||||
key={workspace.id}
|
||||
workspace={workspace}
|
||||
isSelected={isSelected}
|
||||
onOpen={handleOpenWorkspace}
|
||||
archiveWorkspace={handleWorkspaceArchive}
|
||||
/>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</Stack>
|
||||
</>
|
||||
)}
|
||||
|
||||
<CreateWorkspaceModal
|
||||
opened={isCreateModalOpen}
|
||||
onClose={handleCloseCreateModal}
|
||||
onSubmit={handleCreateWorkspace}
|
||||
databaseOptions={databaseOptions}
|
||||
isSubmitting={isCreating}
|
||||
/>
|
||||
</Stack>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -161,6 +422,7 @@ function DataStudioSidebarToggle({
|
||||
return (
|
||||
<Tooltip
|
||||
label={isSidebarOpened ? t`Close sidebar` : t`Open sidebar`}
|
||||
position="right"
|
||||
openDelay={TOOLTIP_OPEN_DELAY}
|
||||
>
|
||||
<UnstyledButton
|
||||
@@ -176,3 +438,101 @@ function DataStudioSidebarToggle({
|
||||
</Tooltip>
|
||||
);
|
||||
}
|
||||
|
||||
type DataStudioNavToggleProps = {
|
||||
isNavExpanded: boolean;
|
||||
onNavToggle: (isExpanded: boolean) => void;
|
||||
};
|
||||
|
||||
function DataStudioNavToggle({
|
||||
isNavExpanded,
|
||||
onNavToggle,
|
||||
}: DataStudioNavToggleProps) {
|
||||
return (
|
||||
<Tooltip
|
||||
label={isNavExpanded ? t`Collapse navigation` : t`Expand navigation`}
|
||||
position="right"
|
||||
openDelay={TOOLTIP_OPEN_DELAY}
|
||||
>
|
||||
<UnstyledButton
|
||||
className={S.toggle}
|
||||
p="0.75rem"
|
||||
bdrs="md"
|
||||
onClick={() => onNavToggle(!isNavExpanded)}
|
||||
>
|
||||
<FixedSizeIcon name={isNavExpanded ? "chevronleft" : "chevronright"} />
|
||||
</UnstyledButton>
|
||||
</Tooltip>
|
||||
);
|
||||
}
|
||||
|
||||
interface WorkspaceItemProps {
|
||||
workspace: { id: number; name: string; updated_at: string };
|
||||
isSelected: boolean;
|
||||
onOpen: (workspaceId: WorkspaceId) => void;
|
||||
archiveWorkspace: (workspaceId: WorkspaceId) => Promise<void>;
|
||||
}
|
||||
|
||||
function WorkspaceItem({
|
||||
workspace,
|
||||
isSelected,
|
||||
onOpen,
|
||||
archiveWorkspace,
|
||||
}: WorkspaceItemProps) {
|
||||
const timeAgo = dayjs(workspace.updated_at).fromNow();
|
||||
const handleArchive = () => {
|
||||
archiveWorkspace(workspace.id);
|
||||
};
|
||||
|
||||
const handleDelete = () => {
|
||||
archiveWorkspace(workspace.id);
|
||||
};
|
||||
|
||||
return (
|
||||
<UnstyledButton
|
||||
className={cx(S.workspaceItem, { [S.selected]: isSelected })}
|
||||
onClick={() => onOpen(workspace.id)}
|
||||
p="0.75rem"
|
||||
bdrs="md"
|
||||
>
|
||||
<Flex align="flex-start" justify="space-between" gap="xs">
|
||||
<Stack gap="xs" style={{ flex: 1, minWidth: 0 }}>
|
||||
<Text size="sm" fw={600} truncate>
|
||||
{workspace.name}
|
||||
</Text>
|
||||
<Text size="xs" c="text-secondary" truncate>
|
||||
{t`Updated ${timeAgo}`}
|
||||
</Text>
|
||||
</Stack>
|
||||
<Menu position="right" withinPortal>
|
||||
<Menu.Target>
|
||||
<ActionIcon
|
||||
className={S.workspaceMenuButton}
|
||||
aria-label={t`More actions`}
|
||||
onClick={(e: React.MouseEvent) => e.stopPropagation()}
|
||||
size="sm"
|
||||
variant="subtle"
|
||||
>
|
||||
<Icon name="ellipsis" size={16} />
|
||||
</ActionIcon>
|
||||
</Menu.Target>
|
||||
<Menu.Dropdown onClick={(e: React.MouseEvent) => e.stopPropagation()}>
|
||||
<Menu.Item
|
||||
leftSection={<Icon name="archive" />}
|
||||
onClick={handleArchive}
|
||||
>
|
||||
{t`Archive`}
|
||||
</Menu.Item>
|
||||
<Menu.Divider />
|
||||
<Menu.Item
|
||||
leftSection={<Icon name="trash" />}
|
||||
onClick={handleDelete}
|
||||
>
|
||||
{t`Move to trash`}
|
||||
</Menu.Item>
|
||||
</Menu.Dropdown>
|
||||
</Menu>
|
||||
</Flex>
|
||||
</UnstyledButton>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import { t } from "ttag";
|
||||
|
||||
import { usePageTitle } from "metabase/hooks/use-page-title";
|
||||
|
||||
import { SectionLayout, SectionTitle } from "../../components/SectionLayout";
|
||||
|
||||
type WorkspacesSectionLayoutProps = {
|
||||
location: Location;
|
||||
children?: React.ReactNode;
|
||||
};
|
||||
|
||||
export function WorkspacesSectionLayout({
|
||||
children,
|
||||
}: WorkspacesSectionLayoutProps) {
|
||||
usePageTitle(t`Workspaces`);
|
||||
|
||||
return (
|
||||
<SectionLayout title={<SectionTitle title={t`Workspaces`} />}>
|
||||
{children}
|
||||
</SectionLayout>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export * from "./WorkspacesSectionLayout";
|
||||
@@ -15,12 +15,14 @@ import { DataStudioLayout } from "./app/pages/DataStudioLayout";
|
||||
import { DependenciesSectionLayout } from "./app/pages/DependenciesSectionLayout";
|
||||
import { ModelingSectionLayout } from "./app/pages/ModelingSectionLayout";
|
||||
import { TransformsSectionLayout } from "./app/pages/TransformsSectionLayout";
|
||||
import { WorkspacesSectionLayout } from "./app/pages/WorkspacesSectionLayout";
|
||||
import { getDataStudioMetadataRoutes } from "./data-model/routes";
|
||||
import { getDataStudioGlossaryRoutes } from "./glossary/routes";
|
||||
import { getDataStudioMetricRoutes } from "./metrics/routes";
|
||||
import { getDataStudioModelingRoutes } from "./modeling/routes";
|
||||
import { getDataStudioModelRoutes } from "./models/routes";
|
||||
import { getDataStudioSnippetRoutes } from "./snippets/routes";
|
||||
import { getDataStudioWorkspaceRoutes } from "./workspaces/routes";
|
||||
|
||||
export function getDataStudioRoutes(
|
||||
store: Store<State>,
|
||||
@@ -55,6 +57,9 @@ export function getDataStudioRoutes(
|
||||
{getDataStudioSnippetRoutes()}
|
||||
{getDataStudioGlossaryRoutes()}
|
||||
</Route>
|
||||
<Route path="workspaces" component={WorkspacesSectionLayout}>
|
||||
{getDataStudioWorkspaceRoutes()}
|
||||
</Route>
|
||||
{PLUGIN_DEPENDENCIES.isEnabled && (
|
||||
<Route path="dependencies" component={DependenciesSectionLayout}>
|
||||
{PLUGIN_DEPENDENCIES.getDataStudioDependencyRoutes()}
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
import { type FormEvent, useEffect, useMemo, useState } from "react";
|
||||
import { t } from "ttag";
|
||||
|
||||
import { Button, Flex, Modal, Select, Stack, TextInput } from "metabase/ui";
|
||||
|
||||
type DatabaseOption = {
|
||||
value: string;
|
||||
label: string;
|
||||
};
|
||||
|
||||
export type CreateWorkspaceModalProps = {
|
||||
opened: boolean;
|
||||
onClose: () => void;
|
||||
onSubmit: (workspace: { name: string; databaseId: string }) => void;
|
||||
databaseOptions: DatabaseOption[];
|
||||
isSubmitting?: boolean;
|
||||
defaultName?: string;
|
||||
defaultDatabaseId?: string | null;
|
||||
};
|
||||
|
||||
export function CreateWorkspaceModal({
|
||||
opened,
|
||||
onClose,
|
||||
onSubmit,
|
||||
databaseOptions,
|
||||
isSubmitting = false,
|
||||
defaultName,
|
||||
defaultDatabaseId = null,
|
||||
}: CreateWorkspaceModalProps) {
|
||||
const initialName = useMemo(
|
||||
() => defaultName ?? t`New workspace`,
|
||||
[defaultName],
|
||||
);
|
||||
const [workspaceName, setWorkspaceName] = useState(initialName);
|
||||
const [selectedDatabaseId, setSelectedDatabaseId] = useState<string | null>(
|
||||
defaultDatabaseId,
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (!opened) {
|
||||
return;
|
||||
}
|
||||
setWorkspaceName(initialName);
|
||||
setSelectedDatabaseId(defaultDatabaseId);
|
||||
}, [opened, initialName, defaultDatabaseId]);
|
||||
|
||||
const handleSubmit = (event: FormEvent) => {
|
||||
event.preventDefault();
|
||||
|
||||
if (!selectedDatabaseId || !workspaceName.trim() || isSubmitting) {
|
||||
return;
|
||||
}
|
||||
|
||||
onSubmit({
|
||||
name: workspaceName.trim(),
|
||||
databaseId: selectedDatabaseId,
|
||||
});
|
||||
};
|
||||
|
||||
return (
|
||||
<Modal
|
||||
opened={opened}
|
||||
onClose={onClose}
|
||||
title={t`Create new workspace`}
|
||||
withinPortal
|
||||
>
|
||||
<form onSubmit={handleSubmit}>
|
||||
<Stack gap="md">
|
||||
<TextInput
|
||||
autoFocus
|
||||
label={t`Workspace name`}
|
||||
placeholder={t`Enter workspace name`}
|
||||
value={workspaceName}
|
||||
onChange={(e) => setWorkspaceName(e.target.value)}
|
||||
required
|
||||
/>
|
||||
<Select
|
||||
label={t`Database`}
|
||||
placeholder={t`Select a database`}
|
||||
data={databaseOptions}
|
||||
value={selectedDatabaseId}
|
||||
onChange={setSelectedDatabaseId}
|
||||
required
|
||||
searchable
|
||||
/>
|
||||
<Flex gap="sm" justify="flex-end">
|
||||
<Button variant="subtle" onClick={onClose}>
|
||||
{t`Cancel`}
|
||||
</Button>
|
||||
<Button
|
||||
variant="filled"
|
||||
disabled={
|
||||
!selectedDatabaseId || !workspaceName.trim() || isSubmitting
|
||||
}
|
||||
type="submit"
|
||||
loading={isSubmitting}
|
||||
>
|
||||
{t`Create`}
|
||||
</Button>
|
||||
</Flex>
|
||||
</Stack>
|
||||
</form>
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
import { useCallback } from "react";
|
||||
import { t } from "ttag";
|
||||
|
||||
import { useMetadataToasts } from "metabase/metadata/hooks";
|
||||
import { Button, Icon, Menu } from "metabase/ui";
|
||||
import { useExecuteWorkspaceMutation } from "metabase-enterprise/api";
|
||||
import type { WorkspaceId } from "metabase-types/api";
|
||||
|
||||
type RunWorkspaceMenuProps = {
|
||||
workspaceId: WorkspaceId;
|
||||
disabled?: boolean;
|
||||
};
|
||||
|
||||
export function RunWorkspaceMenu({
|
||||
workspaceId,
|
||||
disabled = false,
|
||||
}: RunWorkspaceMenuProps) {
|
||||
const { sendErrorToast } = useMetadataToasts();
|
||||
const [executeWorkspace, { isLoading: isExecuting }] =
|
||||
useExecuteWorkspaceMutation();
|
||||
|
||||
const handleExecuteWorkspace = useCallback(
|
||||
async ({ staleOnly } = { staleOnly: false }) => {
|
||||
try {
|
||||
await executeWorkspace({
|
||||
id: workspaceId,
|
||||
stale_only: staleOnly,
|
||||
}).unwrap();
|
||||
} catch (error) {
|
||||
sendErrorToast(t`Failed to run transforms`);
|
||||
}
|
||||
},
|
||||
[workspaceId, executeWorkspace, sendErrorToast],
|
||||
);
|
||||
|
||||
return (
|
||||
<Menu position="bottom-end">
|
||||
<Menu.Target>
|
||||
<Button
|
||||
variant="default"
|
||||
rightSection={<Icon name="chevrondown" />}
|
||||
loading={isExecuting}
|
||||
disabled={disabled}
|
||||
size="xs"
|
||||
>
|
||||
{t`Run transforms`}
|
||||
</Button>
|
||||
</Menu.Target>
|
||||
<Menu.Dropdown>
|
||||
<Menu.Item onClick={() => handleExecuteWorkspace()}>
|
||||
{t`Run all transforms`}
|
||||
</Menu.Item>
|
||||
<Menu.Item onClick={() => handleExecuteWorkspace({ staleOnly: true })}>
|
||||
{t`Run stale transforms`}
|
||||
</Menu.Item>
|
||||
</Menu.Dropdown>
|
||||
</Menu>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import { t } from "ttag";
|
||||
|
||||
import { ForwardRefLink } from "metabase/common/components/Link";
|
||||
import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
|
||||
import * as Urls from "metabase/lib/urls";
|
||||
import { Anchor, Box, Card, Group, Stack, Text, Title } from "metabase/ui";
|
||||
import { useGetWorkspacesQuery } from "metabase-enterprise/api";
|
||||
|
||||
export function WorkspaceListPage() {
|
||||
const { data: workspacesData, error, isLoading } = useGetWorkspacesQuery();
|
||||
|
||||
if (error || isLoading) {
|
||||
return <LoadingAndErrorWrapper error={error} loading={isLoading} />;
|
||||
}
|
||||
|
||||
const workspaces = workspacesData?.items ?? [];
|
||||
|
||||
return (
|
||||
<Box data-testid="workspaces-page" p="lg">
|
||||
<Title order={2} mb="lg">{t`Workspaces`}</Title>
|
||||
{workspaces.length === 0 ? (
|
||||
<Text c="text-secondary">{t`No workspaces yet`}</Text>
|
||||
) : (
|
||||
<Stack gap="md">
|
||||
{workspaces.map((workspace) => (
|
||||
<Card key={workspace.id} p="md" shadow="none" withBorder>
|
||||
<Group justify="space-between">
|
||||
<Anchor
|
||||
component={ForwardRefLink}
|
||||
to={Urls.dataStudioWorkspace(workspace.id)}
|
||||
fw={500}
|
||||
>
|
||||
{workspace.name}
|
||||
</Anchor>
|
||||
<Text c="text-secondary" size="sm">
|
||||
{t`Created ${new Date(workspace.created_at).toLocaleDateString()}`}
|
||||
</Text>
|
||||
</Group>
|
||||
</Card>
|
||||
))}
|
||||
</Stack>
|
||||
)}
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
import { useCallback, useMemo, useState } from "react";
|
||||
import { t } from "ttag";
|
||||
import * as Yup from "yup";
|
||||
|
||||
import { skipToken, useListDatabaseSchemasQuery } from "metabase/api";
|
||||
import { getErrorMessage } from "metabase/api/utils";
|
||||
import { ActionIcon, Icon, Menu } from "metabase/ui";
|
||||
import {
|
||||
useCreateWorkspaceTransformMutation,
|
||||
useValidateTableNameMutation,
|
||||
} from "metabase-enterprise/api";
|
||||
import {
|
||||
CreateTransformModal,
|
||||
type NewTransformValues,
|
||||
} from "metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal";
|
||||
import {
|
||||
getInitialNativeSource,
|
||||
getInitialPythonSource,
|
||||
} from "metabase-enterprise/transforms/pages/NewTransformPage/utils";
|
||||
import type {
|
||||
CreateWorkspaceTransformRequest,
|
||||
DatabaseId,
|
||||
Transform,
|
||||
TransformSource,
|
||||
TransformTarget,
|
||||
WorkspaceId,
|
||||
} from "metabase-types/api";
|
||||
|
||||
type TransformType = "sql" | "python";
|
||||
|
||||
type AddTransformMenuProps = {
|
||||
databaseId: number;
|
||||
workspaceId: number;
|
||||
onCreate: (transform: Transform) => void;
|
||||
};
|
||||
|
||||
export const AddTransformMenu = ({
|
||||
databaseId,
|
||||
workspaceId,
|
||||
onCreate,
|
||||
}: AddTransformMenuProps) => {
|
||||
const [modalType, setModalType] = useState<TransformType | null>(null);
|
||||
const [createWorkspaceTransform] = useCreateWorkspaceTransformMutation();
|
||||
|
||||
const { data: fetchedSchemas = [] } = useListDatabaseSchemasQuery(
|
||||
databaseId ? { id: databaseId, include_hidden: false } : skipToken,
|
||||
);
|
||||
const allowedSchemas = useMemo(
|
||||
() =>
|
||||
fetchedSchemas.filter((schema) => !schema.startsWith("mb__isolation")),
|
||||
[fetchedSchemas],
|
||||
);
|
||||
|
||||
const getSource = useCallback(
|
||||
(type: TransformType): TransformSource => {
|
||||
if (type === "sql") {
|
||||
const source = getInitialNativeSource();
|
||||
return {
|
||||
...source,
|
||||
query: { ...source.query, database: databaseId },
|
||||
};
|
||||
}
|
||||
return {
|
||||
...getInitialPythonSource(),
|
||||
"source-database": databaseId,
|
||||
};
|
||||
},
|
||||
[databaseId],
|
||||
);
|
||||
|
||||
const handleClose = () => setModalType(null);
|
||||
|
||||
const handleSubmit = useCallback(
|
||||
async (values: NewTransformValues): Promise<Transform> => {
|
||||
const source = getSource(modalType!);
|
||||
const request: CreateWorkspaceTransformRequest & { id: WorkspaceId } =
|
||||
values.incremental
|
||||
? {
|
||||
id: workspaceId,
|
||||
name: values.name,
|
||||
description: null,
|
||||
source,
|
||||
target: {
|
||||
type: "table-incremental" as const,
|
||||
name: values.targetName,
|
||||
schema: values.targetSchema,
|
||||
database: databaseId,
|
||||
"target-incremental-strategy": {
|
||||
type: "append" as const,
|
||||
},
|
||||
},
|
||||
}
|
||||
: {
|
||||
id: workspaceId,
|
||||
name: values.name,
|
||||
description: null,
|
||||
source,
|
||||
target: {
|
||||
type: "table" as const,
|
||||
name: values.targetName,
|
||||
schema: values.targetSchema,
|
||||
database: databaseId,
|
||||
},
|
||||
};
|
||||
|
||||
const transform = await createWorkspaceTransform(request).unwrap();
|
||||
onCreate(transform);
|
||||
handleClose();
|
||||
return transform;
|
||||
},
|
||||
[
|
||||
databaseId,
|
||||
workspaceId,
|
||||
modalType,
|
||||
createWorkspaceTransform,
|
||||
onCreate,
|
||||
getSource,
|
||||
],
|
||||
);
|
||||
|
||||
const validationSchemaExtension = useTransformValidation({
|
||||
databaseId,
|
||||
workspaceId,
|
||||
});
|
||||
|
||||
return (
|
||||
<>
|
||||
<Menu position="bottom-end">
|
||||
<Menu.Target>
|
||||
<ActionIcon size="2rem" p="0" ml="auto" aria-label={t`Add transform`}>
|
||||
<Icon name="add" size={16} aria-hidden />
|
||||
</ActionIcon>
|
||||
</Menu.Target>
|
||||
<Menu.Dropdown>
|
||||
<Menu.Item
|
||||
leftSection={<Icon name="sql" />}
|
||||
onClick={() => setModalType("sql")}
|
||||
>
|
||||
{t`SQL Transform`}
|
||||
</Menu.Item>
|
||||
<Menu.Item
|
||||
leftSection={<Icon name="code_block" />}
|
||||
onClick={() => setModalType("python")}
|
||||
>
|
||||
{t`Python Transform`}
|
||||
</Menu.Item>
|
||||
</Menu.Dropdown>
|
||||
</Menu>
|
||||
{modalType && (
|
||||
<CreateTransformModal
|
||||
source={getSource(modalType)}
|
||||
defaultValues={{ name: t`New transform` }}
|
||||
onClose={handleClose}
|
||||
schemas={allowedSchemas}
|
||||
showIncrementalSettings={false}
|
||||
validationSchemaExtension={validationSchemaExtension}
|
||||
handleSubmit={handleSubmit}
|
||||
/>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export const useTransformValidation = ({
|
||||
databaseId,
|
||||
target,
|
||||
workspaceId,
|
||||
}: {
|
||||
databaseId: DatabaseId;
|
||||
target?: TransformTarget;
|
||||
workspaceId: WorkspaceId;
|
||||
}) => {
|
||||
const [validateTableName] = useValidateTableNameMutation();
|
||||
|
||||
const yupSchema = useMemo(
|
||||
() => ({
|
||||
targetName: Yup.string()
|
||||
.required("Target table name is required")
|
||||
.test(async (value, context) => {
|
||||
if (!value) {
|
||||
return context.createError({
|
||||
message: "Target table name is required",
|
||||
});
|
||||
}
|
||||
|
||||
const schema = context.parent.targetSchema;
|
||||
|
||||
if (target && target.name === value && target.schema === schema) {
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
const message = await validateTableName({
|
||||
id: workspaceId,
|
||||
db_id: databaseId,
|
||||
target: { type: "table", name: value, schema },
|
||||
}).unwrap();
|
||||
|
||||
return message === "OK" ? true : context.createError({ message });
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error);
|
||||
return context.createError({ message });
|
||||
}
|
||||
}),
|
||||
}),
|
||||
[databaseId, target, workspaceId, validateTableName],
|
||||
);
|
||||
|
||||
return yupSchema;
|
||||
};
|
||||
@@ -0,0 +1,86 @@
|
||||
import { t } from "ttag";
|
||||
|
||||
import { useDispatch, useSelector } from "metabase/lib/redux";
|
||||
import { Button } from "metabase/ui";
|
||||
import {
|
||||
useLazyGetTransformQuery,
|
||||
useUpdateWorkspaceContentsMutation,
|
||||
} from "metabase-enterprise/api";
|
||||
import {
|
||||
addSuggestedTransform,
|
||||
getMetabotSuggestedTransform,
|
||||
} from "metabase-enterprise/metabot/state";
|
||||
import type { Transform, TransformId, WorkspaceId } from "metabase-types/api";
|
||||
|
||||
import { useWorkspace } from "./WorkspaceProvider";
|
||||
|
||||
interface Props {
|
||||
transform: Transform;
|
||||
workspaceId: WorkspaceId;
|
||||
onOpenTransform: (transformId: TransformId) => void;
|
||||
}
|
||||
|
||||
export const CheckOutTransformButton = ({
|
||||
transform,
|
||||
workspaceId,
|
||||
onOpenTransform,
|
||||
}: Props) => {
|
||||
const dispatch = useDispatch();
|
||||
const suggestedTransform = useSelector((state) =>
|
||||
getMetabotSuggestedTransform(state, transform.id),
|
||||
);
|
||||
const [getTransform] = useLazyGetTransformQuery();
|
||||
const [updateWorkspaceContents] = useUpdateWorkspaceContentsMutation();
|
||||
|
||||
const {
|
||||
addOpenedTransform,
|
||||
removeOpenedTransform,
|
||||
setActiveTransform,
|
||||
removeEditedTransform,
|
||||
} = useWorkspace();
|
||||
|
||||
const handleClick = async () => {
|
||||
const response = await updateWorkspaceContents({
|
||||
id: workspaceId,
|
||||
add: {
|
||||
transforms: [transform.id],
|
||||
},
|
||||
});
|
||||
|
||||
const newTransform = response.data?.contents.transforms.find(
|
||||
(t) => t.upstream_id === transform.id,
|
||||
);
|
||||
const newTransformId = newTransform?.id;
|
||||
|
||||
if (newTransformId != null) {
|
||||
// TODO: remove when backend adds contents hydration to previous request
|
||||
const newTransform = await getTransform(newTransformId).unwrap();
|
||||
|
||||
if (newTransform) {
|
||||
if (suggestedTransform) {
|
||||
dispatch(
|
||||
addSuggestedTransform({
|
||||
...suggestedTransform,
|
||||
id: newTransform.id,
|
||||
active: true,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
removeEditedTransform(transform.id);
|
||||
addOpenedTransform(newTransform);
|
||||
removeOpenedTransform(transform.id);
|
||||
setActiveTransform(newTransform);
|
||||
onOpenTransform(newTransform.id);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<Button
|
||||
size="sm"
|
||||
variant="filled"
|
||||
onClick={handleClick}
|
||||
>{t`Check out`}</Button>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,101 @@
|
||||
import { useMemo } from "react";
|
||||
import { t } from "ttag";
|
||||
|
||||
import EmptyState from "metabase/common/components/EmptyState";
|
||||
import { Stack, Text } from "metabase/ui";
|
||||
import type { Transform, WorkspaceId } from "metabase-types/api";
|
||||
|
||||
import { useWorkspace } from "../WorkspaceProvider";
|
||||
|
||||
import { TransformListItem } from "./TransformListItem";
|
||||
import { TransformListItemMenu } from "./TransformListItemMenu";
|
||||
|
||||
type CodeTabProps = {
|
||||
activeTransformId?: number;
|
||||
transforms: Transform[];
|
||||
workspaceId: WorkspaceId;
|
||||
workspaceTransforms: Transform[];
|
||||
onTransformClick: (transform: Transform) => void;
|
||||
};
|
||||
|
||||
export const CodeTab = ({
|
||||
activeTransformId,
|
||||
transforms,
|
||||
workspaceId,
|
||||
workspaceTransforms,
|
||||
onTransformClick,
|
||||
}: CodeTabProps) => {
|
||||
const { editedTransforms, hasTransformEdits } = useWorkspace();
|
||||
|
||||
const workspaceTransformsUpstreamIds = useMemo(
|
||||
() => new Set(workspaceTransforms.map((t) => t.upstream_id)),
|
||||
[workspaceTransforms],
|
||||
);
|
||||
const availableTransforms = useMemo(() => {
|
||||
return transforms.filter((transform) => {
|
||||
return !workspaceTransformsUpstreamIds.has(transform.id);
|
||||
});
|
||||
}, [workspaceTransformsUpstreamIds, transforms]);
|
||||
|
||||
const handleTransformClick = (transform: Transform) => {
|
||||
const edited = editedTransforms.get(transform.id);
|
||||
const transformToOpen = edited
|
||||
? ({ ...transform, ...edited } as Transform)
|
||||
: transform;
|
||||
onTransformClick(transformToOpen);
|
||||
};
|
||||
|
||||
return (
|
||||
<Stack h="100%" gap={0}>
|
||||
<Stack
|
||||
data-testid="workspace-transforms"
|
||||
gap="xs"
|
||||
pb="sm"
|
||||
style={{
|
||||
borderBottom: "1px solid var(--mb-color-border)",
|
||||
}}
|
||||
>
|
||||
<Stack gap={0}>
|
||||
<Text fw={600}>{t`Workspace transforms`}</Text>
|
||||
{workspaceTransforms.map((transform) => {
|
||||
const isEdited = hasTransformEdits(transform);
|
||||
|
||||
return (
|
||||
<TransformListItem
|
||||
key={transform.id}
|
||||
name={transform.name}
|
||||
icon="pivot_table"
|
||||
fw={600}
|
||||
isActive={activeTransformId === transform.id}
|
||||
isEdited={isEdited}
|
||||
menu={
|
||||
<TransformListItemMenu
|
||||
transform={transform}
|
||||
workspaceId={workspaceId}
|
||||
/>
|
||||
}
|
||||
onClick={() => handleTransformClick(transform)}
|
||||
/>
|
||||
);
|
||||
})}
|
||||
</Stack>
|
||||
|
||||
{workspaceTransforms.length === 0 && (
|
||||
<EmptyState message={t`Workspace is empty`} />
|
||||
)}
|
||||
</Stack>
|
||||
|
||||
<Stack data-testid="mainland-transforms" py="sm" gap="xs">
|
||||
{availableTransforms.map((transform) => (
|
||||
<TransformListItem
|
||||
key={transform.id}
|
||||
name={transform.name}
|
||||
isActive={activeTransformId === transform.id}
|
||||
isEdited={editedTransforms.has(transform.id)}
|
||||
onClick={() => handleTransformClick(transform)}
|
||||
/>
|
||||
))}
|
||||
</Stack>
|
||||
</Stack>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,26 @@
|
||||
.root {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--mantine-spacing-sm);
|
||||
padding: var(--mantine-spacing-xs) var(--mantine-spacing-sm);
|
||||
border-radius: var(--mantine-radius-sm);
|
||||
cursor: pointer;
|
||||
flex-wrap: nowrap;
|
||||
|
||||
&:hover .menu {
|
||||
visibility: visible;
|
||||
}
|
||||
|
||||
&:hover {
|
||||
background-color: var(--mb-color-bg-light);
|
||||
}
|
||||
}
|
||||
|
||||
.name {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.menu {
|
||||
visibility: hidden;
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import type { ReactNode } from "react";
|
||||
|
||||
import { Box, type BoxProps, Icon, type IconName, Text } from "metabase/ui";
|
||||
|
||||
import { StatusDot } from "../components/StatusDot/StatusDot";
|
||||
|
||||
import S from "./TransformListItem.module.css";
|
||||
|
||||
type TransformListItemProps = {
|
||||
name: string;
|
||||
icon?: IconName;
|
||||
isActive?: boolean;
|
||||
isEdited?: boolean;
|
||||
menu?: ReactNode;
|
||||
onClick?: () => void;
|
||||
} & BoxProps;
|
||||
|
||||
export const TransformListItem = ({
|
||||
name,
|
||||
icon = "database",
|
||||
isActive,
|
||||
isEdited,
|
||||
menu,
|
||||
onClick,
|
||||
...props
|
||||
}: TransformListItemProps) => {
|
||||
return (
|
||||
<Box className={S.root} onClick={onClick} {...props}>
|
||||
<Icon name={icon} size={14} c="brand" />
|
||||
<Text className={S.name} c={isActive ? "brand" : "text-dark"} truncate>
|
||||
{name}
|
||||
</Text>
|
||||
{isEdited && <StatusDot status="changed" />}
|
||||
|
||||
<Box className={S.menu} flex="0 0 auto">
|
||||
{menu}
|
||||
</Box>
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,56 @@
|
||||
import { t } from "ttag";
|
||||
|
||||
import { useMetadataToasts } from "metabase/metadata/hooks";
|
||||
import { ActionIcon, Icon, Menu } from "metabase/ui";
|
||||
import { useUpdateWorkspaceContentsMutation } from "metabase-enterprise/api";
|
||||
import type { Transform, WorkspaceId } from "metabase-types/api";
|
||||
|
||||
import { useWorkspace } from "../WorkspaceProvider";
|
||||
|
||||
interface Props {
|
||||
transform: Transform;
|
||||
workspaceId: WorkspaceId;
|
||||
}
|
||||
|
||||
export function TransformListItemMenu({ transform, workspaceId }: Props) {
|
||||
const [updateWorkspaceContents] = useUpdateWorkspaceContentsMutation();
|
||||
const { removeEditedTransform, removeOpenedTransform } = useWorkspace();
|
||||
const { sendErrorToast, sendSuccessToast } = useMetadataToasts();
|
||||
|
||||
const handleRemove = async () => {
|
||||
const response = await updateWorkspaceContents({
|
||||
id: workspaceId,
|
||||
remove: {
|
||||
transforms: [transform.id],
|
||||
},
|
||||
});
|
||||
|
||||
if (response.error) {
|
||||
sendErrorToast(t`Failed to remove transform from the workspace`);
|
||||
} else {
|
||||
sendSuccessToast(t`Transform removed from the workspace`);
|
||||
removeEditedTransform(transform.id);
|
||||
removeOpenedTransform(transform.id);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<Menu position="bottom-end" withinPortal>
|
||||
<Menu.Target>
|
||||
<ActionIcon
|
||||
onClick={(event) => event.stopPropagation()}
|
||||
size="sm"
|
||||
variant="subtle"
|
||||
>
|
||||
<Icon name="ellipsis" />
|
||||
</ActionIcon>
|
||||
</Menu.Target>
|
||||
|
||||
<Menu.Dropdown onClick={(event) => event.stopPropagation()}>
|
||||
<Menu.Item leftSection={<Icon name="trash" />} onClick={handleRemove}>
|
||||
{t`Remove`}
|
||||
</Menu.Item>
|
||||
</Menu.Dropdown>
|
||||
</Menu>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { t } from "ttag";
|
||||
|
||||
import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
|
||||
import CS from "metabase/css/core/index.css";
|
||||
import { Box, Stack, Text } from "metabase/ui";
|
||||
import Visualization from "metabase/visualizations/components/Visualization";
|
||||
import type { DatabaseId, TableId } from "metabase-types/api";
|
||||
|
||||
import { useTablePreview } from "./useTablePreview";
|
||||
|
||||
interface DataTabProps {
|
||||
databaseId: DatabaseId | null;
|
||||
tableId: TableId | null;
|
||||
}
|
||||
|
||||
export function DataTab({ databaseId, tableId }: DataTabProps) {
|
||||
const { rawSeries, isFetching, error } = useTablePreview({
|
||||
databaseId,
|
||||
tableId,
|
||||
});
|
||||
|
||||
if (!databaseId || !tableId) {
|
||||
return (
|
||||
<Stack h="100%" align="center" justify="center">
|
||||
<Text c="text-medium">{t`Select a table to view its data`}</Text>
|
||||
</Stack>
|
||||
);
|
||||
}
|
||||
|
||||
if (isFetching) {
|
||||
return <LoadingAndErrorWrapper loading />;
|
||||
}
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<Stack h="100%" align="center" justify="center">
|
||||
<Text c="error">{t`Error loading data`}</Text>
|
||||
</Stack>
|
||||
);
|
||||
}
|
||||
|
||||
if (!rawSeries) {
|
||||
return (
|
||||
<Stack h="100%" align="center" justify="center">
|
||||
<Text c="text-medium">{t`No data available`}</Text>
|
||||
</Stack>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<Box h="100%" className={CS.relative}>
|
||||
<Visualization queryBuilderMode="dataset" rawSeries={rawSeries} />
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
import { t } from "ttag";
|
||||
|
||||
import { Stack, Text } from "metabase/ui";
|
||||
import type {
|
||||
TableId,
|
||||
Transform,
|
||||
WorkspaceTablesResponse,
|
||||
WorkspaceTransformItem,
|
||||
} from "metabase-types/api";
|
||||
|
||||
import { type OpenTable, useWorkspace } from "../WorkspaceProvider";
|
||||
|
||||
import { TableListItem } from "./TableListItem";
|
||||
|
||||
type DataTabSidebarProps = {
|
||||
tables: WorkspaceTablesResponse;
|
||||
workspaceTransforms: WorkspaceTransformItem[];
|
||||
dbTransforms: Transform[];
|
||||
selectedTableId?: TableId | null;
|
||||
onTransformClick?: (transform: WorkspaceTransformItem) => void;
|
||||
onTableSelect?: (table: OpenTable) => void;
|
||||
};
|
||||
|
||||
export const DataTabSidebar = ({
|
||||
tables,
|
||||
workspaceTransforms,
|
||||
dbTransforms,
|
||||
selectedTableId,
|
||||
onTransformClick,
|
||||
onTableSelect,
|
||||
}: DataTabSidebarProps) => {
|
||||
const { hasTransformEdits } = useWorkspace();
|
||||
return (
|
||||
<Stack h="100%" gap="sm">
|
||||
<Stack
|
||||
gap="xs"
|
||||
pb="sm"
|
||||
style={{ borderBottom: "1px solid var(--mb-color-border)" }}
|
||||
>
|
||||
<Text fw={600}>{t`Data active in this workspace`}</Text>
|
||||
<Stack gap={0}>
|
||||
{tables.outputs.map((table, index: number) => {
|
||||
const workspaceTransform = table.workspace
|
||||
? workspaceTransforms.find(
|
||||
(t) => t.id === table.workspace?.["transform-id"],
|
||||
)
|
||||
: undefined;
|
||||
const originalTransform = workspaceTransform
|
||||
? dbTransforms.find(
|
||||
(t) => t.id === workspaceTransform.upstream_id,
|
||||
)
|
||||
: undefined;
|
||||
const hasChanges = originalTransform
|
||||
? hasTransformEdits(originalTransform)
|
||||
: false;
|
||||
const tableId = table.workspace?.["table-id"];
|
||||
|
||||
return (
|
||||
<TableListItem
|
||||
key={`output-${index}`}
|
||||
name={table.workspace?.table || ""}
|
||||
schema={table.workspace?.schema}
|
||||
icon="pivot_table"
|
||||
type="output"
|
||||
hasChanges={hasChanges}
|
||||
transform={workspaceTransform}
|
||||
tableId={tableId}
|
||||
isSelected={tableId === selectedTableId}
|
||||
onTransformClick={onTransformClick}
|
||||
onTableClick={onTableSelect}
|
||||
/>
|
||||
);
|
||||
})}
|
||||
</Stack>
|
||||
<Stack gap={0}>
|
||||
{tables.inputs.map((table, index) => (
|
||||
<TableListItem
|
||||
key={`input-${index}`}
|
||||
name={table.table || ""}
|
||||
schema={table.schema}
|
||||
icon="table"
|
||||
type="input"
|
||||
tableId={table.id}
|
||||
onTransformClick={onTransformClick}
|
||||
onTableClick={onTableSelect}
|
||||
/>
|
||||
))}
|
||||
</Stack>
|
||||
</Stack>
|
||||
</Stack>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,34 @@
|
||||
.root {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--mantine-spacing-sm);
|
||||
padding: var(--mantine-spacing-xs) var(--mantine-spacing-sm);
|
||||
border-radius: var(--mantine-radius-sm);
|
||||
flex-wrap: nowrap;
|
||||
|
||||
&:hover .menu {
|
||||
visibility: visible;
|
||||
}
|
||||
|
||||
&:hover {
|
||||
background-color: var(--mb-color-bg-light);
|
||||
}
|
||||
}
|
||||
|
||||
.clickable {
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.selected {
|
||||
background-color: var(--mb-color-bg-medium);
|
||||
}
|
||||
.name {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.actionIcon {
|
||||
&:hover {
|
||||
background-color: var(--mb-color-bg-medium) !important;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
import cx from "classnames";
|
||||
import { t } from "ttag";
|
||||
|
||||
import {
|
||||
ActionIcon,
|
||||
Box,
|
||||
Icon,
|
||||
type IconName,
|
||||
Text,
|
||||
Tooltip,
|
||||
} from "metabase/ui";
|
||||
import type { TableId, WorkspaceTransformItem } from "metabase-types/api";
|
||||
|
||||
import type { OpenTable } from "../WorkspaceProvider";
|
||||
import { StatusDot } from "../components/StatusDot/StatusDot";
|
||||
|
||||
import S from "./TableListItem.module.css";
|
||||
|
||||
type TableListItemProps = {
|
||||
name: string;
|
||||
schema?: string | null;
|
||||
icon?: IconName;
|
||||
type: "input" | "output";
|
||||
hasChanges?: boolean;
|
||||
transform?: WorkspaceTransformItem;
|
||||
tableId?: TableId;
|
||||
isSelected?: boolean;
|
||||
onTransformClick?: (transform: WorkspaceTransformItem) => void;
|
||||
onTableClick?: (table: OpenTable) => void;
|
||||
};
|
||||
|
||||
export const TableListItem = ({
|
||||
name,
|
||||
schema,
|
||||
icon = "table",
|
||||
type,
|
||||
hasChanges = false,
|
||||
transform,
|
||||
tableId,
|
||||
isSelected = false,
|
||||
onTransformClick,
|
||||
onTableClick,
|
||||
}: TableListItemProps) => {
|
||||
const displayName = schema ? `${schema}.${name}` : name;
|
||||
|
||||
const handleTransformClick = (e: React.MouseEvent) => {
|
||||
e.stopPropagation();
|
||||
if (transform && onTransformClick) {
|
||||
onTransformClick(transform);
|
||||
}
|
||||
};
|
||||
|
||||
const handleTableClick = () => {
|
||||
if (tableId && onTableClick) {
|
||||
onTableClick({ tableId, name, schema });
|
||||
}
|
||||
};
|
||||
|
||||
const isClickable = tableId && onTableClick;
|
||||
|
||||
return (
|
||||
<Box
|
||||
className={cx(S.root, {
|
||||
[S.selected]: isSelected,
|
||||
[S.clickable]: isClickable,
|
||||
})}
|
||||
aria-label={displayName}
|
||||
onClick={isClickable ? handleTableClick : undefined}
|
||||
>
|
||||
<Icon
|
||||
name={icon}
|
||||
size={14}
|
||||
c={type === "input" ? "saturated-green" : "saturated-yellow"}
|
||||
/>
|
||||
<Text className={S.name} c="text-dark" truncate>
|
||||
{displayName}
|
||||
</Text>
|
||||
{hasChanges && <StatusDot status="changed" />}
|
||||
{type === "output" && transform && (
|
||||
<Tooltip label={t`Open transform`} position="top">
|
||||
<ActionIcon
|
||||
className={S.actionIcon}
|
||||
ml="auto"
|
||||
size="sm"
|
||||
variant="subtle"
|
||||
onClick={handleTransformClick}
|
||||
>
|
||||
<Icon name="code_block" size={14} c="text-medium" />
|
||||
</ActionIcon>
|
||||
</Tooltip>
|
||||
)}
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,4 @@
|
||||
export { DataTab } from "./DataTab";
|
||||
export { DataTabSidebar } from "./DataTabSidebar";
|
||||
export { useTablePreview as useTableQuestion } from "./useTablePreview";
|
||||
export type { OpenTable } from "../WorkspaceProvider";
|
||||
@@ -0,0 +1,101 @@
|
||||
import { useMemo } from "react";
|
||||
import _ from "underscore";
|
||||
|
||||
import {
|
||||
skipToken,
|
||||
useGetAdhocQueryQuery,
|
||||
useGetTableQueryMetadataQuery,
|
||||
} from "metabase/api";
|
||||
import { getErrorMessage } from "metabase/api/utils";
|
||||
import { useSelector } from "metabase/lib/redux";
|
||||
import { getTableMetadataQuery } from "metabase/metadata/pages/shared";
|
||||
import { getMetadata } from "metabase/selectors/metadata";
|
||||
import * as Lib from "metabase-lib";
|
||||
import type { DatabaseId, RawSeries, TableId } from "metabase-types/api";
|
||||
import { createMockCard } from "metabase-types/api/mocks";
|
||||
|
||||
interface UseTableQuestionProps {
|
||||
databaseId: DatabaseId | null;
|
||||
tableId: TableId | null;
|
||||
}
|
||||
|
||||
interface UseTablePreviewResult {
|
||||
rawSeries?: RawSeries;
|
||||
isFetching: boolean;
|
||||
isLoading: boolean;
|
||||
isError: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export function useTablePreview({
|
||||
databaseId,
|
||||
tableId,
|
||||
}: UseTableQuestionProps): UseTablePreviewResult {
|
||||
const metadata = useSelector(getMetadata);
|
||||
// Need to fetch this to make sure metadata is populated with proper table information on
|
||||
// initial page load.
|
||||
useGetTableQueryMetadataQuery(
|
||||
tableId && tableId in metadata.tables
|
||||
? skipToken
|
||||
: getTableMetadataQuery(tableId),
|
||||
);
|
||||
|
||||
const metadataProvider =
|
||||
Object.keys(metadata.tables).length > 0
|
||||
? Lib.metadataProvider(databaseId, metadata)
|
||||
: null;
|
||||
|
||||
const table =
|
||||
metadataProvider && tableId
|
||||
? Lib.tableOrCardMetadata(metadataProvider, tableId)
|
||||
: null;
|
||||
|
||||
const query =
|
||||
table && metadataProvider
|
||||
? Lib.queryFromTableOrCardMetadata(metadataProvider, table)
|
||||
: null;
|
||||
|
||||
const { data, refetch, ...rest } = useGetAdhocQueryQuery(
|
||||
query == null
|
||||
? skipToken
|
||||
: {
|
||||
...Lib.toJsQuery(query),
|
||||
ignore_error: true,
|
||||
},
|
||||
);
|
||||
const base = {
|
||||
...rest,
|
||||
error: rest.error ? getErrorMessage(rest.error) : undefined,
|
||||
rawSeries: undefined,
|
||||
};
|
||||
|
||||
const rawSeries = useMemo(
|
||||
() =>
|
||||
query
|
||||
? [
|
||||
{
|
||||
card: createMockCard({
|
||||
dataset_query: Lib.toJsQuery(query),
|
||||
display: "table",
|
||||
visualization_settings: {},
|
||||
}),
|
||||
data: {
|
||||
...data?.data,
|
||||
rows: _.uniq(data?.data?.rows ?? []),
|
||||
},
|
||||
},
|
||||
]
|
||||
: ([] as RawSeries),
|
||||
[data?.data, query],
|
||||
);
|
||||
|
||||
if (data?.status === "failed") {
|
||||
return { ...base, isError: true, error: getErrorMessage(data) };
|
||||
}
|
||||
|
||||
if (query == null || !data?.data || data.data.cols.length === 0) {
|
||||
return base;
|
||||
}
|
||||
|
||||
return { ...base, rawSeries };
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import { PLUGIN_METABOT } from "metabase/plugins";
|
||||
import { useRegisterMetabotTransformContext } from "metabase-enterprise/transforms/hooks/use-register-transform-metabot-context";
|
||||
import type { DraftTransformSource, Transform } from "metabase-types/api";
|
||||
|
||||
type MetabotTabProps = {
|
||||
transform?: Transform;
|
||||
source?: DraftTransformSource;
|
||||
};
|
||||
|
||||
const MetabotContextRegistration = ({ transform, source }: MetabotTabProps) => {
|
||||
useRegisterMetabotTransformContext(transform, source);
|
||||
return null;
|
||||
};
|
||||
|
||||
export const MetabotTab = ({ transform, source }: MetabotTabProps) => {
|
||||
const MetabotProvider = PLUGIN_METABOT.getMetabotProvider();
|
||||
const MetabotChat = PLUGIN_METABOT.MetabotChat;
|
||||
const metabotConfig = {
|
||||
// Show top-level mention menu for transforms and databases.
|
||||
suggestionModels: ["transform", "database"],
|
||||
};
|
||||
|
||||
return (
|
||||
<MetabotProvider>
|
||||
<MetabotContextRegistration transform={transform} source={source} />
|
||||
<MetabotChat config={metabotConfig} />
|
||||
</MetabotProvider>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,56 @@
|
||||
import { t } from "ttag";
|
||||
|
||||
import { Button } from "metabase/ui";
|
||||
import { useUpdateTransformMutation } from "metabase-enterprise/api";
|
||||
import { isSameSource } from "metabase-enterprise/transforms/utils";
|
||||
import type { DatabaseId, Transform } from "metabase-types/api";
|
||||
|
||||
import { type EditedTransform, useWorkspace } from "./WorkspaceProvider";
|
||||
|
||||
interface Props {
|
||||
databaseId: DatabaseId;
|
||||
editedTransform: EditedTransform;
|
||||
transform: Transform;
|
||||
}
|
||||
|
||||
export const SaveTransformButton = ({
|
||||
databaseId,
|
||||
editedTransform,
|
||||
transform,
|
||||
}: Props) => {
|
||||
const [updateTransform] = useUpdateTransformMutation();
|
||||
const { updateTransformState } = useWorkspace();
|
||||
|
||||
const hasSourceChanged = !isSameSource(
|
||||
editedTransform.source,
|
||||
transform.source,
|
||||
);
|
||||
const hasTargetNameChanged =
|
||||
transform.target.name !== editedTransform.target.name;
|
||||
const hasChanges = hasSourceChanged || hasTargetNameChanged;
|
||||
|
||||
const handleClick = async () => {
|
||||
const updated = await updateTransform({
|
||||
id: transform.id,
|
||||
source: editedTransform.source,
|
||||
name: editedTransform.name,
|
||||
target: {
|
||||
type: "table",
|
||||
name: editedTransform.target.name,
|
||||
schema: transform.target.schema,
|
||||
database: databaseId,
|
||||
},
|
||||
}).unwrap();
|
||||
|
||||
updateTransformState(updated);
|
||||
};
|
||||
|
||||
return (
|
||||
<Button
|
||||
disabled={!hasChanges}
|
||||
size="sm"
|
||||
variant="filled"
|
||||
onClick={handleClick}
|
||||
>{t`Save`}</Button>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,105 @@
|
||||
import { useMemo } from "react";
|
||||
import { match } from "ts-pattern";
|
||||
import { t } from "ttag";
|
||||
|
||||
import LoadingAndGenericErrorWrapper from "metabase/common/components/LoadingAndGenericErrorWrapper";
|
||||
import { Group, Icon, Loader, Stack, Tooltip } from "metabase/ui";
|
||||
import { useGetWorkspaceLogQuery } from "metabase-enterprise/api";
|
||||
import type {
|
||||
WorkspaceId,
|
||||
WorkspaceLogStatus,
|
||||
WorkspaceTask,
|
||||
} from "metabase-types/api";
|
||||
|
||||
interface SetupTabProps {
|
||||
workspaceId: WorkspaceId;
|
||||
}
|
||||
|
||||
const LOGS_POLLING_INTERVAL = 1000;
|
||||
|
||||
export const SetupLog = ({ workspaceId }: SetupTabProps) => {
|
||||
const { data, error, isLoading } = useGetWorkspaceLogQuery(workspaceId, {
|
||||
pollingInterval: LOGS_POLLING_INTERVAL,
|
||||
refetchOnMountOrArgChange: true,
|
||||
});
|
||||
|
||||
const logs = useMemo(() => data?.logs ?? [], [data]);
|
||||
|
||||
if (error || isLoading) {
|
||||
return <LoadingAndGenericErrorWrapper error={error} loading={isLoading} />;
|
||||
}
|
||||
|
||||
return (
|
||||
<Stack
|
||||
bd="1px solid var(--mb-color-border)"
|
||||
bg="background-info"
|
||||
bdrs="md"
|
||||
gap="sm"
|
||||
p="md"
|
||||
>
|
||||
{logs.length === 0 && data?.status === "pending" && <Loader size="xs" />}
|
||||
|
||||
{logs.map((log) => {
|
||||
return (
|
||||
<Group gap="xs" key={log.id}>
|
||||
<Tooltip disabled={!log.message} label={log.message}>
|
||||
<Group>
|
||||
<LogIcon status={log.status} />
|
||||
</Group>
|
||||
</Tooltip>
|
||||
|
||||
<span>
|
||||
{translateTask(log.task)}
|
||||
{log.status === "started" && "…"}
|
||||
</span>
|
||||
</Group>
|
||||
);
|
||||
})}
|
||||
|
||||
{data?.status === "ready" && (
|
||||
<Group gap="xs">
|
||||
<LogIcon status="success" />
|
||||
|
||||
{t`Workspace ready!`}
|
||||
</Group>
|
||||
)}
|
||||
</Stack>
|
||||
);
|
||||
};
|
||||
|
||||
function LogIcon({ status }: { status: WorkspaceLogStatus | null }) {
|
||||
return match(status)
|
||||
.with("failure", () => {
|
||||
return <Icon c="warning" name="warning_triangle_filled" />;
|
||||
})
|
||||
.with("started", () => {
|
||||
return <Loader size="xs" />;
|
||||
})
|
||||
.with("success", () => {
|
||||
return <Icon c="success" name="check" />;
|
||||
})
|
||||
.with(null, () => {
|
||||
return <Icon name="chevronright" />;
|
||||
})
|
||||
.exhaustive();
|
||||
}
|
||||
|
||||
function translateTask(task: WorkspaceTask | string) {
|
||||
if (task === "workspace-setup") {
|
||||
return t`Setting up the workspace`;
|
||||
}
|
||||
|
||||
if (task === "database-isolation") {
|
||||
return t`Provisioning database isolation`;
|
||||
}
|
||||
|
||||
if (task === "mirror-entities") {
|
||||
return t`Mirroring entities`;
|
||||
}
|
||||
|
||||
if (task === "grant-read-access") {
|
||||
return t`Granting permissions`;
|
||||
}
|
||||
|
||||
return task;
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import { t } from "ttag";
|
||||
|
||||
import { Stack, Text, Title } from "metabase/ui";
|
||||
import type { WorkspaceId } from "metabase-types/api";
|
||||
|
||||
import { SetupLog } from "./SetupLog";
|
||||
|
||||
interface SetupTabProps {
|
||||
databaseName?: string;
|
||||
workspaceId: WorkspaceId;
|
||||
}
|
||||
|
||||
export const SetupTab = ({ databaseName, workspaceId }: SetupTabProps) => {
|
||||
return (
|
||||
<Stack gap="lg">
|
||||
<Stack gap="xs">
|
||||
<Title order={3}>{t`Workspace Setup`}</Title>
|
||||
|
||||
<Text c="text-medium">
|
||||
{t`Configure your data warehouse connection for this workspace`}
|
||||
</Text>
|
||||
</Stack>
|
||||
|
||||
<Stack gap="xs">
|
||||
<Text fw="bold">{t`Data warehouse`}</Text>
|
||||
<Text>
|
||||
<code>{databaseName}</code>
|
||||
</Text>
|
||||
|
||||
<Text c="text-light" size="sm">
|
||||
{t`Data warehouse selection is locked after editing or timeout`}
|
||||
</Text>
|
||||
</Stack>
|
||||
|
||||
<Stack gap="xs">
|
||||
<Text fw="bold">{t`Setup log`}</Text>
|
||||
|
||||
<SetupLog
|
||||
key={workspaceId} // avoid showing status of other workspaces (forces RTK query hook remount)
|
||||
workspaceId={workspaceId}
|
||||
/>
|
||||
</Stack>
|
||||
</Stack>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,65 @@
|
||||
import { useMemo, useState } from "react";
|
||||
import _ from "underscore";
|
||||
|
||||
import {
|
||||
TransformQueryPageEditor,
|
||||
type TransformQueryPageEditorUiState,
|
||||
} from "metabase-enterprise/transforms/pages/TransformQueryPage/TransformQueryPage";
|
||||
import type { DraftTransformSource } from "metabase-types/api";
|
||||
|
||||
type TransformEditorProps = {
|
||||
disabled: boolean;
|
||||
source: DraftTransformSource;
|
||||
onChange: (source: DraftTransformSource) => void;
|
||||
proposedSource?: DraftTransformSource;
|
||||
onAcceptProposed?: () => void;
|
||||
onRejectProposed?: () => void;
|
||||
};
|
||||
|
||||
export function TransformEditor({
|
||||
disabled,
|
||||
source,
|
||||
onChange,
|
||||
proposedSource,
|
||||
onAcceptProposed,
|
||||
onRejectProposed,
|
||||
}: TransformEditorProps) {
|
||||
const [uiState, setUiState] = useState(getInitialUiStateForTransform);
|
||||
const uiOptions = useMemo(
|
||||
() => ({
|
||||
canChangeDatabase: false,
|
||||
readOnly: disabled,
|
||||
}),
|
||||
[disabled],
|
||||
);
|
||||
|
||||
const handleSourceChange = (source: DraftTransformSource) => {
|
||||
onChange?.(source);
|
||||
};
|
||||
|
||||
return (
|
||||
<TransformQueryPageEditor
|
||||
source={source}
|
||||
proposedSource={proposedSource}
|
||||
isDirty={false}
|
||||
uiState={uiState}
|
||||
setUiState={setUiState}
|
||||
uiOptions={uiOptions}
|
||||
databases={[]}
|
||||
setSourceAndRejectProposed={handleSourceChange}
|
||||
acceptProposed={onAcceptProposed ?? _.noop}
|
||||
rejectProposed={onRejectProposed ?? _.noop}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
function getInitialUiStateForTransform() {
|
||||
return {
|
||||
lastRunResult: null,
|
||||
lastRunQuery: null,
|
||||
selectionRange: [],
|
||||
modalSnippet: null,
|
||||
sidebarType: null,
|
||||
modalType: null,
|
||||
} as TransformQueryPageEditorUiState;
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
import { useDisclosure } from "@mantine/hooks";
|
||||
import { useCallback } from "react";
|
||||
import { t } from "ttag";
|
||||
|
||||
import { useDispatch, useSelector } from "metabase/lib/redux";
|
||||
import { useMetadataToasts } from "metabase/metadata/hooks";
|
||||
import { getMetadata } from "metabase/selectors/metadata";
|
||||
import { Box, Button, Group, Icon, Stack, rem } from "metabase/ui";
|
||||
import { useRunTransformMutation, workspaceApi } from "metabase-enterprise/api";
|
||||
import {
|
||||
deactivateSuggestedTransform,
|
||||
getMetabotSuggestedTransform,
|
||||
} from "metabase-enterprise/metabot/state";
|
||||
import { UpdateTargetModal } from "metabase-enterprise/transforms/pages/TransformTargetPage/TargetSection/UpdateTargetModal";
|
||||
import {
|
||||
isSameSource,
|
||||
isTransformRunning,
|
||||
} from "metabase-enterprise/transforms/utils";
|
||||
import * as Lib from "metabase-lib";
|
||||
import Question from "metabase-lib/v1/Question";
|
||||
import type {
|
||||
DatabaseId,
|
||||
DraftTransformSource,
|
||||
Transform,
|
||||
TransformId,
|
||||
WorkspaceId,
|
||||
} from "metabase-types/api";
|
||||
|
||||
import { CheckOutTransformButton } from "./CheckOutTransformButton";
|
||||
import { SaveTransformButton } from "./SaveTransformButton";
|
||||
import { TransformEditor } from "./TransformEditor";
|
||||
import { type EditedTransform, useWorkspace } from "./WorkspaceProvider";
|
||||
|
||||
interface Props {
|
||||
databaseId: DatabaseId;
|
||||
editedTransform: EditedTransform;
|
||||
transform: Transform;
|
||||
workspaceId: WorkspaceId;
|
||||
workspaceTransforms: Transform[];
|
||||
onChange: (patch: Partial<EditedTransform>) => void;
|
||||
onOpenTransform: (transformId: TransformId) => void;
|
||||
}
|
||||
|
||||
export const TransformTab = ({
|
||||
databaseId,
|
||||
editedTransform,
|
||||
transform,
|
||||
workspaceId,
|
||||
workspaceTransforms,
|
||||
onChange,
|
||||
onOpenTransform,
|
||||
}: Props) => {
|
||||
const { updateTransformState } = useWorkspace();
|
||||
const { sendSuccessToast, sendErrorToast } = useMetadataToasts();
|
||||
const [
|
||||
isChangeTargetModalOpen,
|
||||
{ open: openChangeTargetModal, close: closeChangeTargetModal },
|
||||
] = useDisclosure();
|
||||
const dispatch = useDispatch();
|
||||
const suggestedTransform = useSelector((state) =>
|
||||
getMetabotSuggestedTransform(state, transform.id),
|
||||
);
|
||||
const metadata = useSelector(getMetadata);
|
||||
|
||||
const normalizeSource = useCallback(
|
||||
(source: DraftTransformSource) => {
|
||||
if (source.type !== "query") {
|
||||
return source;
|
||||
}
|
||||
|
||||
const question = Question.create({
|
||||
dataset_query: source.query,
|
||||
metadata,
|
||||
});
|
||||
const query = question.query();
|
||||
const { isNative } = Lib.queryDisplayInfo(query);
|
||||
const normalizedQuery = isNative
|
||||
? Lib.withNativeQuery(query, Lib.rawNativeQuery(query))
|
||||
: query;
|
||||
|
||||
return {
|
||||
type: "query",
|
||||
query: question.setQuery(normalizedQuery).datasetQuery(),
|
||||
};
|
||||
},
|
||||
[metadata],
|
||||
);
|
||||
|
||||
const proposedSource =
|
||||
suggestedTransform?.source &&
|
||||
!isSameSource(suggestedTransform.source, editedTransform.source)
|
||||
? normalizeSource(suggestedTransform.source)
|
||||
: undefined;
|
||||
|
||||
const hasSourceChanged = !isSameSource(
|
||||
editedTransform.source,
|
||||
transform.source,
|
||||
);
|
||||
const hasChanges = hasSourceChanged;
|
||||
|
||||
const isSaved = workspaceTransforms.some((t) => t.id === transform.id);
|
||||
|
||||
const [runTransform] = useRunTransformMutation();
|
||||
|
||||
const handleRun = async () => {
|
||||
try {
|
||||
await runTransform(transform.id).unwrap();
|
||||
|
||||
// Invalidate the workspace tables cache since transform execution
|
||||
// may affect the list of workspace tables.
|
||||
if (isSaved) {
|
||||
dispatch(
|
||||
workspaceApi.util.invalidateTags([
|
||||
{ type: "workspace", id: workspaceId },
|
||||
]),
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to run transform", error);
|
||||
}
|
||||
};
|
||||
|
||||
const handleSourceChange = (source: DraftTransformSource) => {
|
||||
onChange({ source });
|
||||
};
|
||||
|
||||
const handleAcceptProposed = useCallback(() => {
|
||||
if (proposedSource == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!isSaved) {
|
||||
sendErrorToast(
|
||||
t`Add this transform to the workspace before applying Metabot changes.`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
onChange({ source: proposedSource });
|
||||
dispatch(deactivateSuggestedTransform(suggestedTransform?.id));
|
||||
}, [
|
||||
proposedSource,
|
||||
onChange,
|
||||
dispatch,
|
||||
suggestedTransform?.id,
|
||||
isSaved,
|
||||
sendErrorToast,
|
||||
]);
|
||||
|
||||
const handleRejectProposed = useCallback(() => {
|
||||
if (suggestedTransform) {
|
||||
dispatch(deactivateSuggestedTransform(suggestedTransform.id));
|
||||
}
|
||||
}, [dispatch, suggestedTransform]);
|
||||
|
||||
const handleTargetUpdate = useCallback(
|
||||
(updatedTransform: Transform) => {
|
||||
updateTransformState(updatedTransform);
|
||||
sendSuccessToast(t`Transform target updated`);
|
||||
closeChangeTargetModal();
|
||||
},
|
||||
[closeChangeTargetModal, updateTransformState, sendSuccessToast],
|
||||
);
|
||||
|
||||
const isRunning = isTransformRunning(transform);
|
||||
|
||||
return (
|
||||
<Stack gap={0} h="100%">
|
||||
<Group
|
||||
flex="0 0 auto"
|
||||
justify="space-between"
|
||||
mih={rem(73)} // avoid CLS when showing/hiding output table input
|
||||
p="md"
|
||||
style={{ borderBottom: "1px solid var(--mb-color-border)" }}
|
||||
>
|
||||
<Group>
|
||||
{isSaved && (
|
||||
<Button
|
||||
leftSection={<Icon name="pencil_lines" />}
|
||||
size="sm"
|
||||
disabled={isRunning || hasChanges}
|
||||
onClick={openChangeTargetModal}
|
||||
>{t`Change target`}</Button>
|
||||
)}
|
||||
</Group>
|
||||
|
||||
<Group>
|
||||
{isSaved && (
|
||||
<Button
|
||||
disabled={hasChanges}
|
||||
leftSection={<Icon name="play" />}
|
||||
size="sm"
|
||||
onClick={handleRun}
|
||||
>{t`Run`}</Button>
|
||||
)}
|
||||
|
||||
{isSaved && (
|
||||
<SaveTransformButton
|
||||
databaseId={databaseId}
|
||||
editedTransform={editedTransform}
|
||||
transform={transform}
|
||||
/>
|
||||
)}
|
||||
|
||||
{!isSaved && (
|
||||
<CheckOutTransformButton
|
||||
transform={transform}
|
||||
workspaceId={workspaceId}
|
||||
onOpenTransform={onOpenTransform}
|
||||
/>
|
||||
)}
|
||||
</Group>
|
||||
</Group>
|
||||
|
||||
{editedTransform && (
|
||||
<Box flex="1">
|
||||
<TransformEditor
|
||||
disabled={!isSaved}
|
||||
source={editedTransform.source}
|
||||
proposedSource={proposedSource}
|
||||
onAcceptProposed={handleAcceptProposed}
|
||||
onRejectProposed={handleRejectProposed}
|
||||
onChange={handleSourceChange}
|
||||
/>
|
||||
</Box>
|
||||
)}
|
||||
|
||||
{isChangeTargetModalOpen && (
|
||||
<UpdateTargetModal
|
||||
transform={transform}
|
||||
onUpdate={handleTargetUpdate}
|
||||
onClose={closeChangeTargetModal}
|
||||
/>
|
||||
)}
|
||||
</Stack>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,20 @@
|
||||
.tabsPanel {
|
||||
display: flex;
|
||||
gap: 0.5rem;
|
||||
flex-wrap: nowrap;
|
||||
overflow: auto;
|
||||
scrollbar-width: none;
|
||||
-ms-overflow-style: none;
|
||||
|
||||
&::-webkit-scrollbar {
|
||||
display: none;
|
||||
}
|
||||
|
||||
&::before {
|
||||
display: none;
|
||||
}
|
||||
|
||||
& button {
|
||||
border-radius: 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,690 @@
|
||||
import type { DragEndEvent } from "@dnd-kit/core";
|
||||
import { DndContext, PointerSensor, useSensor } from "@dnd-kit/core";
|
||||
import {
|
||||
restrictToHorizontalAxis,
|
||||
restrictToParentElement,
|
||||
} from "@dnd-kit/modifiers";
|
||||
import {
|
||||
SortableContext,
|
||||
arrayMove,
|
||||
horizontalListSortingStrategy,
|
||||
} from "@dnd-kit/sortable";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { push, replace } from "react-router-redux";
|
||||
import { t } from "ttag";
|
||||
|
||||
import { useListDatabasesQuery } from "metabase/api";
|
||||
import { Sortable } from "metabase/common/components/Sortable";
|
||||
import { useDispatch, useSelector } from "metabase/lib/redux";
|
||||
import { checkNotNull } from "metabase/lib/types";
|
||||
import * as Urls from "metabase/lib/urls";
|
||||
import { useRegisterMetabotContextProvider } from "metabase/metabot";
|
||||
import { useMetadataToasts } from "metabase/metadata/hooks";
|
||||
import { PLUGIN_METABOT } from "metabase/plugins";
|
||||
import {
|
||||
ActionIcon,
|
||||
Box,
|
||||
Button,
|
||||
Flex,
|
||||
Group,
|
||||
Icon,
|
||||
Stack,
|
||||
Tabs,
|
||||
Text,
|
||||
} from "metabase/ui";
|
||||
import {
|
||||
useGetWorkspaceQuery,
|
||||
useGetWorkspaceTablesQuery,
|
||||
useListTransformsQuery,
|
||||
useMergeWorkspaceMutation,
|
||||
useUpdateWorkspaceNameMutation,
|
||||
} from "metabase-enterprise/api";
|
||||
import { PaneHeaderInput } from "metabase-enterprise/data-studio/common/components/PaneHeader";
|
||||
import { RunWorkspaceMenu } from "metabase-enterprise/data-studio/workspaces/components/RunWorkspaceMenu/RunWorkspaceMenu";
|
||||
import { useMetabotAgent } from "metabase-enterprise/metabot/hooks/use-metabot-agent";
|
||||
import { useMetabotReactions } from "metabase-enterprise/metabot/hooks/use-metabot-reactions";
|
||||
import {
|
||||
type MetabotState,
|
||||
metabotActions,
|
||||
} from "metabase-enterprise/metabot/state/reducer";
|
||||
import { getMetabot } from "metabase-enterprise/metabot/state/selectors";
|
||||
import { NAME_MAX_LENGTH } from "metabase-enterprise/transforms/constants";
|
||||
import type { DraftTransformSource, Transform } from "metabase-types/api";
|
||||
|
||||
import { AddTransformMenu } from "./AddTransformMenu";
|
||||
import { CodeTab } from "./CodeTab/CodeTab";
|
||||
import { DataTab, DataTabSidebar } from "./DataTab";
|
||||
import { MetabotTab } from "./MetabotTab";
|
||||
import { SetupTab } from "./SetupTab";
|
||||
import { TransformTab } from "./TransformTab";
|
||||
import styles from "./WorkspacePage.module.css";
|
||||
import {
|
||||
type EditedTransform,
|
||||
type OpenTable,
|
||||
WorkspaceProvider,
|
||||
type WorkspaceTab,
|
||||
useWorkspace,
|
||||
} from "./WorkspaceProvider";
|
||||
|
||||
type WorkspacePageProps = {
|
||||
params: {
|
||||
workspaceId: string;
|
||||
};
|
||||
};
|
||||
|
||||
type MetabotConversationSnapshot = Pick<
|
||||
MetabotState,
|
||||
| "messages"
|
||||
| "history"
|
||||
| "state"
|
||||
| "reactions"
|
||||
| "activeToolCalls"
|
||||
| "errorMessages"
|
||||
| "conversationId"
|
||||
>;
|
||||
|
||||
function WorkspacePageContent({ params }: WorkspacePageProps) {
|
||||
const id = Number(params.workspaceId);
|
||||
const dispatch = useDispatch();
|
||||
const metabotState = useSelector(getMetabot as any) as MetabotState;
|
||||
const { sendErrorToast } = useMetadataToasts();
|
||||
const isMetabotAvailable = PLUGIN_METABOT.isEnabled();
|
||||
const [tab, setTab] = useState<string>("setup");
|
||||
|
||||
const pointerSensor = useSensor(PointerSensor, {
|
||||
activationConstraint: { distance: 10 },
|
||||
});
|
||||
|
||||
const metabotStateRef = useRef<MetabotState>(metabotState);
|
||||
useEffect(() => {
|
||||
metabotStateRef.current = metabotState;
|
||||
}, [metabotState]);
|
||||
const metabotSnapshots = useRef<Map<number, MetabotConversationSnapshot>>(
|
||||
new Map(),
|
||||
);
|
||||
|
||||
const { data: databases = { data: [] } } = useListDatabasesQuery({});
|
||||
|
||||
const { data: allTransforms = [] } = useListTransformsQuery({});
|
||||
const { data: workspace, isLoading: isLoadingWorkspace } =
|
||||
useGetWorkspaceQuery(id);
|
||||
useRegisterMetabotContextProvider(async () => {
|
||||
if (!workspace?.database_id) {
|
||||
return;
|
||||
}
|
||||
return { default_database_id: workspace.database_id };
|
||||
}, [workspace?.database_id]);
|
||||
const { data: workspaceTables = { inputs: [], outputs: [] } } =
|
||||
useGetWorkspaceTablesQuery(id);
|
||||
const { navigateToPath, setNavigateToPath } = useMetabotReactions();
|
||||
const {
|
||||
resetConversation: resetMetabotConversation,
|
||||
visible: isMetabotVisible,
|
||||
} = useMetabotAgent();
|
||||
|
||||
const [mergeWorkspace, { isLoading: isMerging }] =
|
||||
useMergeWorkspaceMutation();
|
||||
|
||||
const [updateWorkspaceName] = useUpdateWorkspaceNameMutation();
|
||||
|
||||
const sourceDb = databases?.data.find(
|
||||
(db) => db.id === workspace?.database_id,
|
||||
);
|
||||
|
||||
const dbTransforms = useMemo(
|
||||
() =>
|
||||
allTransforms.filter((t) => {
|
||||
// TODO: @uladzimirdev add guards
|
||||
if (t.source_type === "python") {
|
||||
return (
|
||||
"source-database" in t.source &&
|
||||
t.source["source-database"] === sourceDb?.id
|
||||
);
|
||||
}
|
||||
if (t.source_type === "native") {
|
||||
return (
|
||||
"query" in t.source && t.source.query.database === sourceDb?.id
|
||||
);
|
||||
}
|
||||
return false;
|
||||
}),
|
||||
[allTransforms, sourceDb],
|
||||
);
|
||||
|
||||
const {
|
||||
openedTabs,
|
||||
activeTab,
|
||||
activeTransform,
|
||||
activeEditedTransform,
|
||||
activeTable,
|
||||
setActiveTransform,
|
||||
setActiveTab,
|
||||
addOpenedTab,
|
||||
removeOpenedTab,
|
||||
setOpenedTabs,
|
||||
addOpenedTransform,
|
||||
patchEditedTransform,
|
||||
hasUnsavedChanges,
|
||||
} = useWorkspace();
|
||||
const [metabotContextTransform, setMetabotContextTransform] = useState<
|
||||
Transform | undefined
|
||||
>();
|
||||
const [metabotContextSource, setMetabotContextSource] = useState<
|
||||
DraftTransformSource | undefined
|
||||
>();
|
||||
|
||||
const workspaceTransforms = useMemo(
|
||||
() => workspace?.contents?.transforms ?? [],
|
||||
[workspace],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (activeTab) {
|
||||
setTab(activeTab.id);
|
||||
}
|
||||
}, [id, activeTab, setTab]);
|
||||
|
||||
const tabsListRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (tabsListRef.current && tab) {
|
||||
const activeTabElement = tabsListRef.current.querySelector(
|
||||
`[data-active="true"]`,
|
||||
) as HTMLElement;
|
||||
|
||||
if (activeTabElement) {
|
||||
activeTabElement.scrollIntoView({
|
||||
behavior: "smooth",
|
||||
block: "nearest",
|
||||
});
|
||||
}
|
||||
}
|
||||
}, [tab]);
|
||||
|
||||
useEffect(() => {
|
||||
if (
|
||||
metabotContextTransform &&
|
||||
!openedTabs.some(
|
||||
(tab) =>
|
||||
tab.type === "transform" &&
|
||||
tab.transform.id === metabotContextTransform.id,
|
||||
)
|
||||
) {
|
||||
setMetabotContextTransform(undefined);
|
||||
setMetabotContextSource(undefined);
|
||||
}
|
||||
}, [openedTabs, metabotContextTransform]);
|
||||
|
||||
useEffect(() => {
|
||||
const snapshots = metabotSnapshots.current;
|
||||
const snapshot = snapshots.get(id);
|
||||
if (snapshot) {
|
||||
dispatch(metabotActions.setConversationSnapshot(snapshot));
|
||||
} else {
|
||||
resetMetabotConversation();
|
||||
}
|
||||
|
||||
return () => {
|
||||
snapshots.set(id, {
|
||||
messages: metabotStateRef.current.messages,
|
||||
history: metabotStateRef.current.history,
|
||||
state: metabotStateRef.current.state,
|
||||
reactions: metabotStateRef.current.reactions,
|
||||
activeToolCalls: metabotStateRef.current.activeToolCalls,
|
||||
errorMessages: metabotStateRef.current.errorMessages,
|
||||
conversationId: metabotStateRef.current.conversationId,
|
||||
});
|
||||
resetMetabotConversation();
|
||||
};
|
||||
}, [id, dispatch, resetMetabotConversation]);
|
||||
|
||||
useEffect(() => {
|
||||
if (isMetabotAvailable && isMetabotVisible) {
|
||||
setTab("metabot");
|
||||
setActiveTab(undefined);
|
||||
}
|
||||
}, [isMetabotAvailable, isMetabotVisible, setActiveTab]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!navigateToPath) {
|
||||
return;
|
||||
}
|
||||
|
||||
const transformIdFromPath = (() => {
|
||||
const match = navigateToPath.match(/\/transform\/(\d+)/);
|
||||
const extracted = Urls.extractEntityId(navigateToPath);
|
||||
const idString = match?.[1] ?? (extracted ? String(extracted) : null);
|
||||
const parsed = idString ? Number(idString) : NaN;
|
||||
return Number.isFinite(parsed) ? parsed : undefined;
|
||||
})();
|
||||
|
||||
if (transformIdFromPath != null) {
|
||||
const targetTransform =
|
||||
workspaceTransforms.find((t) => t.id === transformIdFromPath) ||
|
||||
allTransforms.find((t) => t.id === transformIdFromPath);
|
||||
|
||||
if (targetTransform) {
|
||||
addOpenedTransform(targetTransform);
|
||||
setActiveTransform(targetTransform);
|
||||
setNavigateToPath(null);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
dispatch(push(navigateToPath));
|
||||
setNavigateToPath(null);
|
||||
}, [
|
||||
navigateToPath,
|
||||
workspaceTransforms,
|
||||
allTransforms,
|
||||
addOpenedTransform,
|
||||
setActiveTransform,
|
||||
setNavigateToPath,
|
||||
dispatch,
|
||||
]);
|
||||
|
||||
const handleTransformChange = useCallback(
|
||||
(patch: Partial<EditedTransform>) => {
|
||||
patchEditedTransform(checkNotNull(activeTransform).id, patch);
|
||||
},
|
||||
[activeTransform, patchEditedTransform],
|
||||
);
|
||||
|
||||
const handleTabClose = useCallback(
|
||||
(event: React.MouseEvent, tab: WorkspaceTab, index: number) => {
|
||||
event.stopPropagation();
|
||||
|
||||
const isActive =
|
||||
(tab.type === "transform" &&
|
||||
activeTransform?.id === tab.transform.id) ||
|
||||
(tab.type === "table" && activeTable?.tableId === tab.table.tableId);
|
||||
const remaining = openedTabs.filter((item) => item.id !== tab.id);
|
||||
|
||||
removeOpenedTab(tab.id);
|
||||
|
||||
if (!isActive) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Prefer previous tab, otherwise use next tab (which is now at the same index after filtering)
|
||||
const fallback = remaining[index - 1] ?? remaining[index] ?? undefined;
|
||||
|
||||
if (fallback) {
|
||||
setActiveTab(fallback);
|
||||
setTab(fallback.id);
|
||||
} else {
|
||||
setActiveTab(undefined);
|
||||
setTab("setup");
|
||||
}
|
||||
},
|
||||
[
|
||||
activeTransform,
|
||||
activeTable,
|
||||
removeOpenedTab,
|
||||
setActiveTab,
|
||||
setTab,
|
||||
openedTabs,
|
||||
],
|
||||
);
|
||||
|
||||
const handleMergeWorkspace = useCallback(async () => {
|
||||
try {
|
||||
const response = await mergeWorkspace(id).unwrap();
|
||||
|
||||
if (response.errors && response.errors.length > 0) {
|
||||
sendErrorToast(
|
||||
t`Failed to merge workspace: ${response.errors.map((e) => e.error).join(", ")}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
dispatch(replace(Urls.transformList()));
|
||||
} catch (error) {
|
||||
sendErrorToast(t`Failed to merge workspace`);
|
||||
}
|
||||
}, [id, mergeWorkspace, sendErrorToast, dispatch]);
|
||||
|
||||
const handleWorkspaceNameChange = useCallback(
|
||||
async (newName: string) => {
|
||||
if (!workspace || newName.trim() === workspace.name.trim()) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await updateWorkspaceName({ id, name: newName.trim() }).unwrap();
|
||||
} catch (error) {
|
||||
sendErrorToast(t`Failed to update workspace name`);
|
||||
}
|
||||
},
|
||||
[workspace, id, updateWorkspaceName, sendErrorToast],
|
||||
);
|
||||
|
||||
const handleTableSelect = useCallback(
|
||||
(table: OpenTable) => {
|
||||
const tableTab: WorkspaceTab = {
|
||||
id: `table-${table.tableId}`,
|
||||
name: table.schema ? `${table.schema}.${table.name}` : table.name,
|
||||
type: "table",
|
||||
table,
|
||||
};
|
||||
addOpenedTab(tableTab);
|
||||
},
|
||||
[addOpenedTab],
|
||||
);
|
||||
|
||||
const handleTabDragEnd = useCallback(
|
||||
(event: DragEndEvent) => {
|
||||
const activeId = event.active.id;
|
||||
const overId = event.over?.id;
|
||||
if (typeof activeId === "string" && typeof overId === "string") {
|
||||
const activeIndex = openedTabs.findIndex(({ id }) => id === activeId);
|
||||
const overIndex = openedTabs.findIndex(({ id }) => id === overId);
|
||||
const reorderedTabs = arrayMove(openedTabs, activeIndex, overIndex);
|
||||
setOpenedTabs(reorderedTabs);
|
||||
|
||||
// Activate the dragged tab after reordering
|
||||
const draggedTab = reorderedTabs.find((tab) => tab.id === activeId);
|
||||
if (draggedTab) {
|
||||
setActiveTab(draggedTab);
|
||||
}
|
||||
}
|
||||
},
|
||||
[openedTabs, setOpenedTabs, setActiveTab],
|
||||
);
|
||||
|
||||
if (isLoadingWorkspace) {
|
||||
return (
|
||||
<Box p="lg">
|
||||
<Text>{t`Loading...`}</Text>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
if (!workspace) {
|
||||
return (
|
||||
<Box p="lg">
|
||||
<Text c="text-dark">{t`Workspace not found`}</Text>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<Stack data-testid="workspace-page" h="100%" gap={0}>
|
||||
<Group
|
||||
px="md"
|
||||
py="sm"
|
||||
style={{ borderBottom: "1px solid var(--mb-color-border)" }}
|
||||
justify="space-between"
|
||||
>
|
||||
<Flex gap="xs" align="center">
|
||||
<Icon name="git_branch" size="1rem" aria-hidden />
|
||||
<PaneHeaderInput
|
||||
initialValue={workspace.name}
|
||||
placeholder={t`Workspace name`}
|
||||
maxLength={NAME_MAX_LENGTH}
|
||||
onChange={handleWorkspaceNameChange}
|
||||
/>
|
||||
</Flex>
|
||||
<Flex gap="sm">
|
||||
<RunWorkspaceMenu
|
||||
workspaceId={id}
|
||||
disabled={workspaceTransforms.length === 0}
|
||||
/>
|
||||
<Button
|
||||
variant="filled"
|
||||
onClick={handleMergeWorkspace}
|
||||
loading={isMerging}
|
||||
disabled={hasUnsavedChanges() || workspaceTransforms.length === 0}
|
||||
size="xs"
|
||||
>
|
||||
{t`Merge`}
|
||||
</Button>
|
||||
</Flex>
|
||||
</Group>
|
||||
|
||||
<Group
|
||||
align="flex-start"
|
||||
gap={0}
|
||||
flex="1 1 auto"
|
||||
wrap="nowrap"
|
||||
style={{ overflow: "hidden" }}
|
||||
>
|
||||
<Box
|
||||
data-testid="workspace-content"
|
||||
w="70%"
|
||||
h="100%"
|
||||
style={{
|
||||
borderRight: "1px solid var(--mb-color-border)",
|
||||
}}
|
||||
pos="relative"
|
||||
>
|
||||
<Tabs
|
||||
defaultValue="setup"
|
||||
display="flex"
|
||||
h="100%"
|
||||
style={{ flexDirection: "column" }}
|
||||
value={tab}
|
||||
onChange={(tab) => {
|
||||
if (tab === "metabot") {
|
||||
if (activeTransform) {
|
||||
setMetabotContextTransform(activeTransform);
|
||||
setMetabotContextSource(
|
||||
activeEditedTransform?.source ?? activeTransform.source,
|
||||
);
|
||||
} else {
|
||||
setMetabotContextTransform(undefined);
|
||||
setMetabotContextSource(undefined);
|
||||
}
|
||||
}
|
||||
|
||||
if (tab) {
|
||||
setTab(tab);
|
||||
}
|
||||
if (
|
||||
tab === "setup" ||
|
||||
(tab === "metabot" && (activeTransform || activeTable))
|
||||
) {
|
||||
setActiveTab(undefined);
|
||||
}
|
||||
}}
|
||||
>
|
||||
<Flex
|
||||
wrap="nowrap"
|
||||
flex="0 0 auto"
|
||||
px="md"
|
||||
style={{ borderBottom: "1px solid var(--mb-color-border)" }}
|
||||
>
|
||||
<DndContext
|
||||
onDragEnd={handleTabDragEnd}
|
||||
modifiers={[restrictToHorizontalAxis, restrictToParentElement]}
|
||||
sensors={[pointerSensor]}
|
||||
>
|
||||
<SortableContext
|
||||
items={openedTabs}
|
||||
strategy={horizontalListSortingStrategy}
|
||||
>
|
||||
<Tabs.List ref={tabsListRef} className={styles.tabsPanel}>
|
||||
<Tabs.Tab value="setup">
|
||||
<Group gap="xs" wrap="nowrap">
|
||||
<Icon name="database" aria-hidden />
|
||||
{t`Setup`}
|
||||
</Group>
|
||||
</Tabs.Tab>
|
||||
{isMetabotAvailable && (
|
||||
<Tabs.Tab value="metabot">
|
||||
<Group gap="xs" wrap="nowrap">
|
||||
<Icon name="message_circle" aria-hidden />
|
||||
{t`Agent Chat`}
|
||||
</Group>
|
||||
</Tabs.Tab>
|
||||
)}
|
||||
|
||||
{openedTabs.map((tab, index) => (
|
||||
<Sortable
|
||||
id={tab.id}
|
||||
as="div"
|
||||
key={tab.id}
|
||||
draggingStyle={{ opacity: 0.5 }}
|
||||
>
|
||||
<Tabs.Tab
|
||||
draggable
|
||||
key={tab.id}
|
||||
value={tab.id}
|
||||
onClick={() => {
|
||||
setActiveTab(tab);
|
||||
}}
|
||||
>
|
||||
<Group gap="xs" wrap="nowrap">
|
||||
<Icon
|
||||
name={
|
||||
tab.type === "transform"
|
||||
? "pivot_table"
|
||||
: "table"
|
||||
}
|
||||
aria-hidden
|
||||
/>
|
||||
{tab.name}
|
||||
<ActionIcon
|
||||
size="1rem"
|
||||
p="0"
|
||||
ml="xs"
|
||||
onClick={(event) =>
|
||||
handleTabClose(event, tab, index)
|
||||
}
|
||||
>
|
||||
<Icon name="close" size={10} aria-hidden />
|
||||
</ActionIcon>
|
||||
</Group>
|
||||
</Tabs.Tab>
|
||||
</Sortable>
|
||||
))}
|
||||
</Tabs.List>
|
||||
</SortableContext>
|
||||
</DndContext>
|
||||
</Flex>
|
||||
|
||||
<Box
|
||||
flex={1}
|
||||
mih={0}
|
||||
style={{
|
||||
overflow: tab === "metabot" ? "auto" : undefined,
|
||||
}}
|
||||
>
|
||||
<Tabs.Panel value="setup" h="100%" p="md">
|
||||
<SetupTab
|
||||
databaseName={sourceDb?.name}
|
||||
workspaceId={workspace.id}
|
||||
/>
|
||||
</Tabs.Panel>
|
||||
|
||||
{isMetabotAvailable && (
|
||||
<Tabs.Panel
|
||||
value="metabot"
|
||||
h="100%"
|
||||
mah="100%"
|
||||
pos="relative"
|
||||
style={{ overflow: "auto" }}
|
||||
>
|
||||
<MetabotTab
|
||||
transform={metabotContextTransform}
|
||||
source={metabotContextSource}
|
||||
/>
|
||||
</Tabs.Panel>
|
||||
)}
|
||||
|
||||
<Tabs.Panel value={`table-${activeTable?.tableId}`} h="100%">
|
||||
{!activeTable ? null : (
|
||||
<DataTab
|
||||
databaseId={workspace?.database_id ?? null}
|
||||
tableId={activeTable.tableId}
|
||||
/>
|
||||
)}
|
||||
</Tabs.Panel>
|
||||
|
||||
<Tabs.Panel value={`transform-${activeTransform?.id}`} h="100%">
|
||||
{openedTabs.length === 0 ||
|
||||
!activeTransform ||
|
||||
!activeEditedTransform ? (
|
||||
<Text c="text-medium">
|
||||
{t`Select a transform on the right.`}
|
||||
</Text>
|
||||
) : (
|
||||
<TransformTab
|
||||
databaseId={checkNotNull(workspace.database_id)}
|
||||
transform={activeTransform}
|
||||
editedTransform={activeEditedTransform}
|
||||
workspaceId={id}
|
||||
workspaceTransforms={workspaceTransforms}
|
||||
onChange={handleTransformChange}
|
||||
onOpenTransform={(transformId) =>
|
||||
setTab(String(transformId))
|
||||
}
|
||||
/>
|
||||
)}
|
||||
</Tabs.Panel>
|
||||
</Box>
|
||||
</Tabs>
|
||||
</Box>
|
||||
|
||||
<Box
|
||||
data-testid="workspace-sidebar"
|
||||
style={{ flex: "1 0 auto", width: "30%" }}
|
||||
>
|
||||
<Tabs defaultValue="code">
|
||||
<Flex
|
||||
px="md"
|
||||
align="center"
|
||||
style={{ borderBottom: "1px solid var(--mb-color-border)" }}
|
||||
>
|
||||
<Tabs.List className={styles.tabsPanel}>
|
||||
<Tabs.Tab value="code">{t`Code`}</Tabs.Tab>
|
||||
<Tabs.Tab value="data">{t`Data`}</Tabs.Tab>
|
||||
</Tabs.List>
|
||||
{sourceDb && (
|
||||
<AddTransformMenu
|
||||
databaseId={sourceDb.id}
|
||||
workspaceId={id}
|
||||
onCreate={(transform) => {
|
||||
addOpenedTransform(transform);
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
</Flex>
|
||||
<Tabs.Panel value="code" p="md">
|
||||
<CodeTab
|
||||
activeTransformId={activeTransform?.id}
|
||||
transforms={dbTransforms}
|
||||
workspaceId={workspace.id}
|
||||
workspaceTransforms={workspaceTransforms}
|
||||
onTransformClick={(transform) => {
|
||||
addOpenedTransform(transform);
|
||||
}}
|
||||
/>
|
||||
</Tabs.Panel>
|
||||
<Tabs.Panel value="data" p="md">
|
||||
<DataTabSidebar
|
||||
tables={workspaceTables}
|
||||
workspaceTransforms={workspaceTransforms}
|
||||
dbTransforms={dbTransforms}
|
||||
selectedTableId={activeTable?.tableId}
|
||||
onTransformClick={(transform) => {
|
||||
addOpenedTransform(transform);
|
||||
}}
|
||||
onTableSelect={handleTableSelect}
|
||||
/>
|
||||
</Tabs.Panel>
|
||||
</Tabs>
|
||||
</Box>
|
||||
</Group>
|
||||
</Stack>
|
||||
);
|
||||
}
|
||||
|
||||
export const WorkspacePage = ({ params }: WorkspacePageProps) => {
|
||||
const workspaceId = Number(params.workspaceId);
|
||||
|
||||
return (
|
||||
<WorkspaceProvider workspaceId={workspaceId}>
|
||||
<WorkspacePageContent key={workspaceId} params={params} />
|
||||
</WorkspaceProvider>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,572 @@
|
||||
import type { ReactNode } from "react";
|
||||
import {
|
||||
createContext,
|
||||
useCallback,
|
||||
useContext,
|
||||
useMemo,
|
||||
useState,
|
||||
} from "react";
|
||||
|
||||
import { checkNotNull } from "metabase/lib/types";
|
||||
import { isSameSource } from "metabase-enterprise/transforms/utils";
|
||||
import type {
|
||||
DraftTransformSource,
|
||||
TableId,
|
||||
Transform,
|
||||
TransformTargetType,
|
||||
} from "metabase-types/api";
|
||||
|
||||
export interface OpenTable {
|
||||
tableId: TableId;
|
||||
name: string;
|
||||
schema?: string | null;
|
||||
}
|
||||
|
||||
export interface EditedTransform {
|
||||
name: string;
|
||||
source: DraftTransformSource;
|
||||
target: {
|
||||
name: string;
|
||||
type: TransformTargetType;
|
||||
};
|
||||
}
|
||||
|
||||
export interface Tab {
|
||||
id: string;
|
||||
name: string;
|
||||
type: "transform" | "table";
|
||||
}
|
||||
|
||||
export interface TransformTab extends Tab {
|
||||
type: "transform";
|
||||
transform: Transform;
|
||||
}
|
||||
|
||||
export interface TableTab extends Tab {
|
||||
type: "table";
|
||||
table: OpenTable;
|
||||
}
|
||||
|
||||
export type WorkspaceTab = TransformTab | TableTab;
|
||||
|
||||
export interface WorkspaceContextValue {
|
||||
workspaceId: number;
|
||||
openedTabs: WorkspaceTab[];
|
||||
activeTransform?: Transform;
|
||||
activeEditedTransform?: EditedTransform;
|
||||
activeTable?: OpenTable;
|
||||
activeTab?: WorkspaceTab;
|
||||
setActiveTab: (tab: WorkspaceTab | undefined) => void;
|
||||
setActiveTransform: (transform: Transform | undefined) => void;
|
||||
setActiveTable: (table: OpenTable | undefined) => void;
|
||||
addOpenedTab: (tab: WorkspaceTab) => void;
|
||||
removeOpenedTab: (tabId: string) => void;
|
||||
setOpenedTabs: (tabs: WorkspaceTab[]) => void;
|
||||
addOpenedTransform: (transform: Transform) => void;
|
||||
removeOpenedTransform: (transformId: number) => void;
|
||||
editedTransforms: Map<number, EditedTransform>;
|
||||
patchEditedTransform: (
|
||||
transformId: number,
|
||||
patch: Partial<EditedTransform>,
|
||||
) => void;
|
||||
removeEditedTransform: (transformId: number) => void;
|
||||
runTransforms: Set<number>;
|
||||
updateTransformState: (transform: Transform) => void;
|
||||
hasUnsavedChanges: () => boolean;
|
||||
hasTransformEdits: (originalTransform: Transform) => boolean;
|
||||
}
|
||||
|
||||
const WorkspaceContext = createContext<WorkspaceContextValue | undefined>(
|
||||
undefined,
|
||||
);
|
||||
|
||||
interface WorkspaceState {
|
||||
openedTabs: WorkspaceTab[];
|
||||
activeTransform?: Transform;
|
||||
activeEditedTransform?: Transform;
|
||||
activeTable?: OpenTable;
|
||||
activeTab?: WorkspaceTab;
|
||||
editedTransforms: Map<number, EditedTransform>;
|
||||
runTransforms: Set<number>;
|
||||
}
|
||||
|
||||
interface WorkspaceProviderProps {
|
||||
children: ReactNode;
|
||||
workspaceId: number;
|
||||
}
|
||||
|
||||
const createEmptyWorkspaceState = (): WorkspaceState => ({
|
||||
openedTabs: [],
|
||||
activeTransform: undefined,
|
||||
activeTable: undefined,
|
||||
activeTab: undefined,
|
||||
editedTransforms: new Map(),
|
||||
runTransforms: new Set(),
|
||||
});
|
||||
|
||||
export const WorkspaceProvider = ({
|
||||
children,
|
||||
workspaceId,
|
||||
}: WorkspaceProviderProps) => {
|
||||
const [workspaceStates, setWorkspaceStates] = useState<
|
||||
Map<number, WorkspaceState>
|
||||
>(new Map());
|
||||
|
||||
const currentState = useMemo(() => {
|
||||
const existing = workspaceStates.get(workspaceId);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
const newState = createEmptyWorkspaceState();
|
||||
setWorkspaceStates((prev) => new Map(prev).set(workspaceId, newState));
|
||||
return newState;
|
||||
}, [workspaceId, workspaceStates]);
|
||||
|
||||
const { openedTabs, activeTransform, activeTable, activeTab } = currentState;
|
||||
|
||||
const updateWorkspaceState = useCallback(
|
||||
(updater: (state: WorkspaceState) => WorkspaceState) => {
|
||||
setWorkspaceStates((prev) => {
|
||||
const currentState =
|
||||
prev.get(workspaceId) ?? createEmptyWorkspaceState();
|
||||
const newState = updater(currentState);
|
||||
return new Map(prev).set(workspaceId, newState);
|
||||
});
|
||||
},
|
||||
[workspaceId],
|
||||
);
|
||||
|
||||
const setActiveTab = useCallback(
|
||||
(tab: WorkspaceTab | undefined) => {
|
||||
updateWorkspaceState((state) => {
|
||||
const newActiveTab = tab;
|
||||
let newActiveTransform = state.activeTransform;
|
||||
let newActiveTable = state.activeTable;
|
||||
|
||||
if (tab?.type === "transform") {
|
||||
newActiveTransform = tab.transform;
|
||||
newActiveTable = undefined;
|
||||
} else if (tab?.type === "table") {
|
||||
newActiveTable = tab.table;
|
||||
newActiveTransform = undefined;
|
||||
} else {
|
||||
newActiveTransform = undefined;
|
||||
newActiveTable = undefined;
|
||||
}
|
||||
|
||||
return {
|
||||
...state,
|
||||
activeTab: newActiveTab,
|
||||
activeTransform: newActiveTransform,
|
||||
activeTable: newActiveTable,
|
||||
};
|
||||
});
|
||||
},
|
||||
[updateWorkspaceState],
|
||||
);
|
||||
|
||||
const addOpenedTab = useCallback(
|
||||
(tab: WorkspaceTab) => {
|
||||
updateWorkspaceState((state) => {
|
||||
const exists = state.openedTabs.some((item) => item.id === tab.id);
|
||||
if (exists) {
|
||||
return {
|
||||
...state,
|
||||
activeTab: tab,
|
||||
activeTransform:
|
||||
tab.type === "transform" ? tab.transform : state.activeTransform,
|
||||
activeTable: tab.type === "table" ? tab.table : state.activeTable,
|
||||
};
|
||||
}
|
||||
|
||||
const newOpenedTabs = [...state.openedTabs, tab];
|
||||
return {
|
||||
...state,
|
||||
openedTabs: newOpenedTabs,
|
||||
activeTab: tab,
|
||||
activeTransform:
|
||||
tab.type === "transform" ? tab.transform : state.activeTransform,
|
||||
activeTable: tab.type === "table" ? tab.table : state.activeTable,
|
||||
};
|
||||
});
|
||||
},
|
||||
[updateWorkspaceState],
|
||||
);
|
||||
|
||||
const removeOpenedTab = useCallback(
|
||||
(tabId: string) => {
|
||||
updateWorkspaceState((state) => {
|
||||
const filteredTabs = state.openedTabs.filter(
|
||||
(item) => item.id !== tabId,
|
||||
);
|
||||
|
||||
const removedTab = state.openedTabs.find((item) => item.id === tabId);
|
||||
const wasActive = state.activeTab?.id === tabId;
|
||||
|
||||
let newActiveTab = state.activeTab;
|
||||
let newActiveTransform = state.activeTransform;
|
||||
let newActiveTable = state.activeTable;
|
||||
|
||||
if (wasActive && removedTab) {
|
||||
const currentIndex = state.openedTabs.findIndex(
|
||||
(item) => item.id === tabId,
|
||||
);
|
||||
|
||||
const preferPrevious = currentIndex > 0;
|
||||
const fallbackTab = preferPrevious
|
||||
? state.openedTabs[currentIndex - 1]
|
||||
: filteredTabs[0];
|
||||
|
||||
if (fallbackTab) {
|
||||
newActiveTab = fallbackTab;
|
||||
newActiveTransform =
|
||||
fallbackTab.type === "transform"
|
||||
? fallbackTab.transform
|
||||
: undefined;
|
||||
newActiveTable =
|
||||
fallbackTab.type === "table" ? fallbackTab.table : undefined;
|
||||
} else {
|
||||
newActiveTab = undefined;
|
||||
newActiveTransform = undefined;
|
||||
newActiveTable = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
...state,
|
||||
openedTabs: filteredTabs,
|
||||
activeTab: newActiveTab,
|
||||
activeTransform: newActiveTransform,
|
||||
activeTable: newActiveTable,
|
||||
};
|
||||
});
|
||||
},
|
||||
[updateWorkspaceState],
|
||||
);
|
||||
|
||||
const setOpenedTabs = useCallback(
|
||||
(tabs: WorkspaceTab[]) => {
|
||||
updateWorkspaceState((state) => ({
|
||||
...state,
|
||||
openedTabs: tabs,
|
||||
}));
|
||||
},
|
||||
[updateWorkspaceState],
|
||||
);
|
||||
|
||||
const setActiveTransform = useCallback(
|
||||
(transform: Transform | undefined) => {
|
||||
updateWorkspaceState((state) => {
|
||||
if (transform) {
|
||||
const existingTab = state.openedTabs.find(
|
||||
(tab) =>
|
||||
tab.type === "transform" && tab.transform.id === transform.id,
|
||||
);
|
||||
|
||||
if (existingTab) {
|
||||
return {
|
||||
...state,
|
||||
activeTab: existingTab,
|
||||
activeTransform: transform,
|
||||
activeTable: undefined,
|
||||
};
|
||||
} else {
|
||||
const newTransformTab: TransformTab = {
|
||||
id: `transform-${transform.id}`,
|
||||
name: transform.name,
|
||||
type: "transform",
|
||||
transform,
|
||||
};
|
||||
|
||||
return {
|
||||
...state,
|
||||
openedTabs: [...state.openedTabs, newTransformTab],
|
||||
activeTab: newTransformTab,
|
||||
activeTransform: transform,
|
||||
activeTable: undefined,
|
||||
};
|
||||
}
|
||||
} else {
|
||||
return {
|
||||
...state,
|
||||
activeTab:
|
||||
state.activeTab?.type === "transform"
|
||||
? undefined
|
||||
: state.activeTab,
|
||||
activeTransform: undefined,
|
||||
};
|
||||
}
|
||||
});
|
||||
},
|
||||
[updateWorkspaceState],
|
||||
);
|
||||
|
||||
const addOpenedTransform = useCallback(
|
||||
(transform: Transform) => {
|
||||
const transformTab: TransformTab = {
|
||||
id: `transform-${transform.id}`,
|
||||
name: transform.name,
|
||||
type: "transform",
|
||||
transform,
|
||||
};
|
||||
addOpenedTab(transformTab);
|
||||
},
|
||||
[addOpenedTab],
|
||||
);
|
||||
|
||||
const removeOpenedTransform = useCallback(
|
||||
(transformId: number) => {
|
||||
removeOpenedTab(`transform-${transformId}`);
|
||||
},
|
||||
[removeOpenedTab],
|
||||
);
|
||||
|
||||
const patchEditedTransform = useCallback(
|
||||
(transformId: number, patch: Partial<EditedTransform>) => {
|
||||
updateWorkspaceState((state) => {
|
||||
const activeTransform = checkNotNull(state.activeTransform);
|
||||
const currentTransform =
|
||||
state.editedTransforms.get(transformId) ?? activeTransform;
|
||||
const newEditedTransform = {
|
||||
name: patch.name ? patch.name : currentTransform.name,
|
||||
source: patch.source ? patch.source : currentTransform.source,
|
||||
target: patch.target ? patch.target : currentTransform.target,
|
||||
};
|
||||
|
||||
const hasChanges =
|
||||
!isSameSource(newEditedTransform.source, activeTransform.source) ||
|
||||
newEditedTransform.name !== activeTransform.name ||
|
||||
newEditedTransform.target.name !== activeTransform.target.name;
|
||||
|
||||
const newEditedTransforms = new Map(state.editedTransforms);
|
||||
|
||||
if (hasChanges) {
|
||||
newEditedTransforms.set(transformId, newEditedTransform);
|
||||
} else {
|
||||
newEditedTransforms.delete(transformId);
|
||||
}
|
||||
const newRunTransforms = new Set(state.runTransforms);
|
||||
newRunTransforms.delete(transformId);
|
||||
|
||||
const newOpenedTabs = state.openedTabs.map((tab) => {
|
||||
if (tab.type === "transform" && tab.transform.id === transformId) {
|
||||
return {
|
||||
...tab,
|
||||
name: newEditedTransform.name,
|
||||
};
|
||||
}
|
||||
return tab;
|
||||
});
|
||||
|
||||
return {
|
||||
...state,
|
||||
openedTabs: newOpenedTabs,
|
||||
editedTransforms: newEditedTransforms,
|
||||
runTransforms: newRunTransforms,
|
||||
};
|
||||
});
|
||||
},
|
||||
[updateWorkspaceState],
|
||||
);
|
||||
|
||||
const removeEditedTransform = useCallback(
|
||||
(transformId: number) => {
|
||||
updateWorkspaceState((state) => {
|
||||
const newEditedTransforms = new Map(state.editedTransforms);
|
||||
newEditedTransforms.delete(transformId);
|
||||
return {
|
||||
...state,
|
||||
editedTransforms: newEditedTransforms,
|
||||
};
|
||||
});
|
||||
},
|
||||
[updateWorkspaceState],
|
||||
);
|
||||
|
||||
const updateTransformState = useCallback(
|
||||
(transform: Transform) => {
|
||||
updateWorkspaceState((state) => {
|
||||
const newOpenedTabs = state.openedTabs.map((tab) => {
|
||||
if (tab.type === "transform" && tab.transform.id === transform.id) {
|
||||
return {
|
||||
...tab,
|
||||
transform,
|
||||
name: transform.name,
|
||||
};
|
||||
}
|
||||
return tab;
|
||||
});
|
||||
|
||||
const newEditedTransforms = new Map(state.editedTransforms);
|
||||
const currentEdit = state.editedTransforms.get(transform.id);
|
||||
|
||||
if (currentEdit) {
|
||||
const hasNameChanged = currentEdit.name !== transform.name;
|
||||
const hasSourceChanged = !isSameSource(
|
||||
currentEdit.source,
|
||||
transform.source,
|
||||
);
|
||||
|
||||
if (hasNameChanged || hasSourceChanged) {
|
||||
newEditedTransforms.set(transform.id, {
|
||||
...currentEdit,
|
||||
target: transform.target,
|
||||
});
|
||||
} else {
|
||||
newEditedTransforms.delete(transform.id);
|
||||
}
|
||||
}
|
||||
|
||||
const newRunTransforms = new Set(state.runTransforms);
|
||||
newRunTransforms.delete(transform.id);
|
||||
|
||||
return {
|
||||
...state,
|
||||
openedTabs: newOpenedTabs,
|
||||
activeTransform:
|
||||
state.activeTransform?.id === transform.id
|
||||
? transform
|
||||
: state.activeTransform,
|
||||
editedTransforms: newEditedTransforms,
|
||||
runTransforms: newRunTransforms,
|
||||
};
|
||||
});
|
||||
},
|
||||
[updateWorkspaceState],
|
||||
);
|
||||
|
||||
const hasUnsavedChanges = useCallback(() => {
|
||||
return currentState.editedTransforms.size > 0;
|
||||
}, [currentState.editedTransforms]);
|
||||
|
||||
const hasTransformEdits = useCallback(
|
||||
(originalTransform: Transform) => {
|
||||
const edited = currentState.editedTransforms.get(originalTransform.id);
|
||||
return (
|
||||
edited != null &&
|
||||
(!isSameSource(edited.source, originalTransform.source) ||
|
||||
edited.name !== originalTransform.name ||
|
||||
edited.target.name !== originalTransform.target.name)
|
||||
);
|
||||
},
|
||||
[currentState.editedTransforms],
|
||||
);
|
||||
|
||||
const setActiveTable = useCallback(
|
||||
(table: OpenTable | undefined) => {
|
||||
updateWorkspaceState((state) => {
|
||||
if (table) {
|
||||
const existingTab = state.openedTabs.find(
|
||||
(tab) =>
|
||||
tab.type === "table" && tab.table.tableId === table.tableId,
|
||||
);
|
||||
|
||||
if (existingTab) {
|
||||
return {
|
||||
...state,
|
||||
activeTab: existingTab,
|
||||
activeTable: table,
|
||||
activeTransform: undefined,
|
||||
};
|
||||
} else {
|
||||
const newTableTab: TableTab = {
|
||||
id: `table-${table.tableId}`,
|
||||
name: table.schema ? `${table.schema}.${table.name}` : table.name,
|
||||
type: "table",
|
||||
table,
|
||||
};
|
||||
|
||||
return {
|
||||
...state,
|
||||
openedTabs: [...state.openedTabs, newTableTab],
|
||||
activeTab: newTableTab,
|
||||
activeTable: table,
|
||||
activeTransform: undefined,
|
||||
};
|
||||
}
|
||||
} else {
|
||||
return {
|
||||
...state,
|
||||
activeTab:
|
||||
state.activeTab?.type === "table" ? undefined : state.activeTab,
|
||||
activeTable: undefined,
|
||||
};
|
||||
}
|
||||
});
|
||||
},
|
||||
[updateWorkspaceState],
|
||||
);
|
||||
|
||||
const activeEditedTransform = activeTransform
|
||||
? (currentState.editedTransforms.get(activeTransform.id) ?? activeTransform)
|
||||
: activeTransform;
|
||||
|
||||
const value = useMemo(
|
||||
() => ({
|
||||
workspaceId,
|
||||
openedTabs,
|
||||
activeTransform,
|
||||
activeTable,
|
||||
activeTab,
|
||||
activeEditedTransform,
|
||||
setActiveTab,
|
||||
setActiveTransform,
|
||||
setActiveTable,
|
||||
addOpenedTab,
|
||||
removeOpenedTab,
|
||||
setOpenedTabs,
|
||||
addOpenedTransform,
|
||||
removeOpenedTransform,
|
||||
editedTransforms: currentState.editedTransforms,
|
||||
patchEditedTransform,
|
||||
removeEditedTransform,
|
||||
runTransforms: currentState.runTransforms,
|
||||
updateTransformState,
|
||||
hasUnsavedChanges,
|
||||
hasTransformEdits,
|
||||
}),
|
||||
[
|
||||
workspaceId,
|
||||
openedTabs,
|
||||
activeTransform,
|
||||
activeTable,
|
||||
activeTab,
|
||||
activeEditedTransform,
|
||||
setActiveTab,
|
||||
setActiveTransform,
|
||||
setActiveTable,
|
||||
addOpenedTab,
|
||||
removeOpenedTab,
|
||||
setOpenedTabs,
|
||||
addOpenedTransform,
|
||||
removeOpenedTransform,
|
||||
currentState.editedTransforms,
|
||||
currentState.runTransforms,
|
||||
patchEditedTransform,
|
||||
removeEditedTransform,
|
||||
updateTransformState,
|
||||
hasUnsavedChanges,
|
||||
hasTransformEdits,
|
||||
],
|
||||
);
|
||||
|
||||
return (
|
||||
<WorkspaceContext.Provider value={value}>
|
||||
{children}
|
||||
</WorkspaceContext.Provider>
|
||||
);
|
||||
};
|
||||
|
||||
export const useWorkspace = (): WorkspaceContextValue => {
|
||||
const context = useContext(WorkspaceContext);
|
||||
|
||||
if (!context) {
|
||||
throw new Error("useWorkspace must be used within a WorkspaceProvider");
|
||||
}
|
||||
|
||||
return context;
|
||||
};
|
||||
|
||||
export const useOptionalWorkspace = (): WorkspaceContextValue | undefined => {
|
||||
return useContext(WorkspaceContext);
|
||||
};
|
||||
@@ -0,0 +1,7 @@
|
||||
.statusDot {
|
||||
width: 0.5rem;
|
||||
height: 0.5rem;
|
||||
border-radius: 50%;
|
||||
background-color: var(--mb-color-warning);
|
||||
flex-shrink: 0;
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import { Box } from "metabase/ui";
|
||||
|
||||
import S from "./StatusDot.module.css";
|
||||
|
||||
type StatusDotProps = {
|
||||
status?: "changed";
|
||||
};
|
||||
|
||||
export const StatusDot = ({ status }: StatusDotProps) => {
|
||||
if (status !== "changed") {
|
||||
return null;
|
||||
}
|
||||
|
||||
return <Box className={S.statusDot} />;
|
||||
};
|
||||
@@ -0,0 +1,13 @@
|
||||
import { IndexRoute, Route } from "react-router";
|
||||
|
||||
import { WorkspaceListPage } from "./pages/WorkspaceListPage/WorkspaceListPage";
|
||||
import { WorkspacePage } from "./pages/WorkspacePage/WorkspacePage";
|
||||
|
||||
export function getDataStudioWorkspaceRoutes() {
|
||||
return (
|
||||
<>
|
||||
<IndexRoute component={WorkspaceListPage} />
|
||||
<Route path=":workspaceId" component={WorkspacePage} />
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
.suggestionPaper {
|
||||
min-height: fit-content;
|
||||
max-height: 50vh;
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ import { tinykeys } from "tinykeys";
|
||||
|
||||
import ErrorBoundary from "metabase/ErrorBoundary";
|
||||
import { useSelector } from "metabase/lib/redux";
|
||||
import { Sidebar } from "metabase/nav/containers/MainNavbar/MainNavbar.styled";
|
||||
import { getUser } from "metabase/selectors/user";
|
||||
import type { SuggestionModel } from "metabase-enterprise/rich_text_editing/tiptap/extensions/shared/types";
|
||||
|
||||
@@ -55,7 +56,14 @@ export const MetabotAuthenticated = ({ hide, config }: MetabotProps) => {
|
||||
|
||||
return (
|
||||
<ErrorBoundary errorComponent={() => null}>
|
||||
<MetabotChat config={config} />
|
||||
<Sidebar
|
||||
isOpen={visible}
|
||||
side="right"
|
||||
width="30rem"
|
||||
aria-hidden={!visible}
|
||||
>
|
||||
<MetabotChat config={config} />
|
||||
</Sidebar>
|
||||
</ErrorBoundary>
|
||||
);
|
||||
};
|
||||
|
||||
@@ -2,16 +2,19 @@ import { unifiedMergeView } from "@codemirror/merge";
|
||||
import { useDisclosure } from "@mantine/hooks";
|
||||
import type { UnknownAction } from "@reduxjs/toolkit";
|
||||
import cx from "classnames";
|
||||
import { useMemo } from "react";
|
||||
import { useMemo, useState } from "react";
|
||||
import { push } from "react-router-redux";
|
||||
import { useLocation, useMount } from "react-use";
|
||||
import { P, match } from "ts-pattern";
|
||||
import { t } from "ttag";
|
||||
import _ from "underscore";
|
||||
|
||||
import { getErrorMessage } from "metabase/api/utils";
|
||||
import { CodeMirror } from "metabase/common/components/CodeMirror";
|
||||
import { slugify } from "metabase/lib/formatting";
|
||||
import { useDispatch, useSelector } from "metabase/lib/redux";
|
||||
import * as Urls from "metabase/lib/urls";
|
||||
import { useMetadataToasts } from "metabase/metadata/hooks";
|
||||
import EditorS from "metabase/query_builder/components/NativeQueryEditor/CodeMirrorEditor/CodeMirrorEditor.module.css";
|
||||
import { getMetadata } from "metabase/selectors/metadata";
|
||||
import {
|
||||
@@ -24,17 +27,24 @@ import {
|
||||
Paper,
|
||||
Text,
|
||||
} from "metabase/ui";
|
||||
import { useLazyGetTransformQuery } from "metabase-enterprise/api";
|
||||
import {
|
||||
useCreateTransformMutation,
|
||||
useLazyGetTransformQuery,
|
||||
} from "metabase-enterprise/api";
|
||||
import { useOptionalWorkspace } from "metabase-enterprise/data-studio/workspaces/pages/WorkspacePage/WorkspaceProvider";
|
||||
import {
|
||||
type MetabotAgentEditSuggestionChatMessage,
|
||||
activateSuggestedTransform,
|
||||
getIsSuggestedTransformActive,
|
||||
} from "metabase-enterprise/metabot/state";
|
||||
import * as Lib from "metabase-lib";
|
||||
import Question from "metabase-lib/v1/Question";
|
||||
import type Metadata from "metabase-lib/v1/metadata/Metadata";
|
||||
import type {
|
||||
DraftTransformSource,
|
||||
MetabotTransformInfo,
|
||||
SuggestedTransform,
|
||||
Transform,
|
||||
} from "metabase-types/api";
|
||||
|
||||
import S from "./MetabotAgentSuggestionMessage.module.css";
|
||||
@@ -104,8 +114,17 @@ export const AgentSuggestionMessage = ({
|
||||
}) => {
|
||||
const dispatch = useDispatch();
|
||||
const metadata = useSelector(getMetadata);
|
||||
const workspace = useOptionalWorkspace();
|
||||
const [createTransform] = useCreateTransformMutation();
|
||||
const { sendErrorToast, sendSuccessToast } = useMetadataToasts();
|
||||
const [isApplying, setIsApplying] = useState(false);
|
||||
const [hasAppliedInWorkspace, setHasAppliedInWorkspace] = useState(false);
|
||||
|
||||
const { suggestedTransform, editorTransform } = message.payload;
|
||||
const existingTransformId =
|
||||
typeof suggestedTransform.id === "number"
|
||||
? suggestedTransform.id
|
||||
: undefined;
|
||||
const isActive = useSelector((state) =>
|
||||
getIsSuggestedTransformActive(state, suggestedTransform.suggestionId),
|
||||
);
|
||||
@@ -117,8 +136,10 @@ export const AgentSuggestionMessage = ({
|
||||
getTransformUrl(suggestedTransform),
|
||||
);
|
||||
|
||||
const canApply = !isViewing || !isActive;
|
||||
const isNew = !isViewing && !editorTransform && suggestedTransform.id == null;
|
||||
const canApply = workspace
|
||||
? !hasAppliedInWorkspace && !isApplying
|
||||
: !isViewing || !isActive;
|
||||
const isNew = !isViewing && !editorTransform && existingTransformId == null;
|
||||
|
||||
const {
|
||||
data: originalTransform,
|
||||
@@ -131,8 +152,133 @@ export const AgentSuggestionMessage = ({
|
||||
: "";
|
||||
const newSource = getSourceCode(suggestedTransform, metadata);
|
||||
|
||||
const handleApply = () => {
|
||||
const handleApply = async () => {
|
||||
dispatch(activateSuggestedTransform(suggestedTransform));
|
||||
|
||||
if (workspace) {
|
||||
const targetTransform: Transform | undefined =
|
||||
(editorTransform as Transform | undefined) ??
|
||||
(originalTransform as Transform | undefined) ??
|
||||
(existingTransformId ? (suggestedTransform as Transform) : undefined);
|
||||
|
||||
if (existingTransformId != null && targetTransform) {
|
||||
workspace.addOpenedTransform(targetTransform);
|
||||
workspace.setActiveTransform(targetTransform);
|
||||
setHasAppliedInWorkspace(true);
|
||||
return;
|
||||
}
|
||||
|
||||
if (existingTransformId == null && suggestedTransform.target) {
|
||||
setIsApplying(true);
|
||||
try {
|
||||
const normalizeSource = (
|
||||
source: DraftTransformSource,
|
||||
): DraftTransformSource => {
|
||||
if (source.type !== "query") {
|
||||
return source;
|
||||
}
|
||||
|
||||
const question = Question.create({
|
||||
dataset_query: source.query,
|
||||
metadata,
|
||||
});
|
||||
const query = question.query();
|
||||
const { isNative } = Lib.queryDisplayInfo(query);
|
||||
const normalizedQuery = isNative
|
||||
? Lib.withNativeQuery(query, Lib.rawNativeQuery(query))
|
||||
: query;
|
||||
|
||||
return {
|
||||
type: "query",
|
||||
query: question.setQuery(normalizedQuery).datasetQuery(),
|
||||
};
|
||||
};
|
||||
|
||||
const normalizedSource = normalizeSource(suggestedTransform.source);
|
||||
const targetWithDatabase =
|
||||
suggestedTransform.target &&
|
||||
suggestedTransform.target.type === "table"
|
||||
? {
|
||||
...suggestedTransform.target,
|
||||
database:
|
||||
suggestedTransform.target.database ??
|
||||
(normalizedSource.type === "query"
|
||||
? normalizedSource.query.database
|
||||
: normalizedSource.type === "python"
|
||||
? normalizedSource["source-database"]
|
||||
: undefined),
|
||||
}
|
||||
: suggestedTransform.target;
|
||||
|
||||
const sanitizedTarget =
|
||||
targetWithDatabase?.type === "table"
|
||||
? (() => {
|
||||
const fallbackName = slugify(suggestedTransform.name);
|
||||
const trimmedName =
|
||||
targetWithDatabase.name?.trim() || fallbackName;
|
||||
|
||||
if (!trimmedName) {
|
||||
sendErrorToast(
|
||||
t`Suggestion is missing a target table name to create the transform.`,
|
||||
);
|
||||
setIsApplying(false);
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
...targetWithDatabase,
|
||||
name: trimmedName,
|
||||
schema:
|
||||
targetWithDatabase.schema &&
|
||||
targetWithDatabase.schema.trim() !== ""
|
||||
? targetWithDatabase.schema.trim()
|
||||
: null,
|
||||
};
|
||||
})()
|
||||
: targetWithDatabase;
|
||||
|
||||
if (sanitizedTarget === null) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
targetWithDatabase?.type === "table" &&
|
||||
targetWithDatabase.database == null
|
||||
) {
|
||||
sendErrorToast(
|
||||
t`Suggestion is missing a target database to create the transform.`,
|
||||
);
|
||||
setIsApplying(false);
|
||||
return;
|
||||
}
|
||||
|
||||
const transform = await createTransform({
|
||||
name: suggestedTransform.name,
|
||||
description: suggestedTransform.description ?? null,
|
||||
source: normalizedSource,
|
||||
target: sanitizedTarget,
|
||||
}).unwrap();
|
||||
|
||||
workspace.addOpenedTransform(transform);
|
||||
workspace.setActiveTransform(transform);
|
||||
setHasAppliedInWorkspace(true);
|
||||
sendSuccessToast(t`Transform created`);
|
||||
return;
|
||||
} catch (error) {
|
||||
sendErrorToast(
|
||||
getErrorMessage(error) ??
|
||||
t`Failed to create transform from suggestion`,
|
||||
);
|
||||
} finally {
|
||||
setIsApplying(false);
|
||||
}
|
||||
} else if (existingTransformId == null) {
|
||||
sendErrorToast(t`Suggestion is missing a target table`);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
dispatch(push(getTransformUrl(suggestedTransform)) as UnknownAction);
|
||||
};
|
||||
|
||||
@@ -215,11 +361,13 @@ export const AgentSuggestionMessage = ({
|
||||
disabled={!canApply}
|
||||
onClick={handleApply}
|
||||
>
|
||||
{match({ isNew, canApply })
|
||||
.with({ canApply: false }, () => t`Applied`)
|
||||
.with({ isNew: true }, () => t`Create`)
|
||||
.with({ canApply: true }, () => t`Apply`)
|
||||
.exhaustive()}
|
||||
{isApplying
|
||||
? t`Applying...`
|
||||
: match({ isNew, canApply })
|
||||
.with({ canApply: false }, () => t`Applied`)
|
||||
.with({ isNew: true }, () => t`Create`)
|
||||
.with({ canApply: true }, () => t`Apply`)
|
||||
.exhaustive()}
|
||||
</Button>
|
||||
</Flex>
|
||||
</Group>
|
||||
|
||||
@@ -3,7 +3,6 @@ import { useCallback, useMemo } from "react";
|
||||
import { t } from "ttag";
|
||||
|
||||
import EmptyDashboardBot from "assets/img/dashboard-empty.svg?component";
|
||||
import { Sidebar } from "metabase/nav/containers/MainNavbar/MainNavbar.styled";
|
||||
import {
|
||||
ActionIcon,
|
||||
Box,
|
||||
@@ -38,11 +37,11 @@ const defaultConfig: MetabotConfig = {
|
||||
],
|
||||
};
|
||||
|
||||
export const MetabotChat = ({
|
||||
config = defaultConfig,
|
||||
}: {
|
||||
interface Props {
|
||||
config?: MetabotConfig;
|
||||
}) => {
|
||||
}
|
||||
|
||||
export const MetabotChat = ({ config = defaultConfig }: Props) => {
|
||||
const metabot = useMetabotAgent();
|
||||
const { handleSubmitInput, handleRetryMessage, handleResetInput } =
|
||||
useMetabotChatHandlers(config);
|
||||
@@ -77,137 +76,127 @@ export const MetabotChat = ({
|
||||
}, [handleSubmitInput, metabot.prompt]);
|
||||
|
||||
return (
|
||||
<Sidebar
|
||||
isOpen={metabot.visible}
|
||||
side="right"
|
||||
width="30rem"
|
||||
aria-hidden={!metabot.visible}
|
||||
>
|
||||
<Box className={Styles.container} data-testid="metabot-chat">
|
||||
{/* header */}
|
||||
<Box ref={headerRef} className={Styles.header}>
|
||||
<Flex align-items="center">
|
||||
<Text lh={1} fz="sm" c="text-secondary">
|
||||
{metabot.profileOverride
|
||||
? t`Using profile: ${metabot.profileOverride}`
|
||||
: t`Metabot isn't perfect. Double-check results.`}
|
||||
</Text>
|
||||
</Flex>
|
||||
<Box className={Styles.container} data-testid="metabot-chat">
|
||||
{/* header */}
|
||||
<Box ref={headerRef} className={Styles.header}>
|
||||
<Flex align-items="center">
|
||||
<Text lh={1} fz="sm" c="text-secondary">
|
||||
{metabot.profileOverride
|
||||
? t`Using profile: ${metabot.profileOverride}`
|
||||
: t`Metabot isn't perfect. Double-check results.`}
|
||||
</Text>
|
||||
</Flex>
|
||||
|
||||
<Flex gap="sm">
|
||||
<Tooltip label={t`Clear conversation`} position="bottom">
|
||||
<ActionIcon
|
||||
onClick={() => metabot.resetConversation()}
|
||||
data-testid="metabot-reset-chat"
|
||||
>
|
||||
<Icon c="text-primary" name="revert" />
|
||||
</ActionIcon>
|
||||
</Tooltip>
|
||||
{!config.preventClose && (
|
||||
<ActionIcon
|
||||
onClick={handleClose}
|
||||
data-testid="metabot-close-chat"
|
||||
>
|
||||
<Icon c="text-primary" name="close" />
|
||||
</ActionIcon>
|
||||
)}
|
||||
</Flex>
|
||||
</Box>
|
||||
|
||||
{/* chat messages */}
|
||||
<Box
|
||||
ref={scrollContainerRef}
|
||||
className={Styles.messagesContainer}
|
||||
data-testid="metabot-chat-messages"
|
||||
>
|
||||
{!hasMessages && !metabot.isDoingScience && (
|
||||
<>
|
||||
{/* empty state */}
|
||||
<Flex
|
||||
h="100%"
|
||||
gap="md"
|
||||
direction="column"
|
||||
align="center"
|
||||
justify="center"
|
||||
data-testid="metabot-empty-chat-info"
|
||||
>
|
||||
<Box component={EmptyDashboardBot} w="6rem" />
|
||||
<Text c="text-light" maw="12rem" ta="center">
|
||||
{config.emptyText ??
|
||||
t`I can help you explore your metrics and models.`}
|
||||
</Text>
|
||||
</Flex>
|
||||
{!config.hideSuggestedPrompts && (
|
||||
<Stack
|
||||
gap="sm"
|
||||
className={Styles.promptSuggestionsContainer}
|
||||
data-testid="metabot-prompt-suggestions"
|
||||
>
|
||||
<>
|
||||
{suggestedPrompts.map(({ prompt }, index) => (
|
||||
<Box key={index}>
|
||||
<Button
|
||||
fz="sm"
|
||||
size="xs"
|
||||
onClick={() => handleSubmitInput(prompt)}
|
||||
className={Styles.promptSuggestionButton}
|
||||
>
|
||||
{prompt}
|
||||
</Button>
|
||||
</Box>
|
||||
))}
|
||||
</>
|
||||
</Stack>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
|
||||
{(hasMessages || metabot.isDoingScience) && (
|
||||
<Box
|
||||
className={Styles.messages}
|
||||
data-testid="metabot-chat-inner-messages"
|
||||
<Flex gap="sm">
|
||||
<Tooltip label={t`Clear conversation`} position="bottom">
|
||||
<ActionIcon
|
||||
onClick={() => metabot.resetConversation()}
|
||||
data-testid="metabot-reset-chat"
|
||||
>
|
||||
{/* conversation messages */}
|
||||
<Messages
|
||||
messages={metabot.messages}
|
||||
errorMessages={metabot.errorMessages}
|
||||
onRetryMessage={handleRetryMessage}
|
||||
isDoingScience={metabot.isDoingScience}
|
||||
showFeedbackButtons
|
||||
/>
|
||||
{/* loading */}
|
||||
{metabot.isDoingScience && (
|
||||
<MetabotThinking toolCalls={metabot.activeToolCalls} />
|
||||
)}
|
||||
{/* filler - height gets set via ref mutation */}
|
||||
<div ref={fillerRef} data-testid="metabot-message-filler" />
|
||||
{/* long convo warning */}
|
||||
{metabot.isLongConversation && <MetabotResetLongChatButton />}
|
||||
</Box>
|
||||
<Icon c="text-primary" name="revert" />
|
||||
</ActionIcon>
|
||||
</Tooltip>
|
||||
{!config.preventClose && (
|
||||
<ActionIcon onClick={handleClose} data-testid="metabot-close-chat">
|
||||
<Icon c="text-primary" name="close" />
|
||||
</ActionIcon>
|
||||
)}
|
||||
</Box>
|
||||
|
||||
<Box className={Styles.textInputContainer}>
|
||||
<Paper
|
||||
className={cx(
|
||||
Styles.inputContainer,
|
||||
metabot.isDoingScience && Styles.inputContainerLoading,
|
||||
)}
|
||||
>
|
||||
<MetabotChatEditor
|
||||
ref={metabot.promptInputRef}
|
||||
value={metabot.prompt}
|
||||
autoFocus
|
||||
isResponding={metabot.isDoingScience}
|
||||
placeholder={t`Tell me to do something, or ask a question`}
|
||||
onChange={handleEditorChange}
|
||||
onSubmit={handleEditorSubmit}
|
||||
onStop={metabot.cancelRequest}
|
||||
suggestionModels={config.suggestionModels}
|
||||
/>
|
||||
</Paper>
|
||||
</Box>
|
||||
</Flex>
|
||||
</Box>
|
||||
</Sidebar>
|
||||
|
||||
{/* chat messages */}
|
||||
<Box
|
||||
ref={scrollContainerRef}
|
||||
className={Styles.messagesContainer}
|
||||
data-testid="metabot-chat-messages"
|
||||
>
|
||||
{!hasMessages && !metabot.isDoingScience && (
|
||||
<>
|
||||
{/* empty state */}
|
||||
<Flex
|
||||
h="100%"
|
||||
gap="md"
|
||||
direction="column"
|
||||
align="center"
|
||||
justify="center"
|
||||
data-testid="metabot-empty-chat-info"
|
||||
>
|
||||
<Box component={EmptyDashboardBot} w="6rem" />
|
||||
<Text c="text-light" maw="12rem" ta="center">
|
||||
{config.emptyText ??
|
||||
t`I can help you explore your metrics and models.`}
|
||||
</Text>
|
||||
</Flex>
|
||||
{!config.hideSuggestedPrompts && (
|
||||
<Stack
|
||||
gap="sm"
|
||||
className={Styles.promptSuggestionsContainer}
|
||||
data-testid="metabot-prompt-suggestions"
|
||||
>
|
||||
<>
|
||||
{suggestedPrompts.map(({ prompt }, index) => (
|
||||
<Box key={index}>
|
||||
<Button
|
||||
fz="sm"
|
||||
size="xs"
|
||||
onClick={() => handleSubmitInput(prompt)}
|
||||
className={Styles.promptSuggestionButton}
|
||||
>
|
||||
{prompt}
|
||||
</Button>
|
||||
</Box>
|
||||
))}
|
||||
</>
|
||||
</Stack>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
|
||||
{(hasMessages || metabot.isDoingScience) && (
|
||||
<Box
|
||||
className={Styles.messages}
|
||||
data-testid="metabot-chat-inner-messages"
|
||||
>
|
||||
{/* conversation messages */}
|
||||
<Messages
|
||||
messages={metabot.messages}
|
||||
errorMessages={metabot.errorMessages}
|
||||
onRetryMessage={handleRetryMessage}
|
||||
isDoingScience={metabot.isDoingScience}
|
||||
showFeedbackButtons
|
||||
/>
|
||||
{/* loading */}
|
||||
{metabot.isDoingScience && (
|
||||
<MetabotThinking toolCalls={metabot.activeToolCalls} />
|
||||
)}
|
||||
{/* filler - height gets set via ref mutation */}
|
||||
<div ref={fillerRef} data-testid="metabot-message-filler" />
|
||||
{/* long convo warning */}
|
||||
{metabot.isLongConversation && <MetabotResetLongChatButton />}
|
||||
</Box>
|
||||
)}
|
||||
</Box>
|
||||
|
||||
<Box className={Styles.textInputContainer}>
|
||||
<Paper
|
||||
className={cx(
|
||||
Styles.inputContainer,
|
||||
metabot.isDoingScience && Styles.inputContainerLoading,
|
||||
)}
|
||||
>
|
||||
<MetabotChatEditor
|
||||
ref={metabot.promptInputRef}
|
||||
value={metabot.prompt}
|
||||
autoFocus
|
||||
isResponding={metabot.isDoingScience}
|
||||
placeholder={t`Tell me to do something, or ask a question`}
|
||||
onChange={handleEditorChange}
|
||||
onSubmit={handleEditorSubmit}
|
||||
onStop={metabot.cancelRequest}
|
||||
suggestionModels={config.suggestionModels}
|
||||
/>
|
||||
</Paper>
|
||||
</Box>
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
@@ -14,7 +14,7 @@ export const useMetabotReactions = () => {
|
||||
) as ReturnType<typeof getMetabotReactionsState>;
|
||||
|
||||
const setNavigateToPath = useCallback(
|
||||
async (navigateToPath: string) => {
|
||||
async (navigateToPath: string | null) => {
|
||||
dispatch(setNavigateToPathAction(navigateToPath));
|
||||
},
|
||||
[dispatch],
|
||||
|
||||
@@ -12,6 +12,7 @@ import { Metabot } from "./components/Metabot";
|
||||
import { MetabotAdminPage } from "./components/MetabotAdmin/MetabotAdminPage";
|
||||
import { MetabotTrialPage } from "./components/MetabotAdmin/MetabotTrialPage";
|
||||
import { MetabotAppBarButton } from "./components/MetabotAppBarButton";
|
||||
import { MetabotChat } from "./components/MetabotChat";
|
||||
import { MetabotDataStudioButton } from "./components/MetabotDataStudioButton";
|
||||
import { getMetabotQuickLinks } from "./components/MetabotQuickLinks";
|
||||
import { MetabotToggleButton } from "./components/MetabotToggleButton";
|
||||
@@ -38,6 +39,7 @@ export function initializePlugin() {
|
||||
if (hasPremiumFeature("metabot_v3")) {
|
||||
PLUGIN_METABOT.isEnabled = () => true;
|
||||
PLUGIN_METABOT.Metabot = Metabot;
|
||||
PLUGIN_METABOT.MetabotChat = MetabotChat;
|
||||
PLUGIN_METABOT.getMetabotRoutes = getMetabotQuickLinks;
|
||||
|
||||
PLUGIN_METABOT.getAdminPaths = () => [
|
||||
|
||||
@@ -267,13 +267,45 @@ export const metabot = createSlice({
|
||||
state.reactions.suggestedTransforms = [];
|
||||
state.experimental.metabotReqIdOverride = undefined;
|
||||
},
|
||||
setConversationSnapshot: (
|
||||
state,
|
||||
action: PayloadAction<{
|
||||
messages: MetabotChatMessage[];
|
||||
history: MetabotHistory;
|
||||
state: any;
|
||||
reactions: MetabotReactionsState;
|
||||
activeToolCalls: MetabotToolCall[];
|
||||
errorMessages: MetabotErrorMessage[];
|
||||
conversationId: string;
|
||||
}>,
|
||||
) => {
|
||||
const {
|
||||
messages,
|
||||
history,
|
||||
state: snapshotState,
|
||||
reactions,
|
||||
activeToolCalls,
|
||||
errorMessages,
|
||||
conversationId,
|
||||
} = action.payload;
|
||||
|
||||
state.messages = messages ?? [];
|
||||
state.history = history ?? [];
|
||||
state.state = snapshotState ?? {};
|
||||
state.reactions =
|
||||
reactions ?? ({ navigateToPath: null, suggestedTransforms: [] } as any);
|
||||
state.activeToolCalls = activeToolCalls ?? [];
|
||||
state.errorMessages = errorMessages ?? [];
|
||||
state.conversationId = conversationId ?? uuid();
|
||||
state.isProcessing = false;
|
||||
},
|
||||
resetConversationId: (state) => {
|
||||
state.conversationId = uuid();
|
||||
},
|
||||
setIsProcessing: (state, action: PayloadAction<boolean>) => {
|
||||
state.isProcessing = action.payload;
|
||||
},
|
||||
setNavigateToPath: (state, action: PayloadAction<string>) => {
|
||||
setNavigateToPath: (state, action: PayloadAction<string | null>) => {
|
||||
state.reactions.navigateToPath = action.payload;
|
||||
},
|
||||
setVisible: (state, action: PayloadAction<boolean>) => {
|
||||
@@ -379,3 +411,4 @@ export const metabot = createSlice({
|
||||
});
|
||||
|
||||
export const metabotReducer = metabot.reducer;
|
||||
export const metabotActions = metabot.actions;
|
||||
|
||||
@@ -144,7 +144,8 @@ export const getProfile = createSelector(
|
||||
return profileOverride;
|
||||
}
|
||||
|
||||
return location.pathname.startsWith(Urls.transformList())
|
||||
return location.pathname.startsWith(Urls.transformList()) ||
|
||||
location.pathname.startsWith(Urls.dataStudioWorkspaceList())
|
||||
? "transforms_codegen"
|
||||
: undefined;
|
||||
},
|
||||
|
||||
@@ -162,6 +162,14 @@ export function useEntitySuggestions({
|
||||
const isInModelSelectionMode =
|
||||
!selectedSearchModel && hasSearchModels && hasMatchingFilteredModels;
|
||||
|
||||
// If searchModels are provided (e.g., only transforms), skip recents and always search.
|
||||
const shouldFetchRecents =
|
||||
enabled &&
|
||||
query.length === 0 &&
|
||||
!isInModelSelectionMode &&
|
||||
!selectedSearchModel &&
|
||||
!searchModels;
|
||||
|
||||
const {
|
||||
menuItems: entityMenuItems,
|
||||
isLoading,
|
||||
@@ -172,11 +180,7 @@ export function useEntitySuggestions({
|
||||
onSelectSearchResult: handleSearchResultSelect,
|
||||
onSelectUser: handleUserSelect,
|
||||
enabled: enabled && !isInModelSelectionMode,
|
||||
shouldFetchRecents:
|
||||
enabled &&
|
||||
query.length === 0 &&
|
||||
!isInModelSelectionMode &&
|
||||
!selectedSearchModel,
|
||||
shouldFetchRecents,
|
||||
searchModels: effectiveSearchModels,
|
||||
});
|
||||
|
||||
|
||||
@@ -30,6 +30,8 @@ import {
|
||||
|
||||
type PythonDataPickerProps = {
|
||||
database?: DatabaseId;
|
||||
disabled?: boolean;
|
||||
canChangeDatabase?: boolean;
|
||||
tables: PythonTransformTableAliases;
|
||||
onChange: (
|
||||
database: number,
|
||||
@@ -40,8 +42,10 @@ type PythonDataPickerProps = {
|
||||
|
||||
export function PythonDataPicker({
|
||||
database,
|
||||
disabled,
|
||||
tables,
|
||||
onChange,
|
||||
canChangeDatabase = true,
|
||||
}: PythonDataPickerProps) {
|
||||
const [tableSelections, setTableSelections] = useState<TableSelection[]>(
|
||||
getInitialTableSelections(tables),
|
||||
@@ -150,22 +154,25 @@ export function PythonDataPicker({
|
||||
className={S.dataPicker}
|
||||
data-testid="python-data-picker"
|
||||
>
|
||||
<Box>
|
||||
<Text fw="bold">{t`Source database`}</Text>
|
||||
<Text size="sm" c="text-light" mb="sm">
|
||||
{t`Select the database that contains your source data.`}
|
||||
</Text>
|
||||
{canChangeDatabase && (
|
||||
<Box>
|
||||
<Text fw="bold">{t`Source database`}</Text>
|
||||
<Text size="sm" c="text-light" mb="sm">
|
||||
{t`Select the database that contains your source data.`}
|
||||
</Text>
|
||||
|
||||
<DatabaseDataSelector
|
||||
className={S.databaseSelector}
|
||||
selectedDatabaseId={database}
|
||||
setDatabaseFn={handleDatabaseChange}
|
||||
databases={databases?.data ?? []}
|
||||
databaseIsDisabled={(database: Database) =>
|
||||
!hasFeature(database, "transforms/python")
|
||||
}
|
||||
/>
|
||||
</Box>
|
||||
<DatabaseDataSelector
|
||||
className={S.databaseSelector}
|
||||
readOnly={disabled}
|
||||
selectedDatabaseId={database}
|
||||
setDatabaseFn={handleDatabaseChange}
|
||||
databases={databases?.data ?? []}
|
||||
databaseIsDisabled={(database: Database) =>
|
||||
!hasFeature(database, "transforms/python")
|
||||
}
|
||||
/>
|
||||
</Box>
|
||||
)}
|
||||
{database && (
|
||||
<Box>
|
||||
<Text fw="bold">{t`Pick tables and alias them`}</Text>
|
||||
@@ -185,12 +192,12 @@ export function PythonDataPicker({
|
||||
handleSelectionChange(index, selection)
|
||||
}
|
||||
onRemove={() => handleRemoveTable(index)}
|
||||
disabled={isLoadingTables}
|
||||
disabled={disabled || isLoadingTables}
|
||||
/>
|
||||
))}
|
||||
<AddTableButton
|
||||
onClick={handleAddTable}
|
||||
disabled={availableTables.length === 0}
|
||||
disabled={disabled || availableTables.length === 0}
|
||||
/>
|
||||
</Stack>
|
||||
</Box>
|
||||
|
||||
@@ -98,6 +98,7 @@ export function TableSelector({
|
||||
|
||||
<Tooltip label={t`Remove this table`}>
|
||||
<ActionIcon
|
||||
disabled={disabled}
|
||||
onClick={onRemove}
|
||||
pr="sm"
|
||||
aria-label={t`Remove this table`}
|
||||
|
||||
@@ -24,6 +24,7 @@ import { ResizableBoxHandle } from "./ResizableBoxHandle";
|
||||
import { hasImport, insertImport, removeImport } from "./utils";
|
||||
|
||||
type PythonEditorBodyProps = {
|
||||
disabled?: boolean;
|
||||
source: string;
|
||||
proposedSource?: string;
|
||||
isRunnable: boolean;
|
||||
@@ -41,6 +42,7 @@ type PythonEditorBodyProps = {
|
||||
const EDITOR_HEIGHT = 400;
|
||||
|
||||
export function PythonEditorBody({
|
||||
disabled,
|
||||
source,
|
||||
proposedSource,
|
||||
onChange,
|
||||
@@ -57,6 +59,7 @@ export function PythonEditorBody({
|
||||
<MaybeResizableBox resizable={withDebugger}>
|
||||
<Flex h="100%" align="end" bg="bg-light" pos="relative">
|
||||
<PythonEditor
|
||||
readOnly={disabled}
|
||||
value={source}
|
||||
proposedValue={proposedSource}
|
||||
onChange={onChange}
|
||||
@@ -106,7 +109,11 @@ export function PythonEditorBody({
|
||||
</Box>
|
||||
)}
|
||||
</Stack>
|
||||
<SharedLibraryActions source={source} onChange={onChange} />
|
||||
<SharedLibraryActions
|
||||
disabled={disabled}
|
||||
source={source}
|
||||
onChange={onChange}
|
||||
/>
|
||||
</Flex>
|
||||
</MaybeResizableBox>
|
||||
);
|
||||
@@ -140,24 +147,32 @@ function MaybeResizableBox({
|
||||
}
|
||||
|
||||
function SharedLibraryActions({
|
||||
disabled,
|
||||
source,
|
||||
onChange,
|
||||
}: {
|
||||
disabled?: boolean;
|
||||
source: string;
|
||||
onChange: (source: string) => void;
|
||||
}) {
|
||||
return (
|
||||
<Group className={S.libraryActions} p="md" gap="sm">
|
||||
<SharedLibraryImportButton source={source} onChange={onChange} />
|
||||
<SharedLibraryImportButton
|
||||
disabled={disabled}
|
||||
source={source}
|
||||
onChange={onChange}
|
||||
/>
|
||||
<SharedLibraryEditLink />
|
||||
</Group>
|
||||
);
|
||||
}
|
||||
|
||||
function SharedLibraryImportButton({
|
||||
disabled,
|
||||
source,
|
||||
onChange,
|
||||
}: {
|
||||
disabled?: boolean;
|
||||
source: string;
|
||||
onChange: (source: string) => void;
|
||||
}) {
|
||||
@@ -173,7 +188,11 @@ function SharedLibraryImportButton({
|
||||
|
||||
return (
|
||||
<Tooltip label={label}>
|
||||
<ActionIcon aria-label={label} onClick={handleToggleSharedLib}>
|
||||
<ActionIcon
|
||||
aria-label={label}
|
||||
disabled={disabled}
|
||||
onClick={handleToggleSharedLib}
|
||||
>
|
||||
<Icon name="reference" c="text-dark" />
|
||||
</ActionIcon>
|
||||
</Tooltip>
|
||||
|
||||
@@ -2,7 +2,11 @@ import { useHotkeys } from "@mantine/hooks";
|
||||
|
||||
import type { PythonTransformEditorProps } from "metabase/plugins";
|
||||
import { Flex, Stack } from "metabase/ui";
|
||||
import type { PythonTransformTableAliases, Table } from "metabase-types/api";
|
||||
import type {
|
||||
DatabaseId,
|
||||
PythonTransformTableAliases,
|
||||
Table,
|
||||
} from "metabase-types/api";
|
||||
|
||||
import { PythonDataPicker } from "./PythonDataPicker";
|
||||
import { PythonEditorBody } from "./PythonEditorBody";
|
||||
@@ -17,6 +21,7 @@ import {
|
||||
export function PythonTransformEditor({
|
||||
source,
|
||||
proposedSource,
|
||||
uiOptions,
|
||||
isDirty,
|
||||
onChangeSource,
|
||||
onAcceptProposed,
|
||||
@@ -34,7 +39,7 @@ export function PythonTransformEditor({
|
||||
};
|
||||
|
||||
const handleDataChange = (
|
||||
database: number,
|
||||
database: DatabaseId,
|
||||
sourceTables: PythonTransformTableAliases,
|
||||
tableInfo: Table[],
|
||||
) => {
|
||||
@@ -71,12 +76,15 @@ export function PythonTransformEditor({
|
||||
return (
|
||||
<Flex h="100%" w="100%">
|
||||
<PythonDataPicker
|
||||
disabled={uiOptions?.readOnly}
|
||||
database={source["source-database"]}
|
||||
canChangeDatabase={uiOptions?.canChangeDatabase}
|
||||
tables={source["source-tables"]}
|
||||
onChange={handleDataChange}
|
||||
/>
|
||||
<Stack w="100%" h="100%" gap={0}>
|
||||
<PythonEditorBody
|
||||
disabled={uiOptions?.readOnly}
|
||||
isRunnable={isPythonTransformSource(source)}
|
||||
isRunning={isRunning}
|
||||
isDirty={isDirty}
|
||||
|
||||
@@ -3,6 +3,7 @@ import { useMemo } from "react";
|
||||
import { useSelector } from "metabase/lib/redux";
|
||||
import {
|
||||
QueryEditor,
|
||||
type QueryEditorUiOptions,
|
||||
type QueryEditorUiState,
|
||||
} from "metabase/querying/editor/components/QueryEditor";
|
||||
import { getMetadata } from "metabase/selectors/metadata";
|
||||
@@ -11,9 +12,10 @@ import type { Database, QueryTransformSource } from "metabase-types/api";
|
||||
|
||||
import { getEditorOptions } from "./utils";
|
||||
|
||||
type TransformEditorProps = {
|
||||
export type TransformEditorProps = {
|
||||
source: QueryTransformSource;
|
||||
uiState: QueryEditorUiState;
|
||||
uiOptions?: QueryEditorUiOptions;
|
||||
proposedSource: QueryTransformSource | undefined;
|
||||
databases: Database[];
|
||||
onChangeSource: (source: QueryTransformSource) => void;
|
||||
@@ -27,6 +29,7 @@ export function TransformEditor({
|
||||
proposedSource,
|
||||
databases,
|
||||
uiState,
|
||||
uiOptions,
|
||||
onChangeSource,
|
||||
onChangeUiState,
|
||||
onAcceptProposed,
|
||||
@@ -44,7 +47,10 @@ export function TransformEditor({
|
||||
: undefined,
|
||||
[proposedSource, metadata],
|
||||
);
|
||||
const uiOptions = useMemo(() => getEditorOptions(databases), [databases]);
|
||||
const mergedUiOptions = useMemo(
|
||||
() => ({ ...getEditorOptions(databases), ...uiOptions }),
|
||||
[databases, uiOptions],
|
||||
);
|
||||
|
||||
const handleQueryChange = (query: Lib.Query) => {
|
||||
const newSource: QueryTransformSource = {
|
||||
@@ -60,7 +66,7 @@ export function TransformEditor({
|
||||
<QueryEditor
|
||||
query={query}
|
||||
uiState={uiState}
|
||||
uiOptions={uiOptions}
|
||||
uiOptions={mergedUiOptions}
|
||||
proposedQuery={proposedQuery}
|
||||
onChangeQuery={handleQueryChange}
|
||||
onChangeUiState={onChangeUiState}
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
import dayjs from "dayjs";
|
||||
import { useMemo, useState } from "react";
|
||||
import { push } from "react-router-redux";
|
||||
import { t } from "ttag";
|
||||
|
||||
import { useListDatabasesQuery } from "metabase/api";
|
||||
import { useDispatch } from "metabase/lib/redux";
|
||||
import { useMetadataToasts } from "metabase/metadata/hooks";
|
||||
import {
|
||||
Box,
|
||||
Button,
|
||||
Flex,
|
||||
Icon,
|
||||
Loader,
|
||||
Menu,
|
||||
ScrollArea,
|
||||
Stack,
|
||||
Text,
|
||||
} from "metabase/ui";
|
||||
import {
|
||||
useCreateWorkspaceMutation,
|
||||
useGetTransformDownstreamMappingQuery,
|
||||
useGetWorkspacesQuery,
|
||||
useUpdateWorkspaceContentsMutation,
|
||||
} from "metabase-enterprise/api";
|
||||
import { CreateWorkspaceModal } from "metabase-enterprise/data-studio/workspaces/components/CreateWorkspaceModal/CreateWorkspaceModal";
|
||||
import type { Transform, Workspace } from "metabase-types/api";
|
||||
|
||||
type EditTransformMenuProps = {
|
||||
transform: Transform;
|
||||
};
|
||||
|
||||
export function EditTransformMenu({ transform }: EditTransformMenuProps) {
|
||||
const dispatch = useDispatch();
|
||||
const { sendErrorToast } = useMetadataToasts();
|
||||
|
||||
const sourceDatabaseId = getTransformDatabaseId(transform);
|
||||
|
||||
const { data: workspacesData, isLoading: isLoadingWorkspaces } =
|
||||
useGetWorkspacesQuery();
|
||||
const [updateWorkspaceContents, { isLoading: isUpdatingWorkspace }] =
|
||||
useUpdateWorkspaceContentsMutation();
|
||||
const [createWorkspace, { isLoading: isCreatingWorkspace }] =
|
||||
useCreateWorkspaceMutation();
|
||||
const { data: downstreamMapping } = useGetTransformDownstreamMappingQuery(
|
||||
transform.id,
|
||||
);
|
||||
const [isCreateModalOpen, setIsCreateModalOpen] = useState(false);
|
||||
const [addedWorkspaceIds, setAddedWorkspaceIds] = useState<Set<number>>(
|
||||
() => new Set(),
|
||||
);
|
||||
|
||||
const { data: databasesData } = useListDatabasesQuery({
|
||||
include_analytics: true,
|
||||
});
|
||||
const workspaces = useMemo(
|
||||
() => workspacesData?.items ?? [],
|
||||
[workspacesData],
|
||||
);
|
||||
|
||||
const existingWorkspaceIds = useMemo(() => {
|
||||
const ids = new Set<number>();
|
||||
downstreamMapping?.transforms?.forEach((item) => {
|
||||
if (item.workspace?.id != null) {
|
||||
ids.add(item.workspace.id);
|
||||
}
|
||||
});
|
||||
addedWorkspaceIds.forEach((id) => ids.add(id));
|
||||
return ids;
|
||||
}, [downstreamMapping, addedWorkspaceIds]);
|
||||
|
||||
const matchingWorkspaces = useMemo(
|
||||
() =>
|
||||
workspaces
|
||||
.filter((workspace) => !existingWorkspaceIds.has(workspace.id))
|
||||
.filter((workspace) =>
|
||||
sourceDatabaseId == null
|
||||
? true
|
||||
: workspace.database_id === sourceDatabaseId,
|
||||
),
|
||||
[workspaces, sourceDatabaseId, existingWorkspaceIds],
|
||||
);
|
||||
|
||||
const isBusy = isUpdatingWorkspace || isCreatingWorkspace;
|
||||
const emptyMessage =
|
||||
workspaces.length === 0 || sourceDatabaseId == null
|
||||
? t`No workspaces yet`
|
||||
: t`No workspaces for this database yet`;
|
||||
const databaseOptions = useMemo(
|
||||
() =>
|
||||
(databasesData?.data ?? []).map((db) => ({
|
||||
value: String(db.id),
|
||||
label: db.name,
|
||||
})),
|
||||
[databasesData],
|
||||
);
|
||||
|
||||
const handleOpenCreateModal = () => {
|
||||
setIsCreateModalOpen(true);
|
||||
};
|
||||
|
||||
const handleCloseCreateModal = () => {
|
||||
setIsCreateModalOpen(false);
|
||||
};
|
||||
|
||||
const handleWorkspaceSelect = async (workspace: Workspace) => {
|
||||
try {
|
||||
await updateWorkspaceContents({
|
||||
id: workspace.id,
|
||||
add: { transforms: [transform.id] },
|
||||
}).unwrap();
|
||||
setAddedWorkspaceIds((prev) => new Set(prev).add(workspace.id));
|
||||
} catch (error) {
|
||||
sendErrorToast(t`Failed to add transform to the workspace`);
|
||||
}
|
||||
};
|
||||
|
||||
const handleCreateWorkspace = async ({
|
||||
name,
|
||||
databaseId,
|
||||
}: {
|
||||
name: string;
|
||||
databaseId: string;
|
||||
}) => {
|
||||
try {
|
||||
const workspace = await createWorkspace({
|
||||
name: name.trim() || t`New workspace`,
|
||||
database_id: Number(databaseId),
|
||||
upstream: { transforms: [transform.id] },
|
||||
}).unwrap();
|
||||
|
||||
setAddedWorkspaceIds((prev) => new Set(prev).add(workspace.id));
|
||||
handleCloseCreateModal();
|
||||
dispatch(push(`/data-studio/workspaces/${workspace.id}`));
|
||||
} catch (error) {
|
||||
sendErrorToast(t`Failed to create workspace`);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<Menu position="bottom-end" width={280}>
|
||||
<Menu.Target>
|
||||
<Button
|
||||
size="sm"
|
||||
variant="filled"
|
||||
leftSection={<Icon name="pencil" />}
|
||||
rightSection={<Icon name="chevrondown" />}
|
||||
loading={isBusy}
|
||||
>
|
||||
{t`Edit transform`}
|
||||
</Button>
|
||||
</Menu.Target>
|
||||
<Menu.Dropdown>
|
||||
<Menu.Label>{t`Add to workspace`}</Menu.Label>
|
||||
<Menu.Item
|
||||
disabled={isBusy}
|
||||
leftSection={<Icon name="add" />}
|
||||
onClick={handleOpenCreateModal}
|
||||
>
|
||||
<Stack gap={0} align="flex-start">
|
||||
<Text fw={600}>{t`New workspace`}</Text>
|
||||
<Text size="sm" c="text-light">
|
||||
{t`Create a new workspace`}
|
||||
</Text>
|
||||
</Stack>
|
||||
</Menu.Item>
|
||||
<Menu.Divider />
|
||||
|
||||
{isLoadingWorkspaces ? (
|
||||
<Flex justify="center" align="center" py="md">
|
||||
<Loader size="sm" />
|
||||
</Flex>
|
||||
) : matchingWorkspaces.length === 0 ? (
|
||||
<Box px="md" py="sm">
|
||||
<Text size="sm" c="text-light">
|
||||
{emptyMessage}
|
||||
</Text>
|
||||
</Box>
|
||||
) : (
|
||||
<ScrollArea.Autosize mah={320} type="scroll">
|
||||
<Stack gap={0}>
|
||||
{matchingWorkspaces.map((workspace) => (
|
||||
<Menu.Item
|
||||
key={workspace.id}
|
||||
leftSection={<Icon name="sparkles" />}
|
||||
onClick={() => handleWorkspaceSelect(workspace)}
|
||||
disabled={isBusy}
|
||||
>
|
||||
<Stack gap={2} align="flex-start">
|
||||
<Text fw={600}>{workspace.name}</Text>
|
||||
<Text size="sm" c="text-light">
|
||||
{formatWorkspaceDate(workspace.created_at)}
|
||||
</Text>
|
||||
</Stack>
|
||||
</Menu.Item>
|
||||
))}
|
||||
</Stack>
|
||||
</ScrollArea.Autosize>
|
||||
)}
|
||||
</Menu.Dropdown>
|
||||
|
||||
<CreateWorkspaceModal
|
||||
opened={isCreateModalOpen}
|
||||
onClose={handleCloseCreateModal}
|
||||
onSubmit={handleCreateWorkspace}
|
||||
databaseOptions={databaseOptions}
|
||||
isSubmitting={isBusy}
|
||||
defaultDatabaseId={
|
||||
sourceDatabaseId != null ? String(sourceDatabaseId) : null
|
||||
}
|
||||
/>
|
||||
</Menu>
|
||||
);
|
||||
}
|
||||
|
||||
function getTransformDatabaseId(transform: Transform) {
|
||||
if (
|
||||
transform.source_type === "python" &&
|
||||
"source-database" in transform.source
|
||||
) {
|
||||
return transform.source["source-database"];
|
||||
}
|
||||
|
||||
if ("query" in transform.source) {
|
||||
return transform.source.query.database;
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function formatWorkspaceDate(createdAt: string) {
|
||||
return dayjs(createdAt).format("DD/MM/YYYY");
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export * from "./EditTransformMenu";
|
||||
@@ -1,16 +1,19 @@
|
||||
import { useRegisterMetabotContextProvider } from "metabase/metabot";
|
||||
import type {
|
||||
PythonTransformSourceDraft,
|
||||
DraftTransformSource,
|
||||
SuggestedTransform,
|
||||
Transform,
|
||||
TransformSource,
|
||||
} from "metabase-types/api";
|
||||
|
||||
export const useRegisterMetabotTransformContext = (
|
||||
transform: Transform | SuggestedTransform | undefined,
|
||||
source: TransformSource | PythonTransformSourceDraft,
|
||||
source?: DraftTransformSource,
|
||||
) => {
|
||||
useRegisterMetabotContextProvider(async () => {
|
||||
if (!source) {
|
||||
return {};
|
||||
}
|
||||
|
||||
return {
|
||||
user_is_viewing: [{ type: "transform", ...(transform || {}), source }],
|
||||
};
|
||||
|
||||
@@ -32,30 +32,40 @@ import { trackTransformCreated } from "../../../analytics";
|
||||
|
||||
import { SchemaFormSelect } from "./../../../components/SchemaFormSelect";
|
||||
|
||||
function getValidationSchema() {
|
||||
return Yup.object({
|
||||
name: Yup.string().required(Errors.required),
|
||||
targetName: Yup.string().required(Errors.required),
|
||||
targetSchema: Yup.string().nullable().defined(),
|
||||
incremental: Yup.boolean().required(),
|
||||
// For native queries, use checkpointFilter (plain string)
|
||||
checkpointFilter: Yup.string().nullable(),
|
||||
// For MBQL/Python queries, use checkpointFilterUniqueKey (prefixed format)
|
||||
checkpointFilterUniqueKey: Yup.string().nullable(),
|
||||
sourceStrategy: Yup.mixed<"checkpoint">().oneOf(["checkpoint"]).required(),
|
||||
targetStrategy: Yup.mixed<"append">().oneOf(["append"]).required(),
|
||||
});
|
||||
}
|
||||
const DEFAULT_VALIDATION_SCHEMA = Yup.object({
|
||||
name: Yup.string().required(Errors.required),
|
||||
targetName: Yup.string().required(Errors.required),
|
||||
targetSchema: Yup.string().nullable().defined(),
|
||||
incremental: Yup.boolean().required(),
|
||||
// For native queries, use checkpointFilter (plain string)
|
||||
checkpointFilter: Yup.string().nullable(),
|
||||
// For MBQL/Python queries, use checkpointFilterUniqueKey (prefixed format)
|
||||
checkpointFilterUniqueKey: Yup.string().nullable(),
|
||||
sourceStrategy: Yup.mixed<"checkpoint">().oneOf(["checkpoint"]).required(),
|
||||
targetStrategy: Yup.mixed<"append">().oneOf(["append"]).required(),
|
||||
});
|
||||
|
||||
export type NewTransformValues = Yup.InferType<
|
||||
ReturnType<typeof getValidationSchema>
|
||||
>;
|
||||
export type NewTransformValues = Yup.InferType<typeof DEFAULT_VALIDATION_SCHEMA>;
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
export type ValidationSchemaExtension = Record<string, Yup.AnySchema>;
|
||||
|
||||
function getValidationSchema(extension?: ValidationSchemaExtension) {
|
||||
if (!extension) {
|
||||
return DEFAULT_VALIDATION_SCHEMA;
|
||||
}
|
||||
return DEFAULT_VALIDATION_SCHEMA.shape(extension);
|
||||
}
|
||||
|
||||
type CreateTransformModalProps = {
|
||||
source: TransformSource;
|
||||
defaultValues: Partial<NewTransformValues>;
|
||||
onCreate: (transform: Transform) => void;
|
||||
onCreate?: (transform: Transform) => void;
|
||||
onClose: () => void;
|
||||
schemas?: string[] | null;
|
||||
showIncrementalSettings?: boolean;
|
||||
validationSchemaExtension?: ValidationSchemaExtension;
|
||||
handleSubmit?: (values: NewTransformValues) => Promise<Transform>;
|
||||
};
|
||||
|
||||
export function CreateTransformModal({
|
||||
@@ -63,6 +73,10 @@ export function CreateTransformModal({
|
||||
defaultValues,
|
||||
onCreate,
|
||||
onClose,
|
||||
schemas,
|
||||
showIncrementalSettings = true,
|
||||
validationSchemaExtension,
|
||||
handleSubmit,
|
||||
}: CreateTransformModalProps) {
|
||||
return (
|
||||
<Modal title={t`Save your transform`} opened padding="xl" onClose={onClose}>
|
||||
@@ -71,6 +85,10 @@ export function CreateTransformModal({
|
||||
defaultValues={defaultValues}
|
||||
onCreate={onCreate}
|
||||
onClose={onClose}
|
||||
schemas={schemas}
|
||||
showIncrementalSettings={showIncrementalSettings}
|
||||
validationSchemaExtension={validationSchemaExtension}
|
||||
handleSubmit={handleSubmit}
|
||||
/>
|
||||
</Modal>
|
||||
);
|
||||
@@ -79,8 +97,12 @@ export function CreateTransformModal({
|
||||
type CreateTransformFormProps = {
|
||||
source: TransformSource;
|
||||
defaultValues: Partial<NewTransformValues>;
|
||||
onCreate: (transform: Transform) => void;
|
||||
onCreate?: (transform: Transform) => void;
|
||||
onClose: () => void;
|
||||
schemas?: string[] | null;
|
||||
showIncrementalSettings?: boolean;
|
||||
validationSchemaExtension?: ValidationSchemaExtension;
|
||||
handleSubmit?: (values: NewTransformValues) => Promise<Transform>;
|
||||
};
|
||||
|
||||
function CreateTransformForm({
|
||||
@@ -88,11 +110,18 @@ function CreateTransformForm({
|
||||
defaultValues,
|
||||
onCreate,
|
||||
onClose,
|
||||
schemas: schemasProp,
|
||||
showIncrementalSettings = true,
|
||||
validationSchemaExtension,
|
||||
handleSubmit,
|
||||
}: CreateTransformFormProps) {
|
||||
const [sendToast] = useToast();
|
||||
const databaseId =
|
||||
source.type === "query" ? source.query.database : source["source-database"];
|
||||
|
||||
const shouldFetchSchemas = schemasProp === undefined;
|
||||
const showSchemaField = schemasProp !== null;
|
||||
|
||||
const {
|
||||
data: database,
|
||||
isLoading: isDatabaseLoading,
|
||||
@@ -100,15 +129,22 @@ function CreateTransformForm({
|
||||
} = useGetDatabaseQuery(databaseId ? { id: databaseId } : skipToken);
|
||||
|
||||
const {
|
||||
data: schemas = [],
|
||||
data: fetchedSchemas = [],
|
||||
isLoading: isSchemasLoading,
|
||||
error: schemasError,
|
||||
} = useListDatabaseSchemasQuery(
|
||||
databaseId ? { id: databaseId, include_hidden: true } : skipToken,
|
||||
shouldFetchSchemas && databaseId
|
||||
? { id: databaseId, include_hidden: true }
|
||||
: skipToken,
|
||||
);
|
||||
|
||||
const isLoading = isDatabaseLoading || isSchemasLoading;
|
||||
const error = databaseError ?? schemasError;
|
||||
const schemas = useMemo(
|
||||
() => schemasProp ?? fetchedSchemas ?? [],
|
||||
[schemasProp, fetchedSchemas],
|
||||
);
|
||||
const isLoading =
|
||||
isDatabaseLoading || (shouldFetchSchemas && isSchemasLoading);
|
||||
const error = databaseError ?? (shouldFetchSchemas ? schemasError : null);
|
||||
|
||||
const [createTransform] = useCreateTransformMutation();
|
||||
const supportsSchemas = database && hasFeature(database, "schemas");
|
||||
@@ -118,13 +154,16 @@ function CreateTransformForm({
|
||||
[schemas, defaultValues],
|
||||
);
|
||||
|
||||
const validationSchema = useMemo(() => getValidationSchema(), []);
|
||||
const validationSchema = useMemo(
|
||||
() => getValidationSchema(validationSchemaExtension),
|
||||
[validationSchemaExtension],
|
||||
);
|
||||
|
||||
if (isLoading || error != null) {
|
||||
return <LoadingAndErrorWrapper loading={isLoading} error={error} />;
|
||||
}
|
||||
|
||||
const handleSubmit = async (values: NewTransformValues) => {
|
||||
const defaultHandleSubmit = async (values: NewTransformValues) => {
|
||||
if (!databaseId) {
|
||||
throw new Error("Database ID is required");
|
||||
}
|
||||
@@ -132,7 +171,7 @@ function CreateTransformForm({
|
||||
try {
|
||||
const transform = await createTransform(request).unwrap();
|
||||
trackTransformCreated({ transformId: transform.id });
|
||||
onCreate(transform);
|
||||
onCreate?.(transform);
|
||||
} catch (error) {
|
||||
sendToast({
|
||||
message: getErrorMessage(error, t`Failed to create transform`),
|
||||
@@ -145,7 +184,7 @@ function CreateTransformForm({
|
||||
<FormProvider
|
||||
initialValues={initialValues}
|
||||
validationSchema={validationSchema}
|
||||
onSubmit={handleSubmit}
|
||||
onSubmit={handleSubmit || defaultHandleSubmit}
|
||||
>
|
||||
<Form>
|
||||
<Stack gap="lg" mt="sm">
|
||||
@@ -155,7 +194,7 @@ function CreateTransformForm({
|
||||
placeholder={t`My Great Transform`}
|
||||
data-autofocus
|
||||
/>
|
||||
{supportsSchemas && (
|
||||
{showSchemaField && supportsSchemas && (
|
||||
<SchemaFormSelect
|
||||
name="targetSchema"
|
||||
label={t`Schema`}
|
||||
@@ -167,7 +206,9 @@ function CreateTransformForm({
|
||||
label={t`Table name`}
|
||||
placeholder={t`descriptive_name`}
|
||||
/>
|
||||
<IncrementalTransformSettings source={source} />
|
||||
{showIncrementalSettings && (
|
||||
<IncrementalTransformSettings source={source} />
|
||||
)}
|
||||
<Group>
|
||||
<Box flex={1}>
|
||||
<FormErrorMessage />
|
||||
@@ -182,7 +223,7 @@ function CreateTransformForm({
|
||||
}
|
||||
|
||||
function getInitialValues(
|
||||
schemas: string[],
|
||||
schemas: string[] | null,
|
||||
defaultValues: Partial<NewTransformValues>,
|
||||
): NewTransformValues {
|
||||
return {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user