Compare commits
49 Commits
python-run
...
js-transfo
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4200120ff9 | ||
|
|
0c719a75e2 | ||
|
|
5d8a5cf8ff | ||
|
|
d3b782b375 | ||
|
|
36868b2aab | ||
|
|
e9d01384d6 | ||
|
|
cd2c14aaf8 | ||
|
|
4098645d3a | ||
|
|
24f36f7979 | ||
|
|
5c919b4add | ||
|
|
ee7346ff68 | ||
|
|
876dab1769 | ||
|
|
0ccf067a29 | ||
|
|
cb3cfcfa21 | ||
|
|
d34ab2be7a | ||
|
|
4c265d8954 | ||
|
|
de8afdc8c8 | ||
|
|
000a03edcf | ||
|
|
c2eef6d6d1 | ||
|
|
a16c25656e | ||
|
|
b60cabdd52 | ||
|
|
2384273b28 | ||
|
|
9d534a68e5 | ||
|
|
47a7332fa4 | ||
|
|
02b5960cd5 | ||
|
|
3ec4a405d2 | ||
|
|
85c964f28a | ||
|
|
91670469bd | ||
|
|
8230584157 | ||
|
|
e97fd4f808 | ||
|
|
f6cced8be1 | ||
|
|
6433912341 | ||
|
|
c36f87e720 | ||
|
|
41c31f2442 | ||
|
|
93c44100ab | ||
|
|
a02a8caa95 | ||
|
|
ad160b0816 | ||
|
|
9d5d294bb4 | ||
|
|
8edec1a692 | ||
|
|
a9c515c771 | ||
|
|
61944251e7 | ||
|
|
54f3bc4fb5 | ||
|
|
dc02a9744e | ||
|
|
398a335f2f | ||
|
|
c531cdaa96 | ||
|
|
6175f8e695 | ||
|
|
038aaab787 | ||
|
|
a6723ed1b4 | ||
|
|
c01e6ad83f |
@@ -1597,6 +1597,7 @@
|
||||
util}
|
||||
:friends #{models
|
||||
enterprise/transforms-python
|
||||
enterprise/transforms-runner
|
||||
enterprise/workspaces}}
|
||||
|
||||
transforms-inspector
|
||||
@@ -2257,7 +2258,7 @@
|
||||
util
|
||||
warehouses
|
||||
enterprise/dependencies
|
||||
enterprise/transforms-python}}
|
||||
enterprise/transforms-runner}}
|
||||
|
||||
enterprise/permission-debug
|
||||
{:team "UX West"
|
||||
@@ -2292,7 +2293,7 @@
|
||||
task-history
|
||||
util
|
||||
enterprise/serialization
|
||||
enterprise/transforms-python}}
|
||||
enterprise/transforms-runner}}
|
||||
|
||||
enterprise/sandbox
|
||||
{:team "UX West"
|
||||
@@ -2457,12 +2458,27 @@
|
||||
:api #{}
|
||||
:uses #{premium-features}}
|
||||
|
||||
enterprise/transforms-clojure
|
||||
{:team "Querying"
|
||||
:api #{metabase-enterprise.transforms-clojure.init}
|
||||
:uses #{transforms}}
|
||||
|
||||
enterprise/transforms-javascript
|
||||
{:team "Querying"
|
||||
:api #{metabase-enterprise.transforms-javascript.init}
|
||||
:uses #{transforms}}
|
||||
|
||||
enterprise/transforms-python
|
||||
{:team "Querying"
|
||||
:api #{metabase-enterprise.transforms-python.api
|
||||
metabase-enterprise.transforms-python.core
|
||||
metabase-enterprise.transforms-python.init
|
||||
metabase-enterprise.transforms-python.python-runner}
|
||||
:api #{metabase-enterprise.transforms-python.init}
|
||||
:uses #{transforms}}
|
||||
|
||||
enterprise/transforms-runner
|
||||
{:team "Querying"
|
||||
:api #{metabase-enterprise.transforms-runner.api
|
||||
metabase-enterprise.transforms-runner.init
|
||||
metabase-enterprise.transforms-runner.models.transform-library
|
||||
metabase-enterprise.transforms-runner.runner}
|
||||
:uses #{analytics
|
||||
api
|
||||
app-db
|
||||
@@ -2510,7 +2526,7 @@
|
||||
system
|
||||
transforms
|
||||
util
|
||||
enterprise/transforms-python}}}
|
||||
enterprise/transforms-runner}}}
|
||||
|
||||
;; namespaces matching these patterns (with `re-find`) are excluded from the module linter. Since regex literals
|
||||
;; aren't allowed in EDN just used the `(str <regex>)` version i.e. two slashes instead of one.
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { WRITABLE_DB_ID } from "e2e/support/cypress_data";
|
||||
import type {
|
||||
AdvancedTransformType,
|
||||
Collection,
|
||||
CollectionId,
|
||||
PythonTransformTableAliases,
|
||||
@@ -166,7 +167,9 @@ export function createSqlTransform({
|
||||
);
|
||||
}
|
||||
|
||||
export function createPythonTransform({
|
||||
export function createAdvancedTransform({
|
||||
name = "Python transform",
|
||||
type = "python",
|
||||
body,
|
||||
sourceTables,
|
||||
targetTable,
|
||||
@@ -174,6 +177,8 @@ export function createPythonTransform({
|
||||
tagIds,
|
||||
visitTransform,
|
||||
}: {
|
||||
name?: string;
|
||||
type?: AdvancedTransformType;
|
||||
body: string;
|
||||
sourceTables: PythonTransformTableAliases;
|
||||
targetTable: string;
|
||||
@@ -183,9 +188,9 @@ export function createPythonTransform({
|
||||
}) {
|
||||
return createTransform(
|
||||
{
|
||||
name: "Python transform",
|
||||
name,
|
||||
source: {
|
||||
type: "python",
|
||||
type,
|
||||
"source-database": WRITABLE_DB_ID,
|
||||
"source-tables": sourceTables,
|
||||
body,
|
||||
|
||||
@@ -215,7 +215,7 @@ describe("scenarios > Metabase Analytics Collection (AuditV2) ", () => {
|
||||
H.visitModel(id);
|
||||
});
|
||||
|
||||
H.newButton("SQL query").click();
|
||||
H.newButton(/SQL/).click();
|
||||
|
||||
// sample DB should be the only one
|
||||
cy.findByTestId("gui-builder-data")
|
||||
|
||||
@@ -21,6 +21,7 @@ describe("scenarios > data studio > transforms > python library", () => {
|
||||
|
||||
it("should allow editing the python library", () => {
|
||||
H.DataStudio.Transforms.visit();
|
||||
H.DataStudio.Transforms.list().findByText("Code library").click();
|
||||
cy.findByRole("link", { name: /Python library/ }).click();
|
||||
|
||||
H.DataStudio.PythonLibrary.editor().should("be.visible");
|
||||
@@ -30,7 +31,7 @@ describe("scenarios > data studio > transforms > python library", () => {
|
||||
.findByText(/# This is your Python library/)
|
||||
.should("be.visible");
|
||||
H.DataStudio.PythonLibrary.editor()
|
||||
.findByText(/# You can add functions and classes here/)
|
||||
.findByText(/# You can add code here that can be reused/)
|
||||
.should("be.visible");
|
||||
|
||||
cy.log("modify and save the python library");
|
||||
@@ -42,7 +43,42 @@ describe("scenarios > data studio > transforms > python library", () => {
|
||||
cy.findByRole("button", { name: "Save" }).should("be.visible").click();
|
||||
|
||||
H.undoToast()
|
||||
.findByText(/Python library saved/)
|
||||
.findByText(/library saved/i)
|
||||
.should("be.visible");
|
||||
|
||||
cy.log("refresh the page and check the content is persisted");
|
||||
cy.url().reload();
|
||||
|
||||
H.DataStudio.PythonLibrary.editor()
|
||||
.findByText(/hello world/)
|
||||
.should("be.visible");
|
||||
});
|
||||
|
||||
it("should allow editing the JavaScript library", () => {
|
||||
H.DataStudio.Transforms.visit();
|
||||
H.DataStudio.Transforms.list().findByText("Code library").click();
|
||||
cy.findByRole("link", { name: /JavaScript library/ }).click();
|
||||
|
||||
H.DataStudio.PythonLibrary.editor().should("be.visible");
|
||||
|
||||
cy.log("make sure placeholder with help comment is displayed");
|
||||
H.DataStudio.PythonLibrary.editor()
|
||||
.findByText(/\/\/ This is your JavaScript library/)
|
||||
.should("be.visible");
|
||||
H.DataStudio.PythonLibrary.editor()
|
||||
.findByText(/\/\/ You can add code here that can be reused/)
|
||||
.should("be.visible");
|
||||
|
||||
cy.log("modify and save the JavaScript library");
|
||||
H.DataStudio.PythonLibrary.editor()
|
||||
.findByRole("textbox")
|
||||
.click()
|
||||
.realType("console.log('hello world');");
|
||||
|
||||
cy.findByRole("button", { name: "Save" }).should("be.visible").click();
|
||||
|
||||
H.undoToast()
|
||||
.findByText(/library saved/i)
|
||||
.should("be.visible");
|
||||
|
||||
cy.log("refresh the page and check the content is persisted");
|
||||
|
||||
@@ -18,7 +18,7 @@ describe("issue #68378", () => {
|
||||
it("should show empty schemas when picking a target schema (metabase#68378)", () => {
|
||||
visitTransformListPage();
|
||||
cy.button("Create a transform").click();
|
||||
H.popover().findByText("SQL query").click();
|
||||
H.popover().findByText(/SQL/).click();
|
||||
H.popover().findByText("Writable Postgres12").click();
|
||||
H.NativeEditor.type("SELECT 42", { allowFastSet: true }).blur();
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ describe("scenarios > admin > transforms", () => {
|
||||
|
||||
visitTransformListPage();
|
||||
cy.button("Create a transform").click();
|
||||
H.popover().findByText("SQL query").click();
|
||||
H.popover().findByText(/SQL/).click();
|
||||
H.popover().findByText(DB_NAME).click();
|
||||
|
||||
function testDataReference() {
|
||||
@@ -270,7 +270,7 @@ describe("scenarios > admin > transforms", () => {
|
||||
cy.log("create a new transform");
|
||||
visitTransformListPage();
|
||||
cy.button("Create a transform").click();
|
||||
H.popover().findByText("SQL query").click();
|
||||
H.popover().findByText(/SQL/).click();
|
||||
|
||||
cy.log("Add a query with multiple template tags");
|
||||
H.NativeEditor.clear()
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
import { createLibraryWithItems } from "e2e/support/test-library-data";
|
||||
import { DataPermissionValue } from "metabase/admin/permissions/types";
|
||||
import type {
|
||||
AdvancedTransformType,
|
||||
CardType,
|
||||
CollectionId,
|
||||
PythonTransformTableAliases,
|
||||
@@ -122,7 +123,6 @@ describe("scenarios > admin > transforms", () => {
|
||||
});
|
||||
|
||||
H.goToMainApp();
|
||||
H.modal().button("Discard changes").click();
|
||||
H.newButton("Question").click();
|
||||
|
||||
H.miniPicker().within(() => {
|
||||
@@ -137,7 +137,7 @@ describe("scenarios > admin > transforms", () => {
|
||||
cy.log("create a new transform");
|
||||
visitTransformListPage();
|
||||
cy.button("Create a transform").click();
|
||||
H.popover().findByText("SQL query").click();
|
||||
H.popover().findByText(/SQL/).click();
|
||||
H.expectUnstructuredSnowplowEvent({
|
||||
event: "transform_create",
|
||||
event_detail: "native",
|
||||
@@ -177,10 +177,12 @@ describe("scenarios > admin > transforms", () => {
|
||||
cy.log("create a new transform");
|
||||
visitTransformListPage();
|
||||
cy.button("Create a transform").click();
|
||||
H.popover().findByText("Python script").click();
|
||||
H.popover()
|
||||
.findByText(/Python/)
|
||||
.click();
|
||||
H.expectUnstructuredSnowplowEvent({
|
||||
event: "transform_create",
|
||||
event_detail: "python",
|
||||
event_detail: "advanced",
|
||||
});
|
||||
|
||||
cy.findByTestId("python-transform-top-bar")
|
||||
@@ -560,7 +562,7 @@ LIMIT
|
||||
visitTransformListPage();
|
||||
cy.button("Create a transform").click();
|
||||
|
||||
H.popover().findByText("SQL query").click();
|
||||
H.popover().findByText(/SQL/).click();
|
||||
|
||||
cy.findByTestId("gui-builder-data")
|
||||
.findByText("Writable Postgres12")
|
||||
@@ -1616,7 +1618,7 @@ LIMIT
|
||||
cy.log("create a new transform");
|
||||
H.getTableId({ name: "Animals", databaseId: WRITABLE_DB_ID }).then(
|
||||
(id) => {
|
||||
createPythonTransform({
|
||||
createAdvancedTransform({
|
||||
body: dedent`
|
||||
import pandas as pd
|
||||
|
||||
@@ -1654,7 +1656,7 @@ LIMIT
|
||||
cy.log("create a new Python transform");
|
||||
H.getTableId({ name: "Animals", databaseId: WRITABLE_DB_ID }).then(
|
||||
(id) => {
|
||||
createPythonTransform({
|
||||
createAdvancedTransform({
|
||||
body: dedent`
|
||||
import pandas as pd
|
||||
|
||||
@@ -1694,7 +1696,7 @@ LIMIT
|
||||
cy.log("create a new Python transform");
|
||||
H.getTableId({ name: "Animals", databaseId: WRITABLE_DB_ID }).then(
|
||||
(id) => {
|
||||
createPythonTransform({
|
||||
createAdvancedTransform({
|
||||
body: dedent`
|
||||
import pandas as pd
|
||||
|
||||
@@ -1730,7 +1732,7 @@ LIMIT
|
||||
cy.log("create a new Python transform");
|
||||
H.getTableId({ name: "Animals", databaseId: WRITABLE_DB_ID }).then(
|
||||
(id) => {
|
||||
createPythonTransform({
|
||||
createAdvancedTransform({
|
||||
body: dedent`
|
||||
import pandas as pd
|
||||
|
||||
@@ -1825,7 +1827,7 @@ LIMIT
|
||||
cy.log("create a new SQL transform with a complex query");
|
||||
visitTransformListPage();
|
||||
cy.button("Create a transform").click();
|
||||
H.popover().findByText("SQL query").click();
|
||||
H.popover().findByText(/SQL/).click();
|
||||
H.popover().findByText(DB_NAME).click();
|
||||
|
||||
H.NativeEditor.type(
|
||||
@@ -2169,7 +2171,9 @@ LIMIT
|
||||
|
||||
visitTransformListPage();
|
||||
cy.button("Create a transform").click();
|
||||
H.popover().findByText("Python script").click();
|
||||
H.popover()
|
||||
.findByText(/Python/)
|
||||
.click();
|
||||
|
||||
cy.log("import common should be included by default");
|
||||
H.PythonEditor.value().should("contain", "import common");
|
||||
@@ -2242,9 +2246,10 @@ LIMIT
|
||||
() => {
|
||||
visitTransformListPage();
|
||||
cy.button("Create a transform").click();
|
||||
H.popover().findByText("Python script").click();
|
||||
H.popover()
|
||||
.findByText(/Python/)
|
||||
.click();
|
||||
cy.get(".cm-clickable-token").should("be.visible").click();
|
||||
H.modal().button("Discard changes").click();
|
||||
cy.url().should("include", "/data-studio/transforms/library/common.py");
|
||||
cy.findByTestId("python-library-header").should("be.visible");
|
||||
},
|
||||
@@ -2259,7 +2264,9 @@ LIMIT
|
||||
cy.stub(win, "open").as("windowOpen");
|
||||
});
|
||||
cy.button("Create a transform").click();
|
||||
H.popover().findByText("Python script").click();
|
||||
H.popover()
|
||||
.findByText(/Python/)
|
||||
.click();
|
||||
cy.get(".cm-clickable-token").should("be.visible").click(H.holdMetaKey);
|
||||
|
||||
cy.get("@windowOpen").should(
|
||||
@@ -2277,7 +2284,9 @@ LIMIT
|
||||
|
||||
visitTransformListPage();
|
||||
cy.button("Create a transform").click();
|
||||
H.popover().findByText("Python script").click();
|
||||
H.popover()
|
||||
.findByText(/Python/)
|
||||
.click();
|
||||
|
||||
cy.log("import common should be included by default");
|
||||
H.PythonEditor.value().should("contain", "import common");
|
||||
@@ -2578,8 +2587,8 @@ LIMIT
|
||||
getTransformsList().findByText("Name").click();
|
||||
|
||||
getRowNames().should("deep.equal", [
|
||||
"Code library",
|
||||
"Middle Transform",
|
||||
"Python library",
|
||||
"Reports",
|
||||
"Alpha Transform",
|
||||
"Zebra Transform",
|
||||
@@ -2591,8 +2600,8 @@ LIMIT
|
||||
"Zebra Transform",
|
||||
"Reports",
|
||||
"Alpha Transform",
|
||||
"Python library",
|
||||
"Middle Transform",
|
||||
"Code library",
|
||||
]);
|
||||
|
||||
cy.log("verify sorting by output table column ascending");
|
||||
@@ -2600,7 +2609,7 @@ LIMIT
|
||||
getRowNames().should("deep.equal", [
|
||||
"Reports",
|
||||
"Alpha Transform",
|
||||
"Python library",
|
||||
"Code library",
|
||||
"Middle Transform",
|
||||
"Zebra Transform",
|
||||
]);
|
||||
@@ -2612,7 +2621,7 @@ LIMIT
|
||||
"Middle Transform",
|
||||
"Reports",
|
||||
"Alpha Transform",
|
||||
"Python library",
|
||||
"Code library",
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -2709,6 +2718,7 @@ LIMIT
|
||||
|
||||
cy.log("Python library should be visible in the list");
|
||||
getTransformsList().within(() => {
|
||||
cy.findByText("Code library").click();
|
||||
cy.findByText("Python library").should("be.visible");
|
||||
});
|
||||
|
||||
@@ -2804,12 +2814,13 @@ LIMIT
|
||||
cy.button("Create a transform").should("not.exist");
|
||||
|
||||
cy.log("clicking Python library navigates to the library editor");
|
||||
getTransformsList().findByText("Code library").click();
|
||||
getTransformsList().findByText("Python library").click();
|
||||
|
||||
cy.log("python library editor is read-only");
|
||||
cy.url().should("include", "/data-studio/transforms/library/common.py");
|
||||
cy.findByRole("alert")
|
||||
.contains(/The Python library is not editable/)
|
||||
.contains(/The library is not editable/)
|
||||
.should("be.visible");
|
||||
|
||||
H.DataStudio.PythonLibrary.editor().within(() => {
|
||||
@@ -3668,7 +3679,7 @@ describe(
|
||||
H.expectNoBadSnowplowEvents();
|
||||
});
|
||||
|
||||
it("should be possible to test run a Python script", () => {
|
||||
it("should be possible to test run a Python transform", () => {
|
||||
H.getTableId({ name: "Animals", databaseId: WRITABLE_DB_ID }).then(
|
||||
(id) => {
|
||||
createPythonLibrary(
|
||||
@@ -3679,7 +3690,7 @@ describe(
|
||||
`,
|
||||
);
|
||||
|
||||
createPythonTransform({
|
||||
createAdvancedTransform({
|
||||
body: dedent`
|
||||
import pandas as pd
|
||||
import common
|
||||
@@ -3724,7 +3735,7 @@ describe(
|
||||
it("should display preview notice message", () => {
|
||||
H.getTableId({ name: "Animals", databaseId: WRITABLE_DB_ID }).then(
|
||||
(id) => {
|
||||
createPythonTransform({
|
||||
createAdvancedTransform({
|
||||
body: dedent`
|
||||
import pandas as pd
|
||||
|
||||
@@ -3775,7 +3786,7 @@ describe("scenarios > admin > transforms", () => {
|
||||
cy.log("create a new transform");
|
||||
visitTransformListPage();
|
||||
cy.button("Create a transform").click();
|
||||
H.popover().findByText("SQL query").click();
|
||||
H.popover().findByText(/SQL/).click();
|
||||
|
||||
cy.findByTestId("gui-builder-data")
|
||||
.findByText("Select a database")
|
||||
@@ -3786,7 +3797,9 @@ describe("scenarios > admin > transforms", () => {
|
||||
cy.log("create a new transform");
|
||||
visitTransformListPage();
|
||||
cy.button("Create a transform").click();
|
||||
H.popover().findByText("Python script").click();
|
||||
H.popover()
|
||||
.findByText(/Python/)
|
||||
.click();
|
||||
|
||||
cy.findByTestId("python-transform-top-bar")
|
||||
.findByText("Select a database")
|
||||
@@ -4002,7 +4015,9 @@ function createSqlTransform(opts: {
|
||||
});
|
||||
}
|
||||
|
||||
function createPythonTransform(opts: {
|
||||
function createAdvancedTransform(opts: {
|
||||
name?: string;
|
||||
type?: AdvancedTransformType;
|
||||
body: string;
|
||||
sourceTables: PythonTransformTableAliases;
|
||||
targetTable?: string;
|
||||
@@ -4010,7 +4025,7 @@ function createPythonTransform(opts: {
|
||||
tagIds?: TransformTagId[];
|
||||
visitTransform?: boolean;
|
||||
}) {
|
||||
return H.createPythonTransform({
|
||||
return H.createAdvancedTransform({
|
||||
targetTable: TARGET_TABLE,
|
||||
targetSchema: TARGET_SCHEMA,
|
||||
...opts,
|
||||
@@ -4055,7 +4070,7 @@ function getPythonDataPicker() {
|
||||
}
|
||||
|
||||
function createPythonLibrary(path: string, source: string) {
|
||||
cy.request("PUT", `/api/ee/transforms-python/library/${path}`, {
|
||||
cy.request("PUT", `/api/ee/advanced-transforms/library/${path}`, {
|
||||
source,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1685,7 +1685,7 @@ describe("scenarios > data studio > workspaces", () => {
|
||||
cy.log("Create transform via UI with model reference");
|
||||
Workspaces.visitTransformListPage();
|
||||
cy.button("Create a transform").click();
|
||||
H.popover().findByText("SQL query").click();
|
||||
H.popover().findByText(/SQL/).click();
|
||||
H.popover().findByText("Writable Postgres12").click();
|
||||
|
||||
cy.get("@modelId").then((modelId) => {
|
||||
@@ -1763,7 +1763,7 @@ describe("scenarios > data studio > workspaces", () => {
|
||||
databaseId: WRITABLE_DB_ID,
|
||||
schema: "Schema B",
|
||||
}).then((id2) => {
|
||||
createPythonTransform({
|
||||
createAdvancedTransform({
|
||||
body: TEST_PYTHON_TRANSFORM,
|
||||
sourceTables: { foo: id1, bar: id2 },
|
||||
visitTransform: true,
|
||||
@@ -1906,7 +1906,7 @@ describe("scenarios > data studio > workspaces", () => {
|
||||
databaseId: WRITABLE_DB_ID,
|
||||
schema: "Schema B",
|
||||
}).then((id2) => {
|
||||
createPythonTransform({
|
||||
createAdvancedTransform({
|
||||
body: TEST_PYTHON_TRANSFORM_MULTI_TABLE,
|
||||
sourceTables: { animals_a: id1, animals_b: id2 },
|
||||
visitTransform: true,
|
||||
@@ -2269,7 +2269,7 @@ describe("scenarios > data studio > workspaces", () => {
|
||||
it("should not show error when editing a new transform in a workspace (GDGT-1445)", () => {
|
||||
Workspaces.visitTransformListPage();
|
||||
cy.findByLabelText("Create a transform").click();
|
||||
H.popover().findByText("SQL query").click();
|
||||
H.popover().findByText(/SQL/).click();
|
||||
// to avoid flakiness on editor's focus
|
||||
cy.findByTestId("native-query-top-bar").should(
|
||||
"contain.text",
|
||||
@@ -2515,7 +2515,7 @@ function createTransforms({ visit }: { visit?: boolean } = { visit: false }) {
|
||||
});
|
||||
|
||||
H.getTableId({ name: "Animals", databaseId: WRITABLE_DB_ID }).then((id) => {
|
||||
createPythonTransform({
|
||||
createAdvancedTransform({
|
||||
body: TEST_PYTHON_TRANSFORM,
|
||||
sourceTables: { foo: id },
|
||||
});
|
||||
@@ -2561,7 +2561,7 @@ function createSqlTransform(opts: {
|
||||
});
|
||||
}
|
||||
|
||||
function createPythonTransform(opts: {
|
||||
function createAdvancedTransform(opts: {
|
||||
body: string;
|
||||
sourceTables: PythonTransformTableAliases;
|
||||
targetTable?: string;
|
||||
@@ -2569,7 +2569,7 @@ function createPythonTransform(opts: {
|
||||
tagIds?: TransformTagId[];
|
||||
visitTransform?: boolean;
|
||||
}) {
|
||||
return H.createPythonTransform({
|
||||
return H.createAdvancedTransform({
|
||||
targetTable: TARGET_TABLE_PYTHON,
|
||||
targetSchema: TARGET_SCHEMA,
|
||||
...opts,
|
||||
|
||||
@@ -182,7 +182,10 @@ describe(
|
||||
|
||||
cy.log("Should be able to visit and the see new transform");
|
||||
viewLastSuggestion();
|
||||
cy.url().should("include", "/data-studio/transforms/new/python");
|
||||
cy.url().should(
|
||||
"include",
|
||||
"/data-studio/transforms/new/advanced/python",
|
||||
);
|
||||
assertEditorDiffState({ exists: false }); // nothing to diff, so we shouldn't show the UI for it
|
||||
assertEditorContent("python", "pd.DataFrame({'value': [1]})");
|
||||
|
||||
@@ -375,7 +378,7 @@ describe(
|
||||
it("should update existing Python transform via metabot", () => {
|
||||
H.getTableId({ name: SOURCE_TABLE, databaseId: WRITABLE_DB_ID }).then(
|
||||
(tableId) => {
|
||||
H.createPythonTransform({
|
||||
H.createAdvancedTransform({
|
||||
targetTable: "transform_table",
|
||||
targetSchema: "Schema A",
|
||||
body: dedent`
|
||||
|
||||
@@ -398,7 +398,7 @@ describe("issue 52806", () => {
|
||||
|
||||
it("should remove parameter values from the URL when leaving the query builder and discarding changes (metabase#52806)", () => {
|
||||
cy.visit("/");
|
||||
H.newButton("SQL query").click();
|
||||
H.newButton(/SQL/).click();
|
||||
H.NativeEditor.focus().type("select {{x}}");
|
||||
cy.location().should((location) => expect(location.search).to.eq("?x="));
|
||||
cy.findByTestId("main-logo-link").click();
|
||||
@@ -441,7 +441,7 @@ describe("issue 55951", () => {
|
||||
},
|
||||
);
|
||||
|
||||
H.newButton("SQL query").click();
|
||||
H.newButton(/SQL/).click();
|
||||
H.popover()
|
||||
.should("be.visible")
|
||||
.within(() => {
|
||||
|
||||
@@ -801,7 +801,7 @@ describe("no native access", { tags: ["@external", "@skip"] }, () => {
|
||||
|
||||
cy.log("#32387");
|
||||
cy.findByRole("button", { name: /New/ }).click();
|
||||
H.popover().findByText("SQL query").click();
|
||||
H.popover().findByText(/SQL/).click();
|
||||
|
||||
cy.wait("@database");
|
||||
cy.go("back");
|
||||
|
||||
@@ -39,7 +39,7 @@
|
||||
[metabase-enterprise.stale.api]
|
||||
[metabase-enterprise.support-access-grants.api]
|
||||
[metabase-enterprise.tenants.api]
|
||||
[metabase-enterprise.transforms-python.api]
|
||||
[metabase-enterprise.transforms-runner.api]
|
||||
[metabase-enterprise.upload-management.api]
|
||||
[metabase-enterprise.workspaces.api]
|
||||
[metabase.api.macros :as api.macros]
|
||||
@@ -130,7 +130,7 @@
|
||||
"/metabot-tools" metabase-enterprise.metabot-v3.tools.api/routes
|
||||
"/metabot-v3" (premium-handler metabase-enterprise.metabot-v3.api/routes :metabot-v3)
|
||||
"/permission_debug" (premium-handler metabase-enterprise.permission-debug.api/routes :advanced-permissions)
|
||||
"/transforms-python" (premium-handler metabase-enterprise.transforms-python.api/routes :transforms-python)
|
||||
"/advanced-transforms" (premium-handler metabase-enterprise.transforms-runner.api/routes :transforms-python)
|
||||
"/scim" (premium-handler metabase-enterprise.scim.routes/routes :scim)
|
||||
"/semantic-search" (premium-handler metabase-enterprise.semantic-search.api/routes :semantic-search)
|
||||
"/serialization" (premium-handler metabase-enterprise.serialization.api/routes :serialization)
|
||||
|
||||
@@ -21,5 +21,8 @@
|
||||
[metabase-enterprise.sso.init]
|
||||
[metabase-enterprise.stale.init]
|
||||
[metabase-enterprise.support-access-grants.init]
|
||||
[metabase-enterprise.transforms-clojure.init]
|
||||
[metabase-enterprise.transforms-javascript.init]
|
||||
[metabase-enterprise.transforms-python.init]
|
||||
[metabase-enterprise.transforms-runner.init]
|
||||
[metabase-enterprise.workspaces.init]))
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
(ns metabase-enterprise.metabot-v3.tools.transforms
|
||||
(:require
|
||||
[metabase-enterprise.metabot-v3.tools.util :as metabot-v3.tools.u]
|
||||
[metabase-enterprise.transforms-python.api :as transforms-python.api]
|
||||
[metabase-enterprise.transforms-runner.api :as transforms-runner.api]
|
||||
[metabase.transforms.core :as transforms]
|
||||
[metabase.transforms.schema :as transforms.schema]
|
||||
[metabase.util.malli.registry :as mr]))
|
||||
@@ -15,7 +15,7 @@
|
||||
(try
|
||||
{:structured_output
|
||||
(->> (transforms/get-transforms)
|
||||
(into [] (comp (filter #(or (transforms/python-transform? %)
|
||||
(into [] (comp (filter #(or (transforms/runner-transform? %)
|
||||
(transforms/native-query-transform? %)))
|
||||
(filter :source_readable)
|
||||
(map #(select-keys % [:id :entity_id :name :type :description :source])))))}
|
||||
@@ -36,6 +36,6 @@
|
||||
[{:keys [path]}]
|
||||
(try
|
||||
{:structured_output
|
||||
(transforms-python.api/get-python-library-by-path path)}
|
||||
(transforms-runner.api/get-library-by-path path nil)}
|
||||
(catch Exception e
|
||||
(metabot-v3.tools.u/handle-agent-error e))))
|
||||
|
||||
@@ -24,7 +24,7 @@
|
||||
(:import (metabase_enterprise.remote_sync.source.protocol SourceSnapshot)))
|
||||
|
||||
(defn- snapshot-has-transforms?
|
||||
"Checks if the snapshot contains any Transform, PythonLibrary entities,
|
||||
"Checks if the snapshot contains any Transform, TransformLibrary entities,
|
||||
non-built-in TransformTags, or transforms-namespace collections.
|
||||
Used to auto-enable/disable remote-sync-transforms setting during import.
|
||||
|
||||
@@ -179,7 +179,7 @@
|
||||
Conflict types detected:
|
||||
- :entity-id-conflict - Items with existing entity IDs that are NOT already synced
|
||||
- :library-conflict - First import only, local Library exists, import has Library
|
||||
- :transforms-not-enabled - Import has Transform/TransformTag/PythonLibrary, setting disabled
|
||||
- :transforms-not-enabled - Import has Transform/TransformTag/TransformLibrary, setting disabled
|
||||
- :snippets-without-library - Import has NativeQuerySnippet, Library not remote-synced"
|
||||
[ingestable first-import?]
|
||||
(let [ingest-list (serialization/ingest-list ingestable)
|
||||
@@ -242,7 +242,7 @@
|
||||
last-imported-version (remote-sync.task/last-version)
|
||||
first-import? (nil? last-imported-version)
|
||||
path-filters [#"collections/.*" #"databases/.*" #"actions/.*"
|
||||
#"transforms/.*" #"python-libraries/.*" #"snippets/.*"]
|
||||
#"transforms/.*" #"transform-libraries/.*" #"snippets/.*"]
|
||||
base-ingestable (source.p/->ingestable snapshot {:path-filters path-filters})
|
||||
has-transforms? (snapshot-has-transforms? base-ingestable)
|
||||
{:keys [conflicts summary]} (get-conflicts base-ingestable first-import?)
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
[clojure.string :as str]
|
||||
[clojure.walk :as walk]
|
||||
[metabase-enterprise.remote-sync.settings :as rs-settings]
|
||||
[metabase-enterprise.transforms-python.core :as transforms-python]
|
||||
[metabase-enterprise.transforms-runner.models.transform-library :as transform-library]
|
||||
[metabase.collections.core :as collections]
|
||||
[metabase.collections.models.collection :as collection]
|
||||
[metabase.models.serialization :as serdes]
|
||||
@@ -71,7 +71,7 @@
|
||||
- :export-scope - Export scope for query-export-roots:
|
||||
:root-collections - Query root-level remote-synced + namespace collections (Collection)
|
||||
:root-only - Query root instances with collection_id = nil (Transform)
|
||||
:all - Query all instances (TransformTag, PythonLibrary, NativeQuerySnippet)
|
||||
:all - Query all instances (TransformTag, TransformLibrary, NativeQuerySnippet)
|
||||
nil/:derived - No root query; derived from other models via serdes/descendants
|
||||
- :enabled? - true, or setting keyword (e.g., :remote-sync-transforms, :library-synced).
|
||||
When :library-synced, uses the library-is-remote-synced? setting."
|
||||
@@ -302,18 +302,18 @@
|
||||
:export-scope :all ; query for all instances
|
||||
:enabled? :remote-sync-transforms}
|
||||
|
||||
:model/PythonLibrary
|
||||
{:model-type "PythonLibrary"
|
||||
:model-key :model/PythonLibrary
|
||||
:model/TransformLibrary
|
||||
{:model-type "TransformLibrary"
|
||||
:model-key :model/TransformLibrary
|
||||
:identity :entity-id
|
||||
:events {:prefix :event/python-library
|
||||
:types [:create :update]} ; no delete - upsert only
|
||||
:events {:prefix :event/transform-library
|
||||
:types [:create :update]} ; no delete - upsert only
|
||||
:eligibility {:type :setting
|
||||
:setting :remote-sync-transforms}
|
||||
:archived-key nil ; no archived field
|
||||
:tracking {:select-fields [:path]
|
||||
:field-mappings {:model_name :path}}
|
||||
:conditions {:entity_id [:not= transforms-python/builtin-entity-id]} ; exclude built-in common.py from sync
|
||||
:conditions {:entity_id [:not-in (transform-library/all-builtin-entity-ids)]} ; exclude built-in libraries from sync
|
||||
:removal {:statuses #{"removed" "delete"} ; no scope-key = global deletion
|
||||
:all-on-setting-disable :remote-sync-transforms}
|
||||
:export-scope :all ; query for all instances
|
||||
@@ -506,7 +506,7 @@
|
||||
|
||||
(defn- has-unsynced-entities-for-feature?
|
||||
"Returns true if any model in the feature group has local entities not tracked in RemoteSyncObject.
|
||||
Excludes built-in TransformTags and the built-in PythonLibrary from the count since they are
|
||||
Excludes built-in TransformTags and built-in TransformLibrary entries from the count since they are
|
||||
system-created and not user data. Namespace collections are not checked here because they are
|
||||
organizational containers, not user data that would be lost on import."
|
||||
[specs-for-feature]
|
||||
@@ -516,7 +516,7 @@
|
||||
;; Exclude built-in entities from count (they are system-created, not user data)
|
||||
local-count (case model-key
|
||||
:model/TransformTag (t2/count model-key :built_in_type nil)
|
||||
:model/PythonLibrary (t2/count model-key :entity_id [:not= transforms-python/builtin-entity-id])
|
||||
:model/TransformLibrary (t2/count model-key :entity_id [:not-in (transform-library/all-builtin-entity-ids)])
|
||||
(t2/count model-key))
|
||||
synced-count (t2/count :model/RemoteSyncObject :model_type model-type)]
|
||||
(and (pos? local-count)
|
||||
|
||||
@@ -35,7 +35,7 @@
|
||||
(conj "Setting")
|
||||
|
||||
(not (:no-transforms opts))
|
||||
(conj "Transform" "TransformTag" "TransformJob" "PythonLibrary")))
|
||||
(conj "Transform" "TransformTag" "TransformJob" "TransformLibrary")))
|
||||
|
||||
(defn make-targets-of-type
|
||||
"Returns a targets seq with model type and given ids"
|
||||
|
||||
@@ -74,7 +74,7 @@
|
||||
read-timestamps))
|
||||
|
||||
(def legal-top-level-paths "Known top-level paths for directory with serialization output"
|
||||
#{"actions" "collections" "databases" "glossary" "python-libraries" "snippets" "transforms"})
|
||||
#{"actions" "collections" "databases" "glossary" "transform-libraries" "snippets" "transforms"})
|
||||
|
||||
(defn- ingest-all [^File root-dir]
|
||||
;; This returns a map {unlabeled-hierarchy [original-hierarchy File]}.
|
||||
|
||||
@@ -35,7 +35,7 @@
|
||||
content
|
||||
["FieldValues"
|
||||
"Metabot"
|
||||
"PythonLibrary"
|
||||
"TransformLibrary"
|
||||
"Setting"
|
||||
"Transform"
|
||||
"TransformJob"
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
(ns metabase-enterprise.transforms-clojure.impl
|
||||
"Registers :clojure as a runner language. Loaded by transforms-clojure/init.clj for side effects.
|
||||
See transforms.interface/register-runner! and transforms.interface/lang-config."
|
||||
(:require
|
||||
[metabase.transforms.interface :as transforms.i]))
|
||||
|
||||
(transforms.i/register-runner! :clojure)
|
||||
|
||||
(defmethod transforms.i/lang-config :clojure [_]
|
||||
{:runtime "clojure"
|
||||
:label "Clojure"
|
||||
:timing-key :clojure-execution
|
||||
:extension ".clj"})
|
||||
@@ -0,0 +1,3 @@
|
||||
(ns metabase-enterprise.transforms-clojure.init
|
||||
(:require
|
||||
[metabase-enterprise.transforms-clojure.impl]))
|
||||
@@ -0,0 +1,13 @@
|
||||
(ns metabase-enterprise.transforms-javascript.impl
|
||||
"Registers :javascript as a runner language. Loaded by transforms-javascript/init.clj for side effects.
|
||||
See transforms.interface/register-runner! and transforms.interface/lang-config."
|
||||
(:require
|
||||
[metabase.transforms.interface :as transforms.i]))
|
||||
|
||||
(transforms.i/register-runner! :javascript)
|
||||
|
||||
(defmethod transforms.i/lang-config :javascript [_]
|
||||
{:runtime "javascript"
|
||||
:label "JavaScript"
|
||||
:timing-key :javascript-execution
|
||||
:extension ".js"})
|
||||
@@ -0,0 +1,3 @@
|
||||
(ns metabase-enterprise.transforms-javascript.init
|
||||
(:require
|
||||
[metabase-enterprise.transforms-javascript.impl]))
|
||||
@@ -1,10 +0,0 @@
|
||||
(ns metabase-enterprise.transforms-python.core
|
||||
"Core namespace for Python transforms functionality.
|
||||
Re-exports commonly used vars from internal namespaces."
|
||||
(:require
|
||||
[metabase-enterprise.transforms-python.models.python-library :as python-library]
|
||||
[potemkin :as p]))
|
||||
|
||||
(p/import-vars
|
||||
[python-library
|
||||
builtin-entity-id])
|
||||
@@ -1,369 +0,0 @@
|
||||
(ns metabase-enterprise.transforms-python.execute
|
||||
(:require
|
||||
[clojure.core.async :as a]
|
||||
[clojure.java.io :as io]
|
||||
[clojure.string :as str]
|
||||
[metabase-enterprise.transforms-python.python-runner :as python-runner]
|
||||
[metabase-enterprise.transforms-python.s3 :as s3]
|
||||
[metabase-enterprise.transforms-python.settings :as transforms-python.settings]
|
||||
[metabase.app-db.core :as app-db]
|
||||
[metabase.driver :as driver]
|
||||
[metabase.driver.connection :as driver.conn]
|
||||
[metabase.driver.util :as driver.u]
|
||||
[metabase.transforms.core :as transforms]
|
||||
[metabase.transforms.instrumentation :as transforms.instrumentation]
|
||||
[metabase.transforms.interface :as transforms.i]
|
||||
[metabase.transforms.util :as transforms.util]
|
||||
[metabase.util :as u]
|
||||
[metabase.util.format :as u.format]
|
||||
[metabase.util.i18n :as i18n]
|
||||
[metabase.util.jvm :as u.jvm]
|
||||
[metabase.util.log :as log]
|
||||
[toucan2.core :as t2])
|
||||
(:import
|
||||
(java.io Closeable)
|
||||
(java.net SocketException)
|
||||
(java.nio.file Files)
|
||||
(java.nio.file.attribute FileAttribute)
|
||||
(java.time Duration)))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(defn- empty-message-log
|
||||
"Returns a new message log.
|
||||
|
||||
Python transforms produce output while they run. The message log is a stateful structure that buffers log outputs during the run.
|
||||
The goal is to assist in debugging in the case of python runtime or syntax errors, and to provide some immediate feedback for longer running transforms.
|
||||
|
||||
Important Note: in memory buffering is considered acceptable as we expect /logs output to be truncated or small.
|
||||
|
||||
The intended use of this structure is to produce a string suitable for `transform_run.message`, saving it every few seconds."
|
||||
[]
|
||||
(atom {:pre-python [] ; log! outputs previous to the python execution, i.e table read progress
|
||||
:python nil ; events json structured logs from the /logs endpoint
|
||||
:post-python []})) ; log! outputs after the python execution, i.e. output reads, writes to target
|
||||
|
||||
(defn- log!
|
||||
"Appends a string to the message log, the string is user facing and should be suitable for presentation as part of the `transform_run.message` field."
|
||||
[message-log s]
|
||||
(swap! message-log (fn [m] (update m (if (:python m) :post-python :pre-python) conj s)))
|
||||
nil)
|
||||
|
||||
(defn- replace-python-logs!
|
||||
"Called when events output is received from a /logs call to copy those events into the message-log."
|
||||
[message-log events]
|
||||
(swap! message-log assoc :python events)
|
||||
nil)
|
||||
|
||||
(defn- message-log->transform-run-message
|
||||
"Presents the content of the log as a string suitable for storage in transform_run.message."
|
||||
[message-log]
|
||||
(let [{:keys [pre-python python post-python]} @message-log]
|
||||
(str/join "\n" (concat pre-python
|
||||
(map :message python)
|
||||
post-python))))
|
||||
|
||||
(defn- save-log-to-transform-run-message!
|
||||
"Saves the content of the log into the transform_run's message field. Lossy, string contains user facing information only."
|
||||
[run-id message-log]
|
||||
(t2/update! :model/TransformRun
|
||||
:id run-id
|
||||
{:message (message-log->transform-run-message message-log)}))
|
||||
|
||||
(def ^:private ^Duration python-message-loop-sleep-duration
|
||||
(Duration/ofMillis 1000))
|
||||
|
||||
(defn- python-message-update-loop!
|
||||
"Block while relevant log data is replicated from the runner into the message log.
|
||||
When new logs are received, the log data will be flushed to the transform_run.message field as a string.
|
||||
|
||||
The loop will exit on error or when interrupted."
|
||||
[run-id message-log]
|
||||
(try
|
||||
(loop []
|
||||
(if (.isInterrupted (Thread/currentThread))
|
||||
(log/debug "Message update loop interrupted")
|
||||
(do (let [sleep-ms (.toMillis python-message-loop-sleep-duration)]
|
||||
(when (pos? sleep-ms) (Thread/sleep sleep-ms)))
|
||||
(let [{:keys [status body]} (python-runner/get-logs run-id)]
|
||||
(cond
|
||||
(<= 200 status 299)
|
||||
(let [{:keys [execution_id events]} body]
|
||||
(if-not (= run-id execution_id)
|
||||
(do (log/debugf "Run id did not match expected: %s actual: %s" run-id execution_id)
|
||||
(recur))
|
||||
(do
|
||||
(replace-python-logs! message-log events)
|
||||
(save-log-to-transform-run-message! run-id message-log)
|
||||
(recur))))
|
||||
(= 404 status)
|
||||
(do
|
||||
(log/debugf "No logs yet (or run finished), run-id: %s" run-id)
|
||||
(recur))
|
||||
:else
|
||||
(do
|
||||
(log/warnf "Unexpected status polling for logs %s %s, run-id: %s" status body run-id)
|
||||
(log/debug "Exiting due to poll error")))))))
|
||||
(catch SocketException se (when-not (= "Closed by interrupt" (ex-message se)) (throw se)))
|
||||
(catch InterruptedException _)
|
||||
(catch Throwable e
|
||||
(log/errorf e "An exception was caught during msg update loop, run-id: %s" run-id))))
|
||||
|
||||
(defn- open-python-message-update-future! ^Closeable [run-id message-log]
|
||||
(let [cleanup (fn [fut]
|
||||
(future-cancel fut)
|
||||
(if (= ::timeout (try (deref fut 10000 ::timeout) (catch Throwable _)))
|
||||
(log/fatalf "Log polling task did not respond to interrupt, run-id: %s" run-id)
|
||||
(log/debugf "Log polling task done, run-id: %s" run-id)))
|
||||
fut (u.jvm/in-virtual-thread*
|
||||
(python-message-update-loop! run-id message-log))]
|
||||
(reify Closeable
|
||||
(close [_] (cleanup fut)))))
|
||||
|
||||
(defn- table-schema [table-name metadata]
|
||||
{:name (if (keyword? table-name) table-name (keyword table-name))
|
||||
:columns (mapv (fn [{:keys [name base_type #_database_type]}]
|
||||
{:name name
|
||||
:type (python-runner/restricted-insert-type base_type)
|
||||
;; :database-type database_type
|
||||
:nullable? true})
|
||||
(:fields metadata))})
|
||||
|
||||
(defn- insert-data!
|
||||
"Insert data from source into an existing table."
|
||||
[driver db-id table-schema data-source]
|
||||
(let [data-source (assoc data-source :table-schema table-schema)]
|
||||
(driver/insert-from-source! driver db-id table-schema data-source)))
|
||||
|
||||
(defn- create-table-and-insert-data!
|
||||
"Create a table from metadata and insert data from source."
|
||||
[driver db-id table-schema data-source]
|
||||
(transforms.util/create-table-from-schema! driver db-id table-schema)
|
||||
(insert-data! driver db-id table-schema data-source))
|
||||
|
||||
(defn- transfer-with-rename-tables-strategy!
|
||||
"Transfer data using the rename-tables*! multimethod with atomicity guarantees.
|
||||
Creates new table, then atomically renames target->old and new->target, then drops old."
|
||||
[driver db-id table-name metadata data-source]
|
||||
(let [source-table-name (driver.u/temp-table-name driver table-name)
|
||||
temp-table-name (driver.u/temp-table-name driver table-name)]
|
||||
(log/info "Using rename-tables strategy with atomicity guarantees")
|
||||
(try
|
||||
|
||||
(create-table-and-insert-data! driver db-id (table-schema source-table-name metadata) data-source)
|
||||
(transforms.util/rename-tables! driver db-id {table-name temp-table-name
|
||||
source-table-name table-name})
|
||||
(transforms.util/drop-table! driver db-id temp-table-name)
|
||||
|
||||
(catch Exception e
|
||||
(log/error e "Failed to transfer data using rename-tables strategy")
|
||||
(try
|
||||
(transforms.util/drop-table! driver db-id source-table-name)
|
||||
(catch Exception _))
|
||||
(throw e)))))
|
||||
|
||||
(defn- transfer-with-create-drop-rename-strategy!
|
||||
"Transfer data using create + drop + rename to minimize time without data.
|
||||
Creates new table, drops old table, then renames new->target."
|
||||
[driver db-id table-name metadata data-source]
|
||||
(let [source-table-name (driver.u/temp-table-name driver table-name)]
|
||||
(log/info "Using create-drop-rename strategy to minimize downtime")
|
||||
(try
|
||||
|
||||
(create-table-and-insert-data! driver db-id (table-schema source-table-name metadata) data-source)
|
||||
(transforms.util/drop-table! driver db-id table-name)
|
||||
(driver/rename-table! driver db-id source-table-name table-name)
|
||||
|
||||
(catch Exception e
|
||||
(log/error e "Failed to transfer data using create-drop-rename strategy")
|
||||
(try
|
||||
(transforms.util/drop-table! driver db-id source-table-name)
|
||||
(catch Exception _))
|
||||
(throw e)))))
|
||||
|
||||
(defn- transfer-with-drop-create-fallback-strategy!
|
||||
"Transfer data using drop + create fallback strategy.
|
||||
Drops old table, then creates new table directly with target name."
|
||||
[driver db-id table-name metadata data-source]
|
||||
(log/info "Using drop-create fallback strategy")
|
||||
(try
|
||||
|
||||
(transforms.util/drop-table! driver db-id table-name)
|
||||
(create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source)
|
||||
|
||||
(catch Exception e
|
||||
(log/error e "Failed to transfer data using drop-create fallback strategy")
|
||||
(throw e))))
|
||||
|
||||
(defmulti ^:private transfer-file-to-db
|
||||
{:arglists '([driver db transform metadata temp-file])}
|
||||
(fn [_ _ transform _ _] (-> transform :target :type keyword)))
|
||||
|
||||
(defmethod transfer-file-to-db :table-incremental
|
||||
[driver {db-id :id}
|
||||
{:keys [target] :as transform}
|
||||
metadata temp-file]
|
||||
(let [table-name (transforms.util/qualified-table-name driver target)
|
||||
table-exists? (transforms.util/target-table-exists? transform)
|
||||
data-source {:type :jsonl-file
|
||||
:file temp-file}]
|
||||
|
||||
;; once we have more than just append, dispatch on :target-incremental-strategy
|
||||
|
||||
(if (not table-exists?)
|
||||
(do
|
||||
(log/info "New table")
|
||||
(create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))
|
||||
(insert-data! driver db-id (table-schema table-name metadata) data-source))))
|
||||
|
||||
(defmethod transfer-file-to-db :table
|
||||
[driver {db-id :id :as db}
|
||||
{:keys [target] :as transform}
|
||||
metadata temp-file]
|
||||
(let [table-name (transforms.util/qualified-table-name driver target)
|
||||
table-exists? (transforms.util/target-table-exists? transform)
|
||||
data-source {:type :jsonl-file
|
||||
:file temp-file}]
|
||||
(cond
|
||||
(not table-exists?)
|
||||
(do
|
||||
(log/info "New table")
|
||||
(create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))
|
||||
|
||||
(driver.u/supports? driver :atomic-renames db)
|
||||
(transfer-with-rename-tables-strategy! driver db-id table-name metadata data-source)
|
||||
|
||||
(driver.u/supports? driver :rename db)
|
||||
(transfer-with-create-drop-rename-strategy! driver db-id table-name metadata data-source)
|
||||
|
||||
:else
|
||||
(transfer-with-drop-create-fallback-strategy! driver db-id table-name metadata data-source))))
|
||||
|
||||
(defn- start-cancellation-process!
|
||||
"Starts a core.async process that optimistically sends a cancellation request to the python executor if cancel-chan receives a value.
|
||||
Returns a channel that will receive either the async http call j.u.c.FutureTask in the case of cancellation, or nil when the cancel-chan is closed."
|
||||
[server-url run-id cancel-chan]
|
||||
(a/go (when (a/<! cancel-chan)
|
||||
(python-runner/cancel-python-code-http-call! server-url run-id))))
|
||||
|
||||
(defn- run-python-transform! [{:keys [source] :as transform} db run-id cancel-chan message-log]
|
||||
;; Resolve name-based source table refs to table IDs (throws if any not found)
|
||||
(let [resolved-source-tables (transforms.util/resolve-source-tables (:source-tables source))]
|
||||
;; TODO restructure things such that s3 can we swapped out for other transfer mechanisms
|
||||
(with-open [^Closeable log-future-ref
|
||||
(if (app-db/in-transaction?)
|
||||
;; if in a transaction (such as under mt/with-temp), it is not safe to poll for logs (close race / contention)
|
||||
;; tests that want to test async log behaviour should opt out of thread-local test helpers
|
||||
(reify Closeable (close [_]))
|
||||
(open-python-message-update-future! run-id message-log))
|
||||
shared-storage-ref (s3/open-shared-storage! resolved-source-tables)]
|
||||
(let [driver (:engine db)
|
||||
server-url (transforms-python.settings/python-runner-url)
|
||||
_ (python-runner/copy-tables-to-s3! {:run-id run-id
|
||||
:shared-storage @shared-storage-ref
|
||||
:source (assoc source :source-tables resolved-source-tables)
|
||||
:cancel-chan cancel-chan
|
||||
:limit (:limit source)
|
||||
:transform-id (:id transform)})
|
||||
_ (start-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan
|
||||
{:keys [status body] :as response}
|
||||
(python-runner/execute-python-code-http-call!
|
||||
{:server-url server-url
|
||||
:code (:body source)
|
||||
:run-id run-id
|
||||
:table-name->id resolved-source-tables
|
||||
:shared-storage @shared-storage-ref})
|
||||
|
||||
output-manifest (python-runner/read-output-manifest @shared-storage-ref)
|
||||
events (python-runner/read-events @shared-storage-ref)]
|
||||
(.close ^Closeable log-future-ref) ; early close to force any writes to flush
|
||||
(replace-python-logs! message-log events)
|
||||
(if (not= 200 status)
|
||||
(do
|
||||
(when (:timeout body)
|
||||
(transforms/timeout-run! run-id))
|
||||
(throw (ex-info "Python runner call failed"
|
||||
{:transform-message (i18n/tru "Python execution failure (exit code {0})" (:exit_code body "?"))
|
||||
:status-code 400
|
||||
:api-status-code status
|
||||
:body body
|
||||
:events events})))
|
||||
(try
|
||||
(let [temp-path (Files/createTempFile "transform-output-" ".jsonl" (u/varargs FileAttribute))
|
||||
temp-file (.toFile temp-path)]
|
||||
(when-not (seq (:fields output-manifest))
|
||||
(throw (ex-info "No fields in metadata"
|
||||
{:metadata output-manifest
|
||||
:raw-body body
|
||||
:events events})))
|
||||
(try
|
||||
(with-open [in (python-runner/open-output @shared-storage-ref)]
|
||||
(io/copy in temp-file))
|
||||
(let [file-size (.length temp-file)]
|
||||
(transforms.instrumentation/with-stage-timing [run-id [:import :file-to-dwh]]
|
||||
(transfer-file-to-db driver db transform output-manifest temp-file))
|
||||
(transforms.instrumentation/record-data-transfer! run-id :file-to-dwh file-size nil))
|
||||
(finally
|
||||
(.delete temp-file))))
|
||||
response
|
||||
(catch Exception e
|
||||
(log/error e "Failed to to create resulting table")
|
||||
(throw (ex-info "Failed to create the resulting table"
|
||||
{:transform-message (or (:transform-message (ex-data e))
|
||||
;; TODO keeping messaging the same at this level
|
||||
;; should be more specific in underlying calls
|
||||
(i18n/tru "Failed to create the resulting table"))}
|
||||
e)))))))))
|
||||
|
||||
(defn- exceptional-run-message [message-log ex]
|
||||
(str/join "\n" (remove str/blank? [(message-log->transform-run-message message-log)
|
||||
(or (:transform-message (ex-data ex))
|
||||
(if (instance? InterruptedException ex)
|
||||
(i18n/tru "Transform interrupted")
|
||||
(i18n/tru "Something went wrong")))])))
|
||||
|
||||
(defn execute-python-transform!
|
||||
"Execute a Python transform by calling the python runner.
|
||||
|
||||
Blocks until the transform returns."
|
||||
[transform {:keys [run-method start-promise user-id]}]
|
||||
(assert (transforms.util/python-transform? transform) "Transform must be a python transform")
|
||||
(try
|
||||
(let [message-log (empty-message-log)
|
||||
{:keys [target owner_user_id creator_id] transform-id :id} transform
|
||||
{driver :engine :as db} (t2/select-one :model/Database (transforms.i/target-db-id transform))
|
||||
;; For manual runs, use the triggering user; for cron, use owner/creator
|
||||
run-user-id (if (and (= run-method :manual) user-id)
|
||||
user-id
|
||||
(or owner_user_id creator_id))
|
||||
{run-id :id} (transforms.util/try-start-unless-already-running transform-id run-method run-user-id)]
|
||||
(some-> start-promise (deliver [:started run-id]))
|
||||
(log! message-log (i18n/tru "Executing Python transform"))
|
||||
(driver.conn/with-write-connection
|
||||
(log/info "Executing Python transform" transform-id "with target" (pr-str target)
|
||||
(when (driver.conn/write-connection-requested?)
|
||||
" using write connection"))
|
||||
(let [start-ms (u/start-timer)
|
||||
conn-spec (driver/connection-spec driver db)
|
||||
transform-details {:db-id (:id db)
|
||||
:transform-id transform-id
|
||||
:transform-type (keyword (:type target))
|
||||
:conn-spec conn-spec
|
||||
:output-schema (:schema target)
|
||||
:output-table (transforms.util/qualified-table-name driver target)}
|
||||
run-fn (fn [cancel-chan]
|
||||
(run-python-transform! transform db run-id cancel-chan message-log)
|
||||
(log! message-log (i18n/tru "Python execution finished successfully in {0}" (u.format/format-milliseconds (u/since-ms start-ms))))
|
||||
(save-log-to-transform-run-message! run-id message-log))
|
||||
ex-message-fn #(exceptional-run-message message-log %)
|
||||
result (transforms.instrumentation/with-stage-timing [run-id [:computation :python-execution]]
|
||||
(transforms.util/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn))]
|
||||
(transforms.util/handle-transform-complete!
|
||||
:run-id run-id
|
||||
:transform transform
|
||||
:db db)
|
||||
{:run_id run-id
|
||||
:result result})))
|
||||
(catch Throwable t
|
||||
(log/error t "Error executing Python transform")
|
||||
(throw t))))
|
||||
@@ -1,39 +1,13 @@
|
||||
(ns metabase-enterprise.transforms-python.impl
|
||||
"Registers :python as a runner language. Loaded by transforms-python/init.clj for side effects.
|
||||
See transforms.interface/register-runner! and transforms.interface/lang-config."
|
||||
(:require
|
||||
[metabase-enterprise.transforms-python.execute :as transforms-python.execute]
|
||||
[metabase.transforms.interface :as transforms.i]))
|
||||
|
||||
(defmethod transforms.i/target-db-id :python
|
||||
[transform]
|
||||
(-> transform :target :database))
|
||||
(transforms.i/register-runner! :python)
|
||||
|
||||
(defmethod transforms.i/source-db-id :python
|
||||
[transform]
|
||||
(-> transform :source :source-database))
|
||||
|
||||
#_{:clj-kondo/ignore [:discouraged-var]}
|
||||
(defmethod transforms.i/execute! :python
|
||||
[transform options]
|
||||
(transforms-python.execute/execute-python-transform! transform options))
|
||||
|
||||
(defn- source-table-value->dependency
|
||||
"Convert a source table value (int or ref map) to a dependency map."
|
||||
[v]
|
||||
(cond
|
||||
;; Integer table ID - direct table dependency
|
||||
(int? v)
|
||||
{:table v}
|
||||
|
||||
;; Map with resolved table_id - direct table dependency
|
||||
(:table_id v)
|
||||
{:table (:table_id v)}
|
||||
|
||||
;; Map without table_id - return ref for ordering system to resolve
|
||||
:else
|
||||
{:table-ref (select-keys v [:database_id :schema :table])}))
|
||||
|
||||
(defmethod transforms.i/table-dependencies :python
|
||||
[transform]
|
||||
(into #{}
|
||||
(map source-table-value->dependency)
|
||||
(vals (get-in transform [:source :source-tables]))))
|
||||
(defmethod transforms.i/lang-config :python [_]
|
||||
{:runtime "python"
|
||||
:label "Python"
|
||||
:timing-key :python-execution
|
||||
:extension ".py"})
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
(ns metabase-enterprise.transforms-python.init
|
||||
(:require
|
||||
[metabase-enterprise.transforms-python.impl]
|
||||
[metabase-enterprise.transforms-python.settings]))
|
||||
[metabase-enterprise.transforms-python.impl]))
|
||||
@@ -1,115 +0,0 @@
|
||||
(ns metabase-enterprise.transforms-python.models.python-library
|
||||
(:require
|
||||
[metabase.api.common :as api]
|
||||
[metabase.app-db.core :as app-db]
|
||||
[metabase.events.core :as events]
|
||||
[metabase.models.interface :as mi]
|
||||
[metabase.models.serialization :as serdes]
|
||||
[metabase.permissions.core :as perms]
|
||||
[metabase.util.i18n :refer [tru]]
|
||||
[methodical.core :as methodical]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(methodical/defmethod t2/table-name :model/PythonLibrary [_model] :python_library)
|
||||
|
||||
(defn- normalize-path
|
||||
"Ensure the path ends with .py extension."
|
||||
[path]
|
||||
(if (.endsWith ^String path ".py")
|
||||
path
|
||||
(str path ".py")))
|
||||
|
||||
(t2/define-before-insert :model/PythonLibrary
|
||||
[library]
|
||||
(update library :path normalize-path))
|
||||
|
||||
(t2/define-before-update :model/PythonLibrary
|
||||
[library]
|
||||
(if (:path library)
|
||||
(update library :path normalize-path)
|
||||
library))
|
||||
|
||||
(doseq [trait [:metabase/model :hook/timestamped? :hook/entity-id]]
|
||||
(derive :model/PythonLibrary trait))
|
||||
|
||||
(defmethod mi/can-read? :model/PythonLibrary
|
||||
([_instance]
|
||||
(perms/has-any-transforms-permission? api/*current-user-id*))
|
||||
([_model _pk]
|
||||
(perms/has-any-transforms-permission? api/*current-user-id*)))
|
||||
|
||||
(defmethod mi/can-write? :model/PythonLibrary
|
||||
([_instance]
|
||||
(perms/has-any-transforms-permission? api/*current-user-id*))
|
||||
([_model _pk]
|
||||
(perms/has-any-transforms-permission? api/*current-user-id*)))
|
||||
|
||||
(def ^:private allowed-paths
|
||||
"Set of allowed library paths. Currently only 'common' is supported."
|
||||
#{"common.py"})
|
||||
|
||||
(def builtin-entity-id
|
||||
"The entity_id of the built-in common.py PythonLibrary created by migration.
|
||||
Used to protect it from deletion during remote-sync import."
|
||||
"cWWH9qJPvHNB3rP2vLZrK")
|
||||
|
||||
(defn- validate-path!
|
||||
"Validates that the given path is allowed. Throws an exception if not."
|
||||
[path]
|
||||
(let [normalized-path (normalize-path path)]
|
||||
(when-not (contains? allowed-paths normalized-path)
|
||||
(throw (ex-info (tru "Invalid library path. Only ''common'' is currently supported.")
|
||||
{:status-code 400
|
||||
:path normalized-path
|
||||
:allowed-paths allowed-paths})))))
|
||||
|
||||
(defn get-python-library-by-path
|
||||
"Get the Python library by path."
|
||||
[path]
|
||||
(let [normalized-path (normalize-path path)]
|
||||
(validate-path! normalized-path)
|
||||
(t2/select-one :model/PythonLibrary :path normalized-path)))
|
||||
|
||||
(defn update-python-library-source!
|
||||
"Update the Python library source code. Creates a new record if none exists. Returns the updated library."
|
||||
[path source]
|
||||
(let [normalized-path (normalize-path path)]
|
||||
(validate-path! normalized-path)
|
||||
(let [id (app-db/update-or-insert! :model/PythonLibrary
|
||||
{:path normalized-path}
|
||||
(constantly {:path normalized-path :source source}))]
|
||||
(t2/select-one :model/PythonLibrary id))))
|
||||
|
||||
;;; ------------------------------------------------- Serialization --------------------------------------------------
|
||||
|
||||
(defmethod serdes/make-spec "PythonLibrary"
|
||||
[_model-name _opts]
|
||||
{:copy [:path :source :entity_id]
|
||||
:transform {:created_at (serdes/date)}})
|
||||
|
||||
(defmethod serdes/hash-fields :model/PythonLibrary
|
||||
[_model]
|
||||
[:path])
|
||||
|
||||
(defmethod serdes/storage-path "PythonLibrary" [entity _ctx]
|
||||
(let [{:keys [id label]} (-> entity serdes/path last)]
|
||||
["python-libraries" (serdes/storage-leaf-file-name id label)]))
|
||||
|
||||
;;; ------------------------------------------------ Event Hooks -----------------------------------------------------
|
||||
|
||||
;; Event type hierarchy for remote-sync tracking
|
||||
(derive ::event :metabase/event)
|
||||
(doseq [e [:event/python-library-create :event/python-library-update :event/python-library-delete]]
|
||||
(derive e ::event))
|
||||
|
||||
(t2/define-after-insert :model/PythonLibrary
|
||||
[library]
|
||||
(events/publish-event! :event/python-library-create {:object library})
|
||||
library)
|
||||
|
||||
(t2/define-after-update :model/PythonLibrary
|
||||
[library]
|
||||
(events/publish-event! :event/python-library-update {:object library})
|
||||
library)
|
||||
@@ -1,9 +1,9 @@
|
||||
(ns metabase-enterprise.transforms-python.api
|
||||
(ns metabase-enterprise.transforms-runner.api
|
||||
(:require
|
||||
[clojure.string :as str]
|
||||
[metabase-enterprise.transforms-python.models.python-library :as python-library]
|
||||
[metabase-enterprise.transforms-python.python-runner :as python-runner]
|
||||
[metabase-enterprise.transforms-python.settings :as transforms-python.settings]
|
||||
[metabase-enterprise.transforms-runner.models.transform-library :as transform-library]
|
||||
[metabase-enterprise.transforms-runner.runner :as runner]
|
||||
[metabase-enterprise.transforms-runner.settings :as runner.settings]
|
||||
[metabase.api.common :as api]
|
||||
[metabase.api.macros :as api.macros]
|
||||
[metabase.api.routes.common :refer [+auth]]
|
||||
@@ -13,10 +13,10 @@
|
||||
[metabase.util.malli.schema :as ms]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(defn get-python-library-by-path
|
||||
"Get Python library details by path for use by other APIs."
|
||||
[path]
|
||||
(-> (python-library/get-python-library-by-path path)
|
||||
(defn get-library-by-path
|
||||
"Get library details by path. Type is \"python\" (default), \"javascript\", or \"clojure\"."
|
||||
[path type]
|
||||
(-> (transform-library/get-library-by-path (or type "python") path)
|
||||
api/read-check
|
||||
(select-keys [:source :path :created_at :updated_at])))
|
||||
|
||||
@@ -25,24 +25,26 @@
|
||||
;;
|
||||
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
|
||||
(api.macros/defendpoint :get "/library/:path"
|
||||
"Get the Python library for user modules."
|
||||
"Get the library for user modules. Pass `?type=javascript` for JavaScript, defaults to Python."
|
||||
[{:keys [path]} :- [:map [:path ms/NonBlankString]]
|
||||
_query-params]
|
||||
(get-python-library-by-path path))
|
||||
{:keys [type]} :- [:map [:type {:optional true} [:maybe :string]]]]
|
||||
(get-library-by-path path type))
|
||||
|
||||
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
|
||||
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
|
||||
;;
|
||||
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
|
||||
(api.macros/defendpoint :put "/library/:path"
|
||||
"Update the Python library source code for user modules."
|
||||
"Update the library source code for user modules. Pass `type` in the body for JavaScript."
|
||||
[{:keys [path]} :- [:map [:path ms/NonBlankString]]
|
||||
_query-params
|
||||
body :- [:map {:closed true}
|
||||
[:source :string]]]
|
||||
body :- [:map
|
||||
[:source :string]
|
||||
[:type {:optional true} [:maybe :string]]]]
|
||||
;; Check permission directly since this is an upsert endpoint - the library may not exist yet.
|
||||
(api/check-403 (perms/has-any-transforms-permission? api/*current-user-id*))
|
||||
(python-library/update-python-library-source! path (:source body)))
|
||||
(let [{:keys [source type]} body]
|
||||
(transform-library/update-library-source! (or type "python") path source)))
|
||||
|
||||
(api.macros/defendpoint :post "/test-run"
|
||||
:- [:map
|
||||
@@ -51,11 +53,13 @@
|
||||
[:output {:optional true} [:map
|
||||
[:cols [:sequential [:map [:name :string]]]]
|
||||
[:rows [:sequential :any]]]]]
|
||||
"Evaluate an ad-hoc python transform on a sample of input data.
|
||||
"Evaluate an ad-hoc transform on a sample of input data.
|
||||
Pass `type` as \"javascript\" to run JavaScript; defaults to \"python\".
|
||||
Intended for short runs for early feedback. Input/output/timeout limits apply."
|
||||
[_
|
||||
_
|
||||
{:keys [code
|
||||
type
|
||||
source_tables
|
||||
output_row_limit
|
||||
per_input_row_limit]
|
||||
@@ -63,6 +67,7 @@
|
||||
per_input_row_limit 100}}
|
||||
:- [:map
|
||||
[:code :string]
|
||||
[:type {:optional true} [:maybe :string]]
|
||||
[:source_tables [:map-of {:min 1} :string :int]]
|
||||
[:output_row_limit {:optional true} [:and :int [:> 1] [:<= 100]]]
|
||||
[:per_input_row_limit {:optional true} [:and :int [:> 1] [:<= 100]]]]]
|
||||
@@ -70,12 +75,13 @@
|
||||
(api/check-400 (= (count db-ids) 1) (i18n/deferred-tru "All source tables must belong to the same database."))
|
||||
(api/check-403 (perms/has-db-transforms-permission? api/*current-user-id* (first db-ids))))
|
||||
;; NOTE: we do not test database support, as there is no write target.
|
||||
(let [result (python-runner/execute-and-read-output!
|
||||
(let [result (runner/execute-and-read-output!
|
||||
{:code code
|
||||
:source-tables source_tables
|
||||
:per-input-limit per_input_row_limit
|
||||
:row-limit output_row_limit
|
||||
:timeout-secs (transforms-python.settings/python-runner-test-run-timeout-seconds)})
|
||||
:timeout-secs (runner.settings/python-runner-test-run-timeout-seconds)
|
||||
:runtime (or type "python")})
|
||||
logs (str/join "\n" (map :message (:logs result)))]
|
||||
(if (= :succeeded (:status result))
|
||||
{:logs logs
|
||||
@@ -85,6 +91,6 @@
|
||||
:error {:message (:message result)}})))
|
||||
|
||||
(def ^{:arglists '([request respond raise])} routes
|
||||
"`/api/ee/transforms-python` routes."
|
||||
"`/api/ee/advanced-transforms` routes."
|
||||
(handlers/routes
|
||||
(api.macros/ns-handler *ns* +auth)))
|
||||
@@ -0,0 +1,428 @@
|
||||
(ns metabase-enterprise.transforms-runner.execute
|
||||
"Shared execution logic for runner-based transforms (Python, JavaScript, etc.).
|
||||
|
||||
All runner-based transforms follow the same pattern:
|
||||
1. Upload source tables to S3
|
||||
2. Call the runner service with a `language` parameter
|
||||
3. Read output from S3
|
||||
4. Transfer results to the target database
|
||||
|
||||
Each language provides a small config map; this namespace does the rest."
|
||||
(:require
|
||||
[clojure.core.async :as a]
|
||||
[clojure.java.io :as io]
|
||||
[clojure.string :as str]
|
||||
[metabase-enterprise.transforms-runner.runner :as runner]
|
||||
[metabase-enterprise.transforms-runner.s3 :as s3]
|
||||
[metabase-enterprise.transforms-runner.settings :as runner.settings]
|
||||
[metabase.app-db.core :as app-db]
|
||||
[metabase.driver :as driver]
|
||||
[metabase.driver.connection :as driver.conn]
|
||||
[metabase.driver.util :as driver.u]
|
||||
[metabase.transforms.core :as transforms]
|
||||
[metabase.transforms.instrumentation :as transforms.instrumentation]
|
||||
[metabase.transforms.interface :as transforms.i]
|
||||
[metabase.transforms.util :as transforms.util]
|
||||
[metabase.util :as u]
|
||||
[metabase.util.format :as u.format]
|
||||
[metabase.util.i18n :as i18n]
|
||||
[metabase.util.jvm :as u.jvm]
|
||||
[metabase.util.log :as log]
|
||||
[toucan2.core :as t2])
|
||||
(:import
|
||||
(java.io Closeable)
|
||||
(java.net SocketException)
|
||||
(java.nio.file Files)
|
||||
(java.nio.file.attribute FileAttribute)
|
||||
(java.time Duration)))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
;;; ------------------------------------------------ Message Log -----------------------------------------------------
|
||||
|
||||
(defn- empty-message-log
|
||||
"Returns a new message log for transform execution.
|
||||
Buffers log outputs during the run to produce a string suitable for `transform_run.message`."
|
||||
[]
|
||||
(atom {:pre []
|
||||
:runner nil
|
||||
:post []}))
|
||||
|
||||
(defn- log!
|
||||
"Appends a user-facing string to the message log."
|
||||
[message-log s]
|
||||
(swap! message-log (fn [m] (update m (if (:runner m) :post :pre) conj s)))
|
||||
nil)
|
||||
|
||||
(defn- replace-runner-logs!
|
||||
"Called when events output is received from a /logs call to copy those events into the message-log."
|
||||
[message-log events]
|
||||
(swap! message-log assoc :runner events)
|
||||
nil)
|
||||
|
||||
(defn- message-log->transform-run-message
|
||||
"Presents the content of the log as a string suitable for storage in transform_run.message."
|
||||
[message-log]
|
||||
(let [{:keys [pre runner post]} @message-log]
|
||||
(str/join "\n" (concat pre
|
||||
(map :message runner)
|
||||
post))))
|
||||
|
||||
(defn- save-log-to-transform-run-message!
|
||||
[run-id message-log]
|
||||
(t2/update! :model/TransformRun
|
||||
:id run-id
|
||||
{:message (message-log->transform-run-message message-log)}))
|
||||
|
||||
;;; ------------------------------------------------ Log Polling Loop ------------------------------------------------
|
||||
|
||||
(def ^:private ^Duration message-loop-sleep-duration
|
||||
(Duration/ofMillis 1000))
|
||||
|
||||
(defn- message-update-loop!
|
||||
"Block while relevant log data is replicated from the runner into the message log."
|
||||
[run-id message-log]
|
||||
(try
|
||||
(loop []
|
||||
(if (.isInterrupted (Thread/currentThread))
|
||||
(log/debug "Message update loop interrupted")
|
||||
(do (let [sleep-ms (.toMillis message-loop-sleep-duration)]
|
||||
(when (pos? sleep-ms) (Thread/sleep sleep-ms)))
|
||||
(let [{:keys [status body]} (runner/get-logs run-id)]
|
||||
(cond
|
||||
(<= 200 status 299)
|
||||
(let [{:keys [execution_id events]} body]
|
||||
(if-not (= run-id execution_id)
|
||||
(do (log/debugf "Run id did not match expected: %s actual: %s" run-id execution_id)
|
||||
(recur))
|
||||
(do
|
||||
(replace-runner-logs! message-log events)
|
||||
(save-log-to-transform-run-message! run-id message-log)
|
||||
(recur))))
|
||||
(= 404 status)
|
||||
(do
|
||||
(log/debugf "No logs yet (or run finished), run-id: %s" run-id)
|
||||
(recur))
|
||||
:else
|
||||
(do
|
||||
(log/warnf "Unexpected status polling for logs %s %s, run-id: %s" status body run-id)
|
||||
(log/debug "Exiting due to poll error")))))))
|
||||
(catch SocketException se (when-not (= "Closed by interrupt" (ex-message se)) (throw se)))
|
||||
(catch InterruptedException _)
|
||||
(catch Throwable e
|
||||
(log/errorf e "An exception was caught during msg update loop, run-id: %s" run-id))))
|
||||
|
||||
(defn- open-message-update-future! ^Closeable [run-id message-log]
|
||||
(let [cleanup (fn [fut]
|
||||
(future-cancel fut)
|
||||
(if (= ::timeout (try (deref fut 10000 ::timeout) (catch Throwable _)))
|
||||
(log/fatalf "Log polling task did not respond to interrupt, run-id: %s" run-id)
|
||||
(log/debugf "Log polling task done, run-id: %s" run-id)))
|
||||
fut (u.jvm/in-virtual-thread*
|
||||
(message-update-loop! run-id message-log))]
|
||||
(reify Closeable
|
||||
(close [_] (cleanup fut)))))
|
||||
|
||||
;;; ------------------------------------------------ Table Transfer --------------------------------------------------
|
||||
|
||||
(defn- maybe-retry-with-backoff
|
||||
[driver thunk]
|
||||
(if (= driver :bigquery-cloud-sdk)
|
||||
(loop [attempt 0]
|
||||
(let [ret (try
|
||||
(thunk)
|
||||
(catch Exception e
|
||||
(if (and (< attempt 5)
|
||||
(or (str/includes? (str e) "not found")
|
||||
(str/includes? (str e) "Not found")))
|
||||
(let [delay-ms (* 150 (Math/pow 2 attempt))]
|
||||
(log/debugf "BigQuery operation failed (attempt %d), retrying in %dms" (inc attempt) delay-ms)
|
||||
(Thread/sleep (long delay-ms))
|
||||
::recur)
|
||||
(throw e))))]
|
||||
(if (= ::recur ret)
|
||||
(recur (inc attempt))
|
||||
ret)))
|
||||
(thunk)))
|
||||
|
||||
(defn- table-schema [table-name metadata]
|
||||
{:name (if (keyword? table-name) table-name (keyword table-name))
|
||||
:columns (mapv (fn [{:keys [name base_type]}]
|
||||
{:name name
|
||||
:type (runner/restricted-insert-type base_type)
|
||||
:nullable? true})
|
||||
(:fields metadata))})
|
||||
|
||||
(defn- insert-data!
|
||||
[driver db-id table-schema data-source]
|
||||
(let [data-source (assoc data-source :table-schema table-schema)]
|
||||
(->> #(driver/insert-from-source! driver db-id table-schema data-source)
|
||||
(maybe-retry-with-backoff driver))))
|
||||
|
||||
(defn- create-table-and-insert-data!
|
||||
[driver db-id table-schema data-source]
|
||||
(transforms.util/create-table-from-schema! driver db-id table-schema)
|
||||
(insert-data! driver db-id table-schema data-source))
|
||||
|
||||
(defn- transfer-with-rename-tables-strategy!
|
||||
[driver db-id table-name metadata data-source]
|
||||
(let [source-table-name (driver.u/temp-table-name driver table-name)
|
||||
temp-table-name (driver.u/temp-table-name driver table-name)]
|
||||
(log/info "Using rename-tables strategy with atomicity guarantees")
|
||||
(try
|
||||
(create-table-and-insert-data! driver db-id (table-schema source-table-name metadata) data-source)
|
||||
(transforms.util/rename-tables! driver db-id {table-name temp-table-name
|
||||
source-table-name table-name})
|
||||
(transforms.util/drop-table! driver db-id temp-table-name)
|
||||
(catch Exception e
|
||||
(log/error e "Failed to transfer data using rename-tables strategy")
|
||||
(try
|
||||
(transforms.util/drop-table! driver db-id source-table-name)
|
||||
(catch Exception _))
|
||||
(throw e)))))
|
||||
|
||||
(defn- transfer-with-create-drop-rename-strategy!
|
||||
[driver db-id table-name metadata data-source]
|
||||
(let [source-table-name (driver.u/temp-table-name driver table-name)]
|
||||
(log/info "Using create-drop-rename strategy to minimize downtime")
|
||||
(try
|
||||
(create-table-and-insert-data! driver db-id (table-schema source-table-name metadata) data-source)
|
||||
(transforms.util/drop-table! driver db-id table-name)
|
||||
(driver/rename-table! driver db-id source-table-name table-name)
|
||||
(catch Exception e
|
||||
(log/error e "Failed to transfer data using create-drop-rename strategy")
|
||||
(try
|
||||
(transforms.util/drop-table! driver db-id source-table-name)
|
||||
(catch Exception _))
|
||||
(throw e)))))
|
||||
|
||||
(defn- transfer-with-drop-create-fallback-strategy!
|
||||
[driver db-id table-name metadata data-source]
|
||||
(log/info "Using drop-create fallback strategy")
|
||||
(try
|
||||
(transforms.util/drop-table! driver db-id table-name)
|
||||
(create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source)
|
||||
(catch Exception e
|
||||
(log/error e "Failed to transfer data using drop-create fallback strategy")
|
||||
(throw e))))
|
||||
|
||||
(defmulti ^:private transfer-file-to-db
|
||||
{:arglists '([driver db transform metadata temp-file])}
|
||||
(fn [_ _ transform _ _] (-> transform :target :type keyword)))
|
||||
|
||||
(defmethod transfer-file-to-db :table-incremental
|
||||
[driver {db-id :id}
|
||||
{:keys [target] :as transform}
|
||||
metadata temp-file]
|
||||
(let [table-name (transforms.util/qualified-table-name driver target)
|
||||
table-exists? (transforms.util/target-table-exists? transform)
|
||||
data-source {:type :jsonl-file
|
||||
:file temp-file}]
|
||||
(if (not table-exists?)
|
||||
(do
|
||||
(log/info "New table")
|
||||
(create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))
|
||||
(insert-data! driver db-id (table-schema table-name metadata) data-source))))
|
||||
|
||||
(defmethod transfer-file-to-db :table
|
||||
[driver {db-id :id :as db}
|
||||
{:keys [target] :as transform}
|
||||
metadata temp-file]
|
||||
(let [table-name (transforms.util/qualified-table-name driver target)
|
||||
table-exists? (transforms.util/target-table-exists? transform)
|
||||
data-source {:type :jsonl-file
|
||||
:file temp-file}]
|
||||
(cond
|
||||
(not table-exists?)
|
||||
(do
|
||||
(log/info "New table")
|
||||
(create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))
|
||||
|
||||
(driver.u/supports? driver :atomic-renames db)
|
||||
(transfer-with-rename-tables-strategy! driver db-id table-name metadata data-source)
|
||||
|
||||
(driver.u/supports? driver :rename db)
|
||||
(transfer-with-create-drop-rename-strategy! driver db-id table-name metadata data-source)
|
||||
|
||||
:else
|
||||
(transfer-with-drop-create-fallback-strategy! driver db-id table-name metadata data-source))))
|
||||
|
||||
;;; ------------------------------------------------ Runner Execution ------------------------------------------------
|
||||
|
||||
(defn- start-cancellation-process!
|
||||
[server-url run-id cancel-chan]
|
||||
(a/go (when (a/<! cancel-chan)
|
||||
(runner/cancel-python-code-http-call! server-url run-id))))
|
||||
|
||||
(defn- run-runner-transform!
|
||||
"Core execution: upload tables to S3, call runner, read output, transfer to DB.
|
||||
`runtime` is the language string passed to the runner (e.g. \"python\", \"javascript\")."
|
||||
[{:keys [source] :as transform} db run-id cancel-chan message-log runtime]
|
||||
(let [resolved-source-tables (transforms.util/resolve-source-tables (:source-tables source))]
|
||||
(with-open [^Closeable log-future-ref
|
||||
(if (app-db/in-transaction?)
|
||||
(reify Closeable (close [_]))
|
||||
(open-message-update-future! run-id message-log))
|
||||
shared-storage-ref (s3/open-shared-storage! resolved-source-tables)]
|
||||
(let [driver (:engine db)
|
||||
server-url (runner.settings/python-runner-url)
|
||||
_ (runner/copy-tables-to-s3! {:run-id run-id
|
||||
:shared-storage @shared-storage-ref
|
||||
:source (assoc source :source-tables resolved-source-tables)
|
||||
:cancel-chan cancel-chan
|
||||
:limit (:limit source)
|
||||
:transform-id (:id transform)})
|
||||
_ (start-cancellation-process! server-url run-id cancel-chan)
|
||||
{:keys [status body] :as response}
|
||||
(runner/execute-python-code-http-call!
|
||||
{:server-url server-url
|
||||
:code (:body source)
|
||||
:run-id run-id
|
||||
:table-name->id resolved-source-tables
|
||||
:shared-storage @shared-storage-ref
|
||||
:runtime runtime})
|
||||
|
||||
output-manifest (runner/read-output-manifest @shared-storage-ref)
|
||||
events (runner/read-events @shared-storage-ref)]
|
||||
(.close ^Closeable log-future-ref)
|
||||
(replace-runner-logs! message-log events)
|
||||
(if (not= 200 status)
|
||||
(do
|
||||
(when (:timeout body)
|
||||
(transforms/timeout-run! run-id))
|
||||
(throw (ex-info (str runtime " runner call failed")
|
||||
{:transform-message (i18n/tru "{0} execution failure (exit code {1})"
|
||||
(str/capitalize runtime)
|
||||
(:exit_code body "?"))
|
||||
:status-code 400
|
||||
:api-status-code status
|
||||
:body body
|
||||
:events events})))
|
||||
(try
|
||||
(let [temp-path (Files/createTempFile "transform-output-" ".jsonl" (u/varargs FileAttribute))
|
||||
temp-file (.toFile temp-path)]
|
||||
(when-not (seq (:fields output-manifest))
|
||||
(throw (ex-info "No fields in metadata"
|
||||
{:metadata output-manifest
|
||||
:raw-body body
|
||||
:events events})))
|
||||
(try
|
||||
(with-open [in (runner/open-output @shared-storage-ref)]
|
||||
(io/copy in temp-file))
|
||||
(let [file-size (.length temp-file)]
|
||||
(transforms.instrumentation/with-stage-timing [run-id [:import :file-to-dwh]]
|
||||
(transfer-file-to-db driver db transform output-manifest temp-file))
|
||||
(transforms.instrumentation/record-data-transfer! run-id :file-to-dwh file-size nil))
|
||||
(finally
|
||||
(.delete temp-file))))
|
||||
response
|
||||
(catch Exception e
|
||||
(log/error e "Failed to create resulting table")
|
||||
(throw (ex-info "Failed to create the resulting table"
|
||||
{:transform-message (or (:transform-message (ex-data e))
|
||||
(i18n/tru "Failed to create the resulting table"))}
|
||||
e)))))))))
|
||||
|
||||
;;; ------------------------------------------------ Public API ------------------------------------------------------
|
||||
|
||||
(defn- exceptional-run-message [message-log ex]
|
||||
(str/join "\n" (remove str/blank? [(message-log->transform-run-message message-log)
|
||||
(or (:transform-message (ex-data ex))
|
||||
(if (instance? InterruptedException ex)
|
||||
(i18n/tru "Transform interrupted")
|
||||
(i18n/tru "Something went wrong")))])))
|
||||
|
||||
(defn execute-runner-transform!
|
||||
"Execute a runner-based transform (Python, JavaScript, Clojure, R, Julia, etc.).
|
||||
|
||||
`lang-config` is a map with:
|
||||
:runtime - string passed to runner as language param (e.g. \"python\", \"javascript\")
|
||||
:label - human-readable label for log messages (e.g. \"Python\", \"JavaScript\")
|
||||
:timing-key - keyword for instrumentation (e.g. :python-execution, :javascript-execution)
|
||||
|
||||
Blocks until the transform returns."
|
||||
[transform {:keys [run-method start-promise user-id]} lang-config]
|
||||
(let [{:keys [runtime label timing-key]} lang-config]
|
||||
(assert (transforms.util/runner-transform? transform)
|
||||
(str "Transform must be a runner-based transform, got: "
|
||||
(transforms.util/transform-type transform)))
|
||||
(try
|
||||
(let [message-log (empty-message-log)
|
||||
{:keys [target owner_user_id creator_id] transform-id :id} transform
|
||||
{driver :engine :as db} (t2/select-one :model/Database (transforms.i/target-db-id transform))
|
||||
run-user-id (if (and (= run-method :manual) user-id)
|
||||
user-id
|
||||
(or owner_user_id creator_id))
|
||||
{run-id :id} (transforms.util/try-start-unless-already-running transform-id run-method run-user-id)]
|
||||
(some-> start-promise (deliver [:started run-id]))
|
||||
(log! message-log (i18n/tru "Executing {0} transform" label))
|
||||
(driver.conn/with-write-connection
|
||||
(log/info "Executing" label "transform" transform-id "with target" (pr-str target)
|
||||
(when (driver.conn/write-connection-requested?)
|
||||
" using write connection"))
|
||||
(let [start-ms (u/start-timer)
|
||||
conn-spec (driver/connection-spec driver db)
|
||||
transform-details {:db-id (:id db)
|
||||
:transform-id transform-id
|
||||
:transform-type (keyword (:type target))
|
||||
:conn-spec conn-spec
|
||||
:output-schema (:schema target)
|
||||
:output-table (transforms.util/qualified-table-name driver target)}
|
||||
run-fn (fn [cancel-chan]
|
||||
(run-runner-transform! transform db run-id cancel-chan message-log runtime)
|
||||
(log! message-log (i18n/tru "{0} execution finished successfully in {1}"
|
||||
label (u.format/format-milliseconds (u/since-ms start-ms))))
|
||||
(save-log-to-transform-run-message! run-id message-log))
|
||||
ex-message-fn #(exceptional-run-message message-log %)
|
||||
result (transforms.instrumentation/with-stage-timing [run-id [:computation timing-key]]
|
||||
(transforms.util/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn))]
|
||||
(transforms.util/handle-transform-complete!
|
||||
:run-id run-id
|
||||
:transform transform
|
||||
:db db)
|
||||
{:run_id run-id
|
||||
:result result})))
|
||||
(catch Throwable t
|
||||
(log/error t (str "Error executing " label " transform"))
|
||||
(throw t)))))
|
||||
|
||||
;;; ------------------------------------------------ Impl Helpers ----------------------------------------------------
|
||||
|
||||
(defn source-table-value->dependency
|
||||
"Convert a source table value (int or ref map) to a dependency map.
|
||||
Shared by all runner-based transform implementations."
|
||||
[v]
|
||||
(cond
|
||||
(int? v)
|
||||
{:table v}
|
||||
|
||||
(:table_id v)
|
||||
{:table (:table_id v)}
|
||||
|
||||
:else
|
||||
{:table-ref (select-keys v [:database_id :schema :table])}))
|
||||
|
||||
;;; ---------------------------------------- Shared ::runner defmethods ----------------------------------------
|
||||
;; All runner-based languages share these implementations. Per-language dispatch
|
||||
;; is handled via the hierarchy in transforms.i — each language only needs to
|
||||
;; define (defmethod transforms.i/lang-config :lang-kw ...).
|
||||
|
||||
(defmethod transforms.i/target-db-id ::transforms.i/runner
|
||||
[transform]
|
||||
(-> transform :target :database))
|
||||
|
||||
(defmethod transforms.i/source-db-id ::transforms.i/runner
|
||||
[transform]
|
||||
(-> transform :source :source-database))
|
||||
|
||||
(defmethod transforms.i/table-dependencies ::transforms.i/runner
|
||||
[transform]
|
||||
(into #{}
|
||||
(map source-table-value->dependency)
|
||||
(vals (get-in transform [:source :source-tables]))))
|
||||
|
||||
#_{:clj-kondo/ignore [:discouraged-var]}
|
||||
(defmethod transforms.i/execute! ::transforms.i/runner
|
||||
[transform options]
|
||||
(let [lang-kw (-> transform :source :type keyword)]
|
||||
(execute-runner-transform! transform options (transforms.i/lang-config lang-kw))))
|
||||
@@ -0,0 +1,8 @@
|
||||
(ns metabase-enterprise.transforms-runner.init
|
||||
(:require
|
||||
[metabase-enterprise.transforms-runner.api]
|
||||
[metabase-enterprise.transforms-runner.execute]
|
||||
[metabase-enterprise.transforms-runner.models.transform-library]
|
||||
[metabase-enterprise.transforms-runner.runner]
|
||||
[metabase-enterprise.transforms-runner.s3]
|
||||
[metabase-enterprise.transforms-runner.settings]))
|
||||
@@ -0,0 +1,151 @@
|
||||
(ns metabase-enterprise.transforms-runner.models.transform-library
|
||||
(:require
|
||||
[metabase.api.common :as api]
|
||||
[metabase.app-db.core :as app-db]
|
||||
[metabase.events.core :as events]
|
||||
[metabase.models.interface :as mi]
|
||||
[metabase.models.serialization :as serdes]
|
||||
[metabase.permissions.core :as perms]
|
||||
[metabase.transforms.interface :as transforms.i]
|
||||
[metabase.util.i18n :refer [tru]]
|
||||
[methodical.core :as methodical]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(methodical/defmethod t2/table-name :model/TransformLibrary [_model] :transform_library)
|
||||
|
||||
(doseq [trait [:metabase/model :hook/timestamped? :hook/entity-id]]
|
||||
(derive :model/TransformLibrary trait))
|
||||
|
||||
(defn- language-extension
|
||||
"Return the file extension (e.g. \".py\") for a language string, looked up from lang-config."
|
||||
[language]
|
||||
(:extension (transforms.i/lang-config (keyword language))))
|
||||
|
||||
(defn- normalize-path
|
||||
[language path]
|
||||
(let [ext (language-extension language)]
|
||||
(if (and ext (not (.endsWith ^String path ext)))
|
||||
(str path ext)
|
||||
path)))
|
||||
|
||||
(t2/define-before-insert :model/TransformLibrary
|
||||
[library]
|
||||
(update library :path (partial normalize-path (:language library))))
|
||||
|
||||
(t2/define-before-update :model/TransformLibrary
|
||||
[library]
|
||||
(if (:path library)
|
||||
(update library :path (partial normalize-path (or (:language library)
|
||||
(:language (t2/original library)))))
|
||||
library))
|
||||
|
||||
(defmethod mi/can-read? :model/TransformLibrary
|
||||
([_instance]
|
||||
(perms/has-any-transforms-permission? api/*current-user-id*))
|
||||
([_model _pk]
|
||||
(perms/has-any-transforms-permission? api/*current-user-id*)))
|
||||
|
||||
(defmethod mi/can-write? :model/TransformLibrary
|
||||
([_instance]
|
||||
(perms/has-any-transforms-permission? api/*current-user-id*))
|
||||
([_model _pk]
|
||||
(perms/has-any-transforms-permission? api/*current-user-id*)))
|
||||
|
||||
(def ^:private legacy-entity-ids
|
||||
"Stable entity IDs for built-in libraries created by Liquibase migrations.
|
||||
New languages get deterministic IDs via [[builtin-entity-id]]."
|
||||
{"python" "cWWH9qJPvHNB3rP2vLZrK"
|
||||
"javascript" "aHWo_0yPLwKqpQPlFNzCg"})
|
||||
|
||||
(defn builtin-entity-id
|
||||
"Return the stable entity ID for a language's built-in common library.
|
||||
Legacy languages (python, javascript) use migration-baked IDs; new languages
|
||||
get a deterministic 21-char NanoID-style hash derived from the language name."
|
||||
[language]
|
||||
(or (get legacy-entity-ids language)
|
||||
(let [bytes (.digest (java.security.MessageDigest/getInstance "SHA-256")
|
||||
(.getBytes (str "transform-library:" language) "UTF-8"))
|
||||
encoded (-> (java.util.Base64/getUrlEncoder)
|
||||
(.withoutPadding)
|
||||
(.encodeToString bytes))]
|
||||
(subs encoded 0 21))))
|
||||
|
||||
(defn all-builtin-entity-ids
|
||||
"Return the set of entity IDs for all registered runner languages' built-in libraries."
|
||||
[]
|
||||
(into #{}
|
||||
(map (comp builtin-entity-id name))
|
||||
(transforms.i/runner-languages)))
|
||||
|
||||
(defn- validate-path!
|
||||
[language path]
|
||||
(let [ext (language-extension language)
|
||||
normalized (normalize-path language path)
|
||||
allowed-path (str "common" ext)]
|
||||
(when-not (= normalized allowed-path)
|
||||
(throw (ex-info (tru "Invalid library path. Only ''common'' is currently supported.")
|
||||
{:status-code 400
|
||||
:path normalized
|
||||
:language language
|
||||
:allowed-paths [allowed-path]})))))
|
||||
|
||||
(defn get-library-by-path
|
||||
"Look up a TransformLibrary record by `language` and `path`, validating the path first."
|
||||
[language path]
|
||||
(let [normalized (normalize-path language path)]
|
||||
(validate-path! language normalized)
|
||||
(t2/select-one :model/TransformLibrary :language language :path normalized)))
|
||||
|
||||
(defn update-library-source!
|
||||
"Create or update the TransformLibrary record for `language` and `path` with the given `source`."
|
||||
[language path source]
|
||||
(let [normalized (normalize-path language path)]
|
||||
(validate-path! language normalized)
|
||||
(let [id (app-db/update-or-insert! :model/TransformLibrary
|
||||
{:language language :path normalized}
|
||||
(constantly {:language language :path normalized :source source}))]
|
||||
(t2/select-one :model/TransformLibrary id))))
|
||||
|
||||
(defn ensure-builtin-library!
|
||||
"Ensure the built-in common library row exists for `language`.
|
||||
Inserts idempotently — no-op if the row already exists.
|
||||
New runner languages can call this instead of adding a Liquibase migration.
|
||||
|
||||
NOTE: Must be called after the app DB is initialized and migrations have run.
|
||||
Do NOT call at namespace load time (e.g. from init.clj) — the table may not exist yet."
|
||||
[language]
|
||||
(let [ext (language-extension language)
|
||||
path (str "common" ext)
|
||||
eid (builtin-entity-id language)]
|
||||
(when-not (t2/exists? :model/TransformLibrary :language language :path path)
|
||||
(t2/insert! :model/TransformLibrary
|
||||
{:path path :source "" :language language :entity_id eid}))))
|
||||
|
||||
(defmethod serdes/make-spec "TransformLibrary"
|
||||
[_model-name _opts]
|
||||
{:copy [:language :path :source :entity_id]
|
||||
:transform {:created_at (serdes/date)}})
|
||||
|
||||
(defmethod serdes/hash-fields :model/TransformLibrary
|
||||
[_model]
|
||||
[:language :path])
|
||||
|
||||
(defmethod serdes/storage-path "TransformLibrary" [entity _ctx]
|
||||
(let [{:keys [id label]} (-> entity serdes/path last)]
|
||||
["transform-libraries" (serdes/storage-leaf-file-name id label)]))
|
||||
|
||||
(derive ::event :metabase/event)
|
||||
(doseq [e [:event/transform-library-create :event/transform-library-update :event/transform-library-delete]]
|
||||
(derive e ::event))
|
||||
|
||||
(t2/define-after-insert :model/TransformLibrary
|
||||
[library]
|
||||
(events/publish-event! :event/transform-library-create {:object library})
|
||||
library)
|
||||
|
||||
(t2/define-after-update :model/TransformLibrary
|
||||
[library]
|
||||
(events/publish-event! :event/transform-library-update {:object library})
|
||||
library)
|
||||
@@ -1,12 +1,12 @@
|
||||
(ns metabase-enterprise.transforms-python.python-runner
|
||||
(ns metabase-enterprise.transforms-runner.runner
|
||||
(:require
|
||||
[clj-http.client :as http]
|
||||
[clojure.core.async :as a]
|
||||
[clojure.java.io :as io]
|
||||
[clojure.string :as str]
|
||||
[medley.core :as m]
|
||||
[metabase-enterprise.transforms-python.s3 :as s3]
|
||||
[metabase-enterprise.transforms-python.settings :as transforms-python.settings]
|
||||
[metabase-enterprise.transforms-runner.s3 :as s3]
|
||||
[metabase-enterprise.transforms-runner.settings :as runner.settings]
|
||||
[metabase.analytics.prometheus :as prometheus]
|
||||
[metabase.config.core :as config]
|
||||
[metabase.lib-be.core :as lib-be]
|
||||
@@ -33,7 +33,7 @@
|
||||
"Returns HTTP headers with Authorization bearer token if configured.
|
||||
Throws configuration error in production if token is not set."
|
||||
[]
|
||||
(let [api-token (transforms-python.settings/python-runner-api-token)]
|
||||
(let [api-token (runner.settings/python-runner-api-token)]
|
||||
(if api-token
|
||||
{"Authorization" (str "Bearer " api-token)}
|
||||
(if config/is-prod?
|
||||
@@ -175,7 +175,7 @@
|
||||
(defn get-logs
|
||||
"Return the logs of the current running python process"
|
||||
[run-id]
|
||||
(let [server-url (transforms-python.settings/python-runner-url)]
|
||||
(let [server-url (runner.settings/python-runner-url)]
|
||||
(python-runner-request server-url :get "/logs" {:query-params {:request_id run-id}})))
|
||||
|
||||
(mu/defn record-python-api-call!
|
||||
@@ -200,23 +200,25 @@
|
||||
(defn execute-python-code-http-call!
|
||||
"Calls the /execute endpoint of the python runner. Blocks until the run either succeeds or fails and returns
|
||||
the response from the server."
|
||||
[{:keys [server-url code request-id run-id table-name->id shared-storage timeout-secs]}]
|
||||
(let [{:keys [objects]} shared-storage
|
||||
[{:keys [server-url code request-id run-id table-name->id shared-storage timeout-secs runtime]}]
|
||||
(let [{:keys [objects]} shared-storage
|
||||
{:keys [output output-manifest events]} objects
|
||||
url-for-path (fn [path] (:url (get objects path)))
|
||||
table-name->url (update-vals table-name->id #(url-for-path [:table % :data]))
|
||||
table-name->manifest-url (update-vals table-name->id #(url-for-path [:table % :manifest]))
|
||||
payload {:code code
|
||||
:library (t2/select-fn->fn :path :source :model/PythonLibrary)
|
||||
:timeout (or timeout-secs (transforms-python.settings/python-runner-timeout-seconds))
|
||||
:request_id (or request-id run-id)
|
||||
:output_url (:url output)
|
||||
:output_manifest_url (:url output-manifest)
|
||||
:events_url (:url events)
|
||||
:table_mapping table-name->url
|
||||
:manifest_mapping table-name->manifest-url}
|
||||
response (with-python-api-timing [run-id]
|
||||
(python-runner-request server-url :post "/execute" {:body (json/encode payload)}))]
|
||||
runtime (or runtime "python")
|
||||
url-for-path (fn [path] (:url (get objects path)))
|
||||
table-name->url (update-vals table-name->id #(url-for-path [:table % :data]))
|
||||
table-name->manifest-url (update-vals table-name->id #(url-for-path [:table % :manifest]))
|
||||
payload {:code code
|
||||
:library (t2/select-fn->fn :path :source :model/TransformLibrary :language runtime)
|
||||
:timeout (or timeout-secs (runner.settings/python-runner-timeout-seconds))
|
||||
:request_id (or request-id run-id)
|
||||
:output_url (:url output)
|
||||
:output_manifest_url (:url output-manifest)
|
||||
:events_url (:url events)
|
||||
:table_mapping table-name->url
|
||||
:manifest_mapping table-name->manifest-url
|
||||
:runtime runtime}
|
||||
response (with-python-api-timing [run-id]
|
||||
(python-runner-request server-url :post "/execute" {:body (json/encode payload)}))]
|
||||
;; when a 500 is returned we observe a string in the body (despite the python returning json)
|
||||
;; always try to parse the returned string as json before yielding (could tighten this up at some point)
|
||||
(update response :body (fn [string-if-error]
|
||||
@@ -342,6 +344,7 @@
|
||||
:source-tables - Map of table-name -> table-id (already resolved)
|
||||
:row-limit - Max rows to return (also limits input rows)
|
||||
:timeout-secs - Optional timeout override
|
||||
:runtime - Optional runtime override (default: \"python\")
|
||||
|
||||
Returns:
|
||||
{:status :succeeded/:failed
|
||||
@@ -350,9 +353,9 @@
|
||||
:logs [{:message ...} ...] ; events from Python execution
|
||||
:message \"error message\"} ; on failure
|
||||
"
|
||||
[{:keys [code source-tables per-input-limit row-limit timeout-secs]}]
|
||||
[{:keys [code source-tables per-input-limit row-limit timeout-secs runtime]}]
|
||||
(with-open [shared-storage-ref (s3/open-shared-storage! source-tables)]
|
||||
(let [server-url (transforms-python.settings/python-runner-url)
|
||||
(let [server-url (runner.settings/python-runner-url)
|
||||
_ (copy-tables-to-s3! {:shared-storage @shared-storage-ref
|
||||
:source {:source-tables source-tables}
|
||||
:limit (or per-input-limit row-limit)})
|
||||
@@ -363,7 +366,8 @@
|
||||
:request-id (u/generate-nano-id)
|
||||
:table-name->id source-tables
|
||||
:timeout-secs timeout-secs
|
||||
:shared-storage @shared-storage-ref})
|
||||
:shared-storage @shared-storage-ref
|
||||
:runtime runtime})
|
||||
events (read-events @shared-storage-ref)]
|
||||
(cond
|
||||
(:timeout body)
|
||||
@@ -1,6 +1,6 @@
|
||||
(ns metabase-enterprise.transforms-python.s3
|
||||
(ns metabase-enterprise.transforms-runner.s3
|
||||
(:require
|
||||
[metabase-enterprise.transforms-python.settings :as transforms-python.settings]
|
||||
[metabase-enterprise.transforms-runner.settings :as runner.settings]
|
||||
[metabase.util.log :as log])
|
||||
(:import
|
||||
(clojure.lang IDeref)
|
||||
@@ -29,12 +29,12 @@
|
||||
(defn- working-dir-for-run
|
||||
"The path within the target bucket within which to store all the files for this run."
|
||||
[]
|
||||
(let [shared-prefix (some-> (transforms-python.settings/python-storage-s-3-prefix) (str "/"))]
|
||||
(let [shared-prefix (some-> (runner.settings/python-storage-s-3-prefix) (str "/"))]
|
||||
(str shared-prefix "run-" (System/nanoTime) "-" (rand-int 10000))))
|
||||
|
||||
(defmacro ^:private maybe-with-endpoint* [builder endpoint]
|
||||
`(let [builder# ~builder]
|
||||
(when-let [region# (transforms-python.settings/python-storage-s-3-region)]
|
||||
(when-let [region# (runner.settings/python-storage-s-3-region)]
|
||||
(.region builder# (Region/of region#)))
|
||||
(when ~endpoint (.endpointOverride builder# (URI/create ~endpoint)))
|
||||
builder#))
|
||||
@@ -46,7 +46,7 @@
|
||||
(maybe-with-endpoint* builder endpoint))
|
||||
|
||||
(defn- s3-configuration ^S3Configuration []
|
||||
(let [path-style-access (transforms-python.settings/python-storage-s-3-path-style-access)]
|
||||
(let [path-style-access (runner.settings/python-storage-s-3-path-style-access)]
|
||||
(-> (S3Configuration/builder)
|
||||
(.pathStyleAccessEnabled path-style-access)
|
||||
;; Disable chunked encoding when using path-style access, as most S3-compatible
|
||||
@@ -65,8 +65,8 @@
|
||||
|
||||
(defn- maybe-with-credentials*
|
||||
[credentials-provider]
|
||||
(let [access-key (transforms-python.settings/python-storage-s-3-access-key)
|
||||
secret-key (transforms-python.settings/python-storage-s-3-secret-key)]
|
||||
(let [access-key (runner.settings/python-storage-s-3-access-key)
|
||||
secret-key (runner.settings/python-storage-s-3-secret-key)]
|
||||
(if (or access-key secret-key)
|
||||
(if-not (and access-key secret-key)
|
||||
(do (log/warnf "Ignoring %s because %s is not defined"
|
||||
@@ -90,15 +90,15 @@
|
||||
^S3Client []
|
||||
(.build
|
||||
(doto (S3Client/builder)
|
||||
(maybe-with-endpoint-s3-client (transforms-python.settings/python-storage-s-3-endpoint))
|
||||
(maybe-with-endpoint-s3-client (runner.settings/python-storage-s-3-endpoint))
|
||||
maybe-with-credentials-s3-client
|
||||
(.serviceConfiguration (s3-configuration)))))
|
||||
|
||||
(defn- create-s3-presigner-for-container
|
||||
"Create S3 presigner for container operations (presigned URLs). Uses distinct container-endpoint if relevant."
|
||||
^S3Presigner []
|
||||
(let [container-endpoint (transforms-python.settings/python-storage-s-3-container-endpoint)
|
||||
endpoint (or container-endpoint (transforms-python.settings/python-storage-s-3-endpoint))]
|
||||
(let [container-endpoint (runner.settings/python-storage-s-3-container-endpoint)
|
||||
endpoint (or container-endpoint (runner.settings/python-storage-s-3-endpoint))]
|
||||
(.build
|
||||
(doto (S3Presigner/builder)
|
||||
(maybe-with-endpoint-s3-presigner endpoint)
|
||||
@@ -126,7 +126,7 @@
|
||||
(defn- s3-shared-storage [table-name->id]
|
||||
(let [work-dir-name (working-dir-for-run)
|
||||
container-presigner (create-s3-presigner-for-container)
|
||||
bucket-name (transforms-python.settings/python-storage-s-3-bucket)
|
||||
bucket-name (runner.settings/python-storage-s-3-bucket)
|
||||
ref (fn [method relative-path]
|
||||
(let [path (str work-dir-name "/" relative-path)]
|
||||
{:path path
|
||||
@@ -1,4 +1,5 @@
|
||||
(ns metabase-enterprise.transforms-python.settings
|
||||
;; TODO(Timothy, 02-25-2026): these settings point to a common runner, not python
|
||||
(ns metabase-enterprise.transforms-runner.settings
|
||||
(:require
|
||||
[metabase.config.core :as config]
|
||||
[metabase.settings.core :as setting]
|
||||
@@ -9,7 +9,7 @@
|
||||
decoupled from AppDb."
|
||||
(:require
|
||||
[clojure.string :as str]
|
||||
[metabase-enterprise.transforms-python.python-runner :as python-runner]
|
||||
[metabase-enterprise.transforms-runner.runner :as runner]
|
||||
[metabase.api.common :as api]
|
||||
[metabase.query-processor :as qp]
|
||||
[metabase.sql-tools.core :as sql-tools]
|
||||
@@ -115,7 +115,7 @@
|
||||
remapped-source (remap-python-source table-mapping source)
|
||||
resolved-source-tables (transforms.util/resolve-source-tables (:source-tables remapped-source))
|
||||
{:as result
|
||||
:keys [cols rows]} (python-runner/execute-and-read-output!
|
||||
:keys [cols rows]} (runner/execute-and-read-output!
|
||||
{:code (:body remapped-source)
|
||||
:source-tables resolved-source-tables
|
||||
:row-limit preview-row-limit})
|
||||
|
||||
@@ -1360,9 +1360,9 @@
|
||||
(let [conversation-id (str (random-uuid))
|
||||
rasta-ai-token (ai-session-token)
|
||||
crowberto-ai-token (ai-session-token :crowberto (str (random-uuid)))
|
||||
saved-python-library (t2/select-one :model/PythonLibrary :path "common.py")]
|
||||
saved-python-library (t2/select-one :model/TransformLibrary :language "python" :path "common.py")]
|
||||
(when (seq saved-python-library)
|
||||
(t2/delete! :model/PythonLibrary))
|
||||
(t2/delete! :model/TransformLibrary :language "python"))
|
||||
(try
|
||||
(testing "With no Python library present"
|
||||
(is (= "Not found."
|
||||
@@ -1370,8 +1370,9 @@
|
||||
{:request-options {:headers {"x-metabase-session" crowberto-ai-token}}}
|
||||
{:arguments {:path "common.py"}
|
||||
:conversation_id conversation-id}))))
|
||||
(mt/with-temp [:model/PythonLibrary lib1 {:path "common.py"
|
||||
:source "def hello():\n return 'world'"}]
|
||||
(mt/with-temp [:model/TransformLibrary lib1 {:language "python"
|
||||
:path "common.py"
|
||||
:source "def hello():\n return 'world'"}]
|
||||
(testing "With insufficient permissions"
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :post 403 "ee/metabot-tools/get-transform-python-library-details"
|
||||
@@ -1395,7 +1396,7 @@
|
||||
:conversation_id conversation-id})))))
|
||||
(finally
|
||||
(when (seq saved-python-library)
|
||||
(t2/insert! :model/PythonLibrary saved-python-library)))))))
|
||||
(t2/insert! :model/TransformLibrary saved-python-library)))))))
|
||||
|
||||
(deftest get-snippets-test
|
||||
(mt/with-premium-features #{:metabot-v3}
|
||||
|
||||
@@ -1070,16 +1070,17 @@
|
||||
(is (true? (remote-sync.settings/remote-sync-transforms))
|
||||
"remote-sync-transforms should be auto-enabled after successful import with transforms"))))))))
|
||||
|
||||
(deftest import!-auto-enables-transforms-setting-when-python-libraries-detected-test
|
||||
(testing "import! auto-enables remote-sync-transforms setting only after successful import with python-libraries"
|
||||
(deftest import!-auto-enables-transforms-setting-when-transform-libraries-detected-test
|
||||
(testing "import! auto-enables remote-sync-transforms setting only after successful import with transform-libraries"
|
||||
(mt/with-premium-features #{:transforms}
|
||||
(mt/with-model-cleanup [:model/RemoteSyncTask :model/PythonLibrary :model/RemoteSyncObject]
|
||||
(mt/with-model-cleanup [:model/RemoteSyncTask :model/TransformLibrary :model/RemoteSyncObject]
|
||||
(mt/with-temporary-setting-values [remote-sync-transforms false
|
||||
remote-sync-enabled true]
|
||||
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
|
||||
lib-entity-id "auto-enable-lib-xxxxx"
|
||||
test-files {"main" {(str "python-libraries/" lib-entity-id ".yaml")
|
||||
(format "path: uncommon.py
|
||||
test-files {"main" {(str "transform-libraries/" lib-entity-id ".yaml")
|
||||
(format "language: python
|
||||
path: uncommon.py
|
||||
source: |
|
||||
# shared code
|
||||
def shared_func():
|
||||
@@ -1088,7 +1089,7 @@ entity_id: %s
|
||||
created_at: '2024-08-28T09:46:18.671622Z'
|
||||
serdes/meta:
|
||||
- id: %s
|
||||
model: PythonLibrary
|
||||
model: TransformLibrary
|
||||
" lib-entity-id lib-entity-id)}}
|
||||
mock-source (test-helpers/create-mock-source :initial-files test-files)]
|
||||
(is (false? (remote-sync.settings/remote-sync-transforms))
|
||||
@@ -1097,7 +1098,7 @@ serdes/meta:
|
||||
(is (= :success (:status result))
|
||||
"Import should succeed")
|
||||
(is (true? (remote-sync.settings/remote-sync-transforms))
|
||||
"remote-sync-transforms should be auto-enabled after successful import with python-libraries"))))))))
|
||||
"remote-sync-transforms should be auto-enabled after successful import with transform-libraries"))))))))
|
||||
|
||||
(deftest import!-keeps-transforms-setting-disabled-when-no-transforms-present-test
|
||||
(testing "import! keeps remote-sync-transforms setting disabled when no transforms are present"
|
||||
@@ -1132,7 +1133,7 @@ serdes/meta:
|
||||
"remote-sync-transforms should be disabled when no transforms in remote")))))))))
|
||||
|
||||
(deftest import!-includes-all-optional-paths-regardless-of-settings-test
|
||||
(testing "import! always includes all optional paths (transforms, python-libraries, snippets)"
|
||||
(testing "import! always includes all optional paths (transforms, transform-libraries, snippets)"
|
||||
(mt/with-model-cleanup [:model/RemoteSyncTask]
|
||||
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
|
||||
(mt/with-temporary-setting-values [remote-sync-transforms false]
|
||||
@@ -1151,8 +1152,8 @@ serdes/meta:
|
||||
(let [filter-strs (map str @paths-passed)]
|
||||
(is (some #(str/includes? % "transforms") filter-strs)
|
||||
"transforms path should be included in filters")
|
||||
(is (some #(str/includes? % "python-libraries") filter-strs)
|
||||
"python-libraries path should be included in filters")
|
||||
(is (some #(str/includes? % "transform-libraries") filter-strs)
|
||||
"transform-libraries path should be included in filters")
|
||||
(is (some #(str/includes? % "snippets") filter-strs)
|
||||
"snippets path should be included in filters"))))))))))))
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
[clojure.string :as str]
|
||||
[clojure.test :as t]
|
||||
[metabase-enterprise.remote-sync.source.protocol :as source.p]
|
||||
[metabase-enterprise.transforms-python.core :as transforms-python]
|
||||
[metabase-enterprise.transforms-runner.models.transform-library :as transform-library]
|
||||
[metabase.util :as u]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
@@ -287,44 +287,45 @@ width: fixed
|
||||
(t2/insert! :model/RemoteSyncTask old-models))))))
|
||||
|
||||
(def ^:private builtin-python-library
|
||||
"The built-in PythonLibrary created by migration. Recreated by fixture after cleanup."
|
||||
{:path "common.py" :source "" :entity_id transforms-python/builtin-entity-id})
|
||||
"The built-in TransformLibrary (python) created by migration. Recreated by fixture after cleanup."
|
||||
{:path "common.py" :source "" :language "python"
|
||||
:entity_id (transform-library/builtin-entity-id "python")})
|
||||
|
||||
(defn- ensure-builtin-python-library!
|
||||
"Ensures the built-in common.py PythonLibrary exists, creating it if missing."
|
||||
"Ensures the built-in common.py TransformLibrary exists, creating it if missing."
|
||||
[]
|
||||
(when-not (t2/exists? :model/PythonLibrary :path (:path builtin-python-library))
|
||||
(t2/insert! :model/PythonLibrary builtin-python-library)))
|
||||
(when-not (t2/exists? :model/TransformLibrary :path (:path builtin-python-library) :language "python")
|
||||
(t2/insert! :model/TransformLibrary builtin-python-library)))
|
||||
|
||||
(defn clean-optional-feature-models
|
||||
"Test fixture that cleans Transform, TransformTag, PythonLibrary, and namespace collection
|
||||
"Test fixture that cleans Transform, TransformTag, TransformLibrary, and namespace collection
|
||||
tables to prevent conflict detection during first-import tests. Preserves built-in TransformTags
|
||||
and recreates the built-in common.py PythonLibrary after cleanup."
|
||||
and recreates the built-in common.py TransformLibrary after cleanup."
|
||||
[f]
|
||||
(let [old-transforms (t2/select :model/Transform)
|
||||
old-tags (t2/select :model/TransformTag :built_in_type nil)
|
||||
old-libs (t2/select :model/PythonLibrary)
|
||||
old-libs (t2/select :model/TransformLibrary)
|
||||
old-ns-colls (t2/select :model/Collection :namespace [:in ["transforms" "snippets"]])]
|
||||
(try
|
||||
(t2/delete! :model/TransformTag :built_in_type nil)
|
||||
(t2/delete! :model/Transform)
|
||||
(t2/delete! :model/PythonLibrary)
|
||||
(t2/delete! :model/TransformLibrary)
|
||||
(t2/delete! :model/Collection :namespace [:in ["transforms" "snippets"]])
|
||||
(f)
|
||||
(finally
|
||||
(t2/delete! :model/TransformTag :built_in_type nil)
|
||||
(t2/delete! :model/Transform)
|
||||
(t2/delete! :model/PythonLibrary)
|
||||
(t2/delete! :model/TransformLibrary)
|
||||
(t2/delete! :model/Collection :namespace [:in ["transforms" "snippets"]])
|
||||
(when (seq old-transforms) (t2/insert! :model/Transform old-transforms))
|
||||
(when (seq old-tags) (t2/insert! :model/TransformTag old-tags))
|
||||
(when (seq old-libs) (t2/insert! :model/PythonLibrary old-libs))
|
||||
(when (seq old-libs) (t2/insert! :model/TransformLibrary old-libs))
|
||||
(when (seq old-ns-colls) (t2/insert! :model/Collection old-ns-colls))
|
||||
(ensure-builtin-python-library!)))))
|
||||
|
||||
(def clean-remote-sync-state
|
||||
"Composed test fixture that ensures RemoteSyncObject, RemoteSyncTask, and optional feature
|
||||
model tables (Transform, TransformTag, PythonLibrary) are clean."
|
||||
model tables (Transform, TransformTag, TransformLibrary) are clean."
|
||||
(t/compose-fixtures clean-object (t/compose-fixtures clean-task-table clean-optional-feature-models)))
|
||||
|
||||
(defn generate-table-yaml
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
[metabase-enterprise.remote-sync.source.protocol :as source.p]
|
||||
[metabase-enterprise.remote-sync.spec :as spec]
|
||||
[metabase-enterprise.remote-sync.test-helpers :as test-helpers]
|
||||
[metabase-enterprise.transforms-python.core :as transforms-python]
|
||||
[metabase-enterprise.transforms-runner.models.transform-library :as transform-library]
|
||||
[metabase.collections.models.collection :as collection]
|
||||
[metabase.events.core :as events]
|
||||
[metabase.search.core :as search]
|
||||
@@ -499,163 +499,164 @@ is_sample: false
|
||||
(is (t2/exists? :model/Collection :entity_id remote-coll-entity-id :namespace "transforms")
|
||||
"Remote transforms collection should be imported")))))))))
|
||||
|
||||
;;; ------------------------------------------- PythonLibrary Tests -------------------------------------------
|
||||
;;; ------------------------------------------- TransformLibrary Tests -------------------------------------------
|
||||
|
||||
(defn- generate-python-library-yaml
|
||||
"Generates YAML content for a PythonLibrary."
|
||||
(defn- generate-transform-library-yaml
|
||||
"Generates YAML content for a TransformLibrary."
|
||||
[entity-id path source]
|
||||
(format "path: %s
|
||||
(format "language: python
|
||||
path: %s
|
||||
source: |
|
||||
%s
|
||||
entity_id: %s
|
||||
created_at: '2024-08-28T09:46:18.671622Z'
|
||||
serdes/meta:
|
||||
- id: %s
|
||||
model: PythonLibrary
|
||||
model: TransformLibrary
|
||||
"
|
||||
path
|
||||
(str/join "\n" (map #(str " " %) (str/split-lines source)))
|
||||
entity-id
|
||||
entity-id))
|
||||
|
||||
(deftest python-library-event-creates-sync-object-when-setting-enabled-test
|
||||
(testing "Creating a PythonLibrary creates a RemoteSyncObject entry when remote-sync-transforms is enabled"
|
||||
(deftest transform-library-event-creates-sync-object-when-setting-enabled-test
|
||||
(testing "Creating a TransformLibrary creates a RemoteSyncObject entry when remote-sync-transforms is enabled"
|
||||
(mt/with-premium-features #{:transforms}
|
||||
(mt/with-temporary-setting-values [remote-sync-transforms true
|
||||
remote-sync-enabled true]
|
||||
(let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})]
|
||||
(let [library (t2/insert-returning-instance! :model/TransformLibrary {:path "common.py" :source "# test" :language "python"})]
|
||||
(is (t2/exists? :model/RemoteSyncObject
|
||||
:model_type "PythonLibrary"
|
||||
:model_type "TransformLibrary"
|
||||
:model_id (:id library))
|
||||
"PythonLibrary should be tracked when remote-sync-transforms is enabled"))))))
|
||||
"TransformLibrary should be tracked when remote-sync-transforms is enabled"))))))
|
||||
|
||||
(deftest python-library-event-ignored-when-setting-disabled-test
|
||||
(testing "PythonLibrary events are ignored when remote-sync-transforms is disabled"
|
||||
(deftest transform-library-event-ignored-when-setting-disabled-test
|
||||
(testing "TransformLibrary events are ignored when remote-sync-transforms is disabled"
|
||||
(mt/with-premium-features #{:transforms}
|
||||
(mt/with-temporary-setting-values [remote-sync-transforms false
|
||||
remote-sync-enabled true]
|
||||
(let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})]
|
||||
(let [library (t2/insert-returning-instance! :model/TransformLibrary {:path "common.py" :source "# test" :language "python"})]
|
||||
(is (not (t2/exists? :model/RemoteSyncObject
|
||||
:model_type "PythonLibrary"
|
||||
:model_type "TransformLibrary"
|
||||
:model_id (:id library)))
|
||||
"PythonLibrary should NOT be tracked when remote-sync-transforms is disabled"))))))
|
||||
"TransformLibrary should NOT be tracked when remote-sync-transforms is disabled"))))))
|
||||
|
||||
(deftest python-library-event-updates-sync-object-test
|
||||
(testing "Updating a PythonLibrary updates the RemoteSyncObject entry when setting is enabled"
|
||||
(deftest transform-library-event-updates-sync-object-test
|
||||
(testing "Updating a TransformLibrary updates the RemoteSyncObject entry when setting is enabled"
|
||||
(mt/with-premium-features #{:transforms}
|
||||
(mt/with-temporary-setting-values [remote-sync-transforms true
|
||||
remote-sync-enabled true]
|
||||
(let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})]
|
||||
(t2/update! :model/RemoteSyncObject {:model_type "PythonLibrary" :model_id (:id library)}
|
||||
(let [library (t2/insert-returning-instance! :model/TransformLibrary {:path "common.py" :source "# test" :language "python"})]
|
||||
(t2/update! :model/RemoteSyncObject {:model_type "TransformLibrary" :model_id (:id library)}
|
||||
{:status "synced"})
|
||||
(t2/update! :model/PythonLibrary (:id library) {:source "# updated"})
|
||||
(t2/update! :model/TransformLibrary (:id library) {:source "# updated"})
|
||||
(let [entry (t2/select-one :model/RemoteSyncObject
|
||||
:model_type "PythonLibrary"
|
||||
:model_type "TransformLibrary"
|
||||
:model_id (:id library))]
|
||||
(is (= "update" (:status entry))
|
||||
"PythonLibrary should have 'update' status after modification")))))))
|
||||
"TransformLibrary should have 'update' status after modification")))))))
|
||||
|
||||
(deftest export-includes-python-library-when-setting-enabled-test
|
||||
(testing "Export includes PythonLibrary when remote-sync-transforms is enabled"
|
||||
(deftest export-includes-transform-library-when-setting-enabled-test
|
||||
(testing "Export includes TransformLibrary when remote-sync-transforms is enabled"
|
||||
(mt/with-premium-features #{:transforms}
|
||||
(mt/with-temporary-setting-values [remote-sync-type :read-write
|
||||
remote-sync-transforms true
|
||||
remote-sync-enabled true]
|
||||
(mt/with-model-cleanup [:model/RemoteSyncTask]
|
||||
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})
|
||||
library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# shared code"})]
|
||||
(t2/update! :model/RemoteSyncObject {:model_type "PythonLibrary" :model_id (:id library)}
|
||||
library (t2/insert-returning-instance! :model/TransformLibrary {:path "common.py" :source "# shared code" :language "python"})]
|
||||
(t2/update! :model/RemoteSyncObject {:model_type "TransformLibrary" :model_id (:id library)}
|
||||
{:status "create"})
|
||||
(let [mock-source (test-helpers/create-mock-source)
|
||||
result (impl/export! (source.p/snapshot mock-source) task-id "Test export")]
|
||||
(is (= :success (:status result))
|
||||
(str "Export should succeed. Result: " result))
|
||||
(let [files-after-export (get @(:files-atom mock-source) "main")]
|
||||
(is (some #(str/includes? % "python-libraries/") (keys files-after-export))
|
||||
"Export should include the PythonLibrary file")))))))))
|
||||
(is (some #(str/includes? % "transform-libraries/") (keys files-after-export))
|
||||
"Export should include the TransformLibrary file")))))))))
|
||||
|
||||
(deftest import-python-library-from-yaml-test
|
||||
(testing "Import brings in PythonLibrary from YAML files"
|
||||
(deftest import-transform-library-from-yaml-test
|
||||
(testing "Import brings in TransformLibrary from YAML files"
|
||||
(mt/with-premium-features #{:transforms}
|
||||
(mt/with-temporary-setting-values [remote-sync-transforms true
|
||||
remote-sync-enabled true]
|
||||
(mt/with-model-cleanup [:model/RemoteSyncTask]
|
||||
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
|
||||
lib-entity-id (u/generate-nano-id)
|
||||
test-files {"main" {(str "python-libraries/" lib-entity-id ".yaml")
|
||||
(generate-python-library-yaml lib-entity-id "common.py" "def shared_func():\n return 42")}}
|
||||
test-files {"main" {(str "transform-libraries/" lib-entity-id ".yaml")
|
||||
(generate-transform-library-yaml lib-entity-id "common.py" "def shared_func():\n return 42")}}
|
||||
mock-source (test-helpers/create-mock-source :initial-files test-files)
|
||||
snapshot (source.p/snapshot mock-source)
|
||||
result (impl/import! snapshot task-id)]
|
||||
(is (= :success (:status result))
|
||||
(str "Import should succeed. Result: " result))
|
||||
(is (t2/exists? :model/PythonLibrary :path "common.py")
|
||||
"PythonLibrary should be imported")
|
||||
(when-let [library (t2/select-one :model/PythonLibrary :path "common.py")]
|
||||
(is (t2/exists? :model/TransformLibrary :path "common.py")
|
||||
"TransformLibrary should be imported")
|
||||
(when-let [library (t2/select-one :model/TransformLibrary :path "common.py")]
|
||||
(is (str/includes? (:source library) "def shared_func()")
|
||||
"PythonLibrary source should be imported correctly"))))))))
|
||||
"TransformLibrary source should be imported correctly"))))))))
|
||||
|
||||
(deftest import-removes-python-library-not-on-remote-test
|
||||
(testing "Import removes local PythonLibrary that doesn't exist on the remote"
|
||||
(deftest import-removes-transform-library-not-on-remote-test
|
||||
(testing "Import removes local TransformLibrary that doesn't exist on the remote"
|
||||
(mt/with-premium-features #{:transforms}
|
||||
(mt/with-temporary-setting-values [remote-sync-transforms true
|
||||
remote-sync-enabled true]
|
||||
(mt/with-model-cleanup [:model/RemoteSyncTask]
|
||||
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
|
||||
local-library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# local only"})]
|
||||
(is (t2/exists? :model/PythonLibrary :id (:id local-library))
|
||||
"Local PythonLibrary should exist before import")
|
||||
local-library (t2/insert-returning-instance! :model/TransformLibrary {:path "common.py" :source "# local only" :language "python"})]
|
||||
(is (t2/exists? :model/TransformLibrary :id (:id local-library))
|
||||
"Local TransformLibrary should exist before import")
|
||||
(let [test-files {"main" {}}
|
||||
mock-source (test-helpers/create-mock-source :initial-files test-files)
|
||||
result (impl/import! (source.p/snapshot mock-source) task-id)]
|
||||
(is (= :success (:status result))
|
||||
(str "Import should succeed. Result: " result))
|
||||
(is (not (t2/exists? :model/PythonLibrary :id (:id local-library)))
|
||||
"Local PythonLibrary should be deleted after import since it wasn't on remote"))))))))
|
||||
(is (not (t2/exists? :model/TransformLibrary :id (:id local-library)))
|
||||
"Local TransformLibrary should be deleted after import since it wasn't on remote"))))))))
|
||||
|
||||
(deftest import-preserves-builtin-python-library-test
|
||||
(testing "Import does NOT delete the built-in PythonLibrary even when it's not on remote"
|
||||
(deftest import-preserves-builtin-transform-library-test
|
||||
(testing "Import does NOT delete the built-in TransformLibrary even when it's not on remote"
|
||||
(mt/with-premium-features #{:transforms}
|
||||
(mt/with-temporary-setting-values [remote-sync-transforms true
|
||||
remote-sync-enabled true]
|
||||
(mt/with-model-cleanup [:model/RemoteSyncTask]
|
||||
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
|
||||
;; Create the builtin library explicitly (fixture cleans PythonLibrary before test)
|
||||
(t2/insert-returning-instance! :model/PythonLibrary
|
||||
(t2/insert-returning-instance! :model/TransformLibrary
|
||||
{:path "common.py"
|
||||
:source "# builtin"
|
||||
:entity_id transforms-python/builtin-entity-id})
|
||||
(is (t2/exists? :model/PythonLibrary :entity_id transforms-python/builtin-entity-id)
|
||||
"Built-in PythonLibrary should exist before import")
|
||||
(let [test-files {"main" {}} ;; Empty remote - no python libraries
|
||||
:language "python"
|
||||
:entity_id (transform-library/builtin-entity-id "python")})
|
||||
(is (t2/exists? :model/TransformLibrary :entity_id (transform-library/builtin-entity-id "python"))
|
||||
"Built-in TransformLibrary should exist before import")
|
||||
(let [test-files {"main" {}}
|
||||
mock-source (test-helpers/create-mock-source :initial-files test-files)
|
||||
result (impl/import! (source.p/snapshot mock-source) task-id)]
|
||||
(is (= :success (:status result))
|
||||
(str "Import should succeed. Result: " result))
|
||||
(is (t2/exists? :model/PythonLibrary :entity_id transforms-python/builtin-entity-id)
|
||||
"Built-in PythonLibrary should NOT be deleted after import"))))))))
|
||||
(is (t2/exists? :model/TransformLibrary :entity_id (transform-library/builtin-entity-id "python"))
|
||||
"Built-in TransformLibrary should NOT be deleted after import"))))))))
|
||||
|
||||
(deftest import-replaces-python-library-with-remote-version-test
|
||||
(testing "Import updates local PythonLibrary when remote has same entity_id"
|
||||
(deftest import-replaces-transform-library-with-remote-version-test
|
||||
(testing "Import updates local TransformLibrary when remote has same entity_id"
|
||||
(mt/with-premium-features #{:transforms}
|
||||
(mt/with-temporary-setting-values [remote-sync-transforms true
|
||||
remote-sync-enabled true]
|
||||
(mt/with-model-cleanup [:model/RemoteSyncTask]
|
||||
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
|
||||
local-library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# local version"})
|
||||
local-library (t2/insert-returning-instance! :model/TransformLibrary {:path "common.py" :source "# local version" :language "python"})
|
||||
local-entity-id (:entity_id local-library)]
|
||||
(is (t2/exists? :model/PythonLibrary :path "common.py")
|
||||
"Local PythonLibrary should exist before import")
|
||||
(let [test-files {"main" {(str "python-libraries/" local-entity-id ".yaml")
|
||||
(generate-python-library-yaml local-entity-id "common.py" "# remote version\ndef new_func():\n pass")}}
|
||||
(is (t2/exists? :model/TransformLibrary :path "common.py")
|
||||
"Local TransformLibrary should exist before import")
|
||||
(let [test-files {"main" {(str "transform-libraries/" local-entity-id ".yaml")
|
||||
(generate-transform-library-yaml local-entity-id "common.py" "# remote version\ndef new_func():\n pass")}}
|
||||
mock-source (test-helpers/create-mock-source :initial-files test-files)
|
||||
result (impl/import! (source.p/snapshot mock-source) task-id)]
|
||||
(is (= :success (:status result))
|
||||
(str "Import should succeed. Result: " result))
|
||||
(let [library (t2/select-one :model/PythonLibrary :path "common.py")]
|
||||
(is (some? library) "PythonLibrary should still exist")
|
||||
(let [library (t2/select-one :model/TransformLibrary :path "common.py")]
|
||||
(is (some? library) "TransformLibrary should still exist")
|
||||
(is (str/includes? (:source library) "# remote version")
|
||||
"PythonLibrary source should be updated with remote version")))))))))
|
||||
"TransformLibrary source should be updated with remote version")))))))))
|
||||
|
||||
;;; ------------------------------------------- Transform Sync Behavior Tests -------------------------------------------
|
||||
|
||||
@@ -829,11 +830,11 @@ serdes/meta:
|
||||
:collection_id coll-id}
|
||||
:model/TransformTag {tag-id :id tag-eid :entity_id} {:name "Test Tag"
|
||||
:built_in_type nil}]
|
||||
(let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})
|
||||
(let [library (t2/insert-returning-instance! :model/TransformLibrary {:path "common.py" :source "# test" :language "python"})
|
||||
lib-eid (:entity_id library)]
|
||||
(is (t2/exists? :model/Transform :id transform-id))
|
||||
(is (t2/exists? :model/TransformTag :id tag-id))
|
||||
(is (t2/exists? :model/PythonLibrary :id (:id library)))
|
||||
(is (t2/exists? :model/TransformLibrary :id (:id library)))
|
||||
(let [paths-before (spec/build-all-removal-paths)]
|
||||
(is (not (some #(str/includes? % transform-eid) paths-before))
|
||||
"Transform should not be in removal paths before setting is disabled"))
|
||||
@@ -850,7 +851,7 @@ serdes/meta:
|
||||
(is (some #(str/includes? % tag-eid) paths-after)
|
||||
"TransformTag should be in removal paths after setting is disabled")
|
||||
(is (some #(str/includes? % lib-eid) paths-after)
|
||||
"PythonLibrary should be in removal paths after setting is disabled"))))))))
|
||||
"TransformLibrary should be in removal paths after setting is disabled"))))))))
|
||||
|
||||
(deftest build-all-removal-paths-excludes-builtin-transform-tags-test
|
||||
(testing "build-all-removal-paths respects :conditions and excludes built-in tags"
|
||||
|
||||
@@ -39,7 +39,7 @@
|
||||
#"/databases/.*/schemas/(.*)" :schema
|
||||
#"/databases/(.*)\.yaml" :database
|
||||
#"/transforms/(.*)\.yaml" :transform
|
||||
#"/python-libraries/(.*)\.yaml" :python-library])
|
||||
#"/transform-libraries/(.*)\.yaml" :transform-library])
|
||||
|
||||
(defn- file-type
|
||||
"Find out entity type by file path"
|
||||
@@ -122,13 +122,13 @@
|
||||
(testing "API respects parameters"
|
||||
(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}
|
||||
:all_collections false :data_model false :settings true)]
|
||||
(is (= #{:log :dir :settings :transform :python-library}
|
||||
(is (= #{:log :dir :settings :transform :transform-library}
|
||||
(tar-file-types f)))))
|
||||
|
||||
(testing "We can export just a single collection"
|
||||
(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}
|
||||
:collection (:id coll) :data_model false :settings false)]
|
||||
(is (= #{:log :dir :dashboard :card :collection :transform :python-library}
|
||||
(is (= #{:log :dir :dashboard :card :collection :transform :transform-library}
|
||||
(tar-file-types f)))))
|
||||
|
||||
(testing "We can export two collections"
|
||||
@@ -144,18 +144,18 @@
|
||||
(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}
|
||||
;; eid:... syntax is kept for backward compat
|
||||
:collection (str "eid:" (:entity_id coll)) :data_model false :settings false)]
|
||||
(is (= #{:log :dir :dashboard :card :collection :transform :python-library}
|
||||
(is (= #{:log :dir :dashboard :card :collection :transform :transform-library}
|
||||
(tar-file-types f)))))
|
||||
|
||||
(testing "We can export that collection using entity id"
|
||||
(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}
|
||||
:collection (:entity_id coll) :data_model false :settings false)]
|
||||
(is (= #{:log :dir :dashboard :card :collection :transform :python-library}
|
||||
(is (= #{:log :dir :dashboard :card :collection :transform :transform-library}
|
||||
(tar-file-types f)))))
|
||||
|
||||
(testing "Default export: all-collections, data-model, settings"
|
||||
(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {})]
|
||||
(is (= #{:transform :log :dir :dashboard :card :collection :settings :schema :database :python-library}
|
||||
(is (= #{:transform :log :dir :dashboard :card :collection :settings :schema :database :transform-library}
|
||||
(tar-file-types f)))))
|
||||
|
||||
(testing "On exception API returns log"
|
||||
@@ -221,7 +221,7 @@
|
||||
io/input-stream)
|
||||
ba (#'api.serialization/ba-copy res)]
|
||||
(testing "Archive contains correct number of files with proper log entries"
|
||||
(is (= 13
|
||||
(is (= 14
|
||||
(with-open [tar (open-tar ba)]
|
||||
(count
|
||||
(for [^TarArchiveEntry e (u.compress/entries tar)
|
||||
@@ -229,7 +229,7 @@
|
||||
(do
|
||||
(condp re-find (.getName e)
|
||||
#"/export.log$" (testing "Log contains extract and store entries"
|
||||
(is (= (+ #_extract 12 #_store 12)
|
||||
(is (= (+ #_extract 13 #_store 13)
|
||||
(count (line-seq (io/reader tar))))))
|
||||
nil)
|
||||
(.getName e))))))))
|
||||
@@ -243,7 +243,7 @@
|
||||
"settings" false
|
||||
"field_values" false
|
||||
"duration_ms" (every-pred number? pos?)
|
||||
"count" 12
|
||||
"count" 13
|
||||
"error_count" 0
|
||||
"source" "api"
|
||||
"secrets" false
|
||||
@@ -273,7 +273,7 @@
|
||||
{:request-options {:headers {"content-type" "multipart/form-data"}}}
|
||||
{:file ba}))]
|
||||
(testing "Log contains imported entity types"
|
||||
(is (= #{"Collection" "Dashboard" "Card" "PythonLibrary" "TransformJob" "TransformTag"}
|
||||
(is (= #{"Collection" "Dashboard" "Card" "TransformLibrary" "TransformJob" "TransformTag"}
|
||||
(log-types (line-seq (io/reader (io/input-stream res)))))))
|
||||
|
||||
(testing "Entities are restored in the database"
|
||||
@@ -285,8 +285,8 @@
|
||||
"direction" "import"
|
||||
"duration_ms" pos?
|
||||
"source" "api"
|
||||
"models" "Card,Collection,Dashboard,PythonLibrary,TransformJob,TransformTag"
|
||||
"count" 12
|
||||
"models" "Card,Collection,Dashboard,TransformJob,TransformLibrary,TransformTag"
|
||||
"count" 13
|
||||
"error_count" 0
|
||||
"success" true
|
||||
"error_message" nil}
|
||||
@@ -349,7 +349,7 @@
|
||||
:continue_on_error true)
|
||||
log (slurp (io/input-stream res))]
|
||||
(testing "Log shows loaded entities and error"
|
||||
(is (= #{"Dashboard" "Card" "Collection" "TransformJob" "TransformTag" "PythonLibrary"}
|
||||
(is (= #{"Dashboard" "Card" "Collection" "TransformJob" "TransformTag" "TransformLibrary"}
|
||||
(log-types (str/split-lines log))))
|
||||
(is (re-find #"Collection 'DoesNotExist' was not found" log)))
|
||||
|
||||
@@ -359,9 +359,9 @@
|
||||
"direction" "import"
|
||||
"source" "api"
|
||||
"duration_ms" int?
|
||||
"count" 11
|
||||
"count" 12
|
||||
"error_count" 1
|
||||
"models" "Collection,Dashboard,PythonLibrary,TransformJob,TransformTag"}
|
||||
"models" "Collection,Dashboard,TransformJob,TransformLibrary,TransformTag"}
|
||||
(-> (snowplow-test/pop-event-data-and-user-id!) last :data))))))))))
|
||||
|
||||
(deftest import-invalid-archive-test
|
||||
@@ -429,7 +429,7 @@
|
||||
(doseq [^TarArchiveEntry e (u.compress/entries tar)]
|
||||
(condp re-find (.getName e)
|
||||
#"/export.log$" (testing "Log shows extract entries, error, and store entries"
|
||||
(is (= (+ #_extract 12 #_error 1 #_store 11)
|
||||
(is (= (+ #_extract 13 #_error 1 #_store 12)
|
||||
(count (line-seq (io/reader tar))))))
|
||||
nil))))
|
||||
|
||||
@@ -442,7 +442,7 @@
|
||||
"settings" false
|
||||
"field_values" false
|
||||
"duration_ms" pos?
|
||||
"count" 11
|
||||
"count" 13
|
||||
"error_count" 1
|
||||
"source" "api"
|
||||
"secrets" false
|
||||
|
||||
@@ -45,7 +45,7 @@
|
||||
"settings" true
|
||||
"field_values" false
|
||||
"duration_ms" pos?
|
||||
"count" 12
|
||||
"count" 13
|
||||
"source" "cli"
|
||||
"secrets" false
|
||||
"success" true
|
||||
@@ -60,8 +60,8 @@
|
||||
"direction" "import"
|
||||
"duration_ms" pos?
|
||||
"source" "cli"
|
||||
"models" "Card,Collection,PythonLibrary,Setting,TransformJob,TransformTag"
|
||||
"count" 12
|
||||
"models" "Card,Collection,Setting,TransformJob,TransformLibrary,TransformTag"
|
||||
"count" 13
|
||||
"success" true
|
||||
"error_message" nil}
|
||||
(-> (snowplow-test/pop-event-data-and-user-id!) first :data))))
|
||||
|
||||
@@ -1517,7 +1517,7 @@
|
||||
:dimension dimension}}}}})}}}]
|
||||
|
||||
(testing "selecting a collection includes settings metabot and data model by default"
|
||||
(is (= #{"Card" "Collection" "Dashboard" "Database" "PythonLibrary" "Setting" "TransformTag" "TransformJob"}
|
||||
(is (= #{"Card" "Collection" "Dashboard" "Database" "TransformLibrary" "Setting" "TransformTag" "TransformJob"}
|
||||
(->> (extract/extract {:targets [["Collection" coll1-id]]})
|
||||
(map (comp :model first serdes/path))
|
||||
set))))
|
||||
@@ -2531,23 +2531,23 @@
|
||||
(testing "fields from non-published table are NOT exported"
|
||||
(is (= 2 (count field-ids)))))))))
|
||||
|
||||
(deftest python-library-test
|
||||
(deftest transform-library-test
|
||||
(mt/with-empty-h2-app-db!
|
||||
;; Delete any pre-existing python_library entries from migrations
|
||||
(t2/delete! :model/PythonLibrary)
|
||||
(ts/with-temp-dpc [:model/PythonLibrary {lib-id :id lib-entity-id :entity_id} {:path "common"
|
||||
:source "def helper():\n return 42"}]
|
||||
(testing "python library extraction"
|
||||
(let [ser (serdes/extract-one "PythonLibrary" {} (t2/select-one :model/PythonLibrary :id lib-id))]
|
||||
(is (=? {:serdes/meta [{:model "PythonLibrary"
|
||||
(t2/delete! :model/TransformLibrary)
|
||||
(ts/with-temp-dpc [:model/TransformLibrary {lib-id :id lib-entity-id :entity_id} {:language "python"
|
||||
:path "common"
|
||||
:source "def helper():\n return 42"}]
|
||||
(testing "transform library extraction"
|
||||
(let [ser (serdes/extract-one "TransformLibrary" {} (t2/select-one :model/TransformLibrary :id lib-id))]
|
||||
(is (=? {:serdes/meta [{:model "TransformLibrary"
|
||||
:id lib-entity-id}]
|
||||
:language "python"
|
||||
:path "common.py"
|
||||
:source "def helper():\n return 42"
|
||||
:entity_id lib-entity-id
|
||||
:created_at string?}
|
||||
ser))
|
||||
(is (not (contains? ser :id)))
|
||||
|
||||
(testing "has no dependencies"
|
||||
(is (empty? (serdes/dependencies ser)))))))))
|
||||
|
||||
|
||||
@@ -238,16 +238,16 @@
|
||||
["parent.child.yaml"]}
|
||||
(file-set (io/file dump-dir "databases" "mydb" "tables" "table" "fields")))))))))
|
||||
|
||||
(deftest python-library-storage-test
|
||||
(deftest transform-library-storage-test
|
||||
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
|
||||
(mt/with-empty-h2-app-db!
|
||||
(t2/delete! :model/PythonLibrary)
|
||||
(ts/with-temp-dpc [:model/PythonLibrary lib {:path "common" :source "def test(): pass"}]
|
||||
(t2/delete! :model/TransformLibrary)
|
||||
(ts/with-temp-dpc [:model/TransformLibrary lib {:language "python" :path "common" :source "def test(): pass"}]
|
||||
(let [export (into [] (extract/extract {:no-settings true :no-data-model true}))]
|
||||
(storage/store! export dump-dir)
|
||||
(testing "python library stored at top-level python-libraries/"
|
||||
(testing "transform library stored at top-level transform-libraries/"
|
||||
(is (= #{[(str (:entity_id lib) ".yaml")]}
|
||||
(file-set (io/file dump-dir "python-libraries"))))))))))
|
||||
(file-set (io/file dump-dir "transform-libraries"))))))))))
|
||||
|
||||
(deftest name-too-long-test
|
||||
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
[clojure.string :as str]
|
||||
[clojure.test :refer :all]
|
||||
[honey.sql :as sql]
|
||||
[metabase-enterprise.transforms-python.python-runner :as python-runner]
|
||||
[metabase-enterprise.transforms-runner.runner :as runner]
|
||||
[metabase.driver :as driver]
|
||||
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
|
||||
[metabase.driver.sql.query-processor :as sql.qp]
|
||||
@@ -61,10 +61,10 @@
|
||||
query)))
|
||||
|
||||
(defmacro with-python-order-by!
|
||||
"Execute body with python-runner/build-table-query redef'd to include ORDER BY for deterministic test results."
|
||||
"Execute body with runner/build-table-query redef'd to include ORDER BY for deterministic test results."
|
||||
[checkpoint-field & body]
|
||||
`(let [original-fn# @#'python-runner/build-table-query]
|
||||
(with-redefs [python-runner/build-table-query
|
||||
`(let [original-fn# @#'runner/build-table-query]
|
||||
(with-redefs [runner/build-table-query
|
||||
(fn [table-id# source-incremental-strategy# transform-id# limit#]
|
||||
(build-table-query-with-order-by original-fn# table-id# source-incremental-strategy# transform-id# limit# ~checkpoint-field))]
|
||||
~@body)))
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
(ns metabase-enterprise.transforms-inspector.context-test
|
||||
"Tests for transforms-inspector context that require runner language registration.
|
||||
Pre-existing :python tests moved here because transform-source-type now uses dynamic
|
||||
language registration instead of a hardcoded `case`."
|
||||
(:require
|
||||
[clojure.test :refer :all]
|
||||
[metabase.test :as mt]
|
||||
[metabase.transforms-inspector.context :as context]
|
||||
[metabase.transforms.interface :as transforms.i]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(use-fixtures :once (fn [thunk]
|
||||
(doseq [lang [:python :javascript :clojure]]
|
||||
(transforms.i/register-runner! lang))
|
||||
(thunk)))
|
||||
|
||||
(deftest extract-sources-python-test
|
||||
(testing "Python: extracts source tables from source-tables map"
|
||||
(let [transform {:source {:type :python
|
||||
:source-tables {"orders" (mt/id :orders)}
|
||||
:source-database (mt/id)}
|
||||
:name "test"}
|
||||
sources (context/extract-sources transform)]
|
||||
(is (seq sources))
|
||||
(is (= (mt/id :orders) (:table-id (first sources))))
|
||||
(is (= (mt/id) (:db-id (first sources)))))))
|
||||
|
||||
(deftest extract-sources-python-multiple-tables-test
|
||||
(testing "Python: extracts multiple source tables"
|
||||
(let [transform {:source {:type :python
|
||||
:source-tables {"orders" (mt/id :orders)
|
||||
"products" (mt/id :products)}
|
||||
:source-database (mt/id)}
|
||||
:name "test"}
|
||||
sources (context/extract-sources transform)
|
||||
table-ids (set (map :table-id sources))]
|
||||
(is (= 2 (count sources)))
|
||||
(is (contains? table-ids (mt/id :orders)))
|
||||
(is (contains? table-ids (mt/id :products))))))
|
||||
|
||||
(deftest extract-sources-all-runner-languages-test
|
||||
(testing "extract-sources works for all registered runner languages, not just :python"
|
||||
(doseq [lang [:python :javascript :clojure]]
|
||||
(testing (str lang)
|
||||
(let [transform {:source {:type lang
|
||||
:source-tables {"orders" (mt/id :orders)}
|
||||
:source-database (mt/id)}
|
||||
:name "test"}
|
||||
sources (context/extract-sources transform)]
|
||||
(is (seq sources) (str "Expected sources for " lang))
|
||||
(is (= (mt/id :orders) (:table-id (first sources))))
|
||||
(is (= (mt/id) (:db-id (first sources)))))))))
|
||||
@@ -1,193 +0,0 @@
|
||||
(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.api-test
|
||||
"Tests for /api/ee/transforms-python endpoints."
|
||||
(:require
|
||||
[clojure.string :as str]
|
||||
[clojure.test :refer :all]
|
||||
[metabase-enterprise.transforms-python.models.python-library :as python-library]
|
||||
[metabase.permissions.models.permissions-group :as perms-group]
|
||||
[metabase.test :as mt]
|
||||
[metabase.transforms.test-dataset :as transforms-dataset]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(deftest get-library-path-test
|
||||
(testing "GET /api/ee/transforms-python/library/:path"
|
||||
(mt/with-premium-features #{:transforms-python :transforms}
|
||||
(testing "requires transform permissions"
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :get 403 "ee/transforms-python/library/common"))))
|
||||
(mt/with-data-analyst-role! (mt/user->id :lucky)
|
||||
(mt/with-db-perm-for-group! (perms-group/all-users) (mt/id) :perms/transforms :yes
|
||||
(testing "returns nil when no library exists"
|
||||
(t2/delete! :model/PythonLibrary)
|
||||
(is (= "Not found." (mt/user-http-request :lucky :get 404 "ee/transforms-python/library/common"))))
|
||||
(testing "returns existing library"
|
||||
(t2/delete! :model/PythonLibrary)
|
||||
(python-library/update-python-library-source! "common" "def my_function():\n return 42")
|
||||
(is (=? {:source "def my_function():\n return 42"
|
||||
:path "common.py"
|
||||
:created_at some?
|
||||
:updated_at some?}
|
||||
(mt/user-http-request :lucky :get 200 "ee/transforms-python/library/common"))))
|
||||
(testing "rejects invalid paths"
|
||||
(is (= "Invalid library path. Only 'common' is currently supported."
|
||||
(:message (mt/user-http-request :lucky :get 400 "ee/transforms-python/library/invalid-path"))))))))))
|
||||
|
||||
(deftest put-library-path-test
|
||||
(testing "PUT /api/ee/transforms-python/library/:path"
|
||||
(mt/with-premium-features #{:transforms-python :transforms}
|
||||
(testing "requires transform permissions"
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :put 403 "ee/transforms-python/library/common"
|
||||
{:source "def test(): pass"}))))
|
||||
(mt/with-data-analyst-role! (mt/user->id :lucky)
|
||||
(mt/with-db-perm-for-group! (perms-group/all-users) (mt/id) :perms/transforms :yes
|
||||
(testing "validates request body"
|
||||
(testing "requires source field"
|
||||
(is (=? {:errors
|
||||
{:source "string"}}
|
||||
(mt/user-http-request :lucky :put 400 "ee/transforms-python/library/common"
|
||||
{}))))
|
||||
(testing "allows empty string"
|
||||
(t2/delete! :model/PythonLibrary)
|
||||
(is (=? {:source ""
|
||||
:path "common.py"
|
||||
:id integer?
|
||||
:created_at some?
|
||||
:updated_at some?}
|
||||
(mt/user-http-request :lucky :put 200 "ee/transforms-python/library/common"
|
||||
{:source ""})))))
|
||||
(testing "creates new library when none exists"
|
||||
(t2/delete! :model/PythonLibrary)
|
||||
(is (=? {:source "def new_function():\n return 100"
|
||||
:path "common.py"
|
||||
:id integer?
|
||||
:created_at some?
|
||||
:updated_at some?}
|
||||
(mt/user-http-request :lucky :put 200 "ee/transforms-python/library/common"
|
||||
{:source "def new_function():\n return 100"})))
|
||||
(is (= "def new_function():\n return 100"
|
||||
(t2/select-one-fn :source :model/PythonLibrary))))
|
||||
(testing "updates existing library"
|
||||
(t2/delete! :model/PythonLibrary)
|
||||
(python-library/update-python-library-source! "common" "def old_function():\n return 1")
|
||||
(is (=? {:source "def updated_function():\n return 2"
|
||||
:path "common.py"
|
||||
:id integer?
|
||||
:created_at some?
|
||||
:updated_at some?}
|
||||
(mt/user-http-request :lucky :put 200 "ee/transforms-python/library/common"
|
||||
{:source "def updated_function():\n return 2"})))
|
||||
(is (= "def updated_function():\n return 2"
|
||||
(t2/select-one-fn :source :model/PythonLibrary)))
|
||||
(is (= 1 (t2/count :model/PythonLibrary)) "Should not create duplicate records"))
|
||||
(testing "rejects invalid paths"
|
||||
(is (= "Invalid library path. Only 'common' is currently supported."
|
||||
(:message (mt/user-http-request :lucky :put 400 "ee/transforms-python/library/invalid-path"
|
||||
{:source "def test(): pass"}))))))))))
|
||||
|
||||
(deftest test-run-test
|
||||
(mt/with-premium-features #{:transforms :transforms-python}
|
||||
(let [program ["import pandas as pd"
|
||||
"import sys"
|
||||
"def transform():"
|
||||
" print(\"out1\")"
|
||||
" print(\"err1\", file=sys.stderr)"
|
||||
" print(\"out2\")"
|
||||
" print(\"err2\", file=sys.stderr)"
|
||||
" return pd.DataFrame({'x': [42, 43]})"]
|
||||
body {:source_tables {:test (t2/select-one-pk :model/Table :db_id (mt/id))}, :code (str/join "\n" program)}
|
||||
{:keys [error logs output]} (mt/user-http-request :crowberto :post 200 "ee/transforms-python/test-run" body)]
|
||||
(is (nil? error))
|
||||
(is (str/includes? logs "out1\nerr1\nout2\nerr2"))
|
||||
(is (=? {:cols [{:name "x"}] :rows [{:x 42} {:x 43}]} output)))))
|
||||
|
||||
(defn- test-run [& {:keys [program user features source-tables extra-opts]
|
||||
:or {program ["import pandas as pd" "def transform():" " return pd.DataFrame()"]
|
||||
user :crowberto
|
||||
source-tables {:test (t2/select-one-pk :model/Table :db_id (mt/id) :active true)}
|
||||
features #{:transforms :transforms-python}}}]
|
||||
(let [body (merge {:source_tables source-tables, :code (str/join "\n" program)} extra-opts)]
|
||||
(mt/with-premium-features features
|
||||
(mt/user-http-request-full-response user :post "ee/transforms-python/test-run" body))))
|
||||
|
||||
(deftest test-run-with-inputs-test
|
||||
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
|
||||
(mt/dataset transforms-dataset/transforms-test
|
||||
(let [program ["def transform(customers):" " return customers"]
|
||||
source-tables {"customers" (mt/id :transforms_customers)}
|
||||
{:keys [status body]} (test-run :program program :source-tables source-tables)]
|
||||
(is (= 200 status))
|
||||
(is (nil? (:error body)))
|
||||
(is (seq (:rows (:output body))))))))
|
||||
|
||||
(deftest test-run-empty-results-test
|
||||
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
|
||||
(mt/dataset transforms-dataset/transforms-test
|
||||
(let [program ["import pandas as pd" "def transform(customers):" " return pd.DataFrame()"]
|
||||
source-tables {"customers" (mt/id :transforms_customers)}
|
||||
{:keys [status body]} (test-run :program program :source-tables source-tables)]
|
||||
(is (= 200 status))
|
||||
(is (nil? (:error body)))
|
||||
(is (empty? (:rows (:output body))))))))
|
||||
|
||||
(deftest test-run-error-test
|
||||
(let [program ["import pandas as pd" "def transform():" " totallynotsourcecode"]
|
||||
{:keys [status body]} (test-run :program program)]
|
||||
(is (= 200 status))
|
||||
(is (=? {:message "Python execution failure (exit code 1)"} (:error body)))
|
||||
(is (str/includes? (:logs body "") "Traceback"))))
|
||||
|
||||
(deftest test-run-feature-test
|
||||
(is (=? {:status 402} (test-run :features #{})))
|
||||
(testing "transforms alone is not enough"
|
||||
(is (=? {:status 402} (test-run :features #{:transforms}))))
|
||||
(is (=? {:status 200} (test-run :features #{:transforms :transforms-python}))))
|
||||
|
||||
(deftest test-run-permissions-test
|
||||
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
|
||||
(mt/with-data-analyst-role! (mt/user->id :lucky)
|
||||
(mt/with-group-for-user [lucky-group :lucky {:name "Lucky Transforms Group"}]
|
||||
(mt/with-db-perm-for-group! lucky-group (mt/id) :perms/transforms :yes
|
||||
(is (=? {:status 403} (test-run :user :rasta)))
|
||||
(is (=? {:status 200} (test-run :user :lucky))))))))
|
||||
|
||||
(deftest test-run-input-limit-test
|
||||
(mt/dataset transforms-dataset/transforms-test
|
||||
(testing "maximum"
|
||||
(is (=? {:status 400} (test-run :extra-opts {:per_input_row_limit 10000}))))
|
||||
(testing "minimum"
|
||||
(is (=? {:status 400} (test-run :extra-opts {:per_input_row_limit 0}))))
|
||||
(testing "truncates sources"
|
||||
(let [program ["def transform(customers):" " return customers"]
|
||||
source-tables {"customers" (mt/id :transforms_customers)}
|
||||
response (test-run :program program :source-tables source-tables :extra-opts {:per_input_row_limit 2})]
|
||||
(is (=? {:body {:output {:rows #(= 2 (count %))}}} response))))))
|
||||
|
||||
(deftest test-run-output-limit-test
|
||||
(testing "maximum"
|
||||
(is (=? {:status 400} (test-run :extra-opts {:output_row_limit 10000}))))
|
||||
(testing "minimum"
|
||||
(is (=? {:status 400} (test-run :extra-opts {:output_row_limit 0}))))
|
||||
(testing "truncates outputs"
|
||||
(let [program ["import pandas as pd" "def transform():" " return pd.DataFrame({'x': [1,2,3,4]})"]
|
||||
response (test-run :program program :extra-opts {:output_row_limit 2})]
|
||||
(is (=? {:body {:output {:rows #(= 2 (count %))}}} response)))))
|
||||
|
||||
(deftest test-run-timeout-test
|
||||
(let [program ["import pandas as pd"
|
||||
"import time"
|
||||
"def transform():"
|
||||
" time.sleep(30)"
|
||||
" return pd.DataFrame()"]]
|
||||
(mt/with-temporary-setting-values [python-runner-test-run-timeout-seconds 1]
|
||||
(let [response (test-run :program program)]
|
||||
(is (=? {:body {:error {:message "Python execution timed out"}}} response))))))
|
||||
|
||||
(deftest test-run-cols-meta-empty-return-test
|
||||
(let [program ["import pandas as pd"
|
||||
"def transform():"
|
||||
" return pd.DataFrame(columns=['y', 'z', 'x'])"]
|
||||
response (test-run :program program)]
|
||||
(is (=? {:output {:cols [{:name "y"} {:name "z"} {:name "x"}]}} (:body response)))))
|
||||
@@ -3,9 +3,9 @@
|
||||
(:require
|
||||
[clojure.string :as str]
|
||||
[clojure.test :refer :all]
|
||||
[metabase-enterprise.transforms-python.execute :as transforms-python.execute]
|
||||
[metabase-enterprise.transforms-python.python-runner :as python-runner]
|
||||
[metabase-enterprise.transforms-python.python-runner-test :as python-runner-test]
|
||||
[metabase-enterprise.transforms-runner.execute :as runner.execute]
|
||||
[metabase-enterprise.transforms-runner.runner :as runner]
|
||||
[metabase.driver :as driver]
|
||||
[metabase.driver.mysql :as mysql]
|
||||
[metabase.sync.core :as sync]
|
||||
@@ -38,7 +38,7 @@
|
||||
:target target}]
|
||||
(with-transform-cleanup! [_target target]
|
||||
(mt/with-temp [:model/Transform transform transform-def]
|
||||
(transforms-python.execute/execute-python-transform! transform {:run-method :manual})
|
||||
(runner.execute/execute-runner-transform! transform {:run-method :manual} {:runtime "python" :label "Python" :timing-key :python-execution})
|
||||
(let [table (transforms.tu/wait-for-table table-name 10000)
|
||||
columns (t2/select :model/Field :table_id (:id table) {:order-by [:position]})
|
||||
column-names (filterv (fn [x] (not= x "_id")) ;; for mongo
|
||||
@@ -244,7 +244,7 @@
|
||||
(testing (str "Exotic types for " driver-key)
|
||||
(let [{:keys [metadata]} validation
|
||||
type-map (u/for-map [{:keys [name base_type]} (:fields metadata)]
|
||||
[name (python-runner/restricted-insert-type base_type)])]
|
||||
[name (runner/restricted-insert-type base_type)])]
|
||||
|
||||
(is (isa? :type/Integer (type-map "id")))
|
||||
|
||||
@@ -292,7 +292,7 @@
|
||||
(let [{:keys [metadata]} validation]
|
||||
(testing "Base type preservation"
|
||||
(let [type-map (u/for-map [{:keys [name base_type]} (:fields metadata)]
|
||||
[name (python-runner/restricted-insert-type base_type)])]
|
||||
[name (runner/restricted-insert-type base_type)])]
|
||||
(is (isa? (type-map "id") (if (= driver/*driver* :snowflake) :type/Number :type/Integer)))
|
||||
(is (isa? (type-map "name") :type/Text))
|
||||
(is (isa? (type-map "price") :type/Float))
|
||||
@@ -346,7 +346,7 @@
|
||||
(when validation
|
||||
(let [{:keys [rows metadata]} validation
|
||||
type-map (u/for-map [{:keys [name base_type]} (:fields metadata)]
|
||||
[name (python-runner/restricted-insert-type base_type)])]
|
||||
[name (runner/restricted-insert-type base_type)])]
|
||||
|
||||
(testing "Original columns preserved"
|
||||
(is (isa? (type-map "id") (if (= driver/*driver* :snowflake) :type/Number :type/Integer)))
|
||||
@@ -411,7 +411,7 @@
|
||||
(testing "Computed columns are added correctly"
|
||||
(let [{:keys [metadata]} validation
|
||||
type-map (u/for-map [{:keys [name base_type]} (:fields metadata)]
|
||||
[name (python-runner/restricted-insert-type base_type)])]
|
||||
[name (runner/restricted-insert-type base_type)])]
|
||||
(is (= :type/Float (type-map "computed_field")))
|
||||
(is (= :type/Text (type-map "name_upper"))))))))))))))
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"End-to-end tests for Python library functionality."
|
||||
(:require
|
||||
[clojure.test :refer :all]
|
||||
[metabase-enterprise.transforms-python.models.python-library :as python-library]
|
||||
[metabase-enterprise.transforms-runner.models.transform-library :as transform-library]
|
||||
[metabase.driver :as driver]
|
||||
[metabase.test :as mt]
|
||||
[metabase.test.util :as mt.util]
|
||||
@@ -17,14 +17,14 @@
|
||||
(mt/test-drivers #{:postgres}
|
||||
(mt/with-premium-features #{:transforms :transforms-python}
|
||||
(mt/dataset transforms-dataset/transforms-test
|
||||
(mt.util/with-discard-model-updates! [:model/PythonLibrary]
|
||||
(mt.util/with-discard-model-updates! [:model/TransformLibrary]
|
||||
;; Create or update the python library
|
||||
(python-library/update-python-library-source! "common"
|
||||
(str "def multiply_by_two(x):\n"
|
||||
" return x * 2\n"
|
||||
"\n"
|
||||
"def add_greeting(name):\n"
|
||||
" return f'Hello, {name}!'\n"))
|
||||
(transform-library/update-library-source! "python" "common"
|
||||
(str "def multiply_by_two(x):\n"
|
||||
" return x * 2\n"
|
||||
"\n"
|
||||
"def add_greeting(name):\n"
|
||||
" return f'Hello, {name}!'\n"))
|
||||
(let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
|
||||
;; Create and run a transform that uses the library
|
||||
(with-transform-cleanup! [{table-name :name :as target} {:type "table"
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.execute-test
|
||||
(:require
|
||||
[clojure.test :refer :all]
|
||||
[metabase-enterprise.transforms-python.execute :as transforms-python.execute]
|
||||
[metabase-enterprise.transforms-runner.execute :as runner.execute]
|
||||
[metabase.test :as mt]
|
||||
[metabase.test.util :as test.util]
|
||||
[metabase.transforms.test-dataset :as transforms-dataset]
|
||||
@@ -32,7 +32,7 @@
|
||||
" return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}
|
||||
:target (assoc target :database (mt/id))}]
|
||||
(mt/with-temp [:model/Transform transform initial-transform]
|
||||
(transforms-python.execute/execute-python-transform! transform {:run-method :manual})
|
||||
(runner.execute/execute-runner-transform! transform {:run-method :manual} {:runtime "python" :label "Python" :timing-key :python-execution})
|
||||
(transforms.tu/wait-for-table table-name 10000)
|
||||
|
||||
(let [initial-rows (transforms.tu/table-rows table-name)]
|
||||
@@ -53,9 +53,10 @@
|
||||
(.await swap-latch)
|
||||
(original-rename-tables-atomic! driver db-id rename-pairs))]
|
||||
(let [transform-future (future
|
||||
(transforms-python.execute/execute-python-transform!
|
||||
(runner.execute/execute-runner-transform!
|
||||
(t2/select-one :model/Transform (:id transform))
|
||||
{:run-method :manual}))]
|
||||
{:run-method :manual}
|
||||
{:runtime "python" :label "Python" :timing-key :python-execution}))]
|
||||
(is (= [["Alice" 25] ["Bob" 30]]
|
||||
(transforms.tu/table-rows table-name))
|
||||
"Original data should still be accessible during transform")
|
||||
@@ -84,10 +85,10 @@
|
||||
" return pd.DataFrame({'col1': [1, 2, 3], 'col2': ['a', 'b', 'c']})")}
|
||||
:target (assoc target :database (mt/id))}]
|
||||
(mt/with-temp [:model/Transform transform transform-def]
|
||||
(transforms-python.execute/execute-python-transform! transform {:run-method :manual})
|
||||
(runner.execute/execute-runner-transform! transform {:run-method :manual} {:runtime "python" :label "Python" :timing-key :python-execution})
|
||||
(transforms.tu/wait-for-table table-name 10000)
|
||||
|
||||
(transforms-python.execute/execute-python-transform! transform {:run-method :manual})
|
||||
(runner.execute/execute-runner-transform! transform {:run-method :manual} {:runtime "python" :label "Python" :timing-key :python-execution})
|
||||
|
||||
(let [db-id (mt/id)
|
||||
tables (t2/select :model/Table :db_id db-id :active true)]
|
||||
@@ -121,7 +122,7 @@
|
||||
:target (assoc target :database (mt/id))}]
|
||||
(mt/with-temp [:model/Transform transform transform-def]
|
||||
(let [{:keys [run_id]} (try
|
||||
(transforms-python.execute/execute-python-transform! transform {:run-method :manual})
|
||||
(runner.execute/execute-runner-transform! transform {:run-method :manual} {:runtime "python" :label "Python" :timing-key :python-execution})
|
||||
(catch Exception _
|
||||
;; We expect this to fail due to timeout
|
||||
{:run_id (t2/select-one-fn :id :model/TransformRun
|
||||
@@ -153,7 +154,7 @@
|
||||
(is (thrown-with-msg?
|
||||
clojure.lang.ExceptionInfo
|
||||
#"Tables not found:.*nonexistent_table"
|
||||
(transforms-python.execute/execute-python-transform! transform {:run-method :manual}))))))))))))
|
||||
(runner.execute/execute-runner-transform! transform {:run-method :manual} {:runtime "python" :label "Python" :timing-key :python-execution}))))))))))))
|
||||
|
||||
(deftest python-transform-unresolved-source-table-no-schema-test
|
||||
(testing "Python transform error message omits schema when nil"
|
||||
@@ -174,4 +175,4 @@
|
||||
(is (thrown-with-msg?
|
||||
clojure.lang.ExceptionInfo
|
||||
#"Tables not found: missing_table"
|
||||
(transforms-python.execute/execute-python-transform! transform {:run-method :manual}))))))))))
|
||||
(runner.execute/execute-runner-transform! transform {:run-method :manual} {:runtime "python" :label "Python" :timing-key :python-execution}))))))))))
|
||||
|
||||
@@ -1,78 +0,0 @@
|
||||
(ns metabase-enterprise.transforms-python.models.python-library-test
|
||||
(:require
|
||||
[clojure.test :refer :all]
|
||||
[metabase-enterprise.transforms-python.models.python-library :as python-library]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(deftest update-python-library-source-test
|
||||
(testing "update-python-library-source!"
|
||||
(testing "creates new record when none exists"
|
||||
(t2/delete! :model/PythonLibrary)
|
||||
(is (=? {:source "def new_func(): return 1"
|
||||
:path "common.py"
|
||||
:id integer?
|
||||
:created_at some?
|
||||
:updated_at some?}
|
||||
(python-library/update-python-library-source! "common" "def new_func(): return 1")))
|
||||
(is (= 1 (t2/count :model/PythonLibrary)))
|
||||
(is (= "def new_func(): return 1"
|
||||
(t2/select-one-fn :source :model/PythonLibrary))))
|
||||
|
||||
(testing "updates existing record"
|
||||
(is (= 1 (t2/count :model/PythonLibrary)))
|
||||
(is (=? {:source "def updated_func(): return 2"
|
||||
:path "common.py"
|
||||
:id integer?
|
||||
:created_at some?
|
||||
:updated_at some?}
|
||||
(python-library/update-python-library-source! "common" "def updated_func(): return 2")))
|
||||
(is (= 1 (t2/count :model/PythonLibrary)) "Should not create duplicate")
|
||||
(is (= "def updated_func(): return 2"
|
||||
(t2/select-one-fn :source :model/PythonLibrary))))
|
||||
|
||||
(testing "rejects invalid paths"
|
||||
(is (thrown-with-msg? clojure.lang.ExceptionInfo
|
||||
#"Invalid library path"
|
||||
(python-library/update-python-library-source! "invalid-path" "def test(): pass")))
|
||||
(is (= 1 (t2/count :model/PythonLibrary)) "Should not create library with invalid path"))))
|
||||
|
||||
(deftest get-python-library-by-path-test
|
||||
(testing "get-python-library-by-path"
|
||||
(testing "returns library when path is valid"
|
||||
(t2/delete! :model/PythonLibrary)
|
||||
(python-library/update-python-library-source! "common" "def test(): pass")
|
||||
(is (=? {:source "def test(): pass"
|
||||
:path "common.py"
|
||||
:id integer?
|
||||
:created_at some?
|
||||
:updated_at some?}
|
||||
(python-library/get-python-library-by-path "common"))))
|
||||
|
||||
(testing "rejects invalid paths"
|
||||
(is (thrown-with-msg? clojure.lang.ExceptionInfo
|
||||
#"Invalid library path"
|
||||
(python-library/get-python-library-by-path "invalid-path"))))))
|
||||
|
||||
(deftest normalize-path-test
|
||||
(testing "normalize-path function"
|
||||
(testing "adds .py extension when missing"
|
||||
(is (= "common.py" (#'python-library/normalize-path "common"))))
|
||||
|
||||
(testing "doesn't duplicate .py extension"
|
||||
(is (= "common.py" (#'python-library/normalize-path "common.py"))))
|
||||
|
||||
(testing "works with paths that already have .py extension when updating"
|
||||
(t2/delete! :model/PythonLibrary)
|
||||
(python-library/update-python-library-source! "common.py" "def test(): pass")
|
||||
(is (=? {:source "def test(): pass"
|
||||
:path "common.py"
|
||||
:id integer?
|
||||
:created_at some?
|
||||
:updated_at some?}
|
||||
(python-library/get-python-library-by-path "common.py")))
|
||||
;; Verify we can also access it without .py
|
||||
(is (=? {:source "def test(): pass"
|
||||
:path "common.py"}
|
||||
(python-library/get-python-library-by-path "common"))))))
|
||||
@@ -5,9 +5,9 @@
|
||||
[clojure.string :as str]
|
||||
[clojure.test :refer :all]
|
||||
[java-time.api :as t]
|
||||
[metabase-enterprise.transforms-python.python-runner :as python-runner]
|
||||
[metabase-enterprise.transforms-python.s3 :as s3]
|
||||
[metabase-enterprise.transforms-python.settings :as transforms-python.settings]
|
||||
[metabase-enterprise.transforms-runner.runner :as runner]
|
||||
[metabase-enterprise.transforms-runner.s3 :as s3]
|
||||
[metabase-enterprise.transforms-runner.settings :as runner.settings]
|
||||
[metabase.driver :as driver]
|
||||
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
|
||||
[metabase.sync.core :as sync]
|
||||
@@ -26,7 +26,7 @@
|
||||
|
||||
(defn next-job-run-id [] (swap! last-job-run-id inc))
|
||||
|
||||
(defn- parse-jsonl [s] (map json/decode+kw (str/split-lines s)))
|
||||
(defn- parse-jsonl [s] (when s (map json/decode+kw (str/split-lines s))))
|
||||
|
||||
(defn template->regex
|
||||
"Convert a template string with $var$ placeholders to a regex pattern.
|
||||
@@ -57,24 +57,24 @@
|
||||
|
||||
(defn execute! [{:keys [code tables]}]
|
||||
(with-open [shared-storage-ref (s3/open-shared-storage! (or tables {}))]
|
||||
(let [server-url (transforms-python.settings/python-runner-url)
|
||||
(let [server-url (runner.settings/python-runner-url)
|
||||
cancel-chan (a/promise-chan)
|
||||
table-name->id (or tables {})
|
||||
test-id (next-job-run-id)
|
||||
_ (python-runner/copy-tables-to-s3! {:run-id test-id
|
||||
:shared-storage @shared-storage-ref
|
||||
:source {:source-tables table-name->id}
|
||||
:cancel-chan cancel-chan})
|
||||
response (python-runner/execute-python-code-http-call! {:server-url server-url
|
||||
:code code
|
||||
:run-id test-id
|
||||
:table-name->id table-name->id
|
||||
:shared-storage @shared-storage-ref})
|
||||
events (python-runner/read-events @shared-storage-ref)
|
||||
output-manifest (python-runner/read-output-manifest @shared-storage-ref)]
|
||||
_ (runner/copy-tables-to-s3! {:run-id test-id
|
||||
:shared-storage @shared-storage-ref
|
||||
:source {:source-tables table-name->id}
|
||||
:cancel-chan cancel-chan})
|
||||
response (runner/execute-python-code-http-call! {:server-url server-url
|
||||
:code code
|
||||
:run-id test-id
|
||||
:table-name->id table-name->id
|
||||
:shared-storage @shared-storage-ref})
|
||||
events (runner/read-events @shared-storage-ref)
|
||||
output-manifest (runner/read-output-manifest @shared-storage-ref)]
|
||||
;; not sure about munging this all together but its what tests expect for now
|
||||
(merge (:body response)
|
||||
{:output (when-some [in (python-runner/open-output @shared-storage-ref)] (with-open [in in] (slurp in)))
|
||||
{:output (when-some [in (runner/open-output @shared-storage-ref)] (with-open [in in] (slurp in)))
|
||||
:output-manifest output-manifest
|
||||
:stdout (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
|
||||
:stderr (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))}))))
|
||||
@@ -366,15 +366,17 @@
|
||||
"updated_at" :type/DateTime
|
||||
"scheduled_for" :type/DateTimeWithLocalTZ}
|
||||
(u/for-map [{:keys [name base_type]} (:fields metadata)]
|
||||
[name (python-runner/restricted-insert-type base_type)]))))))))
|
||||
[name (runner/restricted-insert-type base_type)]))))))))
|
||||
|
||||
(deftest transform-function-with-library-test
|
||||
(testing "transform function can use libraries"
|
||||
(mt/test-drivers #{:postgres}
|
||||
(mt/with-temp [:model/PythonLibrary _ {:path "circle"
|
||||
:source "import math\n\ndef calculate_circle_area(radius):\n return math.pi * radius ** 2"}
|
||||
:model/PythonLibrary _ {:path "utils"
|
||||
:source "def format_currency(amount):\n return f\"${amount:,.2f}\""}]
|
||||
(mt/with-temp [:model/TransformLibrary _ {:language "python"
|
||||
:path "circle"
|
||||
:source "import math\n\ndef calculate_circle_area(radius):\n return math.pi * radius ** 2"}
|
||||
:model/TransformLibrary _ {:language "python"
|
||||
:path "utils"
|
||||
:source "def format_currency(amount):\n return f\"${amount:,.2f}\""}]
|
||||
(let [transform-code (str "import pandas as pd\n"
|
||||
"from circle import calculate_circle_area\n"
|
||||
"from utils import format_currency\n"
|
||||
@@ -388,17 +390,17 @@
|
||||
result (execute! {:code transform-code})]
|
||||
(is (=? {:output (jsonl-output [{:radius 5, :area 78.5398163397, :price "$78.54"}
|
||||
{:radius 10, :area 314.159265359, :price "$314.16"}])
|
||||
:stdout (ok-stdout 2 3)
|
||||
#_#_:stderr ""}
|
||||
:stdout (ok-stdout 2 3)}
|
||||
result)))))))
|
||||
|
||||
(deftest transform-function-without-libraries-test
|
||||
(testing "transform function works when no libraries exist"
|
||||
(mt/test-drivers #{:postgres}
|
||||
(with-redefs [t2/select-fn->fn (fn [k v model]
|
||||
(with-redefs [t2/select-fn->fn (fn [k v model & conditions]
|
||||
(when (and (= k :path)
|
||||
(= v :source)
|
||||
(= model :model/PythonLibrary))
|
||||
(= model :model/TransformLibrary)
|
||||
(= conditions [:language "python"]))
|
||||
{}))]
|
||||
(let [transform-code (str "import pandas as pd\n"
|
||||
"\n"
|
||||
@@ -412,10 +414,11 @@
|
||||
(deftest transform-function-library-import-error-test
|
||||
(testing "transform function handles missing library gracefully"
|
||||
(mt/test-drivers #{:postgres}
|
||||
(with-redefs [t2/select-fn->fn (fn [k v model]
|
||||
(with-redefs [t2/select-fn->fn (fn [k v model & conditions]
|
||||
(when (and (= k :path)
|
||||
(= v :source)
|
||||
(= model :model/PythonLibrary))
|
||||
(= model :model/TransformLibrary)
|
||||
(= conditions [:language "python"]))
|
||||
{"utils" "def helper():\n return 42"}))]
|
||||
(let [transform-code (str "import pandas as pd\n"
|
||||
"from common import some_function # This library doesn't exist\n"
|
||||
@@ -423,7 +426,6 @@
|
||||
"def transform():\n"
|
||||
" return pd.DataFrame({'value': [some_function()]})")
|
||||
result (execute! {:code transform-code})]
|
||||
;; TODO this error message could still be improved a lot
|
||||
(is (=? {#_#_:error "Execution failed"
|
||||
:exit_code 1
|
||||
:stderr #(str/includes? % "No module named 'common'")}
|
||||
@@ -491,7 +493,7 @@
|
||||
"created_date" :type/Date
|
||||
"description" :type/Text}
|
||||
(u/for-map [{:keys [name base_type]} (:fields metadata)]
|
||||
[name (python-runner/restricted-insert-type base_type)]))))
|
||||
[name (runner/restricted-insert-type base_type)]))))
|
||||
|
||||
;; cleanup
|
||||
(driver/drop-table! driver db-id qualified-table-name)))))
|
||||
|
||||
@@ -7,8 +7,8 @@
|
||||
[clj-http.client :as http]
|
||||
[clojure.string :as str]
|
||||
[clojure.test :refer :all]
|
||||
[metabase-enterprise.transforms-python.s3 :as s3]
|
||||
[metabase-enterprise.transforms-python.settings :as transforms-python.settings]
|
||||
[metabase-enterprise.transforms-runner.s3 :as s3]
|
||||
[metabase-enterprise.transforms-runner.settings :as runner.settings]
|
||||
[metabase.test :as mt]
|
||||
[metabase.util.log :as log])
|
||||
(:import
|
||||
@@ -24,7 +24,7 @@
|
||||
|
||||
(defn- s3-endpoint-running? []
|
||||
(try
|
||||
(http/head (transforms-python.settings/python-runner-url) {:throw-exceptions false})
|
||||
(http/head (runner.settings/python-runner-url) {:throw-exceptions false})
|
||||
(catch ConnectException _
|
||||
false)))
|
||||
|
||||
@@ -38,7 +38,7 @@
|
||||
(testing "We can open an s3 connection, and read and write things"
|
||||
(with-open [s3-client (s3/create-s3-client)]
|
||||
(is (instance? S3Client s3-client))
|
||||
(let [bucket (transforms-python.settings/python-storage-s-3-bucket)
|
||||
(let [bucket (runner.settings/python-storage-s-3-bucket)
|
||||
key (str "test-object-" (random-uuid) ".txt")
|
||||
body (str "Hello, S3! My secret is:" (random-uuid))]
|
||||
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
(:require
|
||||
[clojure.string :as str]
|
||||
[clojure.test :refer :all]
|
||||
[metabase-enterprise.transforms-python.execute :as transforms-python.execute]
|
||||
[metabase-enterprise.transforms-python.init]
|
||||
[metabase-enterprise.transforms-python.python-runner :as transforms-python.python-runner]
|
||||
[metabase-enterprise.transforms-runner.execute :as runner.execute]
|
||||
[metabase-enterprise.transforms-runner.runner :as runner]
|
||||
[metabase.driver :as driver]
|
||||
[metabase.permissions.models.permissions-group :as perms-group]
|
||||
[metabase.test :as mt]
|
||||
@@ -236,10 +236,10 @@
|
||||
(transforms.tu/wait-for-table (:name target) 5000)))
|
||||
|
||||
(run-scenario [scenario schema]
|
||||
(with-redefs [transforms-python.execute/python-message-loop-sleep-duration (Duration/ofMillis fast-log-polling-ms)
|
||||
transforms-python.execute/transfer-file-to-db (if-some [e (:writeback-ex scenario)]
|
||||
(fn [& _] (throw e))
|
||||
@#'transforms-python.execute/transfer-file-to-db)]
|
||||
(with-redefs [runner.execute/message-loop-sleep-duration (Duration/ofMillis fast-log-polling-ms)
|
||||
runner.execute/transfer-file-to-db (if-some [e (:writeback-ex scenario)]
|
||||
(fn [& _] (throw e))
|
||||
@#'runner.execute/transfer-file-to-db)]
|
||||
(with-transform-cleanup! [target {:type "table"
|
||||
:schema schema
|
||||
:name "result"}]
|
||||
@@ -377,13 +377,13 @@
|
||||
(blocking-redefs [{:keys [block]} ready-signal wait-signal]
|
||||
(case block
|
||||
:read
|
||||
(let [f-ref #'transforms-python.python-runner/write-jsonl-row-to-os-rff
|
||||
(let [f-ref #'runner/write-jsonl-row-to-os-rff
|
||||
f @f-ref]
|
||||
{f-ref
|
||||
(fn [os fields-meta col-meta]
|
||||
(rf-proxy ready-signal wait-signal (f os fields-meta col-meta)))})
|
||||
:write
|
||||
(let [f-ref #'transforms-python.execute/transfer-file-to-db
|
||||
(let [f-ref #'runner.execute/transfer-file-to-db
|
||||
f @f-ref]
|
||||
{f-ref
|
||||
(fn [& args]
|
||||
@@ -396,11 +396,11 @@
|
||||
(let [ready-signal (promise) ; test blocks: until the run is ready to be cancelled
|
||||
wait-signal (promise) ; run blocks: until the test has cancelled
|
||||
finished-signal (promise) ; test blocks: until the run has finished (exceptionally or not)
|
||||
exec-fn @#'transforms-python.execute/execute-python-transform!
|
||||
exec-fn @#'runner.execute/execute-runner-transform!
|
||||
redefs (blocking-redefs scenario ready-signal wait-signal)]
|
||||
(with-redefs-fn
|
||||
(merge
|
||||
{#'transforms-python.execute/execute-python-transform!
|
||||
{#'runner.execute/execute-runner-transform!
|
||||
(fn [& args]
|
||||
(try
|
||||
(apply exec-fn args)
|
||||
@@ -470,7 +470,7 @@
|
||||
(mt/with-premium-features #{:transforms :transforms-python}
|
||||
(mt/dataset transforms-dataset/transforms-test
|
||||
(let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
|
||||
(with-redefs [transforms-python.execute/python-message-loop-sleep-duration (Duration/ofMillis fast-log-polling-ms)]
|
||||
(with-redefs [runner.execute/message-loop-sleep-duration (Duration/ofMillis fast-log-polling-ms)]
|
||||
(with-transform-cleanup! [target {:type "table"
|
||||
:schema schema
|
||||
:name "result"}]
|
||||
|
||||
@@ -0,0 +1,282 @@
|
||||
(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-runner.api-test
|
||||
"Tests for /api/ee/advanced-transforms endpoints and runner-based transform CRUD."
|
||||
(:require
|
||||
[clojure.string :as str]
|
||||
[clojure.test :refer :all]
|
||||
[metabase-enterprise.transforms-clojure.init]
|
||||
[metabase-enterprise.transforms-javascript.init]
|
||||
[metabase-enterprise.transforms-python.init]
|
||||
[metabase-enterprise.transforms-runner.models.transform-library :as transform-library]
|
||||
[metabase.permissions.models.permissions-group :as perms-group]
|
||||
[metabase.test :as mt]
|
||||
[metabase.transforms.crud :as transforms.crud]
|
||||
[metabase.transforms.interface :as transforms.i]
|
||||
[metabase.transforms.test-dataset :as transforms-dataset]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
;;; ------------------------------------------------ CRUD smoke tests ------------------------------------------------
|
||||
|
||||
(deftest create-runner-transform-for-all-languages-test
|
||||
(testing "Creating a transform via the API works for every registered runner language"
|
||||
(mt/with-premium-features #{:transforms :transforms-python}
|
||||
(with-redefs [transforms.crud/check-database-feature (constantly nil)]
|
||||
(doseq [lang (sort (transforms.i/runner-languages))]
|
||||
(testing (str "language: " (name lang))
|
||||
(let [response (mt/user-http-request :crowberto :post "transform"
|
||||
{:name (str "Test " (name lang) " transform")
|
||||
:source {:type (name lang)
|
||||
:body "placeholder"
|
||||
:source-tables {}
|
||||
:source-database (mt/id)}
|
||||
:target {:type "table"
|
||||
:schema nil
|
||||
:name (str (name lang) "_test_output")
|
||||
:database (mt/id)}})]
|
||||
(is (pos-int? (:id response))
|
||||
(str "Expected a transform id for " (name lang) ", got: " (pr-str response)))
|
||||
(when-let [id (:id response)]
|
||||
(mt/user-http-request :crowberto :delete (format "transform/%d" id))))))))))
|
||||
|
||||
;;; ------------------------------------------- Library endpoint tests -----------------------------------------------
|
||||
|
||||
(deftest get-library-path-test
|
||||
(testing "GET /api/ee/advanced-transforms/library/:path"
|
||||
(mt/with-premium-features #{:transforms-python :transforms}
|
||||
(testing "requires transform permissions"
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :get 403 "ee/advanced-transforms/library/common"))))
|
||||
(mt/with-data-analyst-role! (mt/user->id :lucky)
|
||||
(mt/with-db-perm-for-group! (perms-group/all-users) (mt/id) :perms/transforms :yes
|
||||
(testing "returns nil when no library exists"
|
||||
(t2/delete! :model/TransformLibrary :language "python")
|
||||
(is (= "Not found." (mt/user-http-request :lucky :get 404 "ee/advanced-transforms/library/common"))))
|
||||
(testing "returns existing library"
|
||||
(t2/delete! :model/TransformLibrary :language "python")
|
||||
(transform-library/update-library-source! "python" "common" "def my_function():\n return 42")
|
||||
(is (=? {:source "def my_function():\n return 42"
|
||||
:path "common.py"
|
||||
:created_at some?
|
||||
:updated_at some?}
|
||||
(mt/user-http-request :lucky :get 200 "ee/advanced-transforms/library/common"))))
|
||||
(testing "rejects invalid paths"
|
||||
(is (= "Invalid library path. Only 'common' is currently supported."
|
||||
(:message (mt/user-http-request :lucky :get 400 "ee/advanced-transforms/library/invalid-path"))))))))))
|
||||
|
||||
(deftest put-library-path-test
|
||||
(testing "PUT /api/ee/advanced-transforms/library/:path"
|
||||
(mt/with-premium-features #{:transforms-python :transforms}
|
||||
(testing "requires transform permissions"
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :put 403 "ee/advanced-transforms/library/common"
|
||||
{:source "def test(): pass"}))))
|
||||
(mt/with-data-analyst-role! (mt/user->id :lucky)
|
||||
(mt/with-db-perm-for-group! (perms-group/all-users) (mt/id) :perms/transforms :yes
|
||||
(testing "validates request body"
|
||||
(testing "requires source field"
|
||||
(is (=? {:errors
|
||||
{:source "string"}}
|
||||
(mt/user-http-request :lucky :put 400 "ee/advanced-transforms/library/common"
|
||||
{}))))
|
||||
(testing "allows empty string"
|
||||
(t2/delete! :model/TransformLibrary :language "python")
|
||||
(is (=? {:source ""
|
||||
:path "common.py"
|
||||
:id integer?
|
||||
:created_at some?
|
||||
:updated_at some?}
|
||||
(mt/user-http-request :lucky :put 200 "ee/advanced-transforms/library/common"
|
||||
{:source ""})))))
|
||||
(testing "creates new library when none exists"
|
||||
(t2/delete! :model/TransformLibrary :language "python")
|
||||
(is (=? {:source "def new_function():\n return 100"
|
||||
:path "common.py"
|
||||
:id integer?
|
||||
:created_at some?
|
||||
:updated_at some?}
|
||||
(mt/user-http-request :lucky :put 200 "ee/advanced-transforms/library/common"
|
||||
{:source "def new_function():\n return 100"})))
|
||||
(is (= "def new_function():\n return 100"
|
||||
(t2/select-one-fn :source :model/TransformLibrary :language "python"))))
|
||||
(testing "updates existing library"
|
||||
(t2/delete! :model/TransformLibrary :language "python")
|
||||
(transform-library/update-library-source! "python" "common" "def old_function():\n return 1")
|
||||
(is (=? {:source "def updated_function():\n return 2"
|
||||
:path "common.py"
|
||||
:id integer?
|
||||
:created_at some?
|
||||
:updated_at some?}
|
||||
(mt/user-http-request :lucky :put 200 "ee/advanced-transforms/library/common"
|
||||
{:source "def updated_function():\n return 2"})))
|
||||
(is (= "def updated_function():\n return 2"
|
||||
(t2/select-one-fn :source :model/TransformLibrary :language "python")))
|
||||
(is (= 1 (t2/count :model/TransformLibrary :language "python")) "Should not create duplicate records"))
|
||||
(testing "rejects invalid paths"
|
||||
(is (= "Invalid library path. Only 'common' is currently supported."
|
||||
(:message (mt/user-http-request :lucky :put 400 "ee/advanced-transforms/library/invalid-path"
|
||||
{:source "def test(): pass"}))))))))))
|
||||
|
||||
(deftest get-javascript-library-path-test
|
||||
(testing "GET /api/ee/advanced-transforms/library/:path?type=javascript"
|
||||
(mt/with-premium-features #{:transforms-python :transforms}
|
||||
(mt/with-data-analyst-role! (mt/user->id :lucky)
|
||||
(mt/with-db-perm-for-group! (perms-group/all-users) (mt/id) :perms/transforms :yes
|
||||
(testing "returns nil when no JS library exists"
|
||||
(t2/delete! :model/TransformLibrary :language "javascript")
|
||||
(is (= "Not found." (mt/user-http-request :lucky :get 404 "ee/advanced-transforms/library/common"
|
||||
:type "javascript"))))
|
||||
(testing "returns existing JS library"
|
||||
(t2/delete! :model/TransformLibrary :language "javascript")
|
||||
(transform-library/update-library-source! "javascript" "common" "function myFunc() { return 42; }")
|
||||
(is (=? {:source "function myFunc() { return 42; }"
|
||||
:path "common.js"
|
||||
:created_at some?
|
||||
:updated_at some?}
|
||||
(mt/user-http-request :lucky :get 200 "ee/advanced-transforms/library/common"
|
||||
:type "javascript"))))
|
||||
(testing "without type param, returns Python library (not JS)"
|
||||
(t2/delete! :model/TransformLibrary :language "python")
|
||||
(transform-library/update-library-source! "python" "common" "def py_func(): return 1")
|
||||
(is (=? {:source "def py_func(): return 1"
|
||||
:path "common.py"}
|
||||
(mt/user-http-request :lucky :get 200 "ee/advanced-transforms/library/common")))))))))
|
||||
|
||||
(deftest put-javascript-library-path-test
|
||||
(testing "PUT /api/ee/advanced-transforms/library/:path with type=javascript"
|
||||
(mt/with-premium-features #{:transforms-python :transforms}
|
||||
(mt/with-data-analyst-role! (mt/user->id :lucky)
|
||||
(mt/with-db-perm-for-group! (perms-group/all-users) (mt/id) :perms/transforms :yes
|
||||
(testing "creates new JS library"
|
||||
(t2/delete! :model/TransformLibrary :language "javascript")
|
||||
(is (=? {:source "function newFunc() { return 100; }"
|
||||
:path "common.js"
|
||||
:id integer?
|
||||
:created_at some?
|
||||
:updated_at some?}
|
||||
(mt/user-http-request :lucky :put 200 "ee/advanced-transforms/library/common"
|
||||
{:source "function newFunc() { return 100; }"
|
||||
:type "javascript"})))
|
||||
(is (= "function newFunc() { return 100; }"
|
||||
(t2/select-one-fn :source :model/TransformLibrary :language "javascript"))))
|
||||
(testing "updates existing JS library"
|
||||
(is (=? {:source "function updated() { return 2; }"
|
||||
:path "common.js"}
|
||||
(mt/user-http-request :lucky :put 200 "ee/advanced-transforms/library/common"
|
||||
{:source "function updated() { return 2; }"
|
||||
:type "javascript"})))
|
||||
(is (= 1 (t2/count :model/TransformLibrary :language "javascript")) "Should not create duplicate"))
|
||||
(testing "without type param, updates Python library (not JS)"
|
||||
(t2/delete! :model/TransformLibrary :language "python")
|
||||
(is (=? {:source "def py_func(): return 1"
|
||||
:path "common.py"}
|
||||
(mt/user-http-request :lucky :put 200 "ee/advanced-transforms/library/common"
|
||||
{:source "def py_func(): return 1"})))
|
||||
(is (= 1 (t2/count :model/TransformLibrary :language "python")))))))))
|
||||
|
||||
;;; ------------------------------------------- Test-run endpoint tests ----------------------------------------------
|
||||
|
||||
(deftest test-run-test
|
||||
(mt/with-premium-features #{:transforms :transforms-python}
|
||||
(let [program ["import pandas as pd"
|
||||
"import sys"
|
||||
"def transform():"
|
||||
" print(\"out1\")"
|
||||
" print(\"err1\", file=sys.stderr)"
|
||||
" print(\"out2\")"
|
||||
" print(\"err2\", file=sys.stderr)"
|
||||
" return pd.DataFrame({'x': [42, 43]})"]
|
||||
body {:source_tables {:test (t2/select-one-pk :model/Table :db_id (mt/id))}, :code (str/join "\n" program)}
|
||||
{:keys [error logs output]} (mt/user-http-request :crowberto :post 200 "ee/advanced-transforms/test-run" body)]
|
||||
(is (nil? error))
|
||||
(is (str/includes? logs "out1\nerr1\nout2\nerr2"))
|
||||
(is (=? {:cols [{:name "x"}] :rows [{:x 42} {:x 43}]} output)))))
|
||||
|
||||
(defn- test-run [& {:keys [program user features source-tables extra-opts]
|
||||
:or {program ["import pandas as pd" "def transform():" " return pd.DataFrame()"]
|
||||
user :crowberto
|
||||
source-tables {:test (t2/select-one-pk :model/Table :db_id (mt/id) :active true)}
|
||||
features #{:transforms :transforms-python}}}]
|
||||
(let [body (merge {:source_tables source-tables, :code (str/join "\n" program)} extra-opts)]
|
||||
(mt/with-premium-features features
|
||||
(mt/user-http-request-full-response user :post "ee/advanced-transforms/test-run" body))))
|
||||
|
||||
(deftest test-run-with-inputs-test
|
||||
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
|
||||
(mt/dataset transforms-dataset/transforms-test
|
||||
(let [program ["def transform(customers):" " return customers"]
|
||||
source-tables {"customers" (mt/id :transforms_customers)}
|
||||
{:keys [status body]} (test-run :program program :source-tables source-tables)]
|
||||
(is (= 200 status))
|
||||
(is (nil? (:error body)))
|
||||
(is (seq (:rows (:output body))))))))
|
||||
|
||||
(deftest test-run-empty-results-test
|
||||
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
|
||||
(mt/dataset transforms-dataset/transforms-test
|
||||
(let [program ["import pandas as pd" "def transform(customers):" " return pd.DataFrame()"]
|
||||
source-tables {"customers" (mt/id :transforms_customers)}
|
||||
{:keys [status body]} (test-run :program program :source-tables source-tables)]
|
||||
(is (= 200 status))
|
||||
(is (nil? (:error body)))
|
||||
(is (empty? (:rows (:output body))))))))
|
||||
|
||||
(deftest test-run-error-test
|
||||
(let [program ["import pandas as pd" "def transform():" " totallynotsourcecode"]
|
||||
{:keys [status body]} (test-run :program program)]
|
||||
(is (= 200 status))
|
||||
(is (=? {:message "Python execution failure (exit code 1)"} (:error body)))
|
||||
(is (str/includes? (:logs body "") "Traceback"))))
|
||||
|
||||
(deftest test-run-feature-test
|
||||
(is (=? {:status 402} (test-run :features #{})))
|
||||
(testing "transforms alone is not enough"
|
||||
(is (=? {:status 402} (test-run :features #{:transforms}))))
|
||||
(is (=? {:status 200} (test-run :features #{:transforms :transforms-python}))))
|
||||
|
||||
(deftest test-run-permissions-test
|
||||
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
|
||||
(mt/with-data-analyst-role! (mt/user->id :lucky)
|
||||
(mt/with-group-for-user [lucky-group :lucky {:name "Lucky Transforms Group"}]
|
||||
(mt/with-db-perm-for-group! lucky-group (mt/id) :perms/transforms :yes
|
||||
(is (=? {:status 403} (test-run :user :rasta)))
|
||||
(is (=? {:status 200} (test-run :user :lucky))))))))
|
||||
|
||||
(deftest test-run-input-limit-test
|
||||
(mt/dataset transforms-dataset/transforms-test
|
||||
(testing "maximum"
|
||||
(is (=? {:status 400} (test-run :extra-opts {:per_input_row_limit 10000}))))
|
||||
(testing "minimum"
|
||||
(is (=? {:status 400} (test-run :extra-opts {:per_input_row_limit 0}))))
|
||||
(testing "truncates sources"
|
||||
(let [program ["def transform(customers):" " return customers"]
|
||||
source-tables {"customers" (mt/id :transforms_customers)}
|
||||
response (test-run :program program :source-tables source-tables :extra-opts {:per_input_row_limit 2})]
|
||||
(is (=? {:body {:output {:rows #(= 2 (count %))}}} response))))))
|
||||
|
||||
(deftest test-run-output-limit-test
|
||||
(testing "maximum"
|
||||
(is (=? {:status 400} (test-run :extra-opts {:output_row_limit 10000}))))
|
||||
(testing "minimum"
|
||||
(is (=? {:status 400} (test-run :extra-opts {:output_row_limit 0}))))
|
||||
(testing "truncates outputs"
|
||||
(let [program ["import pandas as pd" "def transform():" " return pd.DataFrame({'x': [1,2,3,4]})"]
|
||||
response (test-run :program program :extra-opts {:output_row_limit 2})]
|
||||
(is (=? {:body {:output {:rows #(= 2 (count %))}}} response)))))
|
||||
|
||||
(deftest test-run-timeout-test
|
||||
(let [program ["import pandas as pd"
|
||||
"import time"
|
||||
"def transform():"
|
||||
" time.sleep(30)"
|
||||
" return pd.DataFrame()"]]
|
||||
(mt/with-temporary-setting-values [python-runner-test-run-timeout-seconds 1]
|
||||
(let [response (test-run :program program)]
|
||||
(is (=? {:body {:error {:message "Python execution timed out"}}} response))))))
|
||||
|
||||
(deftest test-run-cols-meta-empty-return-test
|
||||
(let [program ["import pandas as pd"
|
||||
"def transform():"
|
||||
" return pd.DataFrame(columns=['y', 'z', 'x'])"]
|
||||
response (test-run :program program)]
|
||||
(is (=? {:output {:cols [{:name "y"} {:name "z"} {:name "x"}]}} (:body response)))))
|
||||
@@ -0,0 +1,137 @@
|
||||
(ns metabase-enterprise.transforms-runner.execute-test
|
||||
"Tests for the shared runner execution infrastructure.
|
||||
Covers pure functions: source-table-value->dependency, message-log state, table-schema."
|
||||
(:require
|
||||
[clojure.test :refer :all]
|
||||
[clojure.test.check.clojure-test :refer [defspec]]
|
||||
[clojure.test.check.generators :as gen]
|
||||
[clojure.test.check.properties :as prop]
|
||||
[metabase-enterprise.transforms-runner.execute :as runner.execute]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
;;; ------------------------------------------------ source-table-value->dependency --------------------------------
|
||||
|
||||
(deftest source-table-value->dependency-int-test
|
||||
(testing "integer table ID returns {:table id}"
|
||||
(is (= {:table 42} (runner.execute/source-table-value->dependency 42)))
|
||||
(is (= {:table 1} (runner.execute/source-table-value->dependency 1)))))
|
||||
|
||||
(deftest source-table-value->dependency-ref-with-table-id-test
|
||||
(testing "map with :table_id returns {:table table_id}"
|
||||
(is (= {:table 100}
|
||||
(runner.execute/source-table-value->dependency
|
||||
{:database_id 1 :schema "public" :table "foo" :table_id 100})))))
|
||||
|
||||
(deftest source-table-value->dependency-ref-without-table-id-test
|
||||
(testing "map without :table_id returns {:table-ref ...} with only the relevant keys"
|
||||
(is (= {:table-ref {:database_id 1 :schema "public" :table "foo"}}
|
||||
(runner.execute/source-table-value->dependency
|
||||
{:database_id 1 :schema "public" :table "foo"})))
|
||||
(testing "extra keys are stripped"
|
||||
(is (= {:table-ref {:database_id 1 :schema nil :table "bar"}}
|
||||
(runner.execute/source-table-value->dependency
|
||||
{:database_id 1 :schema nil :table "bar" :extra "ignored"}))))))
|
||||
|
||||
(defspec source-table-value->dependency-int-prop 100
|
||||
(prop/for-all [n gen/nat]
|
||||
(let [result (runner.execute/source-table-value->dependency n)]
|
||||
(and (= {:table n} result)
|
||||
(int? (:table result))))))
|
||||
|
||||
(def ^:private gen-source-table-ref
|
||||
"Generator for source table ref maps (without :table_id)."
|
||||
(gen/let [db-id gen/nat
|
||||
schema (gen/one-of [(gen/return nil) gen/string-alphanumeric])
|
||||
table gen/string-alphanumeric]
|
||||
{:database_id db-id :schema schema :table table}))
|
||||
|
||||
(defspec source-table-value->dependency-ref-no-table-id-prop 100
|
||||
(prop/for-all [ref gen-source-table-ref]
|
||||
(let [result (runner.execute/source-table-value->dependency ref)]
|
||||
(and (contains? result :table-ref)
|
||||
(= (select-keys ref [:database_id :schema :table])
|
||||
(:table-ref result))))))
|
||||
|
||||
(defspec source-table-value->dependency-ref-with-table-id-prop 100
|
||||
(prop/for-all [ref gen-source-table-ref
|
||||
tid gen/nat]
|
||||
(let [result (runner.execute/source-table-value->dependency (assoc ref :table_id tid))]
|
||||
(= {:table tid} result))))
|
||||
|
||||
;;; ------------------------------------------------ Message Log ---------------------------------------------------
|
||||
|
||||
(deftest empty-message-log-test
|
||||
(let [ml (#'runner.execute/empty-message-log)]
|
||||
(is (= {:pre [] :runner nil :post []} @ml))))
|
||||
|
||||
(deftest log!-before-runner-test
|
||||
(testing "log! appends to :pre before runner logs arrive"
|
||||
(let [ml (#'runner.execute/empty-message-log)]
|
||||
(#'runner.execute/log! ml "first")
|
||||
(#'runner.execute/log! ml "second")
|
||||
(is (= ["first" "second"] (:pre @ml)))
|
||||
(is (nil? (:runner @ml)))
|
||||
(is (= [] (:post @ml))))))
|
||||
|
||||
(deftest log!-after-runner-test
|
||||
(testing "log! appends to :post after runner logs are set"
|
||||
(let [ml (#'runner.execute/empty-message-log)]
|
||||
(#'runner.execute/log! ml "before")
|
||||
(#'runner.execute/replace-runner-logs! ml [{:message "runner msg"}])
|
||||
(#'runner.execute/log! ml "after")
|
||||
(is (= ["before"] (:pre @ml)))
|
||||
(is (= [{:message "runner msg"}] (:runner @ml)))
|
||||
(is (= ["after"] (:post @ml))))))
|
||||
|
||||
(deftest replace-runner-logs!-test
|
||||
(testing "replace-runner-logs! overwrites previous runner logs"
|
||||
(let [ml (#'runner.execute/empty-message-log)]
|
||||
(#'runner.execute/replace-runner-logs! ml [{:message "old"}])
|
||||
(#'runner.execute/replace-runner-logs! ml [{:message "new1"} {:message "new2"}])
|
||||
(is (= [{:message "new1"} {:message "new2"}] (:runner @ml))))))
|
||||
|
||||
(deftest message-log->transform-run-message-test
|
||||
(testing "concatenates pre, runner messages, and post"
|
||||
(let [ml (#'runner.execute/empty-message-log)]
|
||||
(#'runner.execute/log! ml "Loading tables")
|
||||
(#'runner.execute/replace-runner-logs! ml [{:message "Processing..."} {:message "Done"}])
|
||||
(#'runner.execute/log! ml "Writing output")
|
||||
(is (= "Loading tables\nProcessing...\nDone\nWriting output"
|
||||
(#'runner.execute/message-log->transform-run-message ml)))))
|
||||
(testing "works with empty log"
|
||||
(let [ml (#'runner.execute/empty-message-log)]
|
||||
(is (= "" (#'runner.execute/message-log->transform-run-message ml)))))
|
||||
(testing "works with only pre messages"
|
||||
(let [ml (#'runner.execute/empty-message-log)]
|
||||
(#'runner.execute/log! ml "only pre")
|
||||
(is (= "only pre" (#'runner.execute/message-log->transform-run-message ml))))))
|
||||
|
||||
(deftest exceptional-run-message-test
|
||||
(testing "combines log and exception message"
|
||||
(let [ml (#'runner.execute/empty-message-log)]
|
||||
(#'runner.execute/log! ml "Started")
|
||||
(#'runner.execute/replace-runner-logs! ml [{:message "Running"}])
|
||||
(let [ex (ex-info "boom" {:transform-message "Python execution failure (exit code 1)"})]
|
||||
(is (= "Started\nRunning\nPython execution failure (exit code 1)"
|
||||
(#'runner.execute/exceptional-run-message ml ex))))))
|
||||
(testing "uses generic message when no :transform-message"
|
||||
(let [ml (#'runner.execute/empty-message-log)
|
||||
ex (ex-info "boom" {})]
|
||||
(is (string? (#'runner.execute/exceptional-run-message ml ex))))))
|
||||
|
||||
;;; ------------------------------------------------ table-schema -------------------------------------------------
|
||||
|
||||
(deftest table-schema-test
|
||||
(testing "builds schema from table name and metadata"
|
||||
(let [metadata {:fields [{:name "id" :base_type "Integer"}
|
||||
{:name "name" :base_type "Text"}]}
|
||||
result (#'runner.execute/table-schema "my_table" metadata)]
|
||||
(is (= :my_table (:name result)))
|
||||
(is (= 2 (count (:columns result))))
|
||||
(is (= "id" (-> result :columns first :name)))
|
||||
(is (true? (-> result :columns first :nullable?)))))
|
||||
(testing "keyword table name passes through"
|
||||
(let [result (#'runner.execute/table-schema :already_keyword {:fields []})]
|
||||
(is (= :already_keyword (:name result)))
|
||||
(is (= [] (:columns result))))))
|
||||
@@ -0,0 +1,56 @@
|
||||
(ns metabase-enterprise.transforms-runner.feature-gating-test
|
||||
"Tests for feature gating that depend on runner language registration.
|
||||
Pure predicate tests live in metabase.transforms.feature-gating-test (OSS)."
|
||||
(:require
|
||||
[clojure.test :refer :all]
|
||||
[metabase.test :as mt]
|
||||
[metabase.transforms.feature-gating :as transforms.gating]
|
||||
[metabase.transforms.interface :as transforms.i]
|
||||
[metabase.transforms.util :as transforms.util]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(use-fixtures :once (fn [thunk]
|
||||
(doseq [lang [:python :javascript :clojure]]
|
||||
(transforms.i/register-runner! lang))
|
||||
(thunk)))
|
||||
|
||||
(def ^:private runner-languages
|
||||
[:python :javascript :clojure])
|
||||
|
||||
(defn- make-transform [source-type]
|
||||
{:source {:type (name source-type)}
|
||||
:target {:database 1 :type "table" :name "out" :schema "s"}})
|
||||
|
||||
(deftest enabled-source-types-full-features-test
|
||||
(mt/with-premium-features #{:transforms :transforms-python}
|
||||
(let [types (transforms.gating/enabled-source-types)]
|
||||
(testing "includes query types"
|
||||
(is (contains? types "native"))
|
||||
(is (contains? types "mbql")))
|
||||
(testing "includes python"
|
||||
(is (contains? types "python")))
|
||||
(testing "includes all runner languages"
|
||||
(doseq [lang (transforms.gating/runner-language-types)]
|
||||
(testing (str "includes " lang)
|
||||
(is (contains? types lang))))))))
|
||||
|
||||
(deftest check-feature-enabled-full-features-test
|
||||
(mt/with-premium-features #{:transforms :transforms-python}
|
||||
(testing "query transform enabled"
|
||||
(is (true? (transforms.util/check-feature-enabled (make-transform :query)))))
|
||||
(testing "all runner languages enabled"
|
||||
(doseq [lang runner-languages]
|
||||
(testing (str "language: " lang)
|
||||
(is (true? (transforms.util/check-feature-enabled (make-transform lang)))))))))
|
||||
|
||||
(deftest check-feature-enabled-no-python-feature-test
|
||||
(mt/with-premium-features #{:transforms}
|
||||
(testing "query transform still enabled"
|
||||
(is (true? (transforms.util/check-feature-enabled (make-transform :query)))))
|
||||
(testing "python disabled without :transforms-python"
|
||||
(is (false? (transforms.util/check-feature-enabled (make-transform :python)))))
|
||||
(testing "other runner languages disabled without :transforms-python"
|
||||
(doseq [lang [:javascript :clojure]]
|
||||
(testing (str "language: " lang)
|
||||
(is (false? (transforms.util/check-feature-enabled (make-transform lang)))))))))
|
||||
@@ -0,0 +1,157 @@
|
||||
(ns metabase-enterprise.transforms-runner.models.transform-library-test
|
||||
(:require
|
||||
[clojure.test :refer :all]
|
||||
[metabase-enterprise.transforms-runner.models.transform-library :as transform-library]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(defn- clean-libraries! []
|
||||
(t2/delete! :model/TransformLibrary))
|
||||
|
||||
(deftest update-library-source-python-test
|
||||
(testing "update-library-source! for python"
|
||||
(testing "creates new record when none exists"
|
||||
(clean-libraries!)
|
||||
(is (=? {:source "def new_func(): return 1"
|
||||
:path "common.py"
|
||||
:language "python"
|
||||
:id integer?
|
||||
:created_at some?
|
||||
:updated_at some?}
|
||||
(transform-library/update-library-source! "python" "common" "def new_func(): return 1")))
|
||||
(is (= 1 (t2/count :model/TransformLibrary :language "python"))))
|
||||
|
||||
(testing "updates existing record"
|
||||
(is (=? {:source "def updated(): return 2"
|
||||
:path "common.py"
|
||||
:language "python"}
|
||||
(transform-library/update-library-source! "python" "common" "def updated(): return 2")))
|
||||
(is (= 1 (t2/count :model/TransformLibrary :language "python")) "Should not create duplicate"))
|
||||
|
||||
(testing "rejects invalid paths"
|
||||
(is (thrown-with-msg? clojure.lang.ExceptionInfo
|
||||
#"Invalid library path"
|
||||
(transform-library/update-library-source! "python" "invalid-path" "code"))))))
|
||||
|
||||
(deftest update-library-source-javascript-test
|
||||
(testing "update-library-source! for javascript"
|
||||
(testing "creates new record when none exists"
|
||||
(clean-libraries!)
|
||||
(is (=? {:source "function newFunc() { return 1; }"
|
||||
:path "common.js"
|
||||
:language "javascript"
|
||||
:id integer?
|
||||
:created_at some?
|
||||
:updated_at some?}
|
||||
(transform-library/update-library-source! "javascript" "common" "function newFunc() { return 1; }")))
|
||||
(is (= 1 (t2/count :model/TransformLibrary :language "javascript"))))
|
||||
|
||||
(testing "updates existing record"
|
||||
(is (=? {:source "function updated() { return 2; }"
|
||||
:path "common.js"
|
||||
:language "javascript"}
|
||||
(transform-library/update-library-source! "javascript" "common" "function updated() { return 2; }")))
|
||||
(is (= 1 (t2/count :model/TransformLibrary :language "javascript")) "Should not create duplicate"))
|
||||
|
||||
(testing "rejects invalid paths"
|
||||
(is (thrown-with-msg? clojure.lang.ExceptionInfo
|
||||
#"Invalid library path"
|
||||
(transform-library/update-library-source! "javascript" "invalid-path" "code"))))))
|
||||
|
||||
(deftest get-library-by-path-test
|
||||
(testing "get-library-by-path"
|
||||
(clean-libraries!)
|
||||
(transform-library/update-library-source! "python" "common" "py code")
|
||||
(transform-library/update-library-source! "javascript" "common" "js code")
|
||||
|
||||
(testing "returns python library"
|
||||
(is (=? {:source "py code" :path "common.py" :language "python"}
|
||||
(transform-library/get-library-by-path "python" "common"))))
|
||||
|
||||
(testing "returns javascript library"
|
||||
(is (=? {:source "js code" :path "common.js" :language "javascript"}
|
||||
(transform-library/get-library-by-path "javascript" "common"))))
|
||||
|
||||
(testing "rejects invalid paths"
|
||||
(is (thrown-with-msg? clojure.lang.ExceptionInfo
|
||||
#"Invalid library path"
|
||||
(transform-library/get-library-by-path "python" "invalid")))
|
||||
(is (thrown-with-msg? clojure.lang.ExceptionInfo
|
||||
#"Invalid library path"
|
||||
(transform-library/get-library-by-path "javascript" "invalid"))))))
|
||||
|
||||
(deftest normalize-path-test
|
||||
(testing "normalize-path adds correct extension per language"
|
||||
(is (= "common.py" (#'transform-library/normalize-path "python" "common")))
|
||||
(is (= "common.js" (#'transform-library/normalize-path "javascript" "common")))
|
||||
(is (= "common.clj" (#'transform-library/normalize-path "clojure" "common"))))
|
||||
|
||||
(testing "does not duplicate extension"
|
||||
(is (= "common.py" (#'transform-library/normalize-path "python" "common.py")))
|
||||
(is (= "common.js" (#'transform-library/normalize-path "javascript" "common.js"))))
|
||||
|
||||
(testing "works with extension when fetching"
|
||||
(clean-libraries!)
|
||||
(transform-library/update-library-source! "python" "common.py" "code1")
|
||||
(is (=? {:source "code1" :path "common.py"}
|
||||
(transform-library/get-library-by-path "python" "common.py")))
|
||||
(is (=? {:source "code1" :path "common.py"}
|
||||
(transform-library/get-library-by-path "python" "common")))))
|
||||
|
||||
(deftest language-isolation-test
|
||||
(testing "libraries for different languages are independent"
|
||||
(clean-libraries!)
|
||||
(transform-library/update-library-source! "python" "common" "py-source")
|
||||
(transform-library/update-library-source! "javascript" "common" "js-source")
|
||||
(is (= 2 (t2/count :model/TransformLibrary)))
|
||||
(is (= "py-source" (:source (transform-library/get-library-by-path "python" "common"))))
|
||||
(is (= "js-source" (:source (transform-library/get-library-by-path "javascript" "common"))))
|
||||
|
||||
(testing "updating one language does not affect the other"
|
||||
(transform-library/update-library-source! "python" "common" "py-updated")
|
||||
(is (= "py-updated" (:source (transform-library/get-library-by-path "python" "common"))))
|
||||
(is (= "js-source" (:source (transform-library/get-library-by-path "javascript" "common")))))))
|
||||
|
||||
(deftest builtin-entity-id-test
|
||||
(testing "builtin-entity-id returns expected values for legacy languages"
|
||||
(is (= "cWWH9qJPvHNB3rP2vLZrK" (transform-library/builtin-entity-id "python")))
|
||||
(is (= "aHWo_0yPLwKqpQPlFNzCg" (transform-library/builtin-entity-id "javascript"))))
|
||||
|
||||
(testing "builtin-entity-id returns deterministic hash for new languages"
|
||||
(let [clj-id (transform-library/builtin-entity-id "clojure")]
|
||||
(is (= 21 (count clj-id)))
|
||||
(is (= clj-id (transform-library/builtin-entity-id "clojure"))
|
||||
"Same language should produce the same ID")))
|
||||
|
||||
(testing "all-builtin-entity-ids returns set of all registered language IDs"
|
||||
(let [ids (transform-library/all-builtin-entity-ids)]
|
||||
(is (set? ids))
|
||||
(is (contains? ids "cWWH9qJPvHNB3rP2vLZrK") "Should include python")
|
||||
(is (contains? ids "aHWo_0yPLwKqpQPlFNzCg") "Should include javascript"))))
|
||||
|
||||
(deftest ensure-builtin-library-test
|
||||
(testing "creates a library row when none exists"
|
||||
(clean-libraries!)
|
||||
(transform-library/ensure-builtin-library! "python")
|
||||
(let [lib (t2/select-one :model/TransformLibrary :language "python")]
|
||||
(is (some? lib))
|
||||
(is (= "common.py" (:path lib)))
|
||||
(is (= "" (:source lib)))
|
||||
(is (= "cWWH9qJPvHNB3rP2vLZrK" (:entity_id lib)))))
|
||||
|
||||
(testing "is idempotent — second call is a no-op"
|
||||
(transform-library/ensure-builtin-library! "python")
|
||||
(is (= 1 (t2/count :model/TransformLibrary :language "python"))))
|
||||
|
||||
(testing "does not overwrite existing source"
|
||||
(transform-library/update-library-source! "python" "common" "user code")
|
||||
(transform-library/ensure-builtin-library! "python")
|
||||
(is (= "user code" (:source (t2/select-one :model/TransformLibrary :language "python")))))
|
||||
|
||||
(testing "works for javascript with correct extension and entity_id"
|
||||
(clean-libraries!)
|
||||
(transform-library/ensure-builtin-library! "javascript")
|
||||
(let [lib (t2/select-one :model/TransformLibrary :language "javascript")]
|
||||
(is (= "common.js" (:path lib)))
|
||||
(is (= "aHWo_0yPLwKqpQPlFNzCg" (:entity_id lib))))))
|
||||
@@ -0,0 +1,193 @@
|
||||
(ns metabase-enterprise.transforms-runner.polyglot-contract-test
|
||||
"Contract equivalence tests: all runner-based languages must implement the
|
||||
transforms.interface multimethods identically given the same transform shape.
|
||||
Also covers transforms.util pure functions for all language types."
|
||||
(:require
|
||||
[clojure.test :refer :all]
|
||||
[clojure.test.check.clojure-test :refer [defspec]]
|
||||
[clojure.test.check.generators :as gen]
|
||||
[clojure.test.check.properties :as prop]
|
||||
[metabase-enterprise.transforms-clojure.impl]
|
||||
[metabase-enterprise.transforms-javascript.impl]
|
||||
[metabase-enterprise.transforms-python.impl]
|
||||
[metabase.models.transforms.transform]
|
||||
[metabase.transforms.interface :as transforms.i]
|
||||
[metabase.transforms.util :as transforms.util]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(def ^:private runner-languages
|
||||
"All registered runner-based language dispatch values."
|
||||
[:python :javascript :clojure])
|
||||
|
||||
;;; ------------------------------------------------ Generators ----------------------------------------------------
|
||||
|
||||
(def ^:private gen-source-table-int
|
||||
"Generator for integer table IDs."
|
||||
(gen/fmap inc gen/nat))
|
||||
|
||||
(def ^:private gen-source-table-ref
|
||||
"Generator for source table reference maps."
|
||||
(gen/let [db-id (gen/fmap inc gen/nat)
|
||||
schema (gen/one-of [(gen/return nil) gen/string-alphanumeric])
|
||||
table gen/string-alphanumeric
|
||||
table-id (gen/one-of [(gen/return nil) (gen/fmap inc gen/nat)])]
|
||||
(cond-> {:database_id db-id :schema schema :table table}
|
||||
table-id (assoc :table_id table-id))))
|
||||
|
||||
(def ^:private gen-source-table-value
|
||||
"Generator for either int or ref source table values."
|
||||
(gen/one-of [gen-source-table-int gen-source-table-ref]))
|
||||
|
||||
(def ^:private gen-source-tables
|
||||
"Generator for source-tables maps (string keys -> source table values)."
|
||||
(gen/let [entries (gen/vector
|
||||
(gen/tuple gen/string-alphanumeric gen-source-table-value)
|
||||
0 5)]
|
||||
(into {} entries)))
|
||||
|
||||
(defn- gen-runner-transform
|
||||
"Generator for a runner transform with a specific language type."
|
||||
[lang]
|
||||
(gen/let [source-db (gen/fmap inc gen/nat)
|
||||
target-db (gen/fmap inc gen/nat)
|
||||
source-tables gen-source-tables
|
||||
body gen/string-alphanumeric]
|
||||
{:source {:type (name lang)
|
||||
:source-database source-db
|
||||
:source-tables source-tables
|
||||
:body body}
|
||||
:target {:database target-db
|
||||
:type "table"
|
||||
:name "output"
|
||||
:schema "public"}}))
|
||||
|
||||
;;; --------------------------------- Contract: target-db-id is language-independent --------------------------------
|
||||
|
||||
(deftest target-db-id-contract-test
|
||||
(testing "all runner languages extract target-db-id identically"
|
||||
(let [transform {:source {:type "python" :source-database 1 :source-tables {} :body "x"}
|
||||
:target {:database 99 :type "table" :name "out" :schema "s"}}]
|
||||
(doseq [lang runner-languages]
|
||||
(testing (str "language: " lang)
|
||||
(is (= 99 (transforms.i/target-db-id
|
||||
(assoc-in transform [:source :type] (name lang))))))))))
|
||||
|
||||
(defspec target-db-id-equivalence-prop 50
|
||||
(prop/for-all [transform (gen-runner-transform :python)]
|
||||
(let [expected (transforms.i/target-db-id transform)]
|
||||
(every? (fn [lang]
|
||||
(= expected
|
||||
(transforms.i/target-db-id
|
||||
(assoc-in transform [:source :type] (name lang)))))
|
||||
runner-languages))))
|
||||
|
||||
;;; --------------------------------- Contract: source-db-id is language-independent --------------------------------
|
||||
|
||||
(deftest source-db-id-contract-test
|
||||
(testing "all runner languages extract source-db-id identically"
|
||||
(let [transform {:source {:type "python" :source-database 42 :source-tables {} :body "x"}
|
||||
:target {:database 1 :type "table" :name "out" :schema "s"}}]
|
||||
(doseq [lang runner-languages]
|
||||
(testing (str "language: " lang)
|
||||
(is (= 42 (transforms.i/source-db-id
|
||||
(assoc-in transform [:source :type] (name lang))))))))))
|
||||
|
||||
(defspec source-db-id-equivalence-prop 50
|
||||
(prop/for-all [transform (gen-runner-transform :python)]
|
||||
(let [expected (transforms.i/source-db-id transform)]
|
||||
(every? (fn [lang]
|
||||
(= expected
|
||||
(transforms.i/source-db-id
|
||||
(assoc-in transform [:source :type] (name lang)))))
|
||||
runner-languages))))
|
||||
|
||||
;;; --------------------------------- Contract: table-dependencies is language-independent --------------------------
|
||||
|
||||
(deftest table-dependencies-contract-test
|
||||
(testing "all runner languages compute table-dependencies identically"
|
||||
(let [transform {:source {:type "python"
|
||||
:source-database 1
|
||||
:source-tables {"a" 100
|
||||
"b" {:database_id 1 :schema "s" :table "t"}
|
||||
"c" {:database_id 2 :schema nil :table "u" :table_id 200}}
|
||||
:body "x"}
|
||||
:target {:database 1 :type "table" :name "out" :schema "s"}}
|
||||
expected (transforms.i/table-dependencies transform)]
|
||||
(doseq [lang runner-languages]
|
||||
(testing (str "language: " lang)
|
||||
(is (= expected
|
||||
(transforms.i/table-dependencies
|
||||
(assoc-in transform [:source :type] (name lang))))))))))
|
||||
|
||||
(defspec table-dependencies-equivalence-prop 50
|
||||
(prop/for-all [transform (gen-runner-transform :python)]
|
||||
(let [expected (transforms.i/table-dependencies transform)]
|
||||
(every? (fn [lang]
|
||||
(= expected
|
||||
(transforms.i/table-dependencies
|
||||
(assoc-in transform [:source :type] (name lang)))))
|
||||
runner-languages))))
|
||||
|
||||
;;; --------------------------------- transforms.util pure functions ------------------------------------------------
|
||||
|
||||
(deftest transform-type-test
|
||||
(doseq [lang runner-languages]
|
||||
(testing (str "transform-type returns " lang " for runner transform")
|
||||
(is (= lang (transforms.util/transform-type
|
||||
{:source {:type (name lang)}}))))))
|
||||
|
||||
(deftest runner-transform?-test
|
||||
(doseq [lang runner-languages]
|
||||
(testing (str lang " is a runner transform")
|
||||
(is (true? (transforms.util/runner-transform?
|
||||
{:source {:type (name lang)}})))))
|
||||
(testing "query is not a runner transform"
|
||||
(is (false? (transforms.util/runner-transform?
|
||||
{:source {:type "query"}})))))
|
||||
|
||||
(deftest transform-source-database-test
|
||||
(testing "runner transforms get source-database from :source"
|
||||
(doseq [lang runner-languages]
|
||||
(testing (str "language: " lang)
|
||||
(is (= 42 (transforms.util/transform-source-database
|
||||
{:source {:type (name lang) :source-database 42}}))))))
|
||||
(testing "query transforms get database from :query"
|
||||
(is (= 99 (transforms.util/transform-source-database
|
||||
{:source {:type "query" :query {:database 99}}})))))
|
||||
|
||||
(deftest transform-source-type-test
|
||||
(testing "runner types return themselves"
|
||||
(doseq [lang runner-languages]
|
||||
(testing (str "language: " lang)
|
||||
(is (= lang (transforms.util/transform-source-type
|
||||
{:type (name lang)}))))))
|
||||
(testing "unknown type throws"
|
||||
(is (thrown-with-msg? clojure.lang.ExceptionInfo
|
||||
#"Unknown transform source type"
|
||||
(transforms.util/transform-source-type {:type "cobol"})))))
|
||||
|
||||
(deftest required-database-features-test
|
||||
(testing "all runner transforms require :transforms/python"
|
||||
(doseq [lang runner-languages]
|
||||
(testing (str "language: " lang)
|
||||
(is (= [:transforms/python]
|
||||
(transforms.util/required-database-features
|
||||
{:source {:type (name lang)}}))))))
|
||||
(testing "query transforms require :transforms/table"
|
||||
(is (= [:transforms/table]
|
||||
(transforms.util/required-database-features
|
||||
{:source {:type "query"}})))))
|
||||
|
||||
(deftest search-index-query-text-all-runner-languages-test
|
||||
(testing "maybe-extract-transform-query-text extracts :body for all runner languages, not just :python"
|
||||
(let [extract-fn @#'metabase.models.transforms.transform/maybe-extract-transform-query-text
|
||||
body "print('hello world')"]
|
||||
(doseq [lang runner-languages]
|
||||
(testing (str "language: " lang)
|
||||
(let [transform {:source_type (name lang)
|
||||
:source {:type (name lang)
|
||||
:body body
|
||||
:source-tables {}}}]
|
||||
(is (= body (extract-fn transform))
|
||||
(str "Expected body extraction for " lang))))))))
|
||||
@@ -0,0 +1,105 @@
|
||||
(ns metabase-enterprise.transforms-runner.schema-test
|
||||
"Schema validation tests for runner-based transform sources.
|
||||
These require runner language registration, so they live in enterprise."
|
||||
(:require
|
||||
[clojure.test :refer :all]
|
||||
[metabase.transforms.interface :as transforms.i]
|
||||
[metabase.transforms.schema :as transforms.schema]
|
||||
[metabase.util.malli.registry :as mr]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(use-fixtures :once (fn [thunk]
|
||||
(doseq [lang [:python :javascript :clojure]]
|
||||
(transforms.i/register-runner! lang))
|
||||
(thunk)))
|
||||
|
||||
(def ^:private runner-languages
|
||||
[:python :javascript :clojure])
|
||||
|
||||
(def ^:private valid-runner-source
|
||||
{:source-database 1
|
||||
:source-tables {"t1" 100}
|
||||
:body "some code"})
|
||||
|
||||
;;; ---------------------------------------- Valid runner sources ---------------------------------------------------
|
||||
|
||||
(deftest runner-source-validates-for-all-languages-test
|
||||
(doseq [lang runner-languages]
|
||||
(testing (str lang " source validates")
|
||||
(is (true? (mr/validate ::transforms.schema/transform-source
|
||||
(assoc valid-runner-source :type lang)))))))
|
||||
|
||||
(deftest runner-source-with-table-ref-validates-test
|
||||
(doseq [lang runner-languages]
|
||||
(testing (str lang " with table ref source-tables validates")
|
||||
(is (true? (mr/validate ::transforms.schema/transform-source
|
||||
(assoc valid-runner-source
|
||||
:type lang
|
||||
:source-tables {"t1" {:database_id 1
|
||||
:schema "public"
|
||||
:table "foo"}})))))))
|
||||
|
||||
(deftest runner-source-with-incremental-strategy-validates-test
|
||||
(doseq [lang runner-languages]
|
||||
(testing (str lang " with optional incremental strategy validates")
|
||||
(is (true? (mr/validate ::transforms.schema/transform-source
|
||||
(assoc valid-runner-source
|
||||
:type lang
|
||||
:source-incremental-strategy
|
||||
{:type "checkpoint"
|
||||
:checkpoint {:type "checkpoint"}})))))))
|
||||
|
||||
(deftest runner-source-optional-source-database-test
|
||||
(doseq [lang runner-languages]
|
||||
(testing (str lang " without source-database validates (it's optional)")
|
||||
(is (true? (mr/validate ::transforms.schema/transform-source
|
||||
(-> valid-runner-source
|
||||
(dissoc :source-database)
|
||||
(assoc :type lang))))))))
|
||||
|
||||
;;; ---------------------------------------- Invalid runner sources ------------------------------------------------
|
||||
|
||||
(deftest runner-source-missing-body-invalid-test
|
||||
(doseq [lang runner-languages]
|
||||
(testing (str lang " without :body is invalid")
|
||||
(is (false? (mr/validate ::transforms.schema/transform-source
|
||||
(-> valid-runner-source
|
||||
(dissoc :body)
|
||||
(assoc :type lang))))))))
|
||||
|
||||
(deftest runner-source-missing-source-tables-invalid-test
|
||||
(doseq [lang runner-languages]
|
||||
(testing (str lang " without :source-tables is invalid")
|
||||
(is (false? (mr/validate ::transforms.schema/transform-source
|
||||
(-> valid-runner-source
|
||||
(dissoc :source-tables)
|
||||
(assoc :type lang))))))))
|
||||
|
||||
(deftest runner-source-wrong-body-type-invalid-test
|
||||
(doseq [lang runner-languages]
|
||||
(testing (str lang " with non-string :body is invalid")
|
||||
(is (false? (mr/validate ::transforms.schema/transform-source
|
||||
(assoc valid-runner-source :type lang :body 42)))))))
|
||||
|
||||
(deftest unknown-source-type-invalid-test
|
||||
(testing "unknown source type is invalid"
|
||||
(is (false? (mr/validate ::transforms.schema/transform-source
|
||||
(assoc valid-runner-source :type :cobol))))))
|
||||
|
||||
;;; ---------------------------------------- Query source ----------------------------------------------------------
|
||||
|
||||
(deftest query-source-not-confused-with-runner-test
|
||||
(testing "a runner-shaped source with :type :query does not validate as runner"
|
||||
(is (false? (mr/validate ::transforms.schema/transform-source
|
||||
(assoc valid-runner-source :type :query))))))
|
||||
|
||||
;;; ---------------------------------------- Shape equivalence across languages ------------------------------------
|
||||
|
||||
(deftest all-runner-languages-accept-same-shape-test
|
||||
(testing "the exact same map (minus :type) validates for every runner language"
|
||||
(let [base (dissoc valid-runner-source :type)]
|
||||
(doseq [lang runner-languages]
|
||||
(testing (str lang)
|
||||
(is (true? (mr/validate ::transforms.schema/transform-source
|
||||
(assoc base :type lang)))))))))
|
||||
@@ -10,29 +10,30 @@ import { idTag, invalidateTags, providePythonLibraryTags } from "./tags";
|
||||
export const pythonLibraryApi = EnterpriseApi.injectEndpoints({
|
||||
endpoints: (builder) => ({
|
||||
getPythonLibrary: builder.query<PythonLibrary, GetPythonLibraryRequest>({
|
||||
query: ({ path }) => ({
|
||||
url: `/api/ee/transforms-python/library/${path}`,
|
||||
query: ({ path, type }) => ({
|
||||
url: `/api/ee/advanced-transforms/library/${path}`,
|
||||
method: "GET",
|
||||
params: { type },
|
||||
}),
|
||||
providesTags: (library) =>
|
||||
library ? providePythonLibraryTags(library) : [],
|
||||
}),
|
||||
updatePythonLibrary: builder.mutation<void, UpdatePythonLibraryRequest>({
|
||||
query: ({ path, source }) => ({
|
||||
url: `/api/ee/transforms-python/library/${path}`,
|
||||
query: ({ path, source, type }) => ({
|
||||
url: `/api/ee/advanced-transforms/library/${path}`,
|
||||
method: "PUT",
|
||||
body: { source },
|
||||
body: { source, type },
|
||||
}),
|
||||
invalidatesTags: (_, error, { path }) =>
|
||||
invalidateTags(error, [idTag("python-transform-library", path)]),
|
||||
onQueryStarted: async (
|
||||
{ path, ...patch },
|
||||
{ path, type, ...patch },
|
||||
{ dispatch, queryFulfilled },
|
||||
) => {
|
||||
const patchResult = dispatch(
|
||||
pythonLibraryApi.util.updateQueryData(
|
||||
"getPythonLibrary",
|
||||
{ path },
|
||||
{ path, type },
|
||||
(draft) => {
|
||||
Object.assign(draft, patch);
|
||||
},
|
||||
|
||||
@@ -11,10 +11,11 @@ export const pythonRunnerApi = EnterpriseApi.injectEndpoints({
|
||||
TestPythonTransformResponse,
|
||||
TestPythonTransformRequest
|
||||
>({
|
||||
query: ({ code, source_tables }) => ({
|
||||
url: "/api/ee/transforms-python/test-run",
|
||||
query: ({ type, code, source_tables }) => ({
|
||||
url: "/api/ee/advanced-transforms/test-run",
|
||||
method: "POST",
|
||||
body: {
|
||||
type,
|
||||
code,
|
||||
source_tables,
|
||||
},
|
||||
|
||||
@@ -307,7 +307,10 @@ function getSourceCode(
|
||||
function getTransformUrl(transform: SuggestedTransform): string {
|
||||
return match(transform)
|
||||
.with({ id: P.number }, ({ id }) => Urls.transformEdit(id))
|
||||
.with({ source: { type: "python" } }, () => Urls.newPythonTransform())
|
||||
.with({ source: { type: "python" } }, () => Urls.newAdvancedTransform())
|
||||
.with({ source: { type: "javascript" } }, () =>
|
||||
Urls.newAdvancedTransform("javascript"),
|
||||
)
|
||||
.with({ source: { type: "query" } }, () => Urls.newNativeTransform())
|
||||
.exhaustive();
|
||||
}
|
||||
|
||||
@@ -1,35 +1,36 @@
|
||||
import type { Extension } from "@uiw/react-codemirror";
|
||||
import cx from "classnames";
|
||||
import _ from "underscore";
|
||||
import { useMemo } from "react";
|
||||
|
||||
import { CodeEditor } from "metabase/common/components/CodeEditor";
|
||||
import type { AdvancedTransformType } from "metabase-types/api";
|
||||
|
||||
import S from "./PythonEditor.module.css";
|
||||
import { completion } from "./utils";
|
||||
import { getCompletionExtensions } from "./utils";
|
||||
|
||||
export function PythonEditor({
|
||||
type,
|
||||
value,
|
||||
proposedValue,
|
||||
onChange,
|
||||
withPandasCompletions,
|
||||
className,
|
||||
readOnly,
|
||||
extensions: externalExtensions,
|
||||
...rest
|
||||
}: {
|
||||
type: AdvancedTransformType;
|
||||
value: string;
|
||||
proposedValue?: string;
|
||||
|
||||
onChange?: (value: string) => void;
|
||||
withPandasCompletions?: boolean;
|
||||
className?: string;
|
||||
readOnly?: boolean;
|
||||
extensions?: Extension[];
|
||||
}) {
|
||||
const extensions = _.compact([
|
||||
...(externalExtensions ?? []),
|
||||
withPandasCompletions ? completion : undefined,
|
||||
]);
|
||||
const extensions = useMemo(
|
||||
() => [...(externalExtensions ?? []), ...getCompletionExtensions(type)],
|
||||
[externalExtensions, type],
|
||||
);
|
||||
|
||||
return (
|
||||
<CodeEditor
|
||||
@@ -37,7 +38,7 @@ export function PythonEditor({
|
||||
value={value}
|
||||
proposedValue={proposedValue}
|
||||
onChange={onChange}
|
||||
language="python"
|
||||
language={type === "javascript" ? "typescript" : type}
|
||||
extensions={extensions}
|
||||
readOnly={readOnly}
|
||||
{...rest}
|
||||
|
||||
@@ -4,8 +4,11 @@ import {
|
||||
autocompletion,
|
||||
} from "@codemirror/autocomplete";
|
||||
import { pythonLanguage } from "@codemirror/lang-python";
|
||||
import type { Extension } from "@uiw/react-codemirror";
|
||||
import { t } from "ttag";
|
||||
|
||||
import type { AdvancedTransformType } from "metabase-types/api";
|
||||
|
||||
const pandasDefinitions = (): Record<string, Completion[]> => ({
|
||||
pd: [
|
||||
{
|
||||
@@ -667,9 +670,16 @@ function genericPandasCompletions(context: CompletionContext) {
|
||||
}
|
||||
}
|
||||
|
||||
export const completion = [
|
||||
pythonLanguage.data.of({
|
||||
autocomplete: genericPandasCompletions,
|
||||
}),
|
||||
autocompletion(),
|
||||
];
|
||||
export function getCompletionExtensions(
|
||||
type: AdvancedTransformType,
|
||||
): Extension[] {
|
||||
if (type === "python") {
|
||||
return [
|
||||
pythonLanguage.data.of({
|
||||
autocomplete: genericPandasCompletions,
|
||||
}),
|
||||
autocompletion(),
|
||||
];
|
||||
}
|
||||
return [autocompletion()];
|
||||
}
|
||||
|
||||
@@ -124,7 +124,7 @@ export function PythonDataPicker({
|
||||
<Box>
|
||||
<Text fw="bold">{t`Pick tables and alias them`}</Text>
|
||||
<Text size="sm" c="text-tertiary" mb="sm">
|
||||
{t`Select tables to use as data sources and provide aliases that can be referenced in your Python script.`}
|
||||
{t`Select tables to use as data sources and provide aliases that can be referenced in your transform.`}
|
||||
</Text>
|
||||
<Stack gap="md">
|
||||
{tableSelections.map((selection, index) => (
|
||||
|
||||
@@ -9,14 +9,16 @@ import { useDispatch } from "metabase/lib/redux";
|
||||
import * as Urls from "metabase/lib/urls";
|
||||
import { RunButtonWithTooltip } from "metabase/query_builder/components/RunButtonWithTooltip";
|
||||
import { Button, Flex, Icon, Stack, Tooltip } from "metabase/ui";
|
||||
import type { AdvancedTransformType } from "metabase-types/api";
|
||||
|
||||
import { SHARED_LIB_IMPORT_PATH } from "../../../constants";
|
||||
import { SHARED_LIB_IMPORT_PATHS } from "../../../constants";
|
||||
import { PythonEditor } from "../../PythonEditor";
|
||||
|
||||
import { ResizableBoxHandle } from "./ResizableBoxHandle";
|
||||
import { createPythonImportTokenLocator } from "./utils";
|
||||
import { createImportTokenLocator } from "./utils";
|
||||
|
||||
type PythonEditorBodyProps = {
|
||||
type: AdvancedTransformType;
|
||||
disabled?: boolean;
|
||||
source: string;
|
||||
proposedSource?: string;
|
||||
@@ -34,11 +36,12 @@ type PythonEditorBodyProps = {
|
||||
onRejectProposed?: () => void;
|
||||
};
|
||||
|
||||
const EDITOR_HEIGHT = 400;
|
||||
const EDITOR_HEIGHT = 330;
|
||||
const HEADER_HEIGHT = 65 + 50; // Top bar height + transform header height
|
||||
const PREVIEW_MAX_INITIAL_HEIGHT = 192;
|
||||
|
||||
export function PythonEditorBody({
|
||||
type,
|
||||
disabled,
|
||||
source,
|
||||
proposedSource,
|
||||
@@ -63,7 +66,7 @@ export function PythonEditorBody({
|
||||
(e: MouseEvent) => {
|
||||
const openInNewTab = e.metaKey || e.ctrlKey || e.button === 1;
|
||||
const href = Urls.transformPythonLibrary({
|
||||
path: SHARED_LIB_IMPORT_PATH,
|
||||
path: SHARED_LIB_IMPORT_PATHS[type],
|
||||
});
|
||||
if (openInNewTab) {
|
||||
window.open(href, "_blank", "noopener,noreferrer");
|
||||
@@ -71,27 +74,27 @@ export function PythonEditorBody({
|
||||
dispatch(push(href));
|
||||
}
|
||||
},
|
||||
[dispatch],
|
||||
[dispatch, type],
|
||||
);
|
||||
|
||||
const clickableTokensExtension = useMemo(
|
||||
() =>
|
||||
clickableTokens([
|
||||
{
|
||||
tokenLocator: createPythonImportTokenLocator("common"),
|
||||
tokenLocator: createImportTokenLocator(type, "common"),
|
||||
onClick: (e) => navigateToCommonLibrary(e),
|
||||
},
|
||||
]),
|
||||
[navigateToCommonLibrary],
|
||||
[navigateToCommonLibrary, type],
|
||||
);
|
||||
|
||||
const editorContent = (
|
||||
<Flex h="100%" align="end" bg="background-secondary" pos="relative">
|
||||
<PythonEditor
|
||||
type={type}
|
||||
value={source}
|
||||
proposedValue={proposedSource}
|
||||
onChange={onChange}
|
||||
withPandasCompletions
|
||||
readOnly={!isEditMode || disabled}
|
||||
extensions={[clickableTokensExtension]}
|
||||
data-testid="python-editor"
|
||||
@@ -134,7 +137,7 @@ export function PythonEditorBody({
|
||||
isDirty={isDirty}
|
||||
onRun={onRun}
|
||||
onCancel={onCancel}
|
||||
getTooltip={() => t`Run Python script`}
|
||||
getTooltip={() => t`Run transform`}
|
||||
/>
|
||||
)}
|
||||
</Stack>
|
||||
|
||||
@@ -41,6 +41,7 @@ function setup({
|
||||
onCancel={jest.fn()}
|
||||
onAcceptProposed={onAcceptProposed}
|
||||
onRejectProposed={onRejectProposed}
|
||||
type="python"
|
||||
/>,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import type { EditorView } from "@codemirror/view";
|
||||
import type { SyntaxNodeRef } from "@lezer/common";
|
||||
|
||||
import type { AdvancedTransformType } from "metabase-types/api";
|
||||
|
||||
export function insertImport(source: string, path: string) {
|
||||
const lines = source.split("\n");
|
||||
|
||||
@@ -35,7 +37,7 @@ export function hasImport(source: string, path: string) {
|
||||
return regex.test(source);
|
||||
}
|
||||
|
||||
export const createPythonImportTokenLocator =
|
||||
const createPythonImportTokenLocator =
|
||||
(moduleName: string) =>
|
||||
(view: EditorView, node: SyntaxNodeRef): boolean => {
|
||||
if (node.type.name !== "VariableName") {
|
||||
@@ -54,3 +56,36 @@ export const createPythonImportTokenLocator =
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
const createJavascriptImportTokenLocator =
|
||||
(moduleName: string) =>
|
||||
(view: EditorView, node: SyntaxNodeRef): boolean => {
|
||||
if (node.type.name !== "VariableDefinition") {
|
||||
return false;
|
||||
}
|
||||
const variableName = view.state.doc.sliceString(node.from, node.to);
|
||||
if (variableName !== moduleName) {
|
||||
return false;
|
||||
}
|
||||
let cur: SyntaxNodeRef | null = node;
|
||||
while (cur) {
|
||||
if (cur.type.name === "ImportDeclaration") {
|
||||
return true;
|
||||
}
|
||||
cur = cur.node.parent;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
export function createImportTokenLocator(
|
||||
type: AdvancedTransformType,
|
||||
moduleName: string,
|
||||
) {
|
||||
if (type === "python") {
|
||||
return createPythonImportTokenLocator(moduleName);
|
||||
} else if (type === "javascript") {
|
||||
return createJavascriptImportTokenLocator(moduleName);
|
||||
}
|
||||
const _exhaustive: never = type;
|
||||
return _exhaustive;
|
||||
}
|
||||
|
||||
@@ -161,7 +161,7 @@ function ResultsFooter({
|
||||
<Text
|
||||
c="text-primary"
|
||||
ml="xs"
|
||||
>{t`An error occurred while executing your Python script.`}</Text>
|
||||
>{t`An error occurred while executing your transform.`}</Text>
|
||||
<DismissButton onDismiss={onDismiss} />
|
||||
</Flex>
|
||||
);
|
||||
|
||||
@@ -2,5 +2,26 @@
|
||||
height: 100%;
|
||||
min-height: 0;
|
||||
overflow: auto;
|
||||
position: relative;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.typeLabel {
|
||||
background-color: var(--mb-color-background-primary);
|
||||
border-color: var(--mb-color-border);
|
||||
border-radius: var(--mantine-spacing-sm) 0 0 0;
|
||||
border-style: solid;
|
||||
border-width: 1px 0 0 1px;
|
||||
bottom: 0;
|
||||
color: var(--mb-color-text-secondary);
|
||||
right: 0;
|
||||
padding: 0 0.5rem;
|
||||
position: absolute;
|
||||
|
||||
&.editMode {
|
||||
border-radius: 0 var(--mantine-spacing-sm) 0 0;
|
||||
border-width: 1px 1px 0 0;
|
||||
left: 0;
|
||||
right: auto;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
import { useHotkeys } from "@mantine/hooks";
|
||||
import cx from "classnames";
|
||||
import { useEffect } from "react";
|
||||
import { usePrevious } from "react-use";
|
||||
|
||||
import type { PythonTransformEditorProps } from "metabase/plugins";
|
||||
import { Flex, Stack } from "metabase/ui";
|
||||
import type {
|
||||
DatabaseId,
|
||||
PythonTransformTableAliases,
|
||||
Table,
|
||||
import { Box, Flex, Stack, Text } from "metabase/ui";
|
||||
import {
|
||||
ADVANCED_TRANSFORM_TYPES,
|
||||
type DatabaseId,
|
||||
type PythonTransformTableAliases,
|
||||
type Table,
|
||||
} from "metabase-types/api";
|
||||
|
||||
import { isPythonTransformSource } from "../../utils";
|
||||
@@ -17,6 +19,7 @@ import { PythonEditorBody } from "./PythonEditorBody";
|
||||
import { PythonEditorResults } from "./PythonEditorResults";
|
||||
import S from "./PythonTransformEditor.module.css";
|
||||
import { PythonTransformTopBar } from "./PythonTransformTopBar";
|
||||
import { TransformTypeSelect } from "./TransformTypeSelect";
|
||||
import { useTestPythonTransform } from "./hooks";
|
||||
import { updateTransformSignature } from "./utils";
|
||||
|
||||
@@ -64,6 +67,7 @@ export function PythonTransformEditor({
|
||||
source.body,
|
||||
sourceTables,
|
||||
tableInfo,
|
||||
source.type,
|
||||
);
|
||||
|
||||
const newSource = {
|
||||
@@ -119,6 +123,7 @@ export function PythonTransformEditor({
|
||||
};
|
||||
|
||||
useHotkeys([["mod+Enter", handleCmdEnter]], []);
|
||||
const canChangeType = !transform?.id;
|
||||
|
||||
return (
|
||||
<Flex h="100%" w="100%" direction="column">
|
||||
@@ -141,6 +146,7 @@ export function PythonTransformEditor({
|
||||
)}
|
||||
<Stack w="100%" h="100%" gap={0}>
|
||||
<PythonEditorBody
|
||||
type={source.type}
|
||||
disabled={uiOptions?.readOnly}
|
||||
isRunnable={isPythonTransformSource(source)}
|
||||
isRunning={isRunning}
|
||||
@@ -163,6 +169,17 @@ export function PythonTransformEditor({
|
||||
/>
|
||||
)}
|
||||
</Stack>
|
||||
{!!source?.type && (
|
||||
<Box className={cx(S.typeLabel, { [S.editMode]: isEditMode })}>
|
||||
{canChangeType ? (
|
||||
<TransformTypeSelect value={source.type} />
|
||||
) : (
|
||||
<Text fz="sm" c="text-secondary" px="xs">
|
||||
{ADVANCED_TRANSFORM_TYPES[source.type].displayName}
|
||||
</Text>
|
||||
)}
|
||||
</Box>
|
||||
)}
|
||||
</Flex>
|
||||
</Flex>
|
||||
);
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
import { replace } from "react-router-redux";
|
||||
|
||||
import { useDispatch } from "metabase/lib/redux";
|
||||
import { newAdvancedTransform } from "metabase/lib/urls";
|
||||
import { Button, Combobox, Group, Icon, Text, useCombobox } from "metabase/ui";
|
||||
import {
|
||||
ADVANCED_TRANSFORM_TYPES,
|
||||
type AdvancedTransformType,
|
||||
isAdvancedTransformType,
|
||||
} from "metabase-types/api";
|
||||
|
||||
const transformTypes = Object.keys(
|
||||
ADVANCED_TRANSFORM_TYPES,
|
||||
) as AdvancedTransformType[];
|
||||
|
||||
type AdvancedTransformTypeSelectProps = {
|
||||
value: AdvancedTransformType;
|
||||
};
|
||||
|
||||
export function TransformTypeSelect({
|
||||
value,
|
||||
}: AdvancedTransformTypeSelectProps) {
|
||||
const dispatch = useDispatch();
|
||||
const combobox = useCombobox();
|
||||
const onChange = (type: string) => {
|
||||
if (isAdvancedTransformType(type)) {
|
||||
dispatch(replace(newAdvancedTransform(type)));
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<Combobox onOptionSubmit={onChange} store={combobox} position="top">
|
||||
<Combobox.Target>
|
||||
<Button
|
||||
c="text-secondary"
|
||||
onClick={() => combobox.toggleDropdown()}
|
||||
pl="xs"
|
||||
pr={0}
|
||||
py={0}
|
||||
h="1.5rem"
|
||||
variant="subtle"
|
||||
>
|
||||
<Group fz="sm" gap="sm" align="center" fw="normal">
|
||||
{ADVANCED_TRANSFORM_TYPES[value].displayName}
|
||||
<Icon name="chevrondown" size={8} />
|
||||
</Group>
|
||||
</Button>
|
||||
</Combobox.Target>
|
||||
<Combobox.Dropdown miw="8rem">
|
||||
<Combobox.Options>
|
||||
{transformTypes.map((type) => (
|
||||
<Combobox.Option value={type} key={type} py="xs">
|
||||
<Text>{ADVANCED_TRANSFORM_TYPES[type].displayName}</Text>
|
||||
</Combobox.Option>
|
||||
))}
|
||||
</Combobox.Options>
|
||||
</Combobox.Dropdown>
|
||||
</Combobox>
|
||||
);
|
||||
}
|
||||
@@ -33,6 +33,7 @@ export function useTestPythonTransform(
|
||||
}
|
||||
|
||||
const request = executePython({
|
||||
type: source.type,
|
||||
code: source.body,
|
||||
source_tables: source["source-tables"],
|
||||
});
|
||||
@@ -65,7 +66,7 @@ function getError(error: unknown) {
|
||||
|
||||
if (typeof error === "object" && error !== null) {
|
||||
if ("name" in error && error.name === "AbortError") {
|
||||
return { message: t`Python script execution was canceled` };
|
||||
return { message: t`Transform execution was canceled` };
|
||||
}
|
||||
}
|
||||
return {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type {
|
||||
AdvancedTransformType,
|
||||
PythonTransformTableAliases,
|
||||
Table,
|
||||
TableId,
|
||||
@@ -8,25 +9,26 @@ export function updateTransformSignature(
|
||||
script: string,
|
||||
tables: PythonTransformTableAliases,
|
||||
tableInfo: Table[],
|
||||
type: AdvancedTransformType,
|
||||
): string {
|
||||
return type === "python"
|
||||
? updatePythonTransformSignature(script, tables, tableInfo)
|
||||
: updateJavascriptTransformSignature(script, tables, tableInfo);
|
||||
}
|
||||
|
||||
function updatePythonTransformSignature(
|
||||
script: string,
|
||||
tables: PythonTransformTableAliases,
|
||||
tableInfo: Table[],
|
||||
): string {
|
||||
const tableAliases = Object.keys(tables);
|
||||
|
||||
const transformRegex = /^def\s+transform\s*\([^)]*\)\s*:\s*\n(\s*)/m;
|
||||
|
||||
const signatureOneLine = `def transform(${tableAliases.join(", ")}):`;
|
||||
const newSignature =
|
||||
signatureOneLine.length > 80 && tableAliases.length > 0
|
||||
? `def transform(\n${tableAliases.map((alias) => ` ${alias}`).join(",\n")},\n):`
|
||||
: signatureOneLine;
|
||||
|
||||
function getTableName(tableId: TableId) {
|
||||
const table = tableInfo.find((t) => t.id === tableId);
|
||||
if (!table) {
|
||||
return undefined;
|
||||
}
|
||||
return [table.db?.name, table.schema, table.name].filter(Boolean).join(".");
|
||||
}
|
||||
|
||||
if (transformRegex.test(script)) {
|
||||
// Capture the existing indentation after the colon
|
||||
const match = script.match(transformRegex);
|
||||
@@ -47,7 +49,7 @@ export function updateTransformSignature(
|
||||
.map((alias) => {
|
||||
const padding = " ".repeat(maxAliasLength - alias.length);
|
||||
const tableId = tables[alias];
|
||||
const tableName = getTableName(tableId);
|
||||
const tableName = getTableName(tableInfo, tableId);
|
||||
return ` ${alias}:${padding} DataFrame containing the data from the "${tableName}" table`;
|
||||
})
|
||||
.join("\n");
|
||||
@@ -119,7 +121,7 @@ ${tableAliases
|
||||
.map((alias) => {
|
||||
const padding = " ".repeat(maxAliasLength - alias.length);
|
||||
const tableId = tables[alias];
|
||||
const tableName = getTableName(tableId);
|
||||
const tableName = getTableName(tableInfo, tableId);
|
||||
if (alias === tableName) {
|
||||
return ` ${alias}:${padding} DataFrame containing the data from the corresponding table`;
|
||||
}
|
||||
@@ -138,3 +140,138 @@ ${tableAliases
|
||||
|
||||
return script + functionTemplate;
|
||||
}
|
||||
|
||||
function updateJavascriptTransformSignature(
|
||||
script: string,
|
||||
tables: PythonTransformTableAliases,
|
||||
tableInfo: Table[],
|
||||
): string {
|
||||
const tableAliases = Object.keys(tables);
|
||||
const transformRegex =
|
||||
/^export\sfunction\s+transform\s*\([^)]*\)\s*{\s*\n(\s*)/m;
|
||||
const signatureOneLine = `export function transform(${tableAliases.join(", ")}) {`;
|
||||
const newSignature =
|
||||
signatureOneLine.length > 80 && tableAliases.length > 0
|
||||
? `export function transform(\n${tableAliases.map((alias) => ` ${alias}`).join(",\n")},\n) {`
|
||||
: signatureOneLine;
|
||||
|
||||
if (transformRegex.test(script)) {
|
||||
// Capture the existing indentation after the colon
|
||||
const match = script.match(transformRegex);
|
||||
const originalIndent = match ? match[1] : " ";
|
||||
|
||||
let updatedScript = script.replace(
|
||||
transformRegex,
|
||||
newSignature + "\n" + originalIndent,
|
||||
);
|
||||
|
||||
const argsRegex =
|
||||
/(\s+\/\*\*[\s\S]*?)\s*Args:\s*\n([\s\S]*?)\s*(\n\s+Returns:|\n\s+""")/;
|
||||
const argsMatch = updatedScript.match(argsRegex);
|
||||
|
||||
if (tableAliases.length > 0) {
|
||||
const maxAliasLength = Math.max(...tableAliases.map((a) => a.length));
|
||||
const newArgsSection = tableAliases
|
||||
.map((alias) => {
|
||||
const padding = " ".repeat(maxAliasLength - alias.length);
|
||||
const tableId = tables[alias];
|
||||
const tableName = getTableName(tableInfo, tableId);
|
||||
return ` ${alias}:${padding} Object array containing the data from the "${tableName}" table`;
|
||||
})
|
||||
.join("\n");
|
||||
|
||||
if (argsMatch) {
|
||||
// Replace existing Args section
|
||||
updatedScript = updatedScript.replace(
|
||||
argsRegex,
|
||||
` $1
|
||||
|
||||
Args:
|
||||
${newArgsSection}
|
||||
$3`,
|
||||
);
|
||||
} else {
|
||||
// No Args section exists, look for the pattern before Returns and insert Args
|
||||
// Also remove the instructional text about selecting tables
|
||||
const insertRegex =
|
||||
/(\s+\/\*\*[\s\S]*?)\s*Select tables above to add them as function parameters\.\s*(\n\s+Returns:|\n\s+""")/;
|
||||
const insertMatch = updatedScript.match(insertRegex);
|
||||
if (insertMatch) {
|
||||
updatedScript = updatedScript.replace(
|
||||
insertRegex,
|
||||
`$1
|
||||
|
||||
Args:
|
||||
${newArgsSection}
|
||||
$2`,
|
||||
);
|
||||
} else {
|
||||
// Fallback: just look for Returns section without the specific text
|
||||
// Remove any extra whitespace before Returns and ensure single blank line
|
||||
const fallbackRegex = /(\s+"""[\s\S]*?)\s*(\n\s+Returns:|\n\s+""")/;
|
||||
const fallbackMatch = updatedScript.match(fallbackRegex);
|
||||
if (fallbackMatch) {
|
||||
updatedScript = updatedScript.replace(
|
||||
fallbackRegex,
|
||||
`$1
|
||||
|
||||
Args:
|
||||
${newArgsSection}
|
||||
$2`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (argsMatch) {
|
||||
// Remove existing Args section when no tables
|
||||
updatedScript = updatedScript.replace(argsRegex, "$1$3");
|
||||
}
|
||||
|
||||
return updatedScript;
|
||||
}
|
||||
|
||||
const maxAliasLength =
|
||||
tableAliases.length > 0
|
||||
? Math.max(...tableAliases.map((a) => a.length))
|
||||
: 0;
|
||||
const functionTemplate = `
|
||||
${newSignature}
|
||||
/**
|
||||
Transform function that processes the input data.
|
||||
${
|
||||
tableAliases.length > 0
|
||||
? `
|
||||
|
||||
Args:
|
||||
${tableAliases
|
||||
.map((alias) => {
|
||||
const padding = " ".repeat(maxAliasLength - alias.length);
|
||||
const tableId = tables[alias];
|
||||
const tableName = getTableName(tableInfo, tableId);
|
||||
if (alias === tableName) {
|
||||
return ` ${alias}:${padding} Array containing the data from the corresponding table`;
|
||||
}
|
||||
return ` ${alias}:${padding} Array containing the data from table "${tableName}"`;
|
||||
})
|
||||
.join("\n")}`
|
||||
: ""
|
||||
}
|
||||
|
||||
Returns:
|
||||
The transformed data object array
|
||||
*/
|
||||
return []
|
||||
}
|
||||
`;
|
||||
|
||||
return script + functionTemplate;
|
||||
}
|
||||
|
||||
function getTableName(tableInfo: Table[], tableId: TableId) {
|
||||
const table = tableInfo.find((t) => t.id === tableId);
|
||||
if (!table) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return [table.db?.name, table.schema, table.name].filter(Boolean).join(".");
|
||||
}
|
||||
|
||||
@@ -1 +1,6 @@
|
||||
export const SHARED_LIB_IMPORT_PATH = "common.py";
|
||||
import type { AdvancedTransformType } from "metabase-types/api";
|
||||
|
||||
export const SHARED_LIB_IMPORT_PATHS: Record<AdvancedTransformType, string> = {
|
||||
python: "common.py",
|
||||
javascript: "common.js",
|
||||
};
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import { PLUGIN_TRANSFORMS_PYTHON } from "metabase/plugins";
|
||||
import { hasPremiumFeature } from "metabase-enterprise/settings";
|
||||
|
||||
import { PythonTransformEditor } from "./components/PythonTransformEditor";
|
||||
import { SHARED_LIB_IMPORT_PATH } from "./constants";
|
||||
import { PythonTransformEditor } from "./components/PythonTransformEditor/PythonTransformEditor";
|
||||
import { SHARED_LIB_IMPORT_PATHS } from "./constants";
|
||||
import { PythonRunnerSettingsPage } from "./pages/PythonRunnerSettingsPage";
|
||||
import { getPythonTransformsRoutes, getPythonUpsellRoutes } from "./routes";
|
||||
import { getPythonSourceValidationResult } from "./utils";
|
||||
@@ -24,5 +24,5 @@ export function initializePlugin() {
|
||||
PLUGIN_TRANSFORMS_PYTHON.getPythonTransformsRoutes = getPythonUpsellRoutes;
|
||||
}
|
||||
|
||||
PLUGIN_TRANSFORMS_PYTHON.sharedLibImportPath = SHARED_LIB_IMPORT_PATH;
|
||||
PLUGIN_TRANSFORMS_PYTHON.sharedLibImportPaths = SHARED_LIB_IMPORT_PATHS;
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import * as Urls from "metabase/lib/urls";
|
||||
import { Button, Group } from "metabase/ui";
|
||||
|
||||
type PythonLibraryEditorHeaderProps = {
|
||||
typeDisplayName: string;
|
||||
isDirty?: boolean;
|
||||
isSaving?: boolean;
|
||||
onSave: () => void;
|
||||
@@ -17,6 +18,7 @@ type PythonLibraryEditorHeaderProps = {
|
||||
};
|
||||
|
||||
export const PythonLibraryEditorHeader = ({
|
||||
typeDisplayName,
|
||||
isDirty,
|
||||
isSaving,
|
||||
onSave,
|
||||
@@ -27,10 +29,14 @@ export const PythonLibraryEditorHeader = ({
|
||||
breadcrumbs={
|
||||
<DataStudioBreadcrumbs>
|
||||
<Link to={Urls.transformList()}>{t`Transforms`}</Link>
|
||||
{t`Python library`}
|
||||
{typeDisplayName + " " + t`library`}
|
||||
</DataStudioBreadcrumbs>
|
||||
}
|
||||
title={<PanelHeaderTitle>{t`Python library`}</PanelHeaderTitle>}
|
||||
title={
|
||||
<PanelHeaderTitle>
|
||||
{typeDisplayName + " " + t`library`}
|
||||
</PanelHeaderTitle>
|
||||
}
|
||||
actions={
|
||||
(isDirty || isSaving) && (
|
||||
<Group wrap="nowrap">
|
||||
|
||||
@@ -2,6 +2,7 @@ import { useLayoutEffect, useState } from "react";
|
||||
import type { Route } from "react-router";
|
||||
import { t } from "ttag";
|
||||
|
||||
import { skipToken } from "metabase/api";
|
||||
import { LeaveRouteConfirmModal } from "metabase/common/components/LeaveConfirmModal";
|
||||
import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
|
||||
import { PageContainer } from "metabase/data-studio/common/components/PageContainer";
|
||||
@@ -15,6 +16,10 @@ import {
|
||||
useUpdatePythonLibraryMutation,
|
||||
} from "metabase-enterprise/api/python-transform-library";
|
||||
import { getIsRemoteSyncReadOnly } from "metabase-enterprise/remote_sync/selectors";
|
||||
import {
|
||||
ADVANCED_TRANSFORM_TYPES,
|
||||
type AdvancedTransformType,
|
||||
} from "metabase-types/api";
|
||||
|
||||
import { PythonEditor } from "../../components/PythonEditor";
|
||||
|
||||
@@ -26,26 +31,43 @@ type PythonLibraryEditorPageProps = {
|
||||
route: Route;
|
||||
};
|
||||
|
||||
const EMPTY_LIBRARY_SOURCE = `
|
||||
# This is your Python library.
|
||||
# You can add functions and classes here that can be reused in Python transforms.
|
||||
function getTypeFromPath(path: string): AdvancedTransformType | undefined {
|
||||
if (path.endsWith(".py")) {
|
||||
return "python";
|
||||
}
|
||||
if (path.endsWith(".js")) {
|
||||
return "javascript";
|
||||
}
|
||||
}
|
||||
|
||||
function getEmptyLibrarySource(type?: AdvancedTransformType) {
|
||||
if (type == null) {
|
||||
return "";
|
||||
}
|
||||
const { displayName, commentString } = ADVANCED_TRANSFORM_TYPES[type];
|
||||
return `
|
||||
${commentString} This is your ${displayName} library.
|
||||
${commentString} You can add code here that can be reused in ${displayName} transforms.
|
||||
`
|
||||
.trim()
|
||||
.concat("\n");
|
||||
.trim()
|
||||
.concat("\n");
|
||||
}
|
||||
|
||||
export function PythonLibraryEditorPage({
|
||||
params,
|
||||
route,
|
||||
}: PythonLibraryEditorPageProps) {
|
||||
const { path } = params;
|
||||
const [source, setSource] = useState(EMPTY_LIBRARY_SOURCE);
|
||||
const type = getTypeFromPath(path);
|
||||
const emptyLibrarySource = getEmptyLibrarySource(type);
|
||||
const [source, setSource] = useState(emptyLibrarySource);
|
||||
const isRemoteSyncReadOnly = useSelector(getIsRemoteSyncReadOnly);
|
||||
|
||||
const {
|
||||
data: library,
|
||||
isLoading,
|
||||
error,
|
||||
} = useGetPythonLibraryQuery({ path });
|
||||
} = useGetPythonLibraryQuery(type != null ? { path, type } : skipToken);
|
||||
const [updatePythonLibrary, { isLoading: isSaving }] =
|
||||
useUpdatePythonLibraryMutation();
|
||||
const { sendSuccessToast, sendErrorToast } = useMetadataToasts();
|
||||
@@ -56,7 +78,7 @@ export function PythonLibraryEditorPage({
|
||||
}
|
||||
|
||||
if (isResourceNotFoundError(error)) {
|
||||
setSource(EMPTY_LIBRARY_SOURCE);
|
||||
setSource(emptyLibrarySource);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -66,11 +88,14 @@ export function PythonLibraryEditorPage({
|
||||
}
|
||||
|
||||
async function handleSave() {
|
||||
if (type == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await updatePythonLibrary({ path, source }).unwrap();
|
||||
sendSuccessToast(t`Python library saved`);
|
||||
await updatePythonLibrary({ path, source, type }).unwrap();
|
||||
sendSuccessToast(t`Library saved`);
|
||||
} catch (error) {
|
||||
sendErrorToast(t`Python library could not be saved`);
|
||||
sendErrorToast(t`Library could not be saved`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -79,13 +104,13 @@ export function PythonLibraryEditorPage({
|
||||
if (library?.source) {
|
||||
setSource(library.source);
|
||||
} else {
|
||||
setSource(EMPTY_LIBRARY_SOURCE);
|
||||
setSource(emptyLibrarySource);
|
||||
}
|
||||
}, [library]);
|
||||
}, [library, emptyLibrarySource]);
|
||||
|
||||
const isDirty = source !== (library?.source || EMPTY_LIBRARY_SOURCE);
|
||||
const isDirty = source !== (library?.source || emptyLibrarySource);
|
||||
|
||||
if (isLoading || (error && !isResourceNotFoundError(error))) {
|
||||
if (isLoading || (error && !isResourceNotFoundError(error)) || type == null) {
|
||||
return (
|
||||
<Box p="md">
|
||||
<LoadingAndErrorWrapper loading={isLoading} error={error} />
|
||||
@@ -93,11 +118,14 @@ export function PythonLibraryEditorPage({
|
||||
);
|
||||
}
|
||||
|
||||
const typeDisplayName = ADVANCED_TRANSFORM_TYPES[type].displayName;
|
||||
|
||||
return (
|
||||
<>
|
||||
<PageContainer>
|
||||
<Stack>
|
||||
<PythonLibraryEditorHeader
|
||||
typeDisplayName={typeDisplayName}
|
||||
onSave={handleSave}
|
||||
onRevert={handleRevert}
|
||||
isDirty={isDirty && !isRemoteSyncReadOnly}
|
||||
@@ -109,7 +137,7 @@ export function PythonLibraryEditorPage({
|
||||
className={S.flexStart}
|
||||
color="warning"
|
||||
p="0.75rem"
|
||||
title={t`The Python library is not editable because Remote Sync is in read-only mode.`}
|
||||
title={t`The library is not editable because Remote Sync is in read-only mode.`}
|
||||
variant="outline"
|
||||
w="auto"
|
||||
/>
|
||||
@@ -118,9 +146,9 @@ export function PythonLibraryEditorPage({
|
||||
|
||||
<Card withBorder p={0}>
|
||||
<PythonEditor
|
||||
type={type}
|
||||
value={source}
|
||||
onChange={setSource}
|
||||
withPandasCompletions
|
||||
className={S.editor}
|
||||
data-testid="python-editor"
|
||||
readOnly={isRemoteSyncReadOnly}
|
||||
|
||||
@@ -10,25 +10,25 @@ import { useSetting } from "metabase/common/hooks";
|
||||
export function PythonRunnerSettingsPage() {
|
||||
const isHosted = useSetting("is-hosted?");
|
||||
|
||||
// Python Runner settings are managed by Metabase Cloud for hosted instances
|
||||
// Advanced Transform Runner settings are managed by Metabase Cloud for hosted instances
|
||||
if (isHosted) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<SettingsPageWrapper title={t`Python Runner`}>
|
||||
<SettingsPageWrapper title={t`Advanced Transform Runner`}>
|
||||
<SettingsSection title={t`Service Configuration`}>
|
||||
<AdminSettingInput
|
||||
name="python-runner-url"
|
||||
title={t`Python Execution Server URL`}
|
||||
description={t`URL for the Python execution server that runs transform functions.`}
|
||||
title={t`Advanced Transform Execution Server URL`}
|
||||
description={t`URL for the server that runs advanced transform functions.`}
|
||||
placeholder="http://localhost:5001"
|
||||
inputType="text"
|
||||
/>
|
||||
<AdminSettingInput
|
||||
name="python-runner-timeout-seconds"
|
||||
title={t`Python Script Execution Timeout`}
|
||||
description={t`Timeout in seconds for Python script execution. Defaults to 30 minutes (1800 seconds).`}
|
||||
title={t`Advanced Transform Execution Timeout`}
|
||||
description={t`Timeout in seconds for advanced transform execution. Defaults to 30 minutes (1800 seconds).`}
|
||||
placeholder="1800"
|
||||
inputType="number"
|
||||
/>
|
||||
@@ -37,7 +37,7 @@ export function PythonRunnerSettingsPage() {
|
||||
<AdminSettingInput
|
||||
name="python-storage-s-3-endpoint"
|
||||
title={t`S3 Endpoint`}
|
||||
description={t`S3 endpoint URL for storing Python execution artifacts.`}
|
||||
description={t`S3 endpoint URL for storing advanced transform execution artifacts.`}
|
||||
inputType="text"
|
||||
/>
|
||||
<AdminSettingInput
|
||||
@@ -49,7 +49,7 @@ export function PythonRunnerSettingsPage() {
|
||||
<AdminSettingInput
|
||||
name="python-storage-s-3-bucket"
|
||||
title={t`S3 Bucket`}
|
||||
description={t`S3 bucket name for storing Python execution artifacts.`}
|
||||
description={t`S3 bucket name for storing advanced transform execution artifacts.`}
|
||||
inputType="text"
|
||||
/>
|
||||
<AdminSettingInput
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { Route } from "react-router";
|
||||
import { Redirect, Route } from "react-router";
|
||||
|
||||
import { ModalRoute } from "metabase/hoc/ModalRoute";
|
||||
import { NewPythonTransformPage } from "metabase/transforms/pages/NewTransformPage";
|
||||
import { NewAdvancedTransformPage } from "metabase/transforms/pages/NewTransformPage";
|
||||
import { TransformListPage } from "metabase/transforms/pages/TransformListPage";
|
||||
|
||||
import { PythonLibraryEditorPage } from "./pages/PythonLibraryEditorPage";
|
||||
@@ -11,7 +11,9 @@ export function getPythonTransformsRoutes() {
|
||||
return (
|
||||
<>
|
||||
<Route path="library/:path" component={PythonLibraryEditorPage} />
|
||||
<Route path="new/python" component={NewPythonTransformPage} />
|
||||
<Redirect from="new/python" to="new/advanced/python" />
|
||||
<Redirect from="new/advanced" to="new/advanced/python" />
|
||||
<Route path="new/advanced/:type" component={NewAdvancedTransformPage} />
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -25,8 +27,9 @@ export function getPythonUpsellRoutes() {
|
||||
modal={PythonTransformsUpsellModal}
|
||||
noWrap
|
||||
/>
|
||||
<Redirect from="new/python" to="new/advanced/python" />
|
||||
<ModalRoute
|
||||
path="new/python"
|
||||
path="new/advanced/:type"
|
||||
modal={PythonTransformsUpsellModal}
|
||||
noWrap
|
||||
/>
|
||||
|
||||
@@ -10,13 +10,13 @@ export function getPythonSourceValidationResult(
|
||||
source: PythonTransformSourceDraft,
|
||||
): PythonTransformSourceValidationResult {
|
||||
if (!source["source-database"]) {
|
||||
return { isValid: false, errorMessage: t`Select a source a database` };
|
||||
return { isValid: false, errorMessage: t`Select a source database` };
|
||||
}
|
||||
|
||||
if (source.body.trim() === "") {
|
||||
return {
|
||||
isValid: false,
|
||||
errorMessage: t`The Python script cannot be empty`,
|
||||
errorMessage: t`The transform cannot be empty`,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -33,5 +33,5 @@ export function getPythonSourceValidationResult(
|
||||
export function isPythonTransformSource(
|
||||
source: PythonTransformSourceDraft,
|
||||
): source is PythonTransformSource {
|
||||
return source.type === "python" && source["source-database"] !== undefined;
|
||||
return source["source-database"] !== undefined;
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import type {
|
||||
ConcreteTableId,
|
||||
Engine,
|
||||
RelatedDashboardXRays,
|
||||
Transform,
|
||||
TransformId,
|
||||
VisualizationDisplay,
|
||||
} from "metabase-types/api";
|
||||
@@ -292,12 +293,13 @@ export type TransformJobTriggerManualRunEvent = ValidateEvent<{
|
||||
|
||||
export type TransformCreateEvent = ValidateEvent<{
|
||||
event: "transform_create";
|
||||
event_detail: "query" | "native" | "python" | "saved-question";
|
||||
event_detail: "query" | "native" | "advanced" | "saved-question";
|
||||
}>;
|
||||
|
||||
export type TransformCreatedEvent = ValidateEvent<{
|
||||
event: "transform_created";
|
||||
target_id: number;
|
||||
event_detail: Transform["source_type"];
|
||||
}>;
|
||||
|
||||
export type TransformRunTagsUpdated = ValidateEvent<{
|
||||
|
||||
@@ -23,12 +23,31 @@ export type TransformOwner = Pick<
|
||||
"id" | "email" | "first_name" | "last_name"
|
||||
>;
|
||||
|
||||
export const ADVANCED_TRANSFORM_TYPES = {
|
||||
python: {
|
||||
displayName: "Python",
|
||||
commentString: "#",
|
||||
},
|
||||
javascript: {
|
||||
displayName: "JavaScript",
|
||||
commentString: "//",
|
||||
},
|
||||
} satisfies Record<string, { displayName: string; commentString: string }>;
|
||||
|
||||
export type AdvancedTransformType = keyof typeof ADVANCED_TRANSFORM_TYPES;
|
||||
|
||||
export function isAdvancedTransformType(
|
||||
type: string,
|
||||
): type is AdvancedTransformType {
|
||||
return type in ADVANCED_TRANSFORM_TYPES;
|
||||
}
|
||||
|
||||
export type Transform = {
|
||||
id: TransformId;
|
||||
name: string;
|
||||
description: string | null;
|
||||
source: TransformSource;
|
||||
source_type: "native" | "python" | "mbql";
|
||||
source_type: "native" | "mbql" | AdvancedTransformType;
|
||||
target: TransformTarget;
|
||||
collection_id: CollectionId | null;
|
||||
created_at: string;
|
||||
@@ -70,14 +89,14 @@ export type TransformSourceCheckpointStrategy = {
|
||||
export type SourceIncrementalStrategy = TransformSourceCheckpointStrategy;
|
||||
|
||||
export type PythonTransformSourceDraft = {
|
||||
type: "python";
|
||||
type: AdvancedTransformType;
|
||||
body: string;
|
||||
"source-database": DatabaseId | undefined;
|
||||
"source-tables": PythonTransformTableAliases;
|
||||
};
|
||||
|
||||
export type PythonTransformSource = {
|
||||
type: "python";
|
||||
type: AdvancedTransformType;
|
||||
body: string;
|
||||
"source-database": DatabaseId;
|
||||
"source-tables": PythonTransformTableAliases;
|
||||
@@ -96,9 +115,16 @@ export type TransformTargetAppendStrategy = {
|
||||
type: "append";
|
||||
};
|
||||
export type DraftTransformSource =
|
||||
| Transform["source"]
|
||||
| QueryTransformSource
|
||||
| PythonTransformSource
|
||||
| PythonTransformSourceDraft;
|
||||
|
||||
export function isAdvancedTransformSource(
|
||||
source: DraftTransformSource,
|
||||
): source is PythonTransformSource | PythonTransformSourceDraft {
|
||||
return isAdvancedTransformType(source.type);
|
||||
}
|
||||
|
||||
export type DraftTransform = Partial<
|
||||
Pick<Transform, "id" | "name" | "description" | "target">
|
||||
> & { source: DraftTransformSource };
|
||||
@@ -266,6 +292,7 @@ export type ListTransformRunsResponse = {
|
||||
} & PaginationResponse;
|
||||
|
||||
export type TestPythonTransformRequest = {
|
||||
type: AdvancedTransformType;
|
||||
code: string;
|
||||
source_tables: PythonTransformTableAliases;
|
||||
};
|
||||
@@ -286,11 +313,13 @@ export type PythonLibrary = {
|
||||
|
||||
export type GetPythonLibraryRequest = {
|
||||
path: string;
|
||||
type: AdvancedTransformType;
|
||||
};
|
||||
|
||||
export type UpdatePythonLibraryRequest = {
|
||||
path: string;
|
||||
source: string;
|
||||
type: AdvancedTransformType;
|
||||
};
|
||||
|
||||
export type ExtractColumnsFromQueryRequest = {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type {
|
||||
AdvancedTransformType,
|
||||
CardId,
|
||||
CollectionId,
|
||||
DatabaseId,
|
||||
@@ -45,12 +46,8 @@ export function newNativeTransform() {
|
||||
return `${TRANSFORMS_ROOT_URL}/new/native`;
|
||||
}
|
||||
|
||||
export function newPythonTransform() {
|
||||
return `${TRANSFORMS_ROOT_URL}/new/python`;
|
||||
}
|
||||
|
||||
export function pythonTransformsUpsell() {
|
||||
return `${TRANSFORMS_ROOT_URL}/python-transforms`;
|
||||
export function newAdvancedTransform(type: AdvancedTransformType = "python") {
|
||||
return `${TRANSFORMS_ROOT_URL}/new/advanced/${type}`;
|
||||
}
|
||||
|
||||
export function newTransformFromCard(cardId: CardId) {
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
} from "metabase/plugins/components/PluginPlaceholder";
|
||||
import type Question from "metabase-lib/v1/Question";
|
||||
import type {
|
||||
AdvancedTransformType,
|
||||
CheckDependenciesResponse,
|
||||
GetDependencyGraphRequest,
|
||||
PythonTransformSourceDraft,
|
||||
@@ -68,7 +69,7 @@ export type PythonTransformsPlugin = {
|
||||
PythonRunnerSettingsPage: ComponentType;
|
||||
getAdminRoutes: () => ReactNode;
|
||||
getTransformsNavLinks: () => ReactNode;
|
||||
sharedLibImportPath: string;
|
||||
sharedLibImportPaths: Record<AdvancedTransformType, string>;
|
||||
};
|
||||
|
||||
type DependenciesPlugin = {
|
||||
@@ -154,7 +155,7 @@ const getDefaultPluginTransformsPython = (): PythonTransformsPlugin => ({
|
||||
PythonRunnerSettingsPage: NotFoundPlaceholder,
|
||||
getAdminRoutes: () => null,
|
||||
getTransformsNavLinks: () => null,
|
||||
sharedLibImportPath: "",
|
||||
sharedLibImportPaths: {} as Record<AdvancedTransformType, string>,
|
||||
});
|
||||
|
||||
export const PLUGIN_TRANSFORMS_PYTHON = getDefaultPluginTransformsPython();
|
||||
|
||||
@@ -2,6 +2,7 @@ import { trackSimpleEvent } from "metabase/lib/analytics";
|
||||
import type {
|
||||
InspectorCardId,
|
||||
InspectorLensId,
|
||||
Transform,
|
||||
TransformId,
|
||||
TransformJobId,
|
||||
} from "metabase-types/api";
|
||||
@@ -33,7 +34,7 @@ export function trackTransformJobTriggerManualRun({
|
||||
export function trackTransformCreate({
|
||||
creationType,
|
||||
}: {
|
||||
creationType: "query" | "native" | "python" | "saved-question";
|
||||
creationType: "query" | "native" | "advanced" | "saved-question";
|
||||
}) {
|
||||
trackSimpleEvent({
|
||||
event: "transform_create",
|
||||
@@ -41,14 +42,11 @@ export function trackTransformCreate({
|
||||
});
|
||||
}
|
||||
|
||||
export function trackTransformCreated({
|
||||
transformId,
|
||||
}: {
|
||||
transformId: TransformId;
|
||||
}) {
|
||||
export function trackTransformCreated({ transform }: { transform: Transform }) {
|
||||
trackSimpleEvent({
|
||||
event: "transform_created",
|
||||
target_id: transformId,
|
||||
target_id: transform.id,
|
||||
event_detail: transform.source_type,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import { useDispatch, useSelector } from "metabase/lib/redux";
|
||||
import * as Urls from "metabase/lib/urls";
|
||||
import { getShouldShowPythonTransformsUpsell } from "metabase/transforms/selectors";
|
||||
import { Button, Center, Icon, Loader, Menu, Tooltip } from "metabase/ui";
|
||||
import type { AdvancedTransformType } from "metabase-types/api";
|
||||
|
||||
import { trackTransformCreate } from "../../analytics";
|
||||
import { CreateTransformCollectionModal } from "../CreateTransformCollectionModal";
|
||||
@@ -33,14 +34,14 @@ export const CreateTransformMenu = () => {
|
||||
const { data: databases, isLoading } = useListDatabasesQuery({
|
||||
include_analytics: true,
|
||||
});
|
||||
const shouldShowPythonScriptOption =
|
||||
const shouldShowAdvancedTransformsOptions =
|
||||
hasPythonTransformsFeature || shouldShowPythonTransformsUpsell;
|
||||
|
||||
const handlePythonClick = () => {
|
||||
dispatch(push(Urls.newPythonTransform())); // Route will show upsell modal if feature is not enabled
|
||||
const navigateToNewAdvancedTransform = (type: AdvancedTransformType) => {
|
||||
dispatch(push(Urls.newAdvancedTransform(type))); // Route will show upsell modal if feature is not enabled
|
||||
|
||||
if (hasPythonTransformsFeature) {
|
||||
trackTransformCreate({ creationType: "python" });
|
||||
trackTransformCreate({ creationType: "advanced" });
|
||||
}
|
||||
};
|
||||
|
||||
@@ -63,6 +64,42 @@ export const CreateTransformMenu = () => {
|
||||
) : (
|
||||
<>
|
||||
<Menu.Label>{t`Create your transform with…`}</Menu.Label>
|
||||
<Menu.Item
|
||||
leftSection={<Icon name="sql" />}
|
||||
onClick={() => {
|
||||
trackTransformCreate({ creationType: "native" });
|
||||
dispatch(push(Urls.newNativeTransform()));
|
||||
}}
|
||||
>
|
||||
{t`SQL`}
|
||||
</Menu.Item>
|
||||
|
||||
{shouldShowAdvancedTransformsOptions && (
|
||||
<>
|
||||
<Menu.Item
|
||||
leftSection={<Icon name="code_block" />}
|
||||
rightSection={
|
||||
!hasPythonTransformsFeature ? (
|
||||
<UpsellGem size={14} />
|
||||
) : null
|
||||
}
|
||||
onClick={() => navigateToNewAdvancedTransform("python")}
|
||||
>
|
||||
{t`Python`}
|
||||
</Menu.Item>
|
||||
<Menu.Item
|
||||
leftSection={<Icon name="code_block" />}
|
||||
rightSection={
|
||||
!hasPythonTransformsFeature ? (
|
||||
<UpsellGem size={14} />
|
||||
) : null
|
||||
}
|
||||
onClick={() => navigateToNewAdvancedTransform("javascript")}
|
||||
>
|
||||
{t`JavaScript`}
|
||||
</Menu.Item>
|
||||
</>
|
||||
)}
|
||||
<Menu.Item
|
||||
leftSection={<Icon name="notebook" />}
|
||||
onClick={() => {
|
||||
@@ -72,27 +109,6 @@ export const CreateTransformMenu = () => {
|
||||
>
|
||||
{t`Query builder`}
|
||||
</Menu.Item>
|
||||
<Menu.Item
|
||||
leftSection={<Icon name="sql" />}
|
||||
onClick={() => {
|
||||
trackTransformCreate({ creationType: "native" });
|
||||
dispatch(push(Urls.newNativeTransform()));
|
||||
}}
|
||||
>
|
||||
{t`SQL query`}
|
||||
</Menu.Item>
|
||||
|
||||
{shouldShowPythonScriptOption && (
|
||||
<Menu.Item
|
||||
leftSection={<Icon name="code_block" />}
|
||||
rightSection={
|
||||
!hasPythonTransformsFeature ? <UpsellGem size={14} /> : null
|
||||
}
|
||||
onClick={handlePythonClick}
|
||||
>
|
||||
{t`Python script`}
|
||||
</Menu.Item>
|
||||
)}
|
||||
<Menu.Item
|
||||
leftSection={<Icon name="insight" />}
|
||||
onClick={() => {
|
||||
|
||||
@@ -24,7 +24,11 @@ import {
|
||||
Tooltip,
|
||||
} from "metabase/ui";
|
||||
import type * as Lib from "metabase-lib";
|
||||
import type { TransformSource } from "metabase-types/api";
|
||||
import {
|
||||
type AdvancedTransformType,
|
||||
type TransformSource,
|
||||
isAdvancedTransformSource,
|
||||
} from "metabase-types/api";
|
||||
|
||||
import {
|
||||
MBQLKeysetColumnSelect,
|
||||
@@ -56,7 +60,7 @@ export const IncrementalTransformSettings = ({
|
||||
|
||||
// Check if this is a Python transform with exactly one source table
|
||||
// Incremental transforms are only supported for single-table Python transforms
|
||||
const isPythonTransform = source.type === "python";
|
||||
const isPythonTransform = isAdvancedTransformSource(source);
|
||||
const isMultiTablePythonTransform =
|
||||
isPythonTransform && Object.keys(source["source-tables"]).length > 1;
|
||||
|
||||
@@ -65,7 +69,7 @@ export const IncrementalTransformSettings = ({
|
||||
isPythonTransform,
|
||||
})
|
||||
.with({ isMbqlQuery: true }, () => "query" as const)
|
||||
.with({ isPythonTransform: true }, () => "python" as const)
|
||||
.with({ isPythonTransform: true }, () => source.type)
|
||||
.otherwise(() => "native" as const);
|
||||
const { url: incrementalTransformsDocsUrl, showMetabaseLinks } = useDocsUrl(
|
||||
isPythonTransform
|
||||
@@ -205,7 +209,7 @@ function TargetStrategyFields({
|
||||
type SourceStrategyFieldsProps = {
|
||||
source: TransformSource;
|
||||
query: Lib.Query | null;
|
||||
type: "query" | "native" | "python";
|
||||
type: "query" | "native" | AdvancedTransformType;
|
||||
readOnly?: boolean;
|
||||
};
|
||||
|
||||
@@ -252,7 +256,7 @@ function SourceStrategyFields({
|
||||
disabled={readOnly}
|
||||
/>
|
||||
)}
|
||||
{type === "python" && "source-tables" in source && (
|
||||
{isAdvancedTransformSource(source) && "source-tables" in source && (
|
||||
<PythonKeysetColumnSelect
|
||||
name="checkpointFilterUniqueKey"
|
||||
label={t`Field to check for new values`}
|
||||
|
||||
@@ -88,12 +88,8 @@ export function useSourceState({
|
||||
}, [source, suggestedTransform, metadata]);
|
||||
|
||||
const isDirty = useMemo(() => {
|
||||
return (
|
||||
transformId == null ||
|
||||
proposedSource != null ||
|
||||
!isSameSource(source, initialSource)
|
||||
);
|
||||
}, [source, initialSource, proposedSource, transformId]);
|
||||
return proposedSource != null || !isSameSource(source, initialSource);
|
||||
}, [source, initialSource, proposedSource]);
|
||||
|
||||
const setSourceAndRejectProposed = (source: DraftTransformSource) => {
|
||||
if (suggestedTransform != null) {
|
||||
|
||||
@@ -32,7 +32,9 @@ export const useCreateTransform = (
|
||||
databaseId,
|
||||
);
|
||||
const transform = await createTransformMutation(request).unwrap();
|
||||
trackTransformCreated({ transformId: transform.id });
|
||||
trackTransformCreated({
|
||||
transform,
|
||||
});
|
||||
return transform;
|
||||
};
|
||||
|
||||
|
||||
@@ -22,10 +22,14 @@ import { getMetadata } from "metabase/selectors/metadata";
|
||||
import { useTransformPermissions } from "metabase/transforms/hooks/use-transform-permissions";
|
||||
import { Box, Center } from "metabase/ui";
|
||||
import * as Lib from "metabase-lib";
|
||||
import type {
|
||||
Database,
|
||||
DraftTransformSource,
|
||||
Transform,
|
||||
import {
|
||||
type AdvancedTransformType,
|
||||
type Database,
|
||||
type DraftTransformSource,
|
||||
type PythonTransformSourceDraft,
|
||||
type Transform,
|
||||
isAdvancedTransformSource,
|
||||
isAdvancedTransformType,
|
||||
} from "metabase-types/api";
|
||||
|
||||
import { TransformEditor } from "../../components/TransformEditor";
|
||||
@@ -38,6 +42,7 @@ import { CreateTransformModal } from "./CreateTransformModal";
|
||||
import {
|
||||
getDefaultValues,
|
||||
getInitialCardSource,
|
||||
getInitialJavascriptSource,
|
||||
getInitialNativeSource,
|
||||
getInitialPythonSource,
|
||||
getInitialQuerySource,
|
||||
@@ -156,7 +161,7 @@ function NewTransformPageBody({
|
||||
overflow: "hidden",
|
||||
}}
|
||||
>
|
||||
{source.type === "python" ? (
|
||||
{isAdvancedTransformSource(source) ? (
|
||||
<PLUGIN_TRANSFORMS_PYTHON.TransformEditor
|
||||
source={source}
|
||||
proposedSource={
|
||||
@@ -219,15 +224,38 @@ export function NewNativeTransformPage({ route }: NewNativeTransformPageProps) {
|
||||
return <NewTransformPage initialSource={initialSource} route={route} />;
|
||||
}
|
||||
|
||||
type NewPythonTransformPageProps = {
|
||||
type NewAdvancedTransformPageParams = {
|
||||
type?: string;
|
||||
};
|
||||
|
||||
type NewAdvancedTransformPageProps = {
|
||||
params: NewAdvancedTransformPageParams;
|
||||
route: Route;
|
||||
};
|
||||
|
||||
export function NewPythonTransformPage({ route }: NewPythonTransformPageProps) {
|
||||
const initialSource = useMemo(() => getInitialPythonSource(), []);
|
||||
return <NewTransformPage initialSource={initialSource} route={route} />;
|
||||
export function NewAdvancedTransformPage({
|
||||
params,
|
||||
route,
|
||||
}: NewAdvancedTransformPageProps) {
|
||||
const typeParam = params?.type || "";
|
||||
const type: AdvancedTransformType = isAdvancedTransformType(typeParam)
|
||||
? typeParam
|
||||
: "python";
|
||||
const initialSource = useMemo(() => sourceFunctionMap[type](), [type]);
|
||||
|
||||
return (
|
||||
<NewTransformPage initialSource={initialSource} route={route} key={type} />
|
||||
);
|
||||
}
|
||||
|
||||
const sourceFunctionMap: Record<
|
||||
AdvancedTransformType,
|
||||
() => PythonTransformSourceDraft
|
||||
> = {
|
||||
javascript: getInitialJavascriptSource,
|
||||
python: getInitialPythonSource,
|
||||
};
|
||||
|
||||
type NewCardTransformPageParams = {
|
||||
cardId: string;
|
||||
};
|
||||
|
||||
@@ -47,6 +47,31 @@ def transform():
|
||||
};
|
||||
}
|
||||
|
||||
export function getInitialJavascriptSource(): PythonTransformSourceDraft {
|
||||
return {
|
||||
type: "javascript",
|
||||
"source-database": undefined,
|
||||
"source-tables": {},
|
||||
body: `// Write your Javascript transformation script here
|
||||
import * as common from "./common.js";
|
||||
|
||||
export function transform() {
|
||||
/**
|
||||
* Your transformation function.
|
||||
*
|
||||
* Select tables above to add them as function parameters.
|
||||
*
|
||||
* Returns:
|
||||
* Object array to write to the destination table
|
||||
*/
|
||||
|
||||
// Your transformation logic here
|
||||
|
||||
return [{"message": "Hello from Javascript transform!"}];
|
||||
}`,
|
||||
};
|
||||
}
|
||||
|
||||
export function getInitialCardSource(card: Card): QueryTransformSource {
|
||||
return { type: "query", query: card.dataset_query };
|
||||
}
|
||||
|
||||
@@ -145,20 +145,39 @@ export const TransformListPage = ({
|
||||
const data = buildTreeData(collections, transforms);
|
||||
// Only show Python library item if there's at least one item in the table
|
||||
// It will trigger the upsell modal if the feature isn't enabled.
|
||||
const shouldShowPythonLibraryRow =
|
||||
const shouldShowCodeLibraryRow =
|
||||
data.length > 0 &&
|
||||
(hasPythonTransformsFeature || shouldShowPythonTransformsUpsell);
|
||||
|
||||
if (shouldShowPythonLibraryRow) {
|
||||
if (shouldShowCodeLibraryRow) {
|
||||
data.push({
|
||||
id: "library",
|
||||
name: t`Python library`,
|
||||
nodeType: "library",
|
||||
icon: "snippet",
|
||||
url: Urls.transformPythonLibrary({
|
||||
path: PLUGIN_TRANSFORMS_PYTHON.sharedLibImportPath,
|
||||
}),
|
||||
id: "code-library",
|
||||
name: t`Code library`,
|
||||
nodeType: "folder",
|
||||
icon: "folder",
|
||||
source_readable: transformsDatabases.length > 0,
|
||||
children: [
|
||||
{
|
||||
id: "python-library",
|
||||
name: t`Python library`,
|
||||
nodeType: "library",
|
||||
icon: "snippet",
|
||||
url: Urls.transformPythonLibrary({
|
||||
path: PLUGIN_TRANSFORMS_PYTHON.sharedLibImportPaths["python"],
|
||||
}),
|
||||
source_readable: transformsDatabases.length > 0,
|
||||
},
|
||||
{
|
||||
id: "javascript-library",
|
||||
name: t`JavaScript library`,
|
||||
nodeType: "library",
|
||||
icon: "snippet",
|
||||
url: Urls.transformPythonLibrary({
|
||||
path: PLUGIN_TRANSFORMS_PYTHON.sharedLibImportPaths["javascript"],
|
||||
}),
|
||||
source_readable: transformsDatabases.length > 0,
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
return data;
|
||||
|
||||
@@ -7,7 +7,11 @@ import { getMetadata } from "metabase/selectors/metadata";
|
||||
import { EditDefinitionButton } from "metabase/transforms/components/TransformEditor/EditDefinitionButton";
|
||||
import { getValidationResult } from "metabase/transforms/utils";
|
||||
import * as Lib from "metabase-lib";
|
||||
import type { DraftTransformSource, Transform } from "metabase-types/api";
|
||||
import {
|
||||
type DraftTransformSource,
|
||||
type Transform,
|
||||
isAdvancedTransformType,
|
||||
} from "metabase-types/api";
|
||||
|
||||
type Props = {
|
||||
handleCancel: VoidFunction;
|
||||
@@ -52,7 +56,7 @@ export const TransformPaneHeaderActions = (props: Props) => {
|
||||
isNative: false,
|
||||
};
|
||||
}, [source, metadata]);
|
||||
const isPythonTransform = source.type === "python";
|
||||
const isPythonTransform = isAdvancedTransformType(source.type);
|
||||
|
||||
if (
|
||||
!readOnly &&
|
||||
|
||||
@@ -25,11 +25,12 @@ import {
|
||||
import { getInitialUiState } from "metabase/querying/editor/components/QueryEditor";
|
||||
import { useTransformPermissions } from "metabase/transforms/hooks/use-transform-permissions";
|
||||
import { Box, Center, Group, Icon } from "metabase/ui";
|
||||
import type {
|
||||
Database,
|
||||
DatasetQuery,
|
||||
DraftTransformSource,
|
||||
Transform,
|
||||
import {
|
||||
type Database,
|
||||
type DatasetQuery,
|
||||
type DraftTransformSource,
|
||||
type Transform,
|
||||
isAdvancedTransformSource,
|
||||
} from "metabase-types/api";
|
||||
|
||||
import { useQueryComplexityChecks } from "../../components/QueryComplexityWarning";
|
||||
@@ -233,7 +234,7 @@ function TransformQueryPageBody({
|
||||
illustrationElement={<Icon name="key" size={100} />}
|
||||
/>
|
||||
</Center>
|
||||
) : source.type === "python" ? (
|
||||
) : isAdvancedTransformSource(source) ? (
|
||||
<PLUGIN_TRANSFORMS_PYTHON.TransformEditor
|
||||
source={source}
|
||||
proposedSource={
|
||||
@@ -322,7 +323,7 @@ export function TransformQueryPageEditor({
|
||||
onRunTransform,
|
||||
onRun,
|
||||
}: TransformQueryPageEditorProps) {
|
||||
return source.type === "python" ? (
|
||||
return isAdvancedTransformSource(source) ? (
|
||||
<PLUGIN_TRANSFORMS_PYTHON.TransformEditor
|
||||
source={source}
|
||||
uiOptions={uiOptions}
|
||||
|
||||
@@ -6,17 +6,20 @@ import type { OmniPickerCollectionItem } from "metabase/common/components/Picker
|
||||
import { parseTimestamp } from "metabase/lib/time-dayjs";
|
||||
import * as Lib from "metabase-lib";
|
||||
import type Metadata from "metabase-lib/v1/metadata/Metadata";
|
||||
import type {
|
||||
CollectionNamespace,
|
||||
Database,
|
||||
DatabaseId,
|
||||
DraftTransformSource,
|
||||
TemplateTag,
|
||||
Transform,
|
||||
TransformRun,
|
||||
TransformRunMethod,
|
||||
TransformRunStatus,
|
||||
TransformSource,
|
||||
import {
|
||||
type CollectionNamespace,
|
||||
type Database,
|
||||
type DatabaseId,
|
||||
type DraftTransformSource,
|
||||
type PythonTransformSourceDraft,
|
||||
type TemplateTag,
|
||||
type Transform,
|
||||
type TransformRun,
|
||||
type TransformRunMethod,
|
||||
type TransformRunStatus,
|
||||
type TransformSource,
|
||||
isAdvancedTransformSource,
|
||||
isAdvancedTransformType,
|
||||
} from "metabase-types/api";
|
||||
|
||||
import { CHECKPOINT_TEMPLATE_TAG } from "./constants";
|
||||
@@ -83,7 +86,7 @@ export function sourceDatabaseId(source: TransformSource): DatabaseId | null {
|
||||
return source.query.database;
|
||||
}
|
||||
|
||||
if (source.type === "python") {
|
||||
if (isAdvancedTransformType(source.type)) {
|
||||
return source["source-database"];
|
||||
}
|
||||
|
||||
@@ -134,8 +137,15 @@ export function isSameSource(
|
||||
if (source1.type === "query" && source2.type === "query") {
|
||||
return Lib.areLegacyQueriesEqual(source1.query, source2.query);
|
||||
}
|
||||
if (source1.type === "python" && source2.type === "python") {
|
||||
return _.isEqual(source1, source2);
|
||||
if (
|
||||
isAdvancedTransformSource(source1) &&
|
||||
isAdvancedTransformSource(source2)
|
||||
) {
|
||||
// source-database is not set as part of the initialSource, so we omit it from the comparison
|
||||
return _.isEqual(
|
||||
_.omit(source1, "source-database"),
|
||||
_.omit(source2, "source-database"),
|
||||
);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@@ -164,7 +174,10 @@ export function isSourceEmpty(
|
||||
export function isCompleteSource(
|
||||
source: DraftTransformSource,
|
||||
): source is TransformSource {
|
||||
return source.type !== "python" || source["source-database"] != null;
|
||||
return (
|
||||
!isAdvancedTransformType(source.type) ||
|
||||
(source as PythonTransformSourceDraft)["source-database"] != null
|
||||
);
|
||||
}
|
||||
|
||||
export type ValidationResult = {
|
||||
|
||||
@@ -3574,6 +3574,77 @@ databaseChangeLog:
|
||||
path: instance_analytics_views/query_log/v2/h2-query_log.sql
|
||||
relativeToChangelogFile: true
|
||||
|
||||
- changeSet:
|
||||
id: v59.2026-02-24T12:00:00
|
||||
author: galdre
|
||||
comment: Rename python_library to transform_library for unified multi-language library support
|
||||
changes:
|
||||
- renameTable:
|
||||
oldTableName: python_library
|
||||
newTableName: transform_library
|
||||
|
||||
- changeSet:
|
||||
id: v59.2026-02-24T12:00:01
|
||||
author: galdre
|
||||
comment: Add language column to transform_library (existing rows default to python)
|
||||
changes:
|
||||
- addColumn:
|
||||
tableName: transform_library
|
||||
columns:
|
||||
- column:
|
||||
name: language
|
||||
type: varchar(32)
|
||||
remarks: Language identifier (python, javascript, r, clojure, julia)
|
||||
defaultValue: python
|
||||
constraints:
|
||||
nullable: false
|
||||
rollback:
|
||||
- dropColumn:
|
||||
tableName: transform_library
|
||||
columnName: language
|
||||
|
||||
- changeSet:
|
||||
id: v59.2026-02-24T12:00:02
|
||||
author: galdre
|
||||
comment: Drop old path-only unique constraint (replaced by language+path constraint)
|
||||
changes:
|
||||
- dropUniqueConstraint:
|
||||
tableName: transform_library
|
||||
constraintName: uk_python_library_path
|
||||
rollback:
|
||||
- addUniqueConstraint:
|
||||
tableName: transform_library
|
||||
columnNames: path
|
||||
constraintName: uk_python_library_path
|
||||
|
||||
- changeSet:
|
||||
id: v59.2026-02-24T12:01:00
|
||||
author: galdre
|
||||
comment: Add (language, path) unique constraint to transform_library
|
||||
changes:
|
||||
- addUniqueConstraint:
|
||||
tableName: transform_library
|
||||
columnNames: language, path
|
||||
constraintName: uk_transform_library_language_path
|
||||
rollback:
|
||||
- dropUniqueConstraint:
|
||||
tableName: transform_library
|
||||
constraintName: uk_transform_library_language_path
|
||||
|
||||
- changeSet:
|
||||
id: v59.2026-02-24T12:02:00
|
||||
author: galdre
|
||||
comment: Insert default common.js entry in transform_library
|
||||
preConditions:
|
||||
- onFail: MARK_RAN
|
||||
- sqlCheck:
|
||||
expectedResult: 0
|
||||
sql: SELECT count(*) FROM transform_library WHERE language = 'javascript' AND path = 'common.js'
|
||||
changes:
|
||||
- sql:
|
||||
sql: INSERT INTO transform_library (path, source, entity_id, language) VALUES ('common.js', '', 'aHWo_0yPLwKqpQPlFNzCg', 'javascript')
|
||||
rollback:
|
||||
|
||||
# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
|
||||
|
||||
########################################################################################################################
|
||||
|
||||
@@ -165,7 +165,8 @@
|
||||
:TransformJob []
|
||||
[:TransformJob :job_tags] []
|
||||
:TransformTag []
|
||||
:PythonLibrary [:path
|
||||
:TransformLibrary [:language
|
||||
:path
|
||||
:source
|
||||
:entity_id
|
||||
:created_at
|
||||
|
||||
@@ -74,7 +74,7 @@
|
||||
:model/PermissionsRevision metabase.permissions.models.permissions-revision
|
||||
:model/PersistedInfo metabase.model-persistence.models.persisted-info
|
||||
:model/PremiumFeaturesCache metabase.premium-features.models.premium-features-cache
|
||||
:model/PythonLibrary metabase-enterprise.transforms-python.models.python-library
|
||||
:model/TransformLibrary metabase-enterprise.transforms-runner.models.transform-library
|
||||
:model/Pulse metabase.pulse.models.pulse
|
||||
:model/PulseCard metabase.pulse.models.pulse-card
|
||||
:model/PulseChannel metabase.pulse.models.pulse-channel
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user