Compare commits

...

49 Commits

Author SHA1 Message Date
Timothy Dean
4200120ff9 Oops. 2026-03-03 11:44:41 -08:00
Timothy Dean
0c719a75e2 More extensibility nonsense 2026-03-03 11:44:41 -08:00
Timothy Dean
5d8a5cf8ff OSS test shouldn't use EE feature understanding 2026-03-03 11:44:41 -08:00
Timothy Dean
d3b782b375 Moving tests around 2026-03-03 11:44:41 -08:00
Timothy Dean
36868b2aab counting is hard yo 2026-03-03 11:44:41 -08:00
Timothy Dean
e9d01384d6 Add baseline file for serialization test 2026-03-03 11:44:41 -08:00
Timothy Dean
cd2c14aaf8 serialization count fix 2026-03-03 11:44:41 -08:00
Timothy Dean
4098645d3a allowed-paths fix 2026-03-03 11:44:41 -08:00
Anderson
24f36f7979 Replace endpoints with new ones 2026-03-03 11:44:41 -08:00
Timothy Dean
5c919b4add Newlines 2026-03-03 11:44:41 -08:00
Timothy Dean
ee7346ff68 Fix NPE 2026-03-03 11:44:41 -08:00
Timothy Dean
876dab1769 Test move 2026-03-03 11:44:41 -08:00
Timothy Dean
0ccf067a29 module config tweaks 2026-03-03 11:44:41 -08:00
Timothy Dean
cb3cfcfa21 Update migration 2026-03-03 11:44:41 -08:00
Timothy Dean
d34ab2be7a alias consistency 2026-03-03 11:44:40 -08:00
Timothy Dean
4c265d8954 clj-kondo 2026-03-03 11:44:40 -08:00
Timothy Dean
de8afdc8c8 WIP more cleanup 2026-03-03 11:44:40 -08:00
Timothy Dean
000a03edcf WIP cleaning up backend 2026-03-03 11:44:40 -08:00
Timothy Dean
c2eef6d6d1 More WIP cleanup 2026-03-03 11:44:40 -08:00
Timothy Dean
a16c25656e WIP vibe tests and cleanup 2026-03-03 11:44:40 -08:00
Anderson
b60cabdd52 More e2e clean up 2026-03-03 11:44:40 -08:00
Anderson
2384273b28 Put library rows within a folder 2026-03-03 11:44:40 -08:00
Anderson
9d534a68e5 Fix more e2e tests 2026-03-03 11:44:40 -08:00
Anderson
47a7332fa4 A bit of refinement 2026-03-03 11:44:40 -08:00
Anderson
02b5960cd5 Update create transform options / Change the way we should the type selector/label 2026-03-03 11:44:40 -08:00
Kevin Kelleher
3ec4a405d2 fix isDirty, hardcoded pythons 2026-03-03 11:44:40 -08:00
Anderson
85c964f28a Fix a few transforms e2e tests 2026-03-03 11:44:37 -08:00
Anderson
91670469bd Fix ts 2026-03-03 11:43:31 -08:00
Anderson
8230584157 Lint fix 2026-03-03 11:43:31 -08:00
Kevin Kelleher
e97fd4f808 analytics, isDirty 2026-03-03 11:43:30 -08:00
Timothy Dean
f6cced8be1 transforms crud fix 2026-03-03 11:43:30 -08:00
Timothy Dean
6433912341 transforms-inspector has a hardcoded list of source-types allowed 2026-03-03 11:43:30 -08:00
Anderson
c36f87e720 Fix type select position 2026-03-03 11:43:30 -08:00
Anderson
41c31f2442 show transform type when editing a transform 2026-03-03 11:43:30 -08:00
Timothy Dean
93c44100ab :language -> :runtime in runner http call 2026-03-03 11:43:30 -08:00
Timothy Dean
a02a8caa95 feature flag 2026-03-03 11:43:30 -08:00
Anderson
ad160b0816 Fix js transform signature when we change table 2026-03-03 11:43:30 -08:00
Kevin Kelleher
9d5d294bb4 fixes 2026-03-03 11:43:30 -08:00
Kevin Kelleher
8edec1a692 javascript library 2026-03-03 11:43:30 -08:00
Anderson
a9c515c771 Add new route for advanced transforms / Add type dropdown 2026-03-03 11:43:30 -08:00
Timothy Dean
61944251e7 Julia, R, Clojure 2026-03-03 11:43:25 -08:00
Kevin Kelleher
54f3bc4fb5 update python hardcodes, js syntax highlighting and autocomplete 2026-03-03 11:38:33 -08:00
Anderson
dc02a9744e Update javascript transform initial source 2026-03-03 11:38:33 -08:00
Timothy Dean
398a335f2f :runtime 2026-03-03 11:38:33 -08:00
Timothy Dean
c531cdaa96 Remove new endpoints, use type everywhere 2026-03-03 11:38:33 -08:00
Timothy Dean
6175f8e695 Javascript Library? 2026-03-03 11:38:33 -08:00
Kevin Kelleher
038aaab787 fix types and start updating components 2026-03-03 11:38:32 -08:00
Kevin Kelleher
a6723ed1b4 wip 2026-03-03 11:38:32 -08:00
Timothy Dean
c01e6ad83f scaffolding for transforms-javascript extension 2026-03-03 11:38:32 -08:00
114 changed files with 2999 additions and 1425 deletions

View File

@@ -1597,6 +1597,7 @@
util}
:friends #{models
enterprise/transforms-python
enterprise/transforms-runner
enterprise/workspaces}}
transforms-inspector
@@ -2257,7 +2258,7 @@
util
warehouses
enterprise/dependencies
enterprise/transforms-python}}
enterprise/transforms-runner}}
enterprise/permission-debug
{:team "UX West"
@@ -2292,7 +2293,7 @@
task-history
util
enterprise/serialization
enterprise/transforms-python}}
enterprise/transforms-runner}}
enterprise/sandbox
{:team "UX West"
@@ -2457,12 +2458,27 @@
:api #{}
:uses #{premium-features}}
enterprise/transforms-clojure
{:team "Querying"
:api #{metabase-enterprise.transforms-clojure.init}
:uses #{transforms}}
enterprise/transforms-javascript
{:team "Querying"
:api #{metabase-enterprise.transforms-javascript.init}
:uses #{transforms}}
enterprise/transforms-python
{:team "Querying"
:api #{metabase-enterprise.transforms-python.api
metabase-enterprise.transforms-python.core
metabase-enterprise.transforms-python.init
metabase-enterprise.transforms-python.python-runner}
:api #{metabase-enterprise.transforms-python.init}
:uses #{transforms}}
enterprise/transforms-runner
{:team "Querying"
:api #{metabase-enterprise.transforms-runner.api
metabase-enterprise.transforms-runner.init
metabase-enterprise.transforms-runner.models.transform-library
metabase-enterprise.transforms-runner.runner}
:uses #{analytics
api
app-db
@@ -2510,7 +2526,7 @@
system
transforms
util
enterprise/transforms-python}}}
enterprise/transforms-runner}}}
;; namespaces matching these patterns (with `re-find`) are excluded from the module linter. Since regex literals
;; aren't allowed in EDN just used the `(str <regex>)` version i.e. two slashes instead of one.

View File

@@ -1,5 +1,6 @@
import { WRITABLE_DB_ID } from "e2e/support/cypress_data";
import type {
AdvancedTransformType,
Collection,
CollectionId,
PythonTransformTableAliases,
@@ -166,7 +167,9 @@ export function createSqlTransform({
);
}
export function createPythonTransform({
export function createAdvancedTransform({
name = "Python transform",
type = "python",
body,
sourceTables,
targetTable,
@@ -174,6 +177,8 @@ export function createPythonTransform({
tagIds,
visitTransform,
}: {
name?: string;
type?: AdvancedTransformType;
body: string;
sourceTables: PythonTransformTableAliases;
targetTable: string;
@@ -183,9 +188,9 @@ export function createPythonTransform({
}) {
return createTransform(
{
name: "Python transform",
name,
source: {
type: "python",
type,
"source-database": WRITABLE_DB_ID,
"source-tables": sourceTables,
body,

View File

@@ -215,7 +215,7 @@ describe("scenarios > Metabase Analytics Collection (AuditV2) ", () => {
H.visitModel(id);
});
H.newButton("SQL query").click();
H.newButton(/SQL/).click();
// sample DB should be the only one
cy.findByTestId("gui-builder-data")

View File

@@ -21,6 +21,7 @@ describe("scenarios > data studio > transforms > python library", () => {
it("should allow editing the python library", () => {
H.DataStudio.Transforms.visit();
H.DataStudio.Transforms.list().findByText("Code library").click();
cy.findByRole("link", { name: /Python library/ }).click();
H.DataStudio.PythonLibrary.editor().should("be.visible");
@@ -30,7 +31,7 @@ describe("scenarios > data studio > transforms > python library", () => {
.findByText(/# This is your Python library/)
.should("be.visible");
H.DataStudio.PythonLibrary.editor()
.findByText(/# You can add functions and classes here/)
.findByText(/# You can add code here that can be reused/)
.should("be.visible");
cy.log("modify and save the python library");
@@ -42,7 +43,42 @@ describe("scenarios > data studio > transforms > python library", () => {
cy.findByRole("button", { name: "Save" }).should("be.visible").click();
H.undoToast()
.findByText(/Python library saved/)
.findByText(/library saved/i)
.should("be.visible");
cy.log("refresh the page and check the content is persisted");
cy.url().reload();
H.DataStudio.PythonLibrary.editor()
.findByText(/hello world/)
.should("be.visible");
});
it("should allow editing the JavaScript library", () => {
H.DataStudio.Transforms.visit();
H.DataStudio.Transforms.list().findByText("Code library").click();
cy.findByRole("link", { name: /JavaScript library/ }).click();
H.DataStudio.PythonLibrary.editor().should("be.visible");
cy.log("make sure placeholder with help comment is displayed");
H.DataStudio.PythonLibrary.editor()
.findByText(/\/\/ This is your JavaScript library/)
.should("be.visible");
H.DataStudio.PythonLibrary.editor()
.findByText(/\/\/ You can add code here that can be reused/)
.should("be.visible");
cy.log("modify and save the JavaScript library");
H.DataStudio.PythonLibrary.editor()
.findByRole("textbox")
.click()
.realType("console.log('hello world');");
cy.findByRole("button", { name: "Save" }).should("be.visible").click();
H.undoToast()
.findByText(/library saved/i)
.should("be.visible");
cy.log("refresh the page and check the content is persisted");

View File

@@ -18,7 +18,7 @@ describe("issue #68378", () => {
it("should show empty schemas when picking a target schema (metabase#68378)", () => {
visitTransformListPage();
cy.button("Create a transform").click();
H.popover().findByText("SQL query").click();
H.popover().findByText(/SQL/).click();
H.popover().findByText("Writable Postgres12").click();
H.NativeEditor.type("SELECT 42", { allowFastSet: true }).blur();

View File

@@ -37,7 +37,7 @@ describe("scenarios > admin > transforms", () => {
visitTransformListPage();
cy.button("Create a transform").click();
H.popover().findByText("SQL query").click();
H.popover().findByText(/SQL/).click();
H.popover().findByText(DB_NAME).click();
function testDataReference() {
@@ -270,7 +270,7 @@ describe("scenarios > admin > transforms", () => {
cy.log("create a new transform");
visitTransformListPage();
cy.button("Create a transform").click();
H.popover().findByText("SQL query").click();
H.popover().findByText(/SQL/).click();
cy.log("Add a query with multiple template tags");
H.NativeEditor.clear()

View File

@@ -13,6 +13,7 @@ import {
import { createLibraryWithItems } from "e2e/support/test-library-data";
import { DataPermissionValue } from "metabase/admin/permissions/types";
import type {
AdvancedTransformType,
CardType,
CollectionId,
PythonTransformTableAliases,
@@ -122,7 +123,6 @@ describe("scenarios > admin > transforms", () => {
});
H.goToMainApp();
H.modal().button("Discard changes").click();
H.newButton("Question").click();
H.miniPicker().within(() => {
@@ -137,7 +137,7 @@ describe("scenarios > admin > transforms", () => {
cy.log("create a new transform");
visitTransformListPage();
cy.button("Create a transform").click();
H.popover().findByText("SQL query").click();
H.popover().findByText(/SQL/).click();
H.expectUnstructuredSnowplowEvent({
event: "transform_create",
event_detail: "native",
@@ -177,10 +177,12 @@ describe("scenarios > admin > transforms", () => {
cy.log("create a new transform");
visitTransformListPage();
cy.button("Create a transform").click();
H.popover().findByText("Python script").click();
H.popover()
.findByText(/Python/)
.click();
H.expectUnstructuredSnowplowEvent({
event: "transform_create",
event_detail: "python",
event_detail: "advanced",
});
cy.findByTestId("python-transform-top-bar")
@@ -560,7 +562,7 @@ LIMIT
visitTransformListPage();
cy.button("Create a transform").click();
H.popover().findByText("SQL query").click();
H.popover().findByText(/SQL/).click();
cy.findByTestId("gui-builder-data")
.findByText("Writable Postgres12")
@@ -1616,7 +1618,7 @@ LIMIT
cy.log("create a new transform");
H.getTableId({ name: "Animals", databaseId: WRITABLE_DB_ID }).then(
(id) => {
createPythonTransform({
createAdvancedTransform({
body: dedent`
import pandas as pd
@@ -1654,7 +1656,7 @@ LIMIT
cy.log("create a new Python transform");
H.getTableId({ name: "Animals", databaseId: WRITABLE_DB_ID }).then(
(id) => {
createPythonTransform({
createAdvancedTransform({
body: dedent`
import pandas as pd
@@ -1694,7 +1696,7 @@ LIMIT
cy.log("create a new Python transform");
H.getTableId({ name: "Animals", databaseId: WRITABLE_DB_ID }).then(
(id) => {
createPythonTransform({
createAdvancedTransform({
body: dedent`
import pandas as pd
@@ -1730,7 +1732,7 @@ LIMIT
cy.log("create a new Python transform");
H.getTableId({ name: "Animals", databaseId: WRITABLE_DB_ID }).then(
(id) => {
createPythonTransform({
createAdvancedTransform({
body: dedent`
import pandas as pd
@@ -1825,7 +1827,7 @@ LIMIT
cy.log("create a new SQL transform with a complex query");
visitTransformListPage();
cy.button("Create a transform").click();
H.popover().findByText("SQL query").click();
H.popover().findByText(/SQL/).click();
H.popover().findByText(DB_NAME).click();
H.NativeEditor.type(
@@ -2169,7 +2171,9 @@ LIMIT
visitTransformListPage();
cy.button("Create a transform").click();
H.popover().findByText("Python script").click();
H.popover()
.findByText(/Python/)
.click();
cy.log("import common should be included by default");
H.PythonEditor.value().should("contain", "import common");
@@ -2242,9 +2246,10 @@ LIMIT
() => {
visitTransformListPage();
cy.button("Create a transform").click();
H.popover().findByText("Python script").click();
H.popover()
.findByText(/Python/)
.click();
cy.get(".cm-clickable-token").should("be.visible").click();
H.modal().button("Discard changes").click();
cy.url().should("include", "/data-studio/transforms/library/common.py");
cy.findByTestId("python-library-header").should("be.visible");
},
@@ -2259,7 +2264,9 @@ LIMIT
cy.stub(win, "open").as("windowOpen");
});
cy.button("Create a transform").click();
H.popover().findByText("Python script").click();
H.popover()
.findByText(/Python/)
.click();
cy.get(".cm-clickable-token").should("be.visible").click(H.holdMetaKey);
cy.get("@windowOpen").should(
@@ -2277,7 +2284,9 @@ LIMIT
visitTransformListPage();
cy.button("Create a transform").click();
H.popover().findByText("Python script").click();
H.popover()
.findByText(/Python/)
.click();
cy.log("import common should be included by default");
H.PythonEditor.value().should("contain", "import common");
@@ -2578,8 +2587,8 @@ LIMIT
getTransformsList().findByText("Name").click();
getRowNames().should("deep.equal", [
"Code library",
"Middle Transform",
"Python library",
"Reports",
"Alpha Transform",
"Zebra Transform",
@@ -2591,8 +2600,8 @@ LIMIT
"Zebra Transform",
"Reports",
"Alpha Transform",
"Python library",
"Middle Transform",
"Code library",
]);
cy.log("verify sorting by output table column ascending");
@@ -2600,7 +2609,7 @@ LIMIT
getRowNames().should("deep.equal", [
"Reports",
"Alpha Transform",
"Python library",
"Code library",
"Middle Transform",
"Zebra Transform",
]);
@@ -2612,7 +2621,7 @@ LIMIT
"Middle Transform",
"Reports",
"Alpha Transform",
"Python library",
"Code library",
]);
});
@@ -2709,6 +2718,7 @@ LIMIT
cy.log("Python library should be visible in the list");
getTransformsList().within(() => {
cy.findByText("Code library").click();
cy.findByText("Python library").should("be.visible");
});
@@ -2804,12 +2814,13 @@ LIMIT
cy.button("Create a transform").should("not.exist");
cy.log("clicking Python library navigates to the library editor");
getTransformsList().findByText("Code library").click();
getTransformsList().findByText("Python library").click();
cy.log("python library editor is read-only");
cy.url().should("include", "/data-studio/transforms/library/common.py");
cy.findByRole("alert")
.contains(/The Python library is not editable/)
.contains(/The library is not editable/)
.should("be.visible");
H.DataStudio.PythonLibrary.editor().within(() => {
@@ -3668,7 +3679,7 @@ describe(
H.expectNoBadSnowplowEvents();
});
it("should be possible to test run a Python script", () => {
it("should be possible to test run a Python transform", () => {
H.getTableId({ name: "Animals", databaseId: WRITABLE_DB_ID }).then(
(id) => {
createPythonLibrary(
@@ -3679,7 +3690,7 @@ describe(
`,
);
createPythonTransform({
createAdvancedTransform({
body: dedent`
import pandas as pd
import common
@@ -3724,7 +3735,7 @@ describe(
it("should display preview notice message", () => {
H.getTableId({ name: "Animals", databaseId: WRITABLE_DB_ID }).then(
(id) => {
createPythonTransform({
createAdvancedTransform({
body: dedent`
import pandas as pd
@@ -3775,7 +3786,7 @@ describe("scenarios > admin > transforms", () => {
cy.log("create a new transform");
visitTransformListPage();
cy.button("Create a transform").click();
H.popover().findByText("SQL query").click();
H.popover().findByText(/SQL/).click();
cy.findByTestId("gui-builder-data")
.findByText("Select a database")
@@ -3786,7 +3797,9 @@ describe("scenarios > admin > transforms", () => {
cy.log("create a new transform");
visitTransformListPage();
cy.button("Create a transform").click();
H.popover().findByText("Python script").click();
H.popover()
.findByText(/Python/)
.click();
cy.findByTestId("python-transform-top-bar")
.findByText("Select a database")
@@ -4002,7 +4015,9 @@ function createSqlTransform(opts: {
});
}
function createPythonTransform(opts: {
function createAdvancedTransform(opts: {
name?: string;
type?: AdvancedTransformType;
body: string;
sourceTables: PythonTransformTableAliases;
targetTable?: string;
@@ -4010,7 +4025,7 @@ function createPythonTransform(opts: {
tagIds?: TransformTagId[];
visitTransform?: boolean;
}) {
return H.createPythonTransform({
return H.createAdvancedTransform({
targetTable: TARGET_TABLE,
targetSchema: TARGET_SCHEMA,
...opts,
@@ -4055,7 +4070,7 @@ function getPythonDataPicker() {
}
function createPythonLibrary(path: string, source: string) {
cy.request("PUT", `/api/ee/transforms-python/library/${path}`, {
cy.request("PUT", `/api/ee/advanced-transforms/library/${path}`, {
source,
});
}

View File

@@ -1685,7 +1685,7 @@ describe("scenarios > data studio > workspaces", () => {
cy.log("Create transform via UI with model reference");
Workspaces.visitTransformListPage();
cy.button("Create a transform").click();
H.popover().findByText("SQL query").click();
H.popover().findByText(/SQL/).click();
H.popover().findByText("Writable Postgres12").click();
cy.get("@modelId").then((modelId) => {
@@ -1763,7 +1763,7 @@ describe("scenarios > data studio > workspaces", () => {
databaseId: WRITABLE_DB_ID,
schema: "Schema B",
}).then((id2) => {
createPythonTransform({
createAdvancedTransform({
body: TEST_PYTHON_TRANSFORM,
sourceTables: { foo: id1, bar: id2 },
visitTransform: true,
@@ -1906,7 +1906,7 @@ describe("scenarios > data studio > workspaces", () => {
databaseId: WRITABLE_DB_ID,
schema: "Schema B",
}).then((id2) => {
createPythonTransform({
createAdvancedTransform({
body: TEST_PYTHON_TRANSFORM_MULTI_TABLE,
sourceTables: { animals_a: id1, animals_b: id2 },
visitTransform: true,
@@ -2269,7 +2269,7 @@ describe("scenarios > data studio > workspaces", () => {
it("should not show error when editing a new transform in a workspace (GDGT-1445)", () => {
Workspaces.visitTransformListPage();
cy.findByLabelText("Create a transform").click();
H.popover().findByText("SQL query").click();
H.popover().findByText(/SQL/).click();
// to avoid flakiness on editor's focus
cy.findByTestId("native-query-top-bar").should(
"contain.text",
@@ -2515,7 +2515,7 @@ function createTransforms({ visit }: { visit?: boolean } = { visit: false }) {
});
H.getTableId({ name: "Animals", databaseId: WRITABLE_DB_ID }).then((id) => {
createPythonTransform({
createAdvancedTransform({
body: TEST_PYTHON_TRANSFORM,
sourceTables: { foo: id },
});
@@ -2561,7 +2561,7 @@ function createSqlTransform(opts: {
});
}
function createPythonTransform(opts: {
function createAdvancedTransform(opts: {
body: string;
sourceTables: PythonTransformTableAliases;
targetTable?: string;
@@ -2569,7 +2569,7 @@ function createPythonTransform(opts: {
tagIds?: TransformTagId[];
visitTransform?: boolean;
}) {
return H.createPythonTransform({
return H.createAdvancedTransform({
targetTable: TARGET_TABLE_PYTHON,
targetSchema: TARGET_SCHEMA,
...opts,

View File

@@ -182,7 +182,10 @@ describe(
cy.log("Should be able to visit and the see new transform");
viewLastSuggestion();
cy.url().should("include", "/data-studio/transforms/new/python");
cy.url().should(
"include",
"/data-studio/transforms/new/advanced/python",
);
assertEditorDiffState({ exists: false }); // nothing to diff, so we shouldn't show the UI for it
assertEditorContent("python", "pd.DataFrame({'value': [1]})");
@@ -375,7 +378,7 @@ describe(
it("should update existing Python transform via metabot", () => {
H.getTableId({ name: SOURCE_TABLE, databaseId: WRITABLE_DB_ID }).then(
(tableId) => {
H.createPythonTransform({
H.createAdvancedTransform({
targetTable: "transform_table",
targetSchema: "Schema A",
body: dedent`

View File

@@ -398,7 +398,7 @@ describe("issue 52806", () => {
it("should remove parameter values from the URL when leaving the query builder and discarding changes (metabase#52806)", () => {
cy.visit("/");
H.newButton("SQL query").click();
H.newButton(/SQL/).click();
H.NativeEditor.focus().type("select {{x}}");
cy.location().should((location) => expect(location.search).to.eq("?x="));
cy.findByTestId("main-logo-link").click();
@@ -441,7 +441,7 @@ describe("issue 55951", () => {
},
);
H.newButton("SQL query").click();
H.newButton(/SQL/).click();
H.popover()
.should("be.visible")
.within(() => {

View File

@@ -801,7 +801,7 @@ describe("no native access", { tags: ["@external", "@skip"] }, () => {
cy.log("#32387");
cy.findByRole("button", { name: /New/ }).click();
H.popover().findByText("SQL query").click();
H.popover().findByText(/SQL/).click();
cy.wait("@database");
cy.go("back");

View File

@@ -39,7 +39,7 @@
[metabase-enterprise.stale.api]
[metabase-enterprise.support-access-grants.api]
[metabase-enterprise.tenants.api]
[metabase-enterprise.transforms-python.api]
[metabase-enterprise.transforms-runner.api]
[metabase-enterprise.upload-management.api]
[metabase-enterprise.workspaces.api]
[metabase.api.macros :as api.macros]
@@ -130,7 +130,7 @@
"/metabot-tools" metabase-enterprise.metabot-v3.tools.api/routes
"/metabot-v3" (premium-handler metabase-enterprise.metabot-v3.api/routes :metabot-v3)
"/permission_debug" (premium-handler metabase-enterprise.permission-debug.api/routes :advanced-permissions)
"/transforms-python" (premium-handler metabase-enterprise.transforms-python.api/routes :transforms-python)
"/advanced-transforms" (premium-handler metabase-enterprise.transforms-runner.api/routes :transforms-python)
"/scim" (premium-handler metabase-enterprise.scim.routes/routes :scim)
"/semantic-search" (premium-handler metabase-enterprise.semantic-search.api/routes :semantic-search)
"/serialization" (premium-handler metabase-enterprise.serialization.api/routes :serialization)

View File

@@ -21,5 +21,8 @@
[metabase-enterprise.sso.init]
[metabase-enterprise.stale.init]
[metabase-enterprise.support-access-grants.init]
[metabase-enterprise.transforms-clojure.init]
[metabase-enterprise.transforms-javascript.init]
[metabase-enterprise.transforms-python.init]
[metabase-enterprise.transforms-runner.init]
[metabase-enterprise.workspaces.init]))

View File

@@ -1,7 +1,7 @@
(ns metabase-enterprise.metabot-v3.tools.transforms
(:require
[metabase-enterprise.metabot-v3.tools.util :as metabot-v3.tools.u]
[metabase-enterprise.transforms-python.api :as transforms-python.api]
[metabase-enterprise.transforms-runner.api :as transforms-runner.api]
[metabase.transforms.core :as transforms]
[metabase.transforms.schema :as transforms.schema]
[metabase.util.malli.registry :as mr]))
@@ -15,7 +15,7 @@
(try
{:structured_output
(->> (transforms/get-transforms)
(into [] (comp (filter #(or (transforms/python-transform? %)
(into [] (comp (filter #(or (transforms/runner-transform? %)
(transforms/native-query-transform? %)))
(filter :source_readable)
(map #(select-keys % [:id :entity_id :name :type :description :source])))))}
@@ -36,6 +36,6 @@
[{:keys [path]}]
(try
{:structured_output
(transforms-python.api/get-python-library-by-path path)}
(transforms-runner.api/get-library-by-path path nil)}
(catch Exception e
(metabot-v3.tools.u/handle-agent-error e))))

View File

@@ -24,7 +24,7 @@
(:import (metabase_enterprise.remote_sync.source.protocol SourceSnapshot)))
(defn- snapshot-has-transforms?
"Checks if the snapshot contains any Transform, PythonLibrary entities,
"Checks if the snapshot contains any Transform, TransformLibrary entities,
non-built-in TransformTags, or transforms-namespace collections.
Used to auto-enable/disable remote-sync-transforms setting during import.
@@ -179,7 +179,7 @@
Conflict types detected:
- :entity-id-conflict - Items with existing entity IDs that are NOT already synced
- :library-conflict - First import only, local Library exists, import has Library
- :transforms-not-enabled - Import has Transform/TransformTag/PythonLibrary, setting disabled
- :transforms-not-enabled - Import has Transform/TransformTag/TransformLibrary, setting disabled
- :snippets-without-library - Import has NativeQuerySnippet, Library not remote-synced"
[ingestable first-import?]
(let [ingest-list (serialization/ingest-list ingestable)
@@ -242,7 +242,7 @@
last-imported-version (remote-sync.task/last-version)
first-import? (nil? last-imported-version)
path-filters [#"collections/.*" #"databases/.*" #"actions/.*"
#"transforms/.*" #"python-libraries/.*" #"snippets/.*"]
#"transforms/.*" #"transform-libraries/.*" #"snippets/.*"]
base-ingestable (source.p/->ingestable snapshot {:path-filters path-filters})
has-transforms? (snapshot-has-transforms? base-ingestable)
{:keys [conflicts summary]} (get-conflicts base-ingestable first-import?)

View File

@@ -15,7 +15,7 @@
[clojure.string :as str]
[clojure.walk :as walk]
[metabase-enterprise.remote-sync.settings :as rs-settings]
[metabase-enterprise.transforms-python.core :as transforms-python]
[metabase-enterprise.transforms-runner.models.transform-library :as transform-library]
[metabase.collections.core :as collections]
[metabase.collections.models.collection :as collection]
[metabase.models.serialization :as serdes]
@@ -71,7 +71,7 @@
- :export-scope - Export scope for query-export-roots:
:root-collections - Query root-level remote-synced + namespace collections (Collection)
:root-only - Query root instances with collection_id = nil (Transform)
:all - Query all instances (TransformTag, PythonLibrary, NativeQuerySnippet)
:all - Query all instances (TransformTag, TransformLibrary, NativeQuerySnippet)
nil/:derived - No root query; derived from other models via serdes/descendants
- :enabled? - true, or setting keyword (e.g., :remote-sync-transforms, :library-synced).
When :library-synced, uses the library-is-remote-synced? setting."
@@ -302,18 +302,18 @@
:export-scope :all ; query for all instances
:enabled? :remote-sync-transforms}
:model/PythonLibrary
{:model-type "PythonLibrary"
:model-key :model/PythonLibrary
:model/TransformLibrary
{:model-type "TransformLibrary"
:model-key :model/TransformLibrary
:identity :entity-id
:events {:prefix :event/python-library
:types [:create :update]} ; no delete - upsert only
:events {:prefix :event/transform-library
:types [:create :update]} ; no delete - upsert only
:eligibility {:type :setting
:setting :remote-sync-transforms}
:archived-key nil ; no archived field
:tracking {:select-fields [:path]
:field-mappings {:model_name :path}}
:conditions {:entity_id [:not= transforms-python/builtin-entity-id]} ; exclude built-in common.py from sync
:conditions {:entity_id [:not-in (transform-library/all-builtin-entity-ids)]} ; exclude built-in libraries from sync
:removal {:statuses #{"removed" "delete"} ; no scope-key = global deletion
:all-on-setting-disable :remote-sync-transforms}
:export-scope :all ; query for all instances
@@ -506,7 +506,7 @@
(defn- has-unsynced-entities-for-feature?
"Returns true if any model in the feature group has local entities not tracked in RemoteSyncObject.
Excludes built-in TransformTags and the built-in PythonLibrary from the count since they are
Excludes built-in TransformTags and built-in TransformLibrary entries from the count since they are
system-created and not user data. Namespace collections are not checked here because they are
organizational containers, not user data that would be lost on import."
[specs-for-feature]
@@ -516,7 +516,7 @@
;; Exclude built-in entities from count (they are system-created, not user data)
local-count (case model-key
:model/TransformTag (t2/count model-key :built_in_type nil)
:model/PythonLibrary (t2/count model-key :entity_id [:not= transforms-python/builtin-entity-id])
:model/TransformLibrary (t2/count model-key :entity_id [:not-in (transform-library/all-builtin-entity-ids)])
(t2/count model-key))
synced-count (t2/count :model/RemoteSyncObject :model_type model-type)]
(and (pos? local-count)

View File

@@ -35,7 +35,7 @@
(conj "Setting")
(not (:no-transforms opts))
(conj "Transform" "TransformTag" "TransformJob" "PythonLibrary")))
(conj "Transform" "TransformTag" "TransformJob" "TransformLibrary")))
(defn make-targets-of-type
"Returns a targets seq with model type and given ids"

View File

@@ -74,7 +74,7 @@
read-timestamps))
(def legal-top-level-paths "Known top-level paths for directory with serialization output"
#{"actions" "collections" "databases" "glossary" "python-libraries" "snippets" "transforms"})
#{"actions" "collections" "databases" "glossary" "transform-libraries" "snippets" "transforms"})
(defn- ingest-all [^File root-dir]
;; This returns a map {unlabeled-hierarchy [original-hierarchy File]}.

View File

@@ -35,7 +35,7 @@
content
["FieldValues"
"Metabot"
"PythonLibrary"
"TransformLibrary"
"Setting"
"Transform"
"TransformJob"

View File

@@ -0,0 +1,13 @@
(ns metabase-enterprise.transforms-clojure.impl
"Registers :clojure as a runner language. Loaded by transforms-clojure/init.clj for side effects.
See transforms.interface/register-runner! and transforms.interface/lang-config."
(:require
[metabase.transforms.interface :as transforms.i]))
(transforms.i/register-runner! :clojure)
(defmethod transforms.i/lang-config :clojure [_]
{:runtime "clojure"
:label "Clojure"
:timing-key :clojure-execution
:extension ".clj"})

View File

@@ -0,0 +1,3 @@
(ns metabase-enterprise.transforms-clojure.init
(:require
[metabase-enterprise.transforms-clojure.impl]))

View File

@@ -0,0 +1,13 @@
(ns metabase-enterprise.transforms-javascript.impl
"Registers :javascript as a runner language. Loaded by transforms-javascript/init.clj for side effects.
See transforms.interface/register-runner! and transforms.interface/lang-config."
(:require
[metabase.transforms.interface :as transforms.i]))
(transforms.i/register-runner! :javascript)
(defmethod transforms.i/lang-config :javascript [_]
{:runtime "javascript"
:label "JavaScript"
:timing-key :javascript-execution
:extension ".js"})

View File

@@ -0,0 +1,3 @@
(ns metabase-enterprise.transforms-javascript.init
(:require
[metabase-enterprise.transforms-javascript.impl]))

View File

@@ -1,10 +0,0 @@
(ns metabase-enterprise.transforms-python.core
"Core namespace for Python transforms functionality.
Re-exports commonly used vars from internal namespaces."
(:require
[metabase-enterprise.transforms-python.models.python-library :as python-library]
[potemkin :as p]))
(p/import-vars
[python-library
builtin-entity-id])

View File

@@ -1,369 +0,0 @@
(ns metabase-enterprise.transforms-python.execute
(:require
[clojure.core.async :as a]
[clojure.java.io :as io]
[clojure.string :as str]
[metabase-enterprise.transforms-python.python-runner :as python-runner]
[metabase-enterprise.transforms-python.s3 :as s3]
[metabase-enterprise.transforms-python.settings :as transforms-python.settings]
[metabase.app-db.core :as app-db]
[metabase.driver :as driver]
[metabase.driver.connection :as driver.conn]
[metabase.driver.util :as driver.u]
[metabase.transforms.core :as transforms]
[metabase.transforms.instrumentation :as transforms.instrumentation]
[metabase.transforms.interface :as transforms.i]
[metabase.transforms.util :as transforms.util]
[metabase.util :as u]
[metabase.util.format :as u.format]
[metabase.util.i18n :as i18n]
[metabase.util.jvm :as u.jvm]
[metabase.util.log :as log]
[toucan2.core :as t2])
(:import
(java.io Closeable)
(java.net SocketException)
(java.nio.file Files)
(java.nio.file.attribute FileAttribute)
(java.time Duration)))
(set! *warn-on-reflection* true)
(defn- empty-message-log
"Returns a new message log.
Python transforms produce output while they run. The message log is a stateful structure that buffers log outputs during the run.
The goal is to assist in debugging in the case of python runtime or syntax errors, and to provide some immediate feedback for longer running transforms.
Important Note: in memory buffering is considered acceptable as we expect /logs output to be truncated or small.
The intended use of this structure is to produce a string suitable for `transform_run.message`, saving it every few seconds."
[]
(atom {:pre-python [] ; log! outputs previous to the python execution, i.e table read progress
:python nil ; events json structured logs from the /logs endpoint
:post-python []})) ; log! outputs after the python execution, i.e. output reads, writes to target
(defn- log!
"Appends a string to the message log, the string is user facing and should be suitable for presentation as part of the `transform_run.message` field."
[message-log s]
(swap! message-log (fn [m] (update m (if (:python m) :post-python :pre-python) conj s)))
nil)
(defn- replace-python-logs!
"Called when events output is received from a /logs call to copy those events into the message-log."
[message-log events]
(swap! message-log assoc :python events)
nil)
(defn- message-log->transform-run-message
"Presents the content of the log as a string suitable for storage in transform_run.message."
[message-log]
(let [{:keys [pre-python python post-python]} @message-log]
(str/join "\n" (concat pre-python
(map :message python)
post-python))))
(defn- save-log-to-transform-run-message!
"Saves the content of the log into the transform_run's message field. Lossy, string contains user facing information only."
[run-id message-log]
(t2/update! :model/TransformRun
:id run-id
{:message (message-log->transform-run-message message-log)}))
(def ^:private ^Duration python-message-loop-sleep-duration
(Duration/ofMillis 1000))
(defn- python-message-update-loop!
"Block while relevant log data is replicated from the runner into the message log.
When new logs are received, the log data will be flushed to the transform_run.message field as a string.
The loop will exit on error or when interrupted."
[run-id message-log]
(try
(loop []
(if (.isInterrupted (Thread/currentThread))
(log/debug "Message update loop interrupted")
(do (let [sleep-ms (.toMillis python-message-loop-sleep-duration)]
(when (pos? sleep-ms) (Thread/sleep sleep-ms)))
(let [{:keys [status body]} (python-runner/get-logs run-id)]
(cond
(<= 200 status 299)
(let [{:keys [execution_id events]} body]
(if-not (= run-id execution_id)
(do (log/debugf "Run id did not match expected: %s actual: %s" run-id execution_id)
(recur))
(do
(replace-python-logs! message-log events)
(save-log-to-transform-run-message! run-id message-log)
(recur))))
(= 404 status)
(do
(log/debugf "No logs yet (or run finished), run-id: %s" run-id)
(recur))
:else
(do
(log/warnf "Unexpected status polling for logs %s %s, run-id: %s" status body run-id)
(log/debug "Exiting due to poll error")))))))
(catch SocketException se (when-not (= "Closed by interrupt" (ex-message se)) (throw se)))
(catch InterruptedException _)
(catch Throwable e
(log/errorf e "An exception was caught during msg update loop, run-id: %s" run-id))))
(defn- open-python-message-update-future! ^Closeable [run-id message-log]
(let [cleanup (fn [fut]
(future-cancel fut)
(if (= ::timeout (try (deref fut 10000 ::timeout) (catch Throwable _)))
(log/fatalf "Log polling task did not respond to interrupt, run-id: %s" run-id)
(log/debugf "Log polling task done, run-id: %s" run-id)))
fut (u.jvm/in-virtual-thread*
(python-message-update-loop! run-id message-log))]
(reify Closeable
(close [_] (cleanup fut)))))
(defn- table-schema [table-name metadata]
{:name (if (keyword? table-name) table-name (keyword table-name))
:columns (mapv (fn [{:keys [name base_type #_database_type]}]
{:name name
:type (python-runner/restricted-insert-type base_type)
;; :database-type database_type
:nullable? true})
(:fields metadata))})
(defn- insert-data!
"Insert data from source into an existing table."
[driver db-id table-schema data-source]
(let [data-source (assoc data-source :table-schema table-schema)]
(driver/insert-from-source! driver db-id table-schema data-source)))
(defn- create-table-and-insert-data!
"Create a table from metadata and insert data from source."
[driver db-id table-schema data-source]
(transforms.util/create-table-from-schema! driver db-id table-schema)
(insert-data! driver db-id table-schema data-source))
(defn- transfer-with-rename-tables-strategy!
"Transfer data using the rename-tables*! multimethod with atomicity guarantees.
Creates new table, then atomically renames target->old and new->target, then drops old."
[driver db-id table-name metadata data-source]
(let [source-table-name (driver.u/temp-table-name driver table-name)
temp-table-name (driver.u/temp-table-name driver table-name)]
(log/info "Using rename-tables strategy with atomicity guarantees")
(try
(create-table-and-insert-data! driver db-id (table-schema source-table-name metadata) data-source)
(transforms.util/rename-tables! driver db-id {table-name temp-table-name
source-table-name table-name})
(transforms.util/drop-table! driver db-id temp-table-name)
(catch Exception e
(log/error e "Failed to transfer data using rename-tables strategy")
(try
(transforms.util/drop-table! driver db-id source-table-name)
(catch Exception _))
(throw e)))))
(defn- transfer-with-create-drop-rename-strategy!
"Transfer data using create + drop + rename to minimize time without data.
Creates new table, drops old table, then renames new->target."
[driver db-id table-name metadata data-source]
(let [source-table-name (driver.u/temp-table-name driver table-name)]
(log/info "Using create-drop-rename strategy to minimize downtime")
(try
(create-table-and-insert-data! driver db-id (table-schema source-table-name metadata) data-source)
(transforms.util/drop-table! driver db-id table-name)
(driver/rename-table! driver db-id source-table-name table-name)
(catch Exception e
(log/error e "Failed to transfer data using create-drop-rename strategy")
(try
(transforms.util/drop-table! driver db-id source-table-name)
(catch Exception _))
(throw e)))))
(defn- transfer-with-drop-create-fallback-strategy!
"Transfer data using drop + create fallback strategy.
Drops old table, then creates new table directly with target name."
[driver db-id table-name metadata data-source]
(log/info "Using drop-create fallback strategy")
(try
(transforms.util/drop-table! driver db-id table-name)
(create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source)
(catch Exception e
(log/error e "Failed to transfer data using drop-create fallback strategy")
(throw e))))
(defmulti ^:private transfer-file-to-db
{:arglists '([driver db transform metadata temp-file])}
(fn [_ _ transform _ _] (-> transform :target :type keyword)))
(defmethod transfer-file-to-db :table-incremental
[driver {db-id :id}
{:keys [target] :as transform}
metadata temp-file]
(let [table-name (transforms.util/qualified-table-name driver target)
table-exists? (transforms.util/target-table-exists? transform)
data-source {:type :jsonl-file
:file temp-file}]
;; once we have more than just append, dispatch on :target-incremental-strategy
(if (not table-exists?)
(do
(log/info "New table")
(create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))
(insert-data! driver db-id (table-schema table-name metadata) data-source))))
(defmethod transfer-file-to-db :table
[driver {db-id :id :as db}
{:keys [target] :as transform}
metadata temp-file]
(let [table-name (transforms.util/qualified-table-name driver target)
table-exists? (transforms.util/target-table-exists? transform)
data-source {:type :jsonl-file
:file temp-file}]
(cond
(not table-exists?)
(do
(log/info "New table")
(create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))
(driver.u/supports? driver :atomic-renames db)
(transfer-with-rename-tables-strategy! driver db-id table-name metadata data-source)
(driver.u/supports? driver :rename db)
(transfer-with-create-drop-rename-strategy! driver db-id table-name metadata data-source)
:else
(transfer-with-drop-create-fallback-strategy! driver db-id table-name metadata data-source))))
(defn- start-cancellation-process!
"Starts a core.async process that optimistically sends a cancellation request to the python executor if cancel-chan receives a value.
Returns a channel that will receive either the async http call j.u.c.FutureTask in the case of cancellation, or nil when the cancel-chan is closed."
[server-url run-id cancel-chan]
(a/go (when (a/<! cancel-chan)
(python-runner/cancel-python-code-http-call! server-url run-id))))
(defn- run-python-transform! [{:keys [source] :as transform} db run-id cancel-chan message-log]
;; Resolve name-based source table refs to table IDs (throws if any not found)
(let [resolved-source-tables (transforms.util/resolve-source-tables (:source-tables source))]
;; TODO restructure things such that s3 can we swapped out for other transfer mechanisms
(with-open [^Closeable log-future-ref
(if (app-db/in-transaction?)
;; if in a transaction (such as under mt/with-temp), it is not safe to poll for logs (close race / contention)
;; tests that want to test async log behaviour should opt out of thread-local test helpers
(reify Closeable (close [_]))
(open-python-message-update-future! run-id message-log))
shared-storage-ref (s3/open-shared-storage! resolved-source-tables)]
(let [driver (:engine db)
server-url (transforms-python.settings/python-runner-url)
_ (python-runner/copy-tables-to-s3! {:run-id run-id
:shared-storage @shared-storage-ref
:source (assoc source :source-tables resolved-source-tables)
:cancel-chan cancel-chan
:limit (:limit source)
:transform-id (:id transform)})
_ (start-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan
{:keys [status body] :as response}
(python-runner/execute-python-code-http-call!
{:server-url server-url
:code (:body source)
:run-id run-id
:table-name->id resolved-source-tables
:shared-storage @shared-storage-ref})
output-manifest (python-runner/read-output-manifest @shared-storage-ref)
events (python-runner/read-events @shared-storage-ref)]
(.close ^Closeable log-future-ref) ; early close to force any writes to flush
(replace-python-logs! message-log events)
(if (not= 200 status)
(do
(when (:timeout body)
(transforms/timeout-run! run-id))
(throw (ex-info "Python runner call failed"
{:transform-message (i18n/tru "Python execution failure (exit code {0})" (:exit_code body "?"))
:status-code 400
:api-status-code status
:body body
:events events})))
(try
(let [temp-path (Files/createTempFile "transform-output-" ".jsonl" (u/varargs FileAttribute))
temp-file (.toFile temp-path)]
(when-not (seq (:fields output-manifest))
(throw (ex-info "No fields in metadata"
{:metadata output-manifest
:raw-body body
:events events})))
(try
(with-open [in (python-runner/open-output @shared-storage-ref)]
(io/copy in temp-file))
(let [file-size (.length temp-file)]
(transforms.instrumentation/with-stage-timing [run-id [:import :file-to-dwh]]
(transfer-file-to-db driver db transform output-manifest temp-file))
(transforms.instrumentation/record-data-transfer! run-id :file-to-dwh file-size nil))
(finally
(.delete temp-file))))
response
(catch Exception e
(log/error e "Failed to to create resulting table")
(throw (ex-info "Failed to create the resulting table"
{:transform-message (or (:transform-message (ex-data e))
;; TODO keeping messaging the same at this level
;; should be more specific in underlying calls
(i18n/tru "Failed to create the resulting table"))}
e)))))))))
(defn- exceptional-run-message [message-log ex]
(str/join "\n" (remove str/blank? [(message-log->transform-run-message message-log)
(or (:transform-message (ex-data ex))
(if (instance? InterruptedException ex)
(i18n/tru "Transform interrupted")
(i18n/tru "Something went wrong")))])))
(defn execute-python-transform!
"Execute a Python transform by calling the python runner.
Blocks until the transform returns."
[transform {:keys [run-method start-promise user-id]}]
(assert (transforms.util/python-transform? transform) "Transform must be a python transform")
(try
(let [message-log (empty-message-log)
{:keys [target owner_user_id creator_id] transform-id :id} transform
{driver :engine :as db} (t2/select-one :model/Database (transforms.i/target-db-id transform))
;; For manual runs, use the triggering user; for cron, use owner/creator
run-user-id (if (and (= run-method :manual) user-id)
user-id
(or owner_user_id creator_id))
{run-id :id} (transforms.util/try-start-unless-already-running transform-id run-method run-user-id)]
(some-> start-promise (deliver [:started run-id]))
(log! message-log (i18n/tru "Executing Python transform"))
(driver.conn/with-write-connection
(log/info "Executing Python transform" transform-id "with target" (pr-str target)
(when (driver.conn/write-connection-requested?)
" using write connection"))
(let [start-ms (u/start-timer)
conn-spec (driver/connection-spec driver db)
transform-details {:db-id (:id db)
:transform-id transform-id
:transform-type (keyword (:type target))
:conn-spec conn-spec
:output-schema (:schema target)
:output-table (transforms.util/qualified-table-name driver target)}
run-fn (fn [cancel-chan]
(run-python-transform! transform db run-id cancel-chan message-log)
(log! message-log (i18n/tru "Python execution finished successfully in {0}" (u.format/format-milliseconds (u/since-ms start-ms))))
(save-log-to-transform-run-message! run-id message-log))
ex-message-fn #(exceptional-run-message message-log %)
result (transforms.instrumentation/with-stage-timing [run-id [:computation :python-execution]]
(transforms.util/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn))]
(transforms.util/handle-transform-complete!
:run-id run-id
:transform transform
:db db)
{:run_id run-id
:result result})))
(catch Throwable t
(log/error t "Error executing Python transform")
(throw t))))

View File

@@ -1,39 +1,13 @@
(ns metabase-enterprise.transforms-python.impl
"Registers :python as a runner language. Loaded by transforms-python/init.clj for side effects.
See transforms.interface/register-runner! and transforms.interface/lang-config."
(:require
[metabase-enterprise.transforms-python.execute :as transforms-python.execute]
[metabase.transforms.interface :as transforms.i]))
(defmethod transforms.i/target-db-id :python
[transform]
(-> transform :target :database))
(transforms.i/register-runner! :python)
(defmethod transforms.i/source-db-id :python
[transform]
(-> transform :source :source-database))
#_{:clj-kondo/ignore [:discouraged-var]}
(defmethod transforms.i/execute! :python
[transform options]
(transforms-python.execute/execute-python-transform! transform options))
(defn- source-table-value->dependency
"Convert a source table value (int or ref map) to a dependency map."
[v]
(cond
;; Integer table ID - direct table dependency
(int? v)
{:table v}
;; Map with resolved table_id - direct table dependency
(:table_id v)
{:table (:table_id v)}
;; Map without table_id - return ref for ordering system to resolve
:else
{:table-ref (select-keys v [:database_id :schema :table])}))
(defmethod transforms.i/table-dependencies :python
[transform]
(into #{}
(map source-table-value->dependency)
(vals (get-in transform [:source :source-tables]))))
(defmethod transforms.i/lang-config :python [_]
{:runtime "python"
:label "Python"
:timing-key :python-execution
:extension ".py"})

View File

@@ -1,4 +1,3 @@
(ns metabase-enterprise.transforms-python.init
(:require
[metabase-enterprise.transforms-python.impl]
[metabase-enterprise.transforms-python.settings]))
[metabase-enterprise.transforms-python.impl]))

View File

@@ -1,115 +0,0 @@
(ns metabase-enterprise.transforms-python.models.python-library
(:require
[metabase.api.common :as api]
[metabase.app-db.core :as app-db]
[metabase.events.core :as events]
[metabase.models.interface :as mi]
[metabase.models.serialization :as serdes]
[metabase.permissions.core :as perms]
[metabase.util.i18n :refer [tru]]
[methodical.core :as methodical]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
(methodical/defmethod t2/table-name :model/PythonLibrary [_model] :python_library)
(defn- normalize-path
"Ensure the path ends with .py extension."
[path]
(if (.endsWith ^String path ".py")
path
(str path ".py")))
(t2/define-before-insert :model/PythonLibrary
[library]
(update library :path normalize-path))
(t2/define-before-update :model/PythonLibrary
[library]
(if (:path library)
(update library :path normalize-path)
library))
(doseq [trait [:metabase/model :hook/timestamped? :hook/entity-id]]
(derive :model/PythonLibrary trait))
(defmethod mi/can-read? :model/PythonLibrary
([_instance]
(perms/has-any-transforms-permission? api/*current-user-id*))
([_model _pk]
(perms/has-any-transforms-permission? api/*current-user-id*)))
(defmethod mi/can-write? :model/PythonLibrary
([_instance]
(perms/has-any-transforms-permission? api/*current-user-id*))
([_model _pk]
(perms/has-any-transforms-permission? api/*current-user-id*)))
(def ^:private allowed-paths
"Set of allowed library paths. Currently only 'common' is supported."
#{"common.py"})
(def builtin-entity-id
"The entity_id of the built-in common.py PythonLibrary created by migration.
Used to protect it from deletion during remote-sync import."
"cWWH9qJPvHNB3rP2vLZrK")
(defn- validate-path!
"Validates that the given path is allowed. Throws an exception if not."
[path]
(let [normalized-path (normalize-path path)]
(when-not (contains? allowed-paths normalized-path)
(throw (ex-info (tru "Invalid library path. Only ''common'' is currently supported.")
{:status-code 400
:path normalized-path
:allowed-paths allowed-paths})))))
(defn get-python-library-by-path
"Get the Python library by path."
[path]
(let [normalized-path (normalize-path path)]
(validate-path! normalized-path)
(t2/select-one :model/PythonLibrary :path normalized-path)))
(defn update-python-library-source!
"Update the Python library source code. Creates a new record if none exists. Returns the updated library."
[path source]
(let [normalized-path (normalize-path path)]
(validate-path! normalized-path)
(let [id (app-db/update-or-insert! :model/PythonLibrary
{:path normalized-path}
(constantly {:path normalized-path :source source}))]
(t2/select-one :model/PythonLibrary id))))
;;; ------------------------------------------------- Serialization --------------------------------------------------
(defmethod serdes/make-spec "PythonLibrary"
[_model-name _opts]
{:copy [:path :source :entity_id]
:transform {:created_at (serdes/date)}})
(defmethod serdes/hash-fields :model/PythonLibrary
[_model]
[:path])
(defmethod serdes/storage-path "PythonLibrary" [entity _ctx]
(let [{:keys [id label]} (-> entity serdes/path last)]
["python-libraries" (serdes/storage-leaf-file-name id label)]))
;;; ------------------------------------------------ Event Hooks -----------------------------------------------------
;; Event type hierarchy for remote-sync tracking
(derive ::event :metabase/event)
(doseq [e [:event/python-library-create :event/python-library-update :event/python-library-delete]]
(derive e ::event))
(t2/define-after-insert :model/PythonLibrary
[library]
(events/publish-event! :event/python-library-create {:object library})
library)
(t2/define-after-update :model/PythonLibrary
[library]
(events/publish-event! :event/python-library-update {:object library})
library)

View File

@@ -1,9 +1,9 @@
(ns metabase-enterprise.transforms-python.api
(ns metabase-enterprise.transforms-runner.api
(:require
[clojure.string :as str]
[metabase-enterprise.transforms-python.models.python-library :as python-library]
[metabase-enterprise.transforms-python.python-runner :as python-runner]
[metabase-enterprise.transforms-python.settings :as transforms-python.settings]
[metabase-enterprise.transforms-runner.models.transform-library :as transform-library]
[metabase-enterprise.transforms-runner.runner :as runner]
[metabase-enterprise.transforms-runner.settings :as runner.settings]
[metabase.api.common :as api]
[metabase.api.macros :as api.macros]
[metabase.api.routes.common :refer [+auth]]
@@ -13,10 +13,10 @@
[metabase.util.malli.schema :as ms]
[toucan2.core :as t2]))
(defn get-python-library-by-path
"Get Python library details by path for use by other APIs."
[path]
(-> (python-library/get-python-library-by-path path)
(defn get-library-by-path
"Get library details by path. Type is \"python\" (default), \"javascript\", or \"clojure\"."
[path type]
(-> (transform-library/get-library-by-path (or type "python") path)
api/read-check
(select-keys [:source :path :created_at :updated_at])))
@@ -25,24 +25,26 @@
;;
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
(api.macros/defendpoint :get "/library/:path"
"Get the Python library for user modules."
"Get the library for user modules. Pass `?type=javascript` for JavaScript, defaults to Python."
[{:keys [path]} :- [:map [:path ms/NonBlankString]]
_query-params]
(get-python-library-by-path path))
{:keys [type]} :- [:map [:type {:optional true} [:maybe :string]]]]
(get-library-by-path path type))
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
;;
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
(api.macros/defendpoint :put "/library/:path"
"Update the Python library source code for user modules."
"Update the library source code for user modules. Pass `type` in the body for JavaScript."
[{:keys [path]} :- [:map [:path ms/NonBlankString]]
_query-params
body :- [:map {:closed true}
[:source :string]]]
body :- [:map
[:source :string]
[:type {:optional true} [:maybe :string]]]]
;; Check permission directly since this is an upsert endpoint - the library may not exist yet.
(api/check-403 (perms/has-any-transforms-permission? api/*current-user-id*))
(python-library/update-python-library-source! path (:source body)))
(let [{:keys [source type]} body]
(transform-library/update-library-source! (or type "python") path source)))
(api.macros/defendpoint :post "/test-run"
:- [:map
@@ -51,11 +53,13 @@
[:output {:optional true} [:map
[:cols [:sequential [:map [:name :string]]]]
[:rows [:sequential :any]]]]]
"Evaluate an ad-hoc python transform on a sample of input data.
"Evaluate an ad-hoc transform on a sample of input data.
Pass `type` as \"javascript\" to run JavaScript; defaults to \"python\".
Intended for short runs for early feedback. Input/output/timeout limits apply."
[_
_
{:keys [code
type
source_tables
output_row_limit
per_input_row_limit]
@@ -63,6 +67,7 @@
per_input_row_limit 100}}
:- [:map
[:code :string]
[:type {:optional true} [:maybe :string]]
[:source_tables [:map-of {:min 1} :string :int]]
[:output_row_limit {:optional true} [:and :int [:> 1] [:<= 100]]]
[:per_input_row_limit {:optional true} [:and :int [:> 1] [:<= 100]]]]]
@@ -70,12 +75,13 @@
(api/check-400 (= (count db-ids) 1) (i18n/deferred-tru "All source tables must belong to the same database."))
(api/check-403 (perms/has-db-transforms-permission? api/*current-user-id* (first db-ids))))
;; NOTE: we do not test database support, as there is no write target.
(let [result (python-runner/execute-and-read-output!
(let [result (runner/execute-and-read-output!
{:code code
:source-tables source_tables
:per-input-limit per_input_row_limit
:row-limit output_row_limit
:timeout-secs (transforms-python.settings/python-runner-test-run-timeout-seconds)})
:timeout-secs (runner.settings/python-runner-test-run-timeout-seconds)
:runtime (or type "python")})
logs (str/join "\n" (map :message (:logs result)))]
(if (= :succeeded (:status result))
{:logs logs
@@ -85,6 +91,6 @@
:error {:message (:message result)}})))
(def ^{:arglists '([request respond raise])} routes
"`/api/ee/transforms-python` routes."
"`/api/ee/advanced-transforms` routes."
(handlers/routes
(api.macros/ns-handler *ns* +auth)))

View File

@@ -0,0 +1,428 @@
(ns metabase-enterprise.transforms-runner.execute
"Shared execution logic for runner-based transforms (Python, JavaScript, etc.).
All runner-based transforms follow the same pattern:
1. Upload source tables to S3
2. Call the runner service with a `language` parameter
3. Read output from S3
4. Transfer results to the target database
Each language provides a small config map; this namespace does the rest."
(:require
[clojure.core.async :as a]
[clojure.java.io :as io]
[clojure.string :as str]
[metabase-enterprise.transforms-runner.runner :as runner]
[metabase-enterprise.transforms-runner.s3 :as s3]
[metabase-enterprise.transforms-runner.settings :as runner.settings]
[metabase.app-db.core :as app-db]
[metabase.driver :as driver]
[metabase.driver.connection :as driver.conn]
[metabase.driver.util :as driver.u]
[metabase.transforms.core :as transforms]
[metabase.transforms.instrumentation :as transforms.instrumentation]
[metabase.transforms.interface :as transforms.i]
[metabase.transforms.util :as transforms.util]
[metabase.util :as u]
[metabase.util.format :as u.format]
[metabase.util.i18n :as i18n]
[metabase.util.jvm :as u.jvm]
[metabase.util.log :as log]
[toucan2.core :as t2])
(:import
(java.io Closeable)
(java.net SocketException)
(java.nio.file Files)
(java.nio.file.attribute FileAttribute)
(java.time Duration)))
(set! *warn-on-reflection* true)
;;; ------------------------------------------------ Message Log -----------------------------------------------------
(defn- empty-message-log
"Returns a new message log for transform execution.
Buffers log outputs during the run to produce a string suitable for `transform_run.message`."
[]
(atom {:pre []
:runner nil
:post []}))
(defn- log!
"Appends a user-facing string to the message log."
[message-log s]
(swap! message-log (fn [m] (update m (if (:runner m) :post :pre) conj s)))
nil)
(defn- replace-runner-logs!
"Called when events output is received from a /logs call to copy those events into the message-log."
[message-log events]
(swap! message-log assoc :runner events)
nil)
(defn- message-log->transform-run-message
"Presents the content of the log as a string suitable for storage in transform_run.message."
[message-log]
(let [{:keys [pre runner post]} @message-log]
(str/join "\n" (concat pre
(map :message runner)
post))))
(defn- save-log-to-transform-run-message!
[run-id message-log]
(t2/update! :model/TransformRun
:id run-id
{:message (message-log->transform-run-message message-log)}))
;;; ------------------------------------------------ Log Polling Loop ------------------------------------------------
(def ^:private ^Duration message-loop-sleep-duration
(Duration/ofMillis 1000))
(defn- message-update-loop!
"Block while relevant log data is replicated from the runner into the message log."
[run-id message-log]
(try
(loop []
(if (.isInterrupted (Thread/currentThread))
(log/debug "Message update loop interrupted")
(do (let [sleep-ms (.toMillis message-loop-sleep-duration)]
(when (pos? sleep-ms) (Thread/sleep sleep-ms)))
(let [{:keys [status body]} (runner/get-logs run-id)]
(cond
(<= 200 status 299)
(let [{:keys [execution_id events]} body]
(if-not (= run-id execution_id)
(do (log/debugf "Run id did not match expected: %s actual: %s" run-id execution_id)
(recur))
(do
(replace-runner-logs! message-log events)
(save-log-to-transform-run-message! run-id message-log)
(recur))))
(= 404 status)
(do
(log/debugf "No logs yet (or run finished), run-id: %s" run-id)
(recur))
:else
(do
(log/warnf "Unexpected status polling for logs %s %s, run-id: %s" status body run-id)
(log/debug "Exiting due to poll error")))))))
(catch SocketException se (when-not (= "Closed by interrupt" (ex-message se)) (throw se)))
(catch InterruptedException _)
(catch Throwable e
(log/errorf e "An exception was caught during msg update loop, run-id: %s" run-id))))
(defn- open-message-update-future! ^Closeable [run-id message-log]
(let [cleanup (fn [fut]
(future-cancel fut)
(if (= ::timeout (try (deref fut 10000 ::timeout) (catch Throwable _)))
(log/fatalf "Log polling task did not respond to interrupt, run-id: %s" run-id)
(log/debugf "Log polling task done, run-id: %s" run-id)))
fut (u.jvm/in-virtual-thread*
(message-update-loop! run-id message-log))]
(reify Closeable
(close [_] (cleanup fut)))))
;;; ------------------------------------------------ Table Transfer --------------------------------------------------
(defn- maybe-retry-with-backoff
[driver thunk]
(if (= driver :bigquery-cloud-sdk)
(loop [attempt 0]
(let [ret (try
(thunk)
(catch Exception e
(if (and (< attempt 5)
(or (str/includes? (str e) "not found")
(str/includes? (str e) "Not found")))
(let [delay-ms (* 150 (Math/pow 2 attempt))]
(log/debugf "BigQuery operation failed (attempt %d), retrying in %dms" (inc attempt) delay-ms)
(Thread/sleep (long delay-ms))
::recur)
(throw e))))]
(if (= ::recur ret)
(recur (inc attempt))
ret)))
(thunk)))
(defn- table-schema [table-name metadata]
{:name (if (keyword? table-name) table-name (keyword table-name))
:columns (mapv (fn [{:keys [name base_type]}]
{:name name
:type (runner/restricted-insert-type base_type)
:nullable? true})
(:fields metadata))})
(defn- insert-data!
[driver db-id table-schema data-source]
(let [data-source (assoc data-source :table-schema table-schema)]
(->> #(driver/insert-from-source! driver db-id table-schema data-source)
(maybe-retry-with-backoff driver))))
(defn- create-table-and-insert-data!
[driver db-id table-schema data-source]
(transforms.util/create-table-from-schema! driver db-id table-schema)
(insert-data! driver db-id table-schema data-source))
(defn- transfer-with-rename-tables-strategy!
[driver db-id table-name metadata data-source]
(let [source-table-name (driver.u/temp-table-name driver table-name)
temp-table-name (driver.u/temp-table-name driver table-name)]
(log/info "Using rename-tables strategy with atomicity guarantees")
(try
(create-table-and-insert-data! driver db-id (table-schema source-table-name metadata) data-source)
(transforms.util/rename-tables! driver db-id {table-name temp-table-name
source-table-name table-name})
(transforms.util/drop-table! driver db-id temp-table-name)
(catch Exception e
(log/error e "Failed to transfer data using rename-tables strategy")
(try
(transforms.util/drop-table! driver db-id source-table-name)
(catch Exception _))
(throw e)))))
(defn- transfer-with-create-drop-rename-strategy!
[driver db-id table-name metadata data-source]
(let [source-table-name (driver.u/temp-table-name driver table-name)]
(log/info "Using create-drop-rename strategy to minimize downtime")
(try
(create-table-and-insert-data! driver db-id (table-schema source-table-name metadata) data-source)
(transforms.util/drop-table! driver db-id table-name)
(driver/rename-table! driver db-id source-table-name table-name)
(catch Exception e
(log/error e "Failed to transfer data using create-drop-rename strategy")
(try
(transforms.util/drop-table! driver db-id source-table-name)
(catch Exception _))
(throw e)))))
(defn- transfer-with-drop-create-fallback-strategy!
[driver db-id table-name metadata data-source]
(log/info "Using drop-create fallback strategy")
(try
(transforms.util/drop-table! driver db-id table-name)
(create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source)
(catch Exception e
(log/error e "Failed to transfer data using drop-create fallback strategy")
(throw e))))
(defmulti ^:private transfer-file-to-db
{:arglists '([driver db transform metadata temp-file])}
(fn [_ _ transform _ _] (-> transform :target :type keyword)))
(defmethod transfer-file-to-db :table-incremental
[driver {db-id :id}
{:keys [target] :as transform}
metadata temp-file]
(let [table-name (transforms.util/qualified-table-name driver target)
table-exists? (transforms.util/target-table-exists? transform)
data-source {:type :jsonl-file
:file temp-file}]
(if (not table-exists?)
(do
(log/info "New table")
(create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))
(insert-data! driver db-id (table-schema table-name metadata) data-source))))
(defmethod transfer-file-to-db :table
[driver {db-id :id :as db}
{:keys [target] :as transform}
metadata temp-file]
(let [table-name (transforms.util/qualified-table-name driver target)
table-exists? (transforms.util/target-table-exists? transform)
data-source {:type :jsonl-file
:file temp-file}]
(cond
(not table-exists?)
(do
(log/info "New table")
(create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))
(driver.u/supports? driver :atomic-renames db)
(transfer-with-rename-tables-strategy! driver db-id table-name metadata data-source)
(driver.u/supports? driver :rename db)
(transfer-with-create-drop-rename-strategy! driver db-id table-name metadata data-source)
:else
(transfer-with-drop-create-fallback-strategy! driver db-id table-name metadata data-source))))
;;; ------------------------------------------------ Runner Execution ------------------------------------------------
(defn- start-cancellation-process!
[server-url run-id cancel-chan]
(a/go (when (a/<! cancel-chan)
(runner/cancel-python-code-http-call! server-url run-id))))
(defn- run-runner-transform!
"Core execution: upload tables to S3, call runner, read output, transfer to DB.
`runtime` is the language string passed to the runner (e.g. \"python\", \"javascript\")."
[{:keys [source] :as transform} db run-id cancel-chan message-log runtime]
(let [resolved-source-tables (transforms.util/resolve-source-tables (:source-tables source))]
(with-open [^Closeable log-future-ref
(if (app-db/in-transaction?)
(reify Closeable (close [_]))
(open-message-update-future! run-id message-log))
shared-storage-ref (s3/open-shared-storage! resolved-source-tables)]
(let [driver (:engine db)
server-url (runner.settings/python-runner-url)
_ (runner/copy-tables-to-s3! {:run-id run-id
:shared-storage @shared-storage-ref
:source (assoc source :source-tables resolved-source-tables)
:cancel-chan cancel-chan
:limit (:limit source)
:transform-id (:id transform)})
_ (start-cancellation-process! server-url run-id cancel-chan)
{:keys [status body] :as response}
(runner/execute-python-code-http-call!
{:server-url server-url
:code (:body source)
:run-id run-id
:table-name->id resolved-source-tables
:shared-storage @shared-storage-ref
:runtime runtime})
output-manifest (runner/read-output-manifest @shared-storage-ref)
events (runner/read-events @shared-storage-ref)]
(.close ^Closeable log-future-ref)
(replace-runner-logs! message-log events)
(if (not= 200 status)
(do
(when (:timeout body)
(transforms/timeout-run! run-id))
(throw (ex-info (str runtime " runner call failed")
{:transform-message (i18n/tru "{0} execution failure (exit code {1})"
(str/capitalize runtime)
(:exit_code body "?"))
:status-code 400
:api-status-code status
:body body
:events events})))
(try
(let [temp-path (Files/createTempFile "transform-output-" ".jsonl" (u/varargs FileAttribute))
temp-file (.toFile temp-path)]
(when-not (seq (:fields output-manifest))
(throw (ex-info "No fields in metadata"
{:metadata output-manifest
:raw-body body
:events events})))
(try
(with-open [in (runner/open-output @shared-storage-ref)]
(io/copy in temp-file))
(let [file-size (.length temp-file)]
(transforms.instrumentation/with-stage-timing [run-id [:import :file-to-dwh]]
(transfer-file-to-db driver db transform output-manifest temp-file))
(transforms.instrumentation/record-data-transfer! run-id :file-to-dwh file-size nil))
(finally
(.delete temp-file))))
response
(catch Exception e
(log/error e "Failed to create resulting table")
(throw (ex-info "Failed to create the resulting table"
{:transform-message (or (:transform-message (ex-data e))
(i18n/tru "Failed to create the resulting table"))}
e)))))))))
;;; ------------------------------------------------ Public API ------------------------------------------------------
(defn- exceptional-run-message [message-log ex]
(str/join "\n" (remove str/blank? [(message-log->transform-run-message message-log)
(or (:transform-message (ex-data ex))
(if (instance? InterruptedException ex)
(i18n/tru "Transform interrupted")
(i18n/tru "Something went wrong")))])))
(defn execute-runner-transform!
"Execute a runner-based transform (Python, JavaScript, Clojure, R, Julia, etc.).
`lang-config` is a map with:
:runtime - string passed to runner as language param (e.g. \"python\", \"javascript\")
:label - human-readable label for log messages (e.g. \"Python\", \"JavaScript\")
:timing-key - keyword for instrumentation (e.g. :python-execution, :javascript-execution)
Blocks until the transform returns."
[transform {:keys [run-method start-promise user-id]} lang-config]
(let [{:keys [runtime label timing-key]} lang-config]
(assert (transforms.util/runner-transform? transform)
(str "Transform must be a runner-based transform, got: "
(transforms.util/transform-type transform)))
(try
(let [message-log (empty-message-log)
{:keys [target owner_user_id creator_id] transform-id :id} transform
{driver :engine :as db} (t2/select-one :model/Database (transforms.i/target-db-id transform))
run-user-id (if (and (= run-method :manual) user-id)
user-id
(or owner_user_id creator_id))
{run-id :id} (transforms.util/try-start-unless-already-running transform-id run-method run-user-id)]
(some-> start-promise (deliver [:started run-id]))
(log! message-log (i18n/tru "Executing {0} transform" label))
(driver.conn/with-write-connection
(log/info "Executing" label "transform" transform-id "with target" (pr-str target)
(when (driver.conn/write-connection-requested?)
" using write connection"))
(let [start-ms (u/start-timer)
conn-spec (driver/connection-spec driver db)
transform-details {:db-id (:id db)
:transform-id transform-id
:transform-type (keyword (:type target))
:conn-spec conn-spec
:output-schema (:schema target)
:output-table (transforms.util/qualified-table-name driver target)}
run-fn (fn [cancel-chan]
(run-runner-transform! transform db run-id cancel-chan message-log runtime)
(log! message-log (i18n/tru "{0} execution finished successfully in {1}"
label (u.format/format-milliseconds (u/since-ms start-ms))))
(save-log-to-transform-run-message! run-id message-log))
ex-message-fn #(exceptional-run-message message-log %)
result (transforms.instrumentation/with-stage-timing [run-id [:computation timing-key]]
(transforms.util/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn))]
(transforms.util/handle-transform-complete!
:run-id run-id
:transform transform
:db db)
{:run_id run-id
:result result})))
(catch Throwable t
(log/error t (str "Error executing " label " transform"))
(throw t)))))
;;; ------------------------------------------------ Impl Helpers ----------------------------------------------------
(defn source-table-value->dependency
"Convert a source table value (int or ref map) to a dependency map.
Shared by all runner-based transform implementations."
[v]
(cond
(int? v)
{:table v}
(:table_id v)
{:table (:table_id v)}
:else
{:table-ref (select-keys v [:database_id :schema :table])}))
;;; ---------------------------------------- Shared ::runner defmethods ----------------------------------------
;; All runner-based languages share these implementations. Per-language dispatch
;; is handled via the hierarchy in transforms.i — each language only needs to
;; define (defmethod transforms.i/lang-config :lang-kw ...).
(defmethod transforms.i/target-db-id ::transforms.i/runner
[transform]
(-> transform :target :database))
(defmethod transforms.i/source-db-id ::transforms.i/runner
[transform]
(-> transform :source :source-database))
(defmethod transforms.i/table-dependencies ::transforms.i/runner
[transform]
(into #{}
(map source-table-value->dependency)
(vals (get-in transform [:source :source-tables]))))
#_{:clj-kondo/ignore [:discouraged-var]}
(defmethod transforms.i/execute! ::transforms.i/runner
[transform options]
(let [lang-kw (-> transform :source :type keyword)]
(execute-runner-transform! transform options (transforms.i/lang-config lang-kw))))

View File

@@ -0,0 +1,8 @@
(ns metabase-enterprise.transforms-runner.init
(:require
[metabase-enterprise.transforms-runner.api]
[metabase-enterprise.transforms-runner.execute]
[metabase-enterprise.transforms-runner.models.transform-library]
[metabase-enterprise.transforms-runner.runner]
[metabase-enterprise.transforms-runner.s3]
[metabase-enterprise.transforms-runner.settings]))

View File

@@ -0,0 +1,151 @@
(ns metabase-enterprise.transforms-runner.models.transform-library
(:require
[metabase.api.common :as api]
[metabase.app-db.core :as app-db]
[metabase.events.core :as events]
[metabase.models.interface :as mi]
[metabase.models.serialization :as serdes]
[metabase.permissions.core :as perms]
[metabase.transforms.interface :as transforms.i]
[metabase.util.i18n :refer [tru]]
[methodical.core :as methodical]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
(methodical/defmethod t2/table-name :model/TransformLibrary [_model] :transform_library)
(doseq [trait [:metabase/model :hook/timestamped? :hook/entity-id]]
(derive :model/TransformLibrary trait))
(defn- language-extension
"Return the file extension (e.g. \".py\") for a language string, looked up from lang-config."
[language]
(:extension (transforms.i/lang-config (keyword language))))
(defn- normalize-path
[language path]
(let [ext (language-extension language)]
(if (and ext (not (.endsWith ^String path ext)))
(str path ext)
path)))
(t2/define-before-insert :model/TransformLibrary
[library]
(update library :path (partial normalize-path (:language library))))
(t2/define-before-update :model/TransformLibrary
[library]
(if (:path library)
(update library :path (partial normalize-path (or (:language library)
(:language (t2/original library)))))
library))
(defmethod mi/can-read? :model/TransformLibrary
([_instance]
(perms/has-any-transforms-permission? api/*current-user-id*))
([_model _pk]
(perms/has-any-transforms-permission? api/*current-user-id*)))
(defmethod mi/can-write? :model/TransformLibrary
([_instance]
(perms/has-any-transforms-permission? api/*current-user-id*))
([_model _pk]
(perms/has-any-transforms-permission? api/*current-user-id*)))
(def ^:private legacy-entity-ids
"Stable entity IDs for built-in libraries created by Liquibase migrations.
New languages get deterministic IDs via [[builtin-entity-id]]."
{"python" "cWWH9qJPvHNB3rP2vLZrK"
"javascript" "aHWo_0yPLwKqpQPlFNzCg"})
(defn builtin-entity-id
"Return the stable entity ID for a language's built-in common library.
Legacy languages (python, javascript) use migration-baked IDs; new languages
get a deterministic 21-char NanoID-style hash derived from the language name."
[language]
(or (get legacy-entity-ids language)
(let [bytes (.digest (java.security.MessageDigest/getInstance "SHA-256")
(.getBytes (str "transform-library:" language) "UTF-8"))
encoded (-> (java.util.Base64/getUrlEncoder)
(.withoutPadding)
(.encodeToString bytes))]
(subs encoded 0 21))))
(defn all-builtin-entity-ids
"Return the set of entity IDs for all registered runner languages' built-in libraries."
[]
(into #{}
(map (comp builtin-entity-id name))
(transforms.i/runner-languages)))
(defn- validate-path!
[language path]
(let [ext (language-extension language)
normalized (normalize-path language path)
allowed-path (str "common" ext)]
(when-not (= normalized allowed-path)
(throw (ex-info (tru "Invalid library path. Only ''common'' is currently supported.")
{:status-code 400
:path normalized
:language language
:allowed-paths [allowed-path]})))))
(defn get-library-by-path
"Look up a TransformLibrary record by `language` and `path`, validating the path first."
[language path]
(let [normalized (normalize-path language path)]
(validate-path! language normalized)
(t2/select-one :model/TransformLibrary :language language :path normalized)))
(defn update-library-source!
"Create or update the TransformLibrary record for `language` and `path` with the given `source`."
[language path source]
(let [normalized (normalize-path language path)]
(validate-path! language normalized)
(let [id (app-db/update-or-insert! :model/TransformLibrary
{:language language :path normalized}
(constantly {:language language :path normalized :source source}))]
(t2/select-one :model/TransformLibrary id))))
(defn ensure-builtin-library!
"Ensure the built-in common library row exists for `language`.
Inserts idempotently — no-op if the row already exists.
New runner languages can call this instead of adding a Liquibase migration.
NOTE: Must be called after the app DB is initialized and migrations have run.
Do NOT call at namespace load time (e.g. from init.clj) — the table may not exist yet."
[language]
(let [ext (language-extension language)
path (str "common" ext)
eid (builtin-entity-id language)]
(when-not (t2/exists? :model/TransformLibrary :language language :path path)
(t2/insert! :model/TransformLibrary
{:path path :source "" :language language :entity_id eid}))))
(defmethod serdes/make-spec "TransformLibrary"
[_model-name _opts]
{:copy [:language :path :source :entity_id]
:transform {:created_at (serdes/date)}})
(defmethod serdes/hash-fields :model/TransformLibrary
[_model]
[:language :path])
(defmethod serdes/storage-path "TransformLibrary" [entity _ctx]
(let [{:keys [id label]} (-> entity serdes/path last)]
["transform-libraries" (serdes/storage-leaf-file-name id label)]))
(derive ::event :metabase/event)
(doseq [e [:event/transform-library-create :event/transform-library-update :event/transform-library-delete]]
(derive e ::event))
(t2/define-after-insert :model/TransformLibrary
[library]
(events/publish-event! :event/transform-library-create {:object library})
library)
(t2/define-after-update :model/TransformLibrary
[library]
(events/publish-event! :event/transform-library-update {:object library})
library)

View File

@@ -1,12 +1,12 @@
(ns metabase-enterprise.transforms-python.python-runner
(ns metabase-enterprise.transforms-runner.runner
(:require
[clj-http.client :as http]
[clojure.core.async :as a]
[clojure.java.io :as io]
[clojure.string :as str]
[medley.core :as m]
[metabase-enterprise.transforms-python.s3 :as s3]
[metabase-enterprise.transforms-python.settings :as transforms-python.settings]
[metabase-enterprise.transforms-runner.s3 :as s3]
[metabase-enterprise.transforms-runner.settings :as runner.settings]
[metabase.analytics.prometheus :as prometheus]
[metabase.config.core :as config]
[metabase.lib-be.core :as lib-be]
@@ -33,7 +33,7 @@
"Returns HTTP headers with Authorization bearer token if configured.
Throws configuration error in production if token is not set."
[]
(let [api-token (transforms-python.settings/python-runner-api-token)]
(let [api-token (runner.settings/python-runner-api-token)]
(if api-token
{"Authorization" (str "Bearer " api-token)}
(if config/is-prod?
@@ -175,7 +175,7 @@
(defn get-logs
"Return the logs of the current running python process"
[run-id]
(let [server-url (transforms-python.settings/python-runner-url)]
(let [server-url (runner.settings/python-runner-url)]
(python-runner-request server-url :get "/logs" {:query-params {:request_id run-id}})))
(mu/defn record-python-api-call!
@@ -200,23 +200,25 @@
(defn execute-python-code-http-call!
"Calls the /execute endpoint of the python runner. Blocks until the run either succeeds or fails and returns
the response from the server."
[{:keys [server-url code request-id run-id table-name->id shared-storage timeout-secs]}]
(let [{:keys [objects]} shared-storage
[{:keys [server-url code request-id run-id table-name->id shared-storage timeout-secs runtime]}]
(let [{:keys [objects]} shared-storage
{:keys [output output-manifest events]} objects
url-for-path (fn [path] (:url (get objects path)))
table-name->url (update-vals table-name->id #(url-for-path [:table % :data]))
table-name->manifest-url (update-vals table-name->id #(url-for-path [:table % :manifest]))
payload {:code code
:library (t2/select-fn->fn :path :source :model/PythonLibrary)
:timeout (or timeout-secs (transforms-python.settings/python-runner-timeout-seconds))
:request_id (or request-id run-id)
:output_url (:url output)
:output_manifest_url (:url output-manifest)
:events_url (:url events)
:table_mapping table-name->url
:manifest_mapping table-name->manifest-url}
response (with-python-api-timing [run-id]
(python-runner-request server-url :post "/execute" {:body (json/encode payload)}))]
runtime (or runtime "python")
url-for-path (fn [path] (:url (get objects path)))
table-name->url (update-vals table-name->id #(url-for-path [:table % :data]))
table-name->manifest-url (update-vals table-name->id #(url-for-path [:table % :manifest]))
payload {:code code
:library (t2/select-fn->fn :path :source :model/TransformLibrary :language runtime)
:timeout (or timeout-secs (runner.settings/python-runner-timeout-seconds))
:request_id (or request-id run-id)
:output_url (:url output)
:output_manifest_url (:url output-manifest)
:events_url (:url events)
:table_mapping table-name->url
:manifest_mapping table-name->manifest-url
:runtime runtime}
response (with-python-api-timing [run-id]
(python-runner-request server-url :post "/execute" {:body (json/encode payload)}))]
;; when a 500 is returned we observe a string in the body (despite the python returning json)
;; always try to parse the returned string as json before yielding (could tighten this up at some point)
(update response :body (fn [string-if-error]
@@ -342,6 +344,7 @@
:source-tables - Map of table-name -> table-id (already resolved)
:row-limit - Max rows to return (also limits input rows)
:timeout-secs - Optional timeout override
:runtime - Optional runtime override (default: \"python\")
Returns:
{:status :succeeded/:failed
@@ -350,9 +353,9 @@
:logs [{:message ...} ...] ; events from Python execution
:message \"error message\"} ; on failure
"
[{:keys [code source-tables per-input-limit row-limit timeout-secs]}]
[{:keys [code source-tables per-input-limit row-limit timeout-secs runtime]}]
(with-open [shared-storage-ref (s3/open-shared-storage! source-tables)]
(let [server-url (transforms-python.settings/python-runner-url)
(let [server-url (runner.settings/python-runner-url)
_ (copy-tables-to-s3! {:shared-storage @shared-storage-ref
:source {:source-tables source-tables}
:limit (or per-input-limit row-limit)})
@@ -363,7 +366,8 @@
:request-id (u/generate-nano-id)
:table-name->id source-tables
:timeout-secs timeout-secs
:shared-storage @shared-storage-ref})
:shared-storage @shared-storage-ref
:runtime runtime})
events (read-events @shared-storage-ref)]
(cond
(:timeout body)

View File

@@ -1,6 +1,6 @@
(ns metabase-enterprise.transforms-python.s3
(ns metabase-enterprise.transforms-runner.s3
(:require
[metabase-enterprise.transforms-python.settings :as transforms-python.settings]
[metabase-enterprise.transforms-runner.settings :as runner.settings]
[metabase.util.log :as log])
(:import
(clojure.lang IDeref)
@@ -29,12 +29,12 @@
(defn- working-dir-for-run
"The path within the target bucket within which to store all the files for this run."
[]
(let [shared-prefix (some-> (transforms-python.settings/python-storage-s-3-prefix) (str "/"))]
(let [shared-prefix (some-> (runner.settings/python-storage-s-3-prefix) (str "/"))]
(str shared-prefix "run-" (System/nanoTime) "-" (rand-int 10000))))
(defmacro ^:private maybe-with-endpoint* [builder endpoint]
`(let [builder# ~builder]
(when-let [region# (transforms-python.settings/python-storage-s-3-region)]
(when-let [region# (runner.settings/python-storage-s-3-region)]
(.region builder# (Region/of region#)))
(when ~endpoint (.endpointOverride builder# (URI/create ~endpoint)))
builder#))
@@ -46,7 +46,7 @@
(maybe-with-endpoint* builder endpoint))
(defn- s3-configuration ^S3Configuration []
(let [path-style-access (transforms-python.settings/python-storage-s-3-path-style-access)]
(let [path-style-access (runner.settings/python-storage-s-3-path-style-access)]
(-> (S3Configuration/builder)
(.pathStyleAccessEnabled path-style-access)
;; Disable chunked encoding when using path-style access, as most S3-compatible
@@ -65,8 +65,8 @@
(defn- maybe-with-credentials*
[credentials-provider]
(let [access-key (transforms-python.settings/python-storage-s-3-access-key)
secret-key (transforms-python.settings/python-storage-s-3-secret-key)]
(let [access-key (runner.settings/python-storage-s-3-access-key)
secret-key (runner.settings/python-storage-s-3-secret-key)]
(if (or access-key secret-key)
(if-not (and access-key secret-key)
(do (log/warnf "Ignoring %s because %s is not defined"
@@ -90,15 +90,15 @@
^S3Client []
(.build
(doto (S3Client/builder)
(maybe-with-endpoint-s3-client (transforms-python.settings/python-storage-s-3-endpoint))
(maybe-with-endpoint-s3-client (runner.settings/python-storage-s-3-endpoint))
maybe-with-credentials-s3-client
(.serviceConfiguration (s3-configuration)))))
(defn- create-s3-presigner-for-container
"Create S3 presigner for container operations (presigned URLs). Uses distinct container-endpoint if relevant."
^S3Presigner []
(let [container-endpoint (transforms-python.settings/python-storage-s-3-container-endpoint)
endpoint (or container-endpoint (transforms-python.settings/python-storage-s-3-endpoint))]
(let [container-endpoint (runner.settings/python-storage-s-3-container-endpoint)
endpoint (or container-endpoint (runner.settings/python-storage-s-3-endpoint))]
(.build
(doto (S3Presigner/builder)
(maybe-with-endpoint-s3-presigner endpoint)
@@ -126,7 +126,7 @@
(defn- s3-shared-storage [table-name->id]
(let [work-dir-name (working-dir-for-run)
container-presigner (create-s3-presigner-for-container)
bucket-name (transforms-python.settings/python-storage-s-3-bucket)
bucket-name (runner.settings/python-storage-s-3-bucket)
ref (fn [method relative-path]
(let [path (str work-dir-name "/" relative-path)]
{:path path

View File

@@ -1,4 +1,5 @@
(ns metabase-enterprise.transforms-python.settings
;; TODO(Timothy, 02-25-2026): these settings point to a common runner, not python
(ns metabase-enterprise.transforms-runner.settings
(:require
[metabase.config.core :as config]
[metabase.settings.core :as setting]

View File

@@ -9,7 +9,7 @@
decoupled from AppDb."
(:require
[clojure.string :as str]
[metabase-enterprise.transforms-python.python-runner :as python-runner]
[metabase-enterprise.transforms-runner.runner :as runner]
[metabase.api.common :as api]
[metabase.query-processor :as qp]
[metabase.sql-tools.core :as sql-tools]
@@ -115,7 +115,7 @@
remapped-source (remap-python-source table-mapping source)
resolved-source-tables (transforms.util/resolve-source-tables (:source-tables remapped-source))
{:as result
:keys [cols rows]} (python-runner/execute-and-read-output!
:keys [cols rows]} (runner/execute-and-read-output!
{:code (:body remapped-source)
:source-tables resolved-source-tables
:row-limit preview-row-limit})

View File

@@ -1360,9 +1360,9 @@
(let [conversation-id (str (random-uuid))
rasta-ai-token (ai-session-token)
crowberto-ai-token (ai-session-token :crowberto (str (random-uuid)))
saved-python-library (t2/select-one :model/PythonLibrary :path "common.py")]
saved-python-library (t2/select-one :model/TransformLibrary :language "python" :path "common.py")]
(when (seq saved-python-library)
(t2/delete! :model/PythonLibrary))
(t2/delete! :model/TransformLibrary :language "python"))
(try
(testing "With no Python library present"
(is (= "Not found."
@@ -1370,8 +1370,9 @@
{:request-options {:headers {"x-metabase-session" crowberto-ai-token}}}
{:arguments {:path "common.py"}
:conversation_id conversation-id}))))
(mt/with-temp [:model/PythonLibrary lib1 {:path "common.py"
:source "def hello():\n return 'world'"}]
(mt/with-temp [:model/TransformLibrary lib1 {:language "python"
:path "common.py"
:source "def hello():\n return 'world'"}]
(testing "With insufficient permissions"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :post 403 "ee/metabot-tools/get-transform-python-library-details"
@@ -1395,7 +1396,7 @@
:conversation_id conversation-id})))))
(finally
(when (seq saved-python-library)
(t2/insert! :model/PythonLibrary saved-python-library)))))))
(t2/insert! :model/TransformLibrary saved-python-library)))))))
(deftest get-snippets-test
(mt/with-premium-features #{:metabot-v3}

View File

@@ -1070,16 +1070,17 @@
(is (true? (remote-sync.settings/remote-sync-transforms))
"remote-sync-transforms should be auto-enabled after successful import with transforms"))))))))
(deftest import!-auto-enables-transforms-setting-when-python-libraries-detected-test
(testing "import! auto-enables remote-sync-transforms setting only after successful import with python-libraries"
(deftest import!-auto-enables-transforms-setting-when-transform-libraries-detected-test
(testing "import! auto-enables remote-sync-transforms setting only after successful import with transform-libraries"
(mt/with-premium-features #{:transforms}
(mt/with-model-cleanup [:model/RemoteSyncTask :model/PythonLibrary :model/RemoteSyncObject]
(mt/with-model-cleanup [:model/RemoteSyncTask :model/TransformLibrary :model/RemoteSyncObject]
(mt/with-temporary-setting-values [remote-sync-transforms false
remote-sync-enabled true]
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
lib-entity-id "auto-enable-lib-xxxxx"
test-files {"main" {(str "python-libraries/" lib-entity-id ".yaml")
(format "path: uncommon.py
test-files {"main" {(str "transform-libraries/" lib-entity-id ".yaml")
(format "language: python
path: uncommon.py
source: |
# shared code
def shared_func():
@@ -1088,7 +1089,7 @@ entity_id: %s
created_at: '2024-08-28T09:46:18.671622Z'
serdes/meta:
- id: %s
model: PythonLibrary
model: TransformLibrary
" lib-entity-id lib-entity-id)}}
mock-source (test-helpers/create-mock-source :initial-files test-files)]
(is (false? (remote-sync.settings/remote-sync-transforms))
@@ -1097,7 +1098,7 @@ serdes/meta:
(is (= :success (:status result))
"Import should succeed")
(is (true? (remote-sync.settings/remote-sync-transforms))
"remote-sync-transforms should be auto-enabled after successful import with python-libraries"))))))))
"remote-sync-transforms should be auto-enabled after successful import with transform-libraries"))))))))
(deftest import!-keeps-transforms-setting-disabled-when-no-transforms-present-test
(testing "import! keeps remote-sync-transforms setting disabled when no transforms are present"
@@ -1132,7 +1133,7 @@ serdes/meta:
"remote-sync-transforms should be disabled when no transforms in remote")))))))))
(deftest import!-includes-all-optional-paths-regardless-of-settings-test
(testing "import! always includes all optional paths (transforms, python-libraries, snippets)"
(testing "import! always includes all optional paths (transforms, transform-libraries, snippets)"
(mt/with-model-cleanup [:model/RemoteSyncTask]
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
(mt/with-temporary-setting-values [remote-sync-transforms false]
@@ -1151,8 +1152,8 @@ serdes/meta:
(let [filter-strs (map str @paths-passed)]
(is (some #(str/includes? % "transforms") filter-strs)
"transforms path should be included in filters")
(is (some #(str/includes? % "python-libraries") filter-strs)
"python-libraries path should be included in filters")
(is (some #(str/includes? % "transform-libraries") filter-strs)
"transform-libraries path should be included in filters")
(is (some #(str/includes? % "snippets") filter-strs)
"snippets path should be included in filters"))))))))))))

View File

@@ -4,7 +4,7 @@
[clojure.string :as str]
[clojure.test :as t]
[metabase-enterprise.remote-sync.source.protocol :as source.p]
[metabase-enterprise.transforms-python.core :as transforms-python]
[metabase-enterprise.transforms-runner.models.transform-library :as transform-library]
[metabase.util :as u]
[toucan2.core :as t2]))
@@ -287,44 +287,45 @@ width: fixed
(t2/insert! :model/RemoteSyncTask old-models))))))
(def ^:private builtin-python-library
"The built-in PythonLibrary created by migration. Recreated by fixture after cleanup."
{:path "common.py" :source "" :entity_id transforms-python/builtin-entity-id})
"The built-in TransformLibrary (python) created by migration. Recreated by fixture after cleanup."
{:path "common.py" :source "" :language "python"
:entity_id (transform-library/builtin-entity-id "python")})
(defn- ensure-builtin-python-library!
"Ensures the built-in common.py PythonLibrary exists, creating it if missing."
"Ensures the built-in common.py TransformLibrary exists, creating it if missing."
[]
(when-not (t2/exists? :model/PythonLibrary :path (:path builtin-python-library))
(t2/insert! :model/PythonLibrary builtin-python-library)))
(when-not (t2/exists? :model/TransformLibrary :path (:path builtin-python-library) :language "python")
(t2/insert! :model/TransformLibrary builtin-python-library)))
(defn clean-optional-feature-models
"Test fixture that cleans Transform, TransformTag, PythonLibrary, and namespace collection
"Test fixture that cleans Transform, TransformTag, TransformLibrary, and namespace collection
tables to prevent conflict detection during first-import tests. Preserves built-in TransformTags
and recreates the built-in common.py PythonLibrary after cleanup."
and recreates the built-in common.py TransformLibrary after cleanup."
[f]
(let [old-transforms (t2/select :model/Transform)
old-tags (t2/select :model/TransformTag :built_in_type nil)
old-libs (t2/select :model/PythonLibrary)
old-libs (t2/select :model/TransformLibrary)
old-ns-colls (t2/select :model/Collection :namespace [:in ["transforms" "snippets"]])]
(try
(t2/delete! :model/TransformTag :built_in_type nil)
(t2/delete! :model/Transform)
(t2/delete! :model/PythonLibrary)
(t2/delete! :model/TransformLibrary)
(t2/delete! :model/Collection :namespace [:in ["transforms" "snippets"]])
(f)
(finally
(t2/delete! :model/TransformTag :built_in_type nil)
(t2/delete! :model/Transform)
(t2/delete! :model/PythonLibrary)
(t2/delete! :model/TransformLibrary)
(t2/delete! :model/Collection :namespace [:in ["transforms" "snippets"]])
(when (seq old-transforms) (t2/insert! :model/Transform old-transforms))
(when (seq old-tags) (t2/insert! :model/TransformTag old-tags))
(when (seq old-libs) (t2/insert! :model/PythonLibrary old-libs))
(when (seq old-libs) (t2/insert! :model/TransformLibrary old-libs))
(when (seq old-ns-colls) (t2/insert! :model/Collection old-ns-colls))
(ensure-builtin-python-library!)))))
(def clean-remote-sync-state
"Composed test fixture that ensures RemoteSyncObject, RemoteSyncTask, and optional feature
model tables (Transform, TransformTag, PythonLibrary) are clean."
model tables (Transform, TransformTag, TransformLibrary) are clean."
(t/compose-fixtures clean-object (t/compose-fixtures clean-task-table clean-optional-feature-models)))
(defn generate-table-yaml

View File

@@ -10,7 +10,7 @@
[metabase-enterprise.remote-sync.source.protocol :as source.p]
[metabase-enterprise.remote-sync.spec :as spec]
[metabase-enterprise.remote-sync.test-helpers :as test-helpers]
[metabase-enterprise.transforms-python.core :as transforms-python]
[metabase-enterprise.transforms-runner.models.transform-library :as transform-library]
[metabase.collections.models.collection :as collection]
[metabase.events.core :as events]
[metabase.search.core :as search]
@@ -499,163 +499,164 @@ is_sample: false
(is (t2/exists? :model/Collection :entity_id remote-coll-entity-id :namespace "transforms")
"Remote transforms collection should be imported")))))))))
;;; ------------------------------------------- PythonLibrary Tests -------------------------------------------
;;; ------------------------------------------- TransformLibrary Tests -------------------------------------------
(defn- generate-python-library-yaml
"Generates YAML content for a PythonLibrary."
(defn- generate-transform-library-yaml
"Generates YAML content for a TransformLibrary."
[entity-id path source]
(format "path: %s
(format "language: python
path: %s
source: |
%s
entity_id: %s
created_at: '2024-08-28T09:46:18.671622Z'
serdes/meta:
- id: %s
model: PythonLibrary
model: TransformLibrary
"
path
(str/join "\n" (map #(str " " %) (str/split-lines source)))
entity-id
entity-id))
(deftest python-library-event-creates-sync-object-when-setting-enabled-test
(testing "Creating a PythonLibrary creates a RemoteSyncObject entry when remote-sync-transforms is enabled"
(deftest transform-library-event-creates-sync-object-when-setting-enabled-test
(testing "Creating a TransformLibrary creates a RemoteSyncObject entry when remote-sync-transforms is enabled"
(mt/with-premium-features #{:transforms}
(mt/with-temporary-setting-values [remote-sync-transforms true
remote-sync-enabled true]
(let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})]
(let [library (t2/insert-returning-instance! :model/TransformLibrary {:path "common.py" :source "# test" :language "python"})]
(is (t2/exists? :model/RemoteSyncObject
:model_type "PythonLibrary"
:model_type "TransformLibrary"
:model_id (:id library))
"PythonLibrary should be tracked when remote-sync-transforms is enabled"))))))
"TransformLibrary should be tracked when remote-sync-transforms is enabled"))))))
(deftest python-library-event-ignored-when-setting-disabled-test
(testing "PythonLibrary events are ignored when remote-sync-transforms is disabled"
(deftest transform-library-event-ignored-when-setting-disabled-test
(testing "TransformLibrary events are ignored when remote-sync-transforms is disabled"
(mt/with-premium-features #{:transforms}
(mt/with-temporary-setting-values [remote-sync-transforms false
remote-sync-enabled true]
(let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})]
(let [library (t2/insert-returning-instance! :model/TransformLibrary {:path "common.py" :source "# test" :language "python"})]
(is (not (t2/exists? :model/RemoteSyncObject
:model_type "PythonLibrary"
:model_type "TransformLibrary"
:model_id (:id library)))
"PythonLibrary should NOT be tracked when remote-sync-transforms is disabled"))))))
"TransformLibrary should NOT be tracked when remote-sync-transforms is disabled"))))))
(deftest python-library-event-updates-sync-object-test
(testing "Updating a PythonLibrary updates the RemoteSyncObject entry when setting is enabled"
(deftest transform-library-event-updates-sync-object-test
(testing "Updating a TransformLibrary updates the RemoteSyncObject entry when setting is enabled"
(mt/with-premium-features #{:transforms}
(mt/with-temporary-setting-values [remote-sync-transforms true
remote-sync-enabled true]
(let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})]
(t2/update! :model/RemoteSyncObject {:model_type "PythonLibrary" :model_id (:id library)}
(let [library (t2/insert-returning-instance! :model/TransformLibrary {:path "common.py" :source "# test" :language "python"})]
(t2/update! :model/RemoteSyncObject {:model_type "TransformLibrary" :model_id (:id library)}
{:status "synced"})
(t2/update! :model/PythonLibrary (:id library) {:source "# updated"})
(t2/update! :model/TransformLibrary (:id library) {:source "# updated"})
(let [entry (t2/select-one :model/RemoteSyncObject
:model_type "PythonLibrary"
:model_type "TransformLibrary"
:model_id (:id library))]
(is (= "update" (:status entry))
"PythonLibrary should have 'update' status after modification")))))))
"TransformLibrary should have 'update' status after modification")))))))
(deftest export-includes-python-library-when-setting-enabled-test
(testing "Export includes PythonLibrary when remote-sync-transforms is enabled"
(deftest export-includes-transform-library-when-setting-enabled-test
(testing "Export includes TransformLibrary when remote-sync-transforms is enabled"
(mt/with-premium-features #{:transforms}
(mt/with-temporary-setting-values [remote-sync-type :read-write
remote-sync-transforms true
remote-sync-enabled true]
(mt/with-model-cleanup [:model/RemoteSyncTask]
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})
library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# shared code"})]
(t2/update! :model/RemoteSyncObject {:model_type "PythonLibrary" :model_id (:id library)}
library (t2/insert-returning-instance! :model/TransformLibrary {:path "common.py" :source "# shared code" :language "python"})]
(t2/update! :model/RemoteSyncObject {:model_type "TransformLibrary" :model_id (:id library)}
{:status "create"})
(let [mock-source (test-helpers/create-mock-source)
result (impl/export! (source.p/snapshot mock-source) task-id "Test export")]
(is (= :success (:status result))
(str "Export should succeed. Result: " result))
(let [files-after-export (get @(:files-atom mock-source) "main")]
(is (some #(str/includes? % "python-libraries/") (keys files-after-export))
"Export should include the PythonLibrary file")))))))))
(is (some #(str/includes? % "transform-libraries/") (keys files-after-export))
"Export should include the TransformLibrary file")))))))))
(deftest import-python-library-from-yaml-test
(testing "Import brings in PythonLibrary from YAML files"
(deftest import-transform-library-from-yaml-test
(testing "Import brings in TransformLibrary from YAML files"
(mt/with-premium-features #{:transforms}
(mt/with-temporary-setting-values [remote-sync-transforms true
remote-sync-enabled true]
(mt/with-model-cleanup [:model/RemoteSyncTask]
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
lib-entity-id (u/generate-nano-id)
test-files {"main" {(str "python-libraries/" lib-entity-id ".yaml")
(generate-python-library-yaml lib-entity-id "common.py" "def shared_func():\n return 42")}}
test-files {"main" {(str "transform-libraries/" lib-entity-id ".yaml")
(generate-transform-library-yaml lib-entity-id "common.py" "def shared_func():\n return 42")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
snapshot (source.p/snapshot mock-source)
result (impl/import! snapshot task-id)]
(is (= :success (:status result))
(str "Import should succeed. Result: " result))
(is (t2/exists? :model/PythonLibrary :path "common.py")
"PythonLibrary should be imported")
(when-let [library (t2/select-one :model/PythonLibrary :path "common.py")]
(is (t2/exists? :model/TransformLibrary :path "common.py")
"TransformLibrary should be imported")
(when-let [library (t2/select-one :model/TransformLibrary :path "common.py")]
(is (str/includes? (:source library) "def shared_func()")
"PythonLibrary source should be imported correctly"))))))))
"TransformLibrary source should be imported correctly"))))))))
(deftest import-removes-python-library-not-on-remote-test
(testing "Import removes local PythonLibrary that doesn't exist on the remote"
(deftest import-removes-transform-library-not-on-remote-test
(testing "Import removes local TransformLibrary that doesn't exist on the remote"
(mt/with-premium-features #{:transforms}
(mt/with-temporary-setting-values [remote-sync-transforms true
remote-sync-enabled true]
(mt/with-model-cleanup [:model/RemoteSyncTask]
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
local-library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# local only"})]
(is (t2/exists? :model/PythonLibrary :id (:id local-library))
"Local PythonLibrary should exist before import")
local-library (t2/insert-returning-instance! :model/TransformLibrary {:path "common.py" :source "# local only" :language "python"})]
(is (t2/exists? :model/TransformLibrary :id (:id local-library))
"Local TransformLibrary should exist before import")
(let [test-files {"main" {}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
(is (= :success (:status result))
(str "Import should succeed. Result: " result))
(is (not (t2/exists? :model/PythonLibrary :id (:id local-library)))
"Local PythonLibrary should be deleted after import since it wasn't on remote"))))))))
(is (not (t2/exists? :model/TransformLibrary :id (:id local-library)))
"Local TransformLibrary should be deleted after import since it wasn't on remote"))))))))
(deftest import-preserves-builtin-python-library-test
(testing "Import does NOT delete the built-in PythonLibrary even when it's not on remote"
(deftest import-preserves-builtin-transform-library-test
(testing "Import does NOT delete the built-in TransformLibrary even when it's not on remote"
(mt/with-premium-features #{:transforms}
(mt/with-temporary-setting-values [remote-sync-transforms true
remote-sync-enabled true]
(mt/with-model-cleanup [:model/RemoteSyncTask]
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
;; Create the builtin library explicitly (fixture cleans PythonLibrary before test)
(t2/insert-returning-instance! :model/PythonLibrary
(t2/insert-returning-instance! :model/TransformLibrary
{:path "common.py"
:source "# builtin"
:entity_id transforms-python/builtin-entity-id})
(is (t2/exists? :model/PythonLibrary :entity_id transforms-python/builtin-entity-id)
"Built-in PythonLibrary should exist before import")
(let [test-files {"main" {}} ;; Empty remote - no python libraries
:language "python"
:entity_id (transform-library/builtin-entity-id "python")})
(is (t2/exists? :model/TransformLibrary :entity_id (transform-library/builtin-entity-id "python"))
"Built-in TransformLibrary should exist before import")
(let [test-files {"main" {}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
(is (= :success (:status result))
(str "Import should succeed. Result: " result))
(is (t2/exists? :model/PythonLibrary :entity_id transforms-python/builtin-entity-id)
"Built-in PythonLibrary should NOT be deleted after import"))))))))
(is (t2/exists? :model/TransformLibrary :entity_id (transform-library/builtin-entity-id "python"))
"Built-in TransformLibrary should NOT be deleted after import"))))))))
(deftest import-replaces-python-library-with-remote-version-test
(testing "Import updates local PythonLibrary when remote has same entity_id"
(deftest import-replaces-transform-library-with-remote-version-test
(testing "Import updates local TransformLibrary when remote has same entity_id"
(mt/with-premium-features #{:transforms}
(mt/with-temporary-setting-values [remote-sync-transforms true
remote-sync-enabled true]
(mt/with-model-cleanup [:model/RemoteSyncTask]
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
local-library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# local version"})
local-library (t2/insert-returning-instance! :model/TransformLibrary {:path "common.py" :source "# local version" :language "python"})
local-entity-id (:entity_id local-library)]
(is (t2/exists? :model/PythonLibrary :path "common.py")
"Local PythonLibrary should exist before import")
(let [test-files {"main" {(str "python-libraries/" local-entity-id ".yaml")
(generate-python-library-yaml local-entity-id "common.py" "# remote version\ndef new_func():\n pass")}}
(is (t2/exists? :model/TransformLibrary :path "common.py")
"Local TransformLibrary should exist before import")
(let [test-files {"main" {(str "transform-libraries/" local-entity-id ".yaml")
(generate-transform-library-yaml local-entity-id "common.py" "# remote version\ndef new_func():\n pass")}}
mock-source (test-helpers/create-mock-source :initial-files test-files)
result (impl/import! (source.p/snapshot mock-source) task-id)]
(is (= :success (:status result))
(str "Import should succeed. Result: " result))
(let [library (t2/select-one :model/PythonLibrary :path "common.py")]
(is (some? library) "PythonLibrary should still exist")
(let [library (t2/select-one :model/TransformLibrary :path "common.py")]
(is (some? library) "TransformLibrary should still exist")
(is (str/includes? (:source library) "# remote version")
"PythonLibrary source should be updated with remote version")))))))))
"TransformLibrary source should be updated with remote version")))))))))
;;; ------------------------------------------- Transform Sync Behavior Tests -------------------------------------------
@@ -829,11 +830,11 @@ serdes/meta:
:collection_id coll-id}
:model/TransformTag {tag-id :id tag-eid :entity_id} {:name "Test Tag"
:built_in_type nil}]
(let [library (t2/insert-returning-instance! :model/PythonLibrary {:path "common.py" :source "# test"})
(let [library (t2/insert-returning-instance! :model/TransformLibrary {:path "common.py" :source "# test" :language "python"})
lib-eid (:entity_id library)]
(is (t2/exists? :model/Transform :id transform-id))
(is (t2/exists? :model/TransformTag :id tag-id))
(is (t2/exists? :model/PythonLibrary :id (:id library)))
(is (t2/exists? :model/TransformLibrary :id (:id library)))
(let [paths-before (spec/build-all-removal-paths)]
(is (not (some #(str/includes? % transform-eid) paths-before))
"Transform should not be in removal paths before setting is disabled"))
@@ -850,7 +851,7 @@ serdes/meta:
(is (some #(str/includes? % tag-eid) paths-after)
"TransformTag should be in removal paths after setting is disabled")
(is (some #(str/includes? % lib-eid) paths-after)
"PythonLibrary should be in removal paths after setting is disabled"))))))))
"TransformLibrary should be in removal paths after setting is disabled"))))))))
(deftest build-all-removal-paths-excludes-builtin-transform-tags-test
(testing "build-all-removal-paths respects :conditions and excludes built-in tags"

View File

@@ -39,7 +39,7 @@
#"/databases/.*/schemas/(.*)" :schema
#"/databases/(.*)\.yaml" :database
#"/transforms/(.*)\.yaml" :transform
#"/python-libraries/(.*)\.yaml" :python-library])
#"/transform-libraries/(.*)\.yaml" :transform-library])
(defn- file-type
"Find out entity type by file path"
@@ -122,13 +122,13 @@
(testing "API respects parameters"
(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}
:all_collections false :data_model false :settings true)]
(is (= #{:log :dir :settings :transform :python-library}
(is (= #{:log :dir :settings :transform :transform-library}
(tar-file-types f)))))
(testing "We can export just a single collection"
(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}
:collection (:id coll) :data_model false :settings false)]
(is (= #{:log :dir :dashboard :card :collection :transform :python-library}
(is (= #{:log :dir :dashboard :card :collection :transform :transform-library}
(tar-file-types f)))))
(testing "We can export two collections"
@@ -144,18 +144,18 @@
(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}
;; eid:... syntax is kept for backward compat
:collection (str "eid:" (:entity_id coll)) :data_model false :settings false)]
(is (= #{:log :dir :dashboard :card :collection :transform :python-library}
(is (= #{:log :dir :dashboard :card :collection :transform :transform-library}
(tar-file-types f)))))
(testing "We can export that collection using entity id"
(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}
:collection (:entity_id coll) :data_model false :settings false)]
(is (= #{:log :dir :dashboard :card :collection :transform :python-library}
(is (= #{:log :dir :dashboard :card :collection :transform :transform-library}
(tar-file-types f)))))
(testing "Default export: all-collections, data-model, settings"
(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {})]
(is (= #{:transform :log :dir :dashboard :card :collection :settings :schema :database :python-library}
(is (= #{:transform :log :dir :dashboard :card :collection :settings :schema :database :transform-library}
(tar-file-types f)))))
(testing "On exception API returns log"
@@ -221,7 +221,7 @@
io/input-stream)
ba (#'api.serialization/ba-copy res)]
(testing "Archive contains correct number of files with proper log entries"
(is (= 13
(is (= 14
(with-open [tar (open-tar ba)]
(count
(for [^TarArchiveEntry e (u.compress/entries tar)
@@ -229,7 +229,7 @@
(do
(condp re-find (.getName e)
#"/export.log$" (testing "Log contains extract and store entries"
(is (= (+ #_extract 12 #_store 12)
(is (= (+ #_extract 13 #_store 13)
(count (line-seq (io/reader tar))))))
nil)
(.getName e))))))))
@@ -243,7 +243,7 @@
"settings" false
"field_values" false
"duration_ms" (every-pred number? pos?)
"count" 12
"count" 13
"error_count" 0
"source" "api"
"secrets" false
@@ -273,7 +273,7 @@
{:request-options {:headers {"content-type" "multipart/form-data"}}}
{:file ba}))]
(testing "Log contains imported entity types"
(is (= #{"Collection" "Dashboard" "Card" "PythonLibrary" "TransformJob" "TransformTag"}
(is (= #{"Collection" "Dashboard" "Card" "TransformLibrary" "TransformJob" "TransformTag"}
(log-types (line-seq (io/reader (io/input-stream res)))))))
(testing "Entities are restored in the database"
@@ -285,8 +285,8 @@
"direction" "import"
"duration_ms" pos?
"source" "api"
"models" "Card,Collection,Dashboard,PythonLibrary,TransformJob,TransformTag"
"count" 12
"models" "Card,Collection,Dashboard,TransformJob,TransformLibrary,TransformTag"
"count" 13
"error_count" 0
"success" true
"error_message" nil}
@@ -349,7 +349,7 @@
:continue_on_error true)
log (slurp (io/input-stream res))]
(testing "Log shows loaded entities and error"
(is (= #{"Dashboard" "Card" "Collection" "TransformJob" "TransformTag" "PythonLibrary"}
(is (= #{"Dashboard" "Card" "Collection" "TransformJob" "TransformTag" "TransformLibrary"}
(log-types (str/split-lines log))))
(is (re-find #"Collection 'DoesNotExist' was not found" log)))
@@ -359,9 +359,9 @@
"direction" "import"
"source" "api"
"duration_ms" int?
"count" 11
"count" 12
"error_count" 1
"models" "Collection,Dashboard,PythonLibrary,TransformJob,TransformTag"}
"models" "Collection,Dashboard,TransformJob,TransformLibrary,TransformTag"}
(-> (snowplow-test/pop-event-data-and-user-id!) last :data))))))))))
(deftest import-invalid-archive-test
@@ -429,7 +429,7 @@
(doseq [^TarArchiveEntry e (u.compress/entries tar)]
(condp re-find (.getName e)
#"/export.log$" (testing "Log shows extract entries, error, and store entries"
(is (= (+ #_extract 12 #_error 1 #_store 11)
(is (= (+ #_extract 13 #_error 1 #_store 12)
(count (line-seq (io/reader tar))))))
nil))))
@@ -442,7 +442,7 @@
"settings" false
"field_values" false
"duration_ms" pos?
"count" 11
"count" 13
"error_count" 1
"source" "api"
"secrets" false

View File

@@ -45,7 +45,7 @@
"settings" true
"field_values" false
"duration_ms" pos?
"count" 12
"count" 13
"source" "cli"
"secrets" false
"success" true
@@ -60,8 +60,8 @@
"direction" "import"
"duration_ms" pos?
"source" "cli"
"models" "Card,Collection,PythonLibrary,Setting,TransformJob,TransformTag"
"count" 12
"models" "Card,Collection,Setting,TransformJob,TransformLibrary,TransformTag"
"count" 13
"success" true
"error_message" nil}
(-> (snowplow-test/pop-event-data-and-user-id!) first :data))))

View File

@@ -1517,7 +1517,7 @@
:dimension dimension}}}}})}}}]
(testing "selecting a collection includes settings metabot and data model by default"
(is (= #{"Card" "Collection" "Dashboard" "Database" "PythonLibrary" "Setting" "TransformTag" "TransformJob"}
(is (= #{"Card" "Collection" "Dashboard" "Database" "TransformLibrary" "Setting" "TransformTag" "TransformJob"}
(->> (extract/extract {:targets [["Collection" coll1-id]]})
(map (comp :model first serdes/path))
set))))
@@ -2531,23 +2531,23 @@
(testing "fields from non-published table are NOT exported"
(is (= 2 (count field-ids)))))))))
(deftest python-library-test
(deftest transform-library-test
(mt/with-empty-h2-app-db!
;; Delete any pre-existing python_library entries from migrations
(t2/delete! :model/PythonLibrary)
(ts/with-temp-dpc [:model/PythonLibrary {lib-id :id lib-entity-id :entity_id} {:path "common"
:source "def helper():\n return 42"}]
(testing "python library extraction"
(let [ser (serdes/extract-one "PythonLibrary" {} (t2/select-one :model/PythonLibrary :id lib-id))]
(is (=? {:serdes/meta [{:model "PythonLibrary"
(t2/delete! :model/TransformLibrary)
(ts/with-temp-dpc [:model/TransformLibrary {lib-id :id lib-entity-id :entity_id} {:language "python"
:path "common"
:source "def helper():\n return 42"}]
(testing "transform library extraction"
(let [ser (serdes/extract-one "TransformLibrary" {} (t2/select-one :model/TransformLibrary :id lib-id))]
(is (=? {:serdes/meta [{:model "TransformLibrary"
:id lib-entity-id}]
:language "python"
:path "common.py"
:source "def helper():\n return 42"
:entity_id lib-entity-id
:created_at string?}
ser))
(is (not (contains? ser :id)))
(testing "has no dependencies"
(is (empty? (serdes/dependencies ser)))))))))

View File

@@ -238,16 +238,16 @@
["parent.child.yaml"]}
(file-set (io/file dump-dir "databases" "mydb" "tables" "table" "fields")))))))))
(deftest python-library-storage-test
(deftest transform-library-storage-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
(mt/with-empty-h2-app-db!
(t2/delete! :model/PythonLibrary)
(ts/with-temp-dpc [:model/PythonLibrary lib {:path "common" :source "def test(): pass"}]
(t2/delete! :model/TransformLibrary)
(ts/with-temp-dpc [:model/TransformLibrary lib {:language "python" :path "common" :source "def test(): pass"}]
(let [export (into [] (extract/extract {:no-settings true :no-data-model true}))]
(storage/store! export dump-dir)
(testing "python library stored at top-level python-libraries/"
(testing "transform library stored at top-level transform-libraries/"
(is (= #{[(str (:entity_id lib) ".yaml")]}
(file-set (io/file dump-dir "python-libraries"))))))))))
(file-set (io/file dump-dir "transform-libraries"))))))))))
(deftest name-too-long-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]

View File

@@ -5,7 +5,7 @@
[clojure.string :as str]
[clojure.test :refer :all]
[honey.sql :as sql]
[metabase-enterprise.transforms-python.python-runner :as python-runner]
[metabase-enterprise.transforms-runner.runner :as runner]
[metabase.driver :as driver]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql.query-processor :as sql.qp]
@@ -61,10 +61,10 @@
query)))
(defmacro with-python-order-by!
"Execute body with python-runner/build-table-query redef'd to include ORDER BY for deterministic test results."
"Execute body with runner/build-table-query redef'd to include ORDER BY for deterministic test results."
[checkpoint-field & body]
`(let [original-fn# @#'python-runner/build-table-query]
(with-redefs [python-runner/build-table-query
`(let [original-fn# @#'runner/build-table-query]
(with-redefs [runner/build-table-query
(fn [table-id# source-incremental-strategy# transform-id# limit#]
(build-table-query-with-order-by original-fn# table-id# source-incremental-strategy# transform-id# limit# ~checkpoint-field))]
~@body)))

View File

@@ -0,0 +1,53 @@
(ns metabase-enterprise.transforms-inspector.context-test
"Tests for transforms-inspector context that require runner language registration.
Pre-existing :python tests moved here because transform-source-type now uses dynamic
language registration instead of a hardcoded `case`."
(:require
[clojure.test :refer :all]
[metabase.test :as mt]
[metabase.transforms-inspector.context :as context]
[metabase.transforms.interface :as transforms.i]))
(set! *warn-on-reflection* true)
(use-fixtures :once (fn [thunk]
(doseq [lang [:python :javascript :clojure]]
(transforms.i/register-runner! lang))
(thunk)))
(deftest extract-sources-python-test
(testing "Python: extracts source tables from source-tables map"
(let [transform {:source {:type :python
:source-tables {"orders" (mt/id :orders)}
:source-database (mt/id)}
:name "test"}
sources (context/extract-sources transform)]
(is (seq sources))
(is (= (mt/id :orders) (:table-id (first sources))))
(is (= (mt/id) (:db-id (first sources)))))))
(deftest extract-sources-python-multiple-tables-test
(testing "Python: extracts multiple source tables"
(let [transform {:source {:type :python
:source-tables {"orders" (mt/id :orders)
"products" (mt/id :products)}
:source-database (mt/id)}
:name "test"}
sources (context/extract-sources transform)
table-ids (set (map :table-id sources))]
(is (= 2 (count sources)))
(is (contains? table-ids (mt/id :orders)))
(is (contains? table-ids (mt/id :products))))))
(deftest extract-sources-all-runner-languages-test
(testing "extract-sources works for all registered runner languages, not just :python"
(doseq [lang [:python :javascript :clojure]]
(testing (str lang)
(let [transform {:source {:type lang
:source-tables {"orders" (mt/id :orders)}
:source-database (mt/id)}
:name "test"}
sources (context/extract-sources transform)]
(is (seq sources) (str "Expected sources for " lang))
(is (= (mt/id :orders) (:table-id (first sources))))
(is (= (mt/id) (:db-id (first sources)))))))))

View File

@@ -1,193 +0,0 @@
(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.api-test
"Tests for /api/ee/transforms-python endpoints."
(:require
[clojure.string :as str]
[clojure.test :refer :all]
[metabase-enterprise.transforms-python.models.python-library :as python-library]
[metabase.permissions.models.permissions-group :as perms-group]
[metabase.test :as mt]
[metabase.transforms.test-dataset :as transforms-dataset]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
(deftest get-library-path-test
(testing "GET /api/ee/transforms-python/library/:path"
(mt/with-premium-features #{:transforms-python :transforms}
(testing "requires transform permissions"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :get 403 "ee/transforms-python/library/common"))))
(mt/with-data-analyst-role! (mt/user->id :lucky)
(mt/with-db-perm-for-group! (perms-group/all-users) (mt/id) :perms/transforms :yes
(testing "returns nil when no library exists"
(t2/delete! :model/PythonLibrary)
(is (= "Not found." (mt/user-http-request :lucky :get 404 "ee/transforms-python/library/common"))))
(testing "returns existing library"
(t2/delete! :model/PythonLibrary)
(python-library/update-python-library-source! "common" "def my_function():\n return 42")
(is (=? {:source "def my_function():\n return 42"
:path "common.py"
:created_at some?
:updated_at some?}
(mt/user-http-request :lucky :get 200 "ee/transforms-python/library/common"))))
(testing "rejects invalid paths"
(is (= "Invalid library path. Only 'common' is currently supported."
(:message (mt/user-http-request :lucky :get 400 "ee/transforms-python/library/invalid-path"))))))))))
(deftest put-library-path-test
(testing "PUT /api/ee/transforms-python/library/:path"
(mt/with-premium-features #{:transforms-python :transforms}
(testing "requires transform permissions"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :put 403 "ee/transforms-python/library/common"
{:source "def test(): pass"}))))
(mt/with-data-analyst-role! (mt/user->id :lucky)
(mt/with-db-perm-for-group! (perms-group/all-users) (mt/id) :perms/transforms :yes
(testing "validates request body"
(testing "requires source field"
(is (=? {:errors
{:source "string"}}
(mt/user-http-request :lucky :put 400 "ee/transforms-python/library/common"
{}))))
(testing "allows empty string"
(t2/delete! :model/PythonLibrary)
(is (=? {:source ""
:path "common.py"
:id integer?
:created_at some?
:updated_at some?}
(mt/user-http-request :lucky :put 200 "ee/transforms-python/library/common"
{:source ""})))))
(testing "creates new library when none exists"
(t2/delete! :model/PythonLibrary)
(is (=? {:source "def new_function():\n return 100"
:path "common.py"
:id integer?
:created_at some?
:updated_at some?}
(mt/user-http-request :lucky :put 200 "ee/transforms-python/library/common"
{:source "def new_function():\n return 100"})))
(is (= "def new_function():\n return 100"
(t2/select-one-fn :source :model/PythonLibrary))))
(testing "updates existing library"
(t2/delete! :model/PythonLibrary)
(python-library/update-python-library-source! "common" "def old_function():\n return 1")
(is (=? {:source "def updated_function():\n return 2"
:path "common.py"
:id integer?
:created_at some?
:updated_at some?}
(mt/user-http-request :lucky :put 200 "ee/transforms-python/library/common"
{:source "def updated_function():\n return 2"})))
(is (= "def updated_function():\n return 2"
(t2/select-one-fn :source :model/PythonLibrary)))
(is (= 1 (t2/count :model/PythonLibrary)) "Should not create duplicate records"))
(testing "rejects invalid paths"
(is (= "Invalid library path. Only 'common' is currently supported."
(:message (mt/user-http-request :lucky :put 400 "ee/transforms-python/library/invalid-path"
{:source "def test(): pass"}))))))))))
(deftest test-run-test
(mt/with-premium-features #{:transforms :transforms-python}
(let [program ["import pandas as pd"
"import sys"
"def transform():"
" print(\"out1\")"
" print(\"err1\", file=sys.stderr)"
" print(\"out2\")"
" print(\"err2\", file=sys.stderr)"
" return pd.DataFrame({'x': [42, 43]})"]
body {:source_tables {:test (t2/select-one-pk :model/Table :db_id (mt/id))}, :code (str/join "\n" program)}
{:keys [error logs output]} (mt/user-http-request :crowberto :post 200 "ee/transforms-python/test-run" body)]
(is (nil? error))
(is (str/includes? logs "out1\nerr1\nout2\nerr2"))
(is (=? {:cols [{:name "x"}] :rows [{:x 42} {:x 43}]} output)))))
(defn- test-run [& {:keys [program user features source-tables extra-opts]
:or {program ["import pandas as pd" "def transform():" " return pd.DataFrame()"]
user :crowberto
source-tables {:test (t2/select-one-pk :model/Table :db_id (mt/id) :active true)}
features #{:transforms :transforms-python}}}]
(let [body (merge {:source_tables source-tables, :code (str/join "\n" program)} extra-opts)]
(mt/with-premium-features features
(mt/user-http-request-full-response user :post "ee/transforms-python/test-run" body))))
(deftest test-run-with-inputs-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
(mt/dataset transforms-dataset/transforms-test
(let [program ["def transform(customers):" " return customers"]
source-tables {"customers" (mt/id :transforms_customers)}
{:keys [status body]} (test-run :program program :source-tables source-tables)]
(is (= 200 status))
(is (nil? (:error body)))
(is (seq (:rows (:output body))))))))
(deftest test-run-empty-results-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
(mt/dataset transforms-dataset/transforms-test
(let [program ["import pandas as pd" "def transform(customers):" " return pd.DataFrame()"]
source-tables {"customers" (mt/id :transforms_customers)}
{:keys [status body]} (test-run :program program :source-tables source-tables)]
(is (= 200 status))
(is (nil? (:error body)))
(is (empty? (:rows (:output body))))))))
(deftest test-run-error-test
(let [program ["import pandas as pd" "def transform():" " totallynotsourcecode"]
{:keys [status body]} (test-run :program program)]
(is (= 200 status))
(is (=? {:message "Python execution failure (exit code 1)"} (:error body)))
(is (str/includes? (:logs body "") "Traceback"))))
(deftest test-run-feature-test
(is (=? {:status 402} (test-run :features #{})))
(testing "transforms alone is not enough"
(is (=? {:status 402} (test-run :features #{:transforms}))))
(is (=? {:status 200} (test-run :features #{:transforms :transforms-python}))))
(deftest test-run-permissions-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
(mt/with-data-analyst-role! (mt/user->id :lucky)
(mt/with-group-for-user [lucky-group :lucky {:name "Lucky Transforms Group"}]
(mt/with-db-perm-for-group! lucky-group (mt/id) :perms/transforms :yes
(is (=? {:status 403} (test-run :user :rasta)))
(is (=? {:status 200} (test-run :user :lucky))))))))
(deftest test-run-input-limit-test
(mt/dataset transforms-dataset/transforms-test
(testing "maximum"
(is (=? {:status 400} (test-run :extra-opts {:per_input_row_limit 10000}))))
(testing "minimum"
(is (=? {:status 400} (test-run :extra-opts {:per_input_row_limit 0}))))
(testing "truncates sources"
(let [program ["def transform(customers):" " return customers"]
source-tables {"customers" (mt/id :transforms_customers)}
response (test-run :program program :source-tables source-tables :extra-opts {:per_input_row_limit 2})]
(is (=? {:body {:output {:rows #(= 2 (count %))}}} response))))))
(deftest test-run-output-limit-test
(testing "maximum"
(is (=? {:status 400} (test-run :extra-opts {:output_row_limit 10000}))))
(testing "minimum"
(is (=? {:status 400} (test-run :extra-opts {:output_row_limit 0}))))
(testing "truncates outputs"
(let [program ["import pandas as pd" "def transform():" " return pd.DataFrame({'x': [1,2,3,4]})"]
response (test-run :program program :extra-opts {:output_row_limit 2})]
(is (=? {:body {:output {:rows #(= 2 (count %))}}} response)))))
(deftest test-run-timeout-test
(let [program ["import pandas as pd"
"import time"
"def transform():"
" time.sleep(30)"
" return pd.DataFrame()"]]
(mt/with-temporary-setting-values [python-runner-test-run-timeout-seconds 1]
(let [response (test-run :program program)]
(is (=? {:body {:error {:message "Python execution timed out"}}} response))))))
(deftest test-run-cols-meta-empty-return-test
(let [program ["import pandas as pd"
"def transform():"
" return pd.DataFrame(columns=['y', 'z', 'x'])"]
response (test-run :program program)]
(is (=? {:output {:cols [{:name "y"} {:name "z"} {:name "x"}]}} (:body response)))))

View File

@@ -3,9 +3,9 @@
(:require
[clojure.string :as str]
[clojure.test :refer :all]
[metabase-enterprise.transforms-python.execute :as transforms-python.execute]
[metabase-enterprise.transforms-python.python-runner :as python-runner]
[metabase-enterprise.transforms-python.python-runner-test :as python-runner-test]
[metabase-enterprise.transforms-runner.execute :as runner.execute]
[metabase-enterprise.transforms-runner.runner :as runner]
[metabase.driver :as driver]
[metabase.driver.mysql :as mysql]
[metabase.sync.core :as sync]
@@ -38,7 +38,7 @@
:target target}]
(with-transform-cleanup! [_target target]
(mt/with-temp [:model/Transform transform transform-def]
(transforms-python.execute/execute-python-transform! transform {:run-method :manual})
(runner.execute/execute-runner-transform! transform {:run-method :manual} {:runtime "python" :label "Python" :timing-key :python-execution})
(let [table (transforms.tu/wait-for-table table-name 10000)
columns (t2/select :model/Field :table_id (:id table) {:order-by [:position]})
column-names (filterv (fn [x] (not= x "_id")) ;; for mongo
@@ -244,7 +244,7 @@
(testing (str "Exotic types for " driver-key)
(let [{:keys [metadata]} validation
type-map (u/for-map [{:keys [name base_type]} (:fields metadata)]
[name (python-runner/restricted-insert-type base_type)])]
[name (runner/restricted-insert-type base_type)])]
(is (isa? :type/Integer (type-map "id")))
@@ -292,7 +292,7 @@
(let [{:keys [metadata]} validation]
(testing "Base type preservation"
(let [type-map (u/for-map [{:keys [name base_type]} (:fields metadata)]
[name (python-runner/restricted-insert-type base_type)])]
[name (runner/restricted-insert-type base_type)])]
(is (isa? (type-map "id") (if (= driver/*driver* :snowflake) :type/Number :type/Integer)))
(is (isa? (type-map "name") :type/Text))
(is (isa? (type-map "price") :type/Float))
@@ -346,7 +346,7 @@
(when validation
(let [{:keys [rows metadata]} validation
type-map (u/for-map [{:keys [name base_type]} (:fields metadata)]
[name (python-runner/restricted-insert-type base_type)])]
[name (runner/restricted-insert-type base_type)])]
(testing "Original columns preserved"
(is (isa? (type-map "id") (if (= driver/*driver* :snowflake) :type/Number :type/Integer)))
@@ -411,7 +411,7 @@
(testing "Computed columns are added correctly"
(let [{:keys [metadata]} validation
type-map (u/for-map [{:keys [name base_type]} (:fields metadata)]
[name (python-runner/restricted-insert-type base_type)])]
[name (runner/restricted-insert-type base_type)])]
(is (= :type/Float (type-map "computed_field")))
(is (= :type/Text (type-map "name_upper"))))))))))))))

View File

@@ -2,7 +2,7 @@
"End-to-end tests for Python library functionality."
(:require
[clojure.test :refer :all]
[metabase-enterprise.transforms-python.models.python-library :as python-library]
[metabase-enterprise.transforms-runner.models.transform-library :as transform-library]
[metabase.driver :as driver]
[metabase.test :as mt]
[metabase.test.util :as mt.util]
@@ -17,14 +17,14 @@
(mt/test-drivers #{:postgres}
(mt/with-premium-features #{:transforms :transforms-python}
(mt/dataset transforms-dataset/transforms-test
(mt.util/with-discard-model-updates! [:model/PythonLibrary]
(mt.util/with-discard-model-updates! [:model/TransformLibrary]
;; Create or update the python library
(python-library/update-python-library-source! "common"
(str "def multiply_by_two(x):\n"
" return x * 2\n"
"\n"
"def add_greeting(name):\n"
" return f'Hello, {name}!'\n"))
(transform-library/update-library-source! "python" "common"
(str "def multiply_by_two(x):\n"
" return x * 2\n"
"\n"
"def add_greeting(name):\n"
" return f'Hello, {name}!'\n"))
(let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
;; Create and run a transform that uses the library
(with-transform-cleanup! [{table-name :name :as target} {:type "table"

View File

@@ -1,7 +1,7 @@
(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-python.execute-test
(:require
[clojure.test :refer :all]
[metabase-enterprise.transforms-python.execute :as transforms-python.execute]
[metabase-enterprise.transforms-runner.execute :as runner.execute]
[metabase.test :as mt]
[metabase.test.util :as test.util]
[metabase.transforms.test-dataset :as transforms-dataset]
@@ -32,7 +32,7 @@
" return pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})")}
:target (assoc target :database (mt/id))}]
(mt/with-temp [:model/Transform transform initial-transform]
(transforms-python.execute/execute-python-transform! transform {:run-method :manual})
(runner.execute/execute-runner-transform! transform {:run-method :manual} {:runtime "python" :label "Python" :timing-key :python-execution})
(transforms.tu/wait-for-table table-name 10000)
(let [initial-rows (transforms.tu/table-rows table-name)]
@@ -53,9 +53,10 @@
(.await swap-latch)
(original-rename-tables-atomic! driver db-id rename-pairs))]
(let [transform-future (future
(transforms-python.execute/execute-python-transform!
(runner.execute/execute-runner-transform!
(t2/select-one :model/Transform (:id transform))
{:run-method :manual}))]
{:run-method :manual}
{:runtime "python" :label "Python" :timing-key :python-execution}))]
(is (= [["Alice" 25] ["Bob" 30]]
(transforms.tu/table-rows table-name))
"Original data should still be accessible during transform")
@@ -84,10 +85,10 @@
" return pd.DataFrame({'col1': [1, 2, 3], 'col2': ['a', 'b', 'c']})")}
:target (assoc target :database (mt/id))}]
(mt/with-temp [:model/Transform transform transform-def]
(transforms-python.execute/execute-python-transform! transform {:run-method :manual})
(runner.execute/execute-runner-transform! transform {:run-method :manual} {:runtime "python" :label "Python" :timing-key :python-execution})
(transforms.tu/wait-for-table table-name 10000)
(transforms-python.execute/execute-python-transform! transform {:run-method :manual})
(runner.execute/execute-runner-transform! transform {:run-method :manual} {:runtime "python" :label "Python" :timing-key :python-execution})
(let [db-id (mt/id)
tables (t2/select :model/Table :db_id db-id :active true)]
@@ -121,7 +122,7 @@
:target (assoc target :database (mt/id))}]
(mt/with-temp [:model/Transform transform transform-def]
(let [{:keys [run_id]} (try
(transforms-python.execute/execute-python-transform! transform {:run-method :manual})
(runner.execute/execute-runner-transform! transform {:run-method :manual} {:runtime "python" :label "Python" :timing-key :python-execution})
(catch Exception _
;; We expect this to fail due to timeout
{:run_id (t2/select-one-fn :id :model/TransformRun
@@ -153,7 +154,7 @@
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
#"Tables not found:.*nonexistent_table"
(transforms-python.execute/execute-python-transform! transform {:run-method :manual}))))))))))))
(runner.execute/execute-runner-transform! transform {:run-method :manual} {:runtime "python" :label "Python" :timing-key :python-execution}))))))))))))
(deftest python-transform-unresolved-source-table-no-schema-test
(testing "Python transform error message omits schema when nil"
@@ -174,4 +175,4 @@
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
#"Tables not found: missing_table"
(transforms-python.execute/execute-python-transform! transform {:run-method :manual}))))))))))
(runner.execute/execute-runner-transform! transform {:run-method :manual} {:runtime "python" :label "Python" :timing-key :python-execution}))))))))))

View File

@@ -1,78 +0,0 @@
(ns metabase-enterprise.transforms-python.models.python-library-test
(:require
[clojure.test :refer :all]
[metabase-enterprise.transforms-python.models.python-library :as python-library]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
(deftest update-python-library-source-test
(testing "update-python-library-source!"
(testing "creates new record when none exists"
(t2/delete! :model/PythonLibrary)
(is (=? {:source "def new_func(): return 1"
:path "common.py"
:id integer?
:created_at some?
:updated_at some?}
(python-library/update-python-library-source! "common" "def new_func(): return 1")))
(is (= 1 (t2/count :model/PythonLibrary)))
(is (= "def new_func(): return 1"
(t2/select-one-fn :source :model/PythonLibrary))))
(testing "updates existing record"
(is (= 1 (t2/count :model/PythonLibrary)))
(is (=? {:source "def updated_func(): return 2"
:path "common.py"
:id integer?
:created_at some?
:updated_at some?}
(python-library/update-python-library-source! "common" "def updated_func(): return 2")))
(is (= 1 (t2/count :model/PythonLibrary)) "Should not create duplicate")
(is (= "def updated_func(): return 2"
(t2/select-one-fn :source :model/PythonLibrary))))
(testing "rejects invalid paths"
(is (thrown-with-msg? clojure.lang.ExceptionInfo
#"Invalid library path"
(python-library/update-python-library-source! "invalid-path" "def test(): pass")))
(is (= 1 (t2/count :model/PythonLibrary)) "Should not create library with invalid path"))))
(deftest get-python-library-by-path-test
(testing "get-python-library-by-path"
(testing "returns library when path is valid"
(t2/delete! :model/PythonLibrary)
(python-library/update-python-library-source! "common" "def test(): pass")
(is (=? {:source "def test(): pass"
:path "common.py"
:id integer?
:created_at some?
:updated_at some?}
(python-library/get-python-library-by-path "common"))))
(testing "rejects invalid paths"
(is (thrown-with-msg? clojure.lang.ExceptionInfo
#"Invalid library path"
(python-library/get-python-library-by-path "invalid-path"))))))
(deftest normalize-path-test
(testing "normalize-path function"
(testing "adds .py extension when missing"
(is (= "common.py" (#'python-library/normalize-path "common"))))
(testing "doesn't duplicate .py extension"
(is (= "common.py" (#'python-library/normalize-path "common.py"))))
(testing "works with paths that already have .py extension when updating"
(t2/delete! :model/PythonLibrary)
(python-library/update-python-library-source! "common.py" "def test(): pass")
(is (=? {:source "def test(): pass"
:path "common.py"
:id integer?
:created_at some?
:updated_at some?}
(python-library/get-python-library-by-path "common.py")))
;; Verify we can also access it without .py
(is (=? {:source "def test(): pass"
:path "common.py"}
(python-library/get-python-library-by-path "common"))))))

View File

@@ -5,9 +5,9 @@
[clojure.string :as str]
[clojure.test :refer :all]
[java-time.api :as t]
[metabase-enterprise.transforms-python.python-runner :as python-runner]
[metabase-enterprise.transforms-python.s3 :as s3]
[metabase-enterprise.transforms-python.settings :as transforms-python.settings]
[metabase-enterprise.transforms-runner.runner :as runner]
[metabase-enterprise.transforms-runner.s3 :as s3]
[metabase-enterprise.transforms-runner.settings :as runner.settings]
[metabase.driver :as driver]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.sync.core :as sync]
@@ -26,7 +26,7 @@
(defn next-job-run-id [] (swap! last-job-run-id inc))
(defn- parse-jsonl [s] (map json/decode+kw (str/split-lines s)))
(defn- parse-jsonl [s] (when s (map json/decode+kw (str/split-lines s))))
(defn template->regex
"Convert a template string with $var$ placeholders to a regex pattern.
@@ -57,24 +57,24 @@
(defn execute! [{:keys [code tables]}]
(with-open [shared-storage-ref (s3/open-shared-storage! (or tables {}))]
(let [server-url (transforms-python.settings/python-runner-url)
(let [server-url (runner.settings/python-runner-url)
cancel-chan (a/promise-chan)
table-name->id (or tables {})
test-id (next-job-run-id)
_ (python-runner/copy-tables-to-s3! {:run-id test-id
:shared-storage @shared-storage-ref
:source {:source-tables table-name->id}
:cancel-chan cancel-chan})
response (python-runner/execute-python-code-http-call! {:server-url server-url
:code code
:run-id test-id
:table-name->id table-name->id
:shared-storage @shared-storage-ref})
events (python-runner/read-events @shared-storage-ref)
output-manifest (python-runner/read-output-manifest @shared-storage-ref)]
_ (runner/copy-tables-to-s3! {:run-id test-id
:shared-storage @shared-storage-ref
:source {:source-tables table-name->id}
:cancel-chan cancel-chan})
response (runner/execute-python-code-http-call! {:server-url server-url
:code code
:run-id test-id
:table-name->id table-name->id
:shared-storage @shared-storage-ref})
events (runner/read-events @shared-storage-ref)
output-manifest (runner/read-output-manifest @shared-storage-ref)]
;; not sure about munging this all together but its what tests expect for now
(merge (:body response)
{:output (when-some [in (python-runner/open-output @shared-storage-ref)] (with-open [in in] (slurp in)))
{:output (when-some [in (runner/open-output @shared-storage-ref)] (with-open [in in] (slurp in)))
:output-manifest output-manifest
:stdout (->> events (filter #(= "stdout" (:stream %))) (map :message) (str/join "\n"))
:stderr (->> events (filter #(= "stderr" (:stream %))) (map :message) (str/join "\n"))}))))
@@ -366,15 +366,17 @@
"updated_at" :type/DateTime
"scheduled_for" :type/DateTimeWithLocalTZ}
(u/for-map [{:keys [name base_type]} (:fields metadata)]
[name (python-runner/restricted-insert-type base_type)]))))))))
[name (runner/restricted-insert-type base_type)]))))))))
(deftest transform-function-with-library-test
(testing "transform function can use libraries"
(mt/test-drivers #{:postgres}
(mt/with-temp [:model/PythonLibrary _ {:path "circle"
:source "import math\n\ndef calculate_circle_area(radius):\n return math.pi * radius ** 2"}
:model/PythonLibrary _ {:path "utils"
:source "def format_currency(amount):\n return f\"${amount:,.2f}\""}]
(mt/with-temp [:model/TransformLibrary _ {:language "python"
:path "circle"
:source "import math\n\ndef calculate_circle_area(radius):\n return math.pi * radius ** 2"}
:model/TransformLibrary _ {:language "python"
:path "utils"
:source "def format_currency(amount):\n return f\"${amount:,.2f}\""}]
(let [transform-code (str "import pandas as pd\n"
"from circle import calculate_circle_area\n"
"from utils import format_currency\n"
@@ -388,17 +390,17 @@
result (execute! {:code transform-code})]
(is (=? {:output (jsonl-output [{:radius 5, :area 78.5398163397, :price "$78.54"}
{:radius 10, :area 314.159265359, :price "$314.16"}])
:stdout (ok-stdout 2 3)
#_#_:stderr ""}
:stdout (ok-stdout 2 3)}
result)))))))
(deftest transform-function-without-libraries-test
(testing "transform function works when no libraries exist"
(mt/test-drivers #{:postgres}
(with-redefs [t2/select-fn->fn (fn [k v model]
(with-redefs [t2/select-fn->fn (fn [k v model & conditions]
(when (and (= k :path)
(= v :source)
(= model :model/PythonLibrary))
(= model :model/TransformLibrary)
(= conditions [:language "python"]))
{}))]
(let [transform-code (str "import pandas as pd\n"
"\n"
@@ -412,10 +414,11 @@
(deftest transform-function-library-import-error-test
(testing "transform function handles missing library gracefully"
(mt/test-drivers #{:postgres}
(with-redefs [t2/select-fn->fn (fn [k v model]
(with-redefs [t2/select-fn->fn (fn [k v model & conditions]
(when (and (= k :path)
(= v :source)
(= model :model/PythonLibrary))
(= model :model/TransformLibrary)
(= conditions [:language "python"]))
{"utils" "def helper():\n return 42"}))]
(let [transform-code (str "import pandas as pd\n"
"from common import some_function # This library doesn't exist\n"
@@ -423,7 +426,6 @@
"def transform():\n"
" return pd.DataFrame({'value': [some_function()]})")
result (execute! {:code transform-code})]
;; TODO this error message could still be improved a lot
(is (=? {#_#_:error "Execution failed"
:exit_code 1
:stderr #(str/includes? % "No module named 'common'")}
@@ -491,7 +493,7 @@
"created_date" :type/Date
"description" :type/Text}
(u/for-map [{:keys [name base_type]} (:fields metadata)]
[name (python-runner/restricted-insert-type base_type)]))))
[name (runner/restricted-insert-type base_type)]))))
;; cleanup
(driver/drop-table! driver db-id qualified-table-name)))))

View File

@@ -7,8 +7,8 @@
[clj-http.client :as http]
[clojure.string :as str]
[clojure.test :refer :all]
[metabase-enterprise.transforms-python.s3 :as s3]
[metabase-enterprise.transforms-python.settings :as transforms-python.settings]
[metabase-enterprise.transforms-runner.s3 :as s3]
[metabase-enterprise.transforms-runner.settings :as runner.settings]
[metabase.test :as mt]
[metabase.util.log :as log])
(:import
@@ -24,7 +24,7 @@
(defn- s3-endpoint-running? []
(try
(http/head (transforms-python.settings/python-runner-url) {:throw-exceptions false})
(http/head (runner.settings/python-runner-url) {:throw-exceptions false})
(catch ConnectException _
false)))
@@ -38,7 +38,7 @@
(testing "We can open an s3 connection, and read and write things"
(with-open [s3-client (s3/create-s3-client)]
(is (instance? S3Client s3-client))
(let [bucket (transforms-python.settings/python-storage-s-3-bucket)
(let [bucket (runner.settings/python-storage-s-3-bucket)
key (str "test-object-" (random-uuid) ".txt")
body (str "Hello, S3! My secret is:" (random-uuid))]

View File

@@ -2,9 +2,9 @@
(:require
[clojure.string :as str]
[clojure.test :refer :all]
[metabase-enterprise.transforms-python.execute :as transforms-python.execute]
[metabase-enterprise.transforms-python.init]
[metabase-enterprise.transforms-python.python-runner :as transforms-python.python-runner]
[metabase-enterprise.transforms-runner.execute :as runner.execute]
[metabase-enterprise.transforms-runner.runner :as runner]
[metabase.driver :as driver]
[metabase.permissions.models.permissions-group :as perms-group]
[metabase.test :as mt]
@@ -236,10 +236,10 @@
(transforms.tu/wait-for-table (:name target) 5000)))
(run-scenario [scenario schema]
(with-redefs [transforms-python.execute/python-message-loop-sleep-duration (Duration/ofMillis fast-log-polling-ms)
transforms-python.execute/transfer-file-to-db (if-some [e (:writeback-ex scenario)]
(fn [& _] (throw e))
@#'transforms-python.execute/transfer-file-to-db)]
(with-redefs [runner.execute/message-loop-sleep-duration (Duration/ofMillis fast-log-polling-ms)
runner.execute/transfer-file-to-db (if-some [e (:writeback-ex scenario)]
(fn [& _] (throw e))
@#'runner.execute/transfer-file-to-db)]
(with-transform-cleanup! [target {:type "table"
:schema schema
:name "result"}]
@@ -377,13 +377,13 @@
(blocking-redefs [{:keys [block]} ready-signal wait-signal]
(case block
:read
(let [f-ref #'transforms-python.python-runner/write-jsonl-row-to-os-rff
(let [f-ref #'runner/write-jsonl-row-to-os-rff
f @f-ref]
{f-ref
(fn [os fields-meta col-meta]
(rf-proxy ready-signal wait-signal (f os fields-meta col-meta)))})
:write
(let [f-ref #'transforms-python.execute/transfer-file-to-db
(let [f-ref #'runner.execute/transfer-file-to-db
f @f-ref]
{f-ref
(fn [& args]
@@ -396,11 +396,11 @@
(let [ready-signal (promise) ; test blocks: until the run is ready to be cancelled
wait-signal (promise) ; run blocks: until the test has cancelled
finished-signal (promise) ; test blocks: until the run has finished (exceptionally or not)
exec-fn @#'transforms-python.execute/execute-python-transform!
exec-fn @#'runner.execute/execute-runner-transform!
redefs (blocking-redefs scenario ready-signal wait-signal)]
(with-redefs-fn
(merge
{#'transforms-python.execute/execute-python-transform!
{#'runner.execute/execute-runner-transform!
(fn [& args]
(try
(apply exec-fn args)
@@ -470,7 +470,7 @@
(mt/with-premium-features #{:transforms :transforms-python}
(mt/dataset transforms-dataset/transforms-test
(let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
(with-redefs [transforms-python.execute/python-message-loop-sleep-duration (Duration/ofMillis fast-log-polling-ms)]
(with-redefs [runner.execute/message-loop-sleep-duration (Duration/ofMillis fast-log-polling-ms)]
(with-transform-cleanup! [target {:type "table"
:schema schema
:name "result"}]

View File

@@ -0,0 +1,282 @@
(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms-runner.api-test
"Tests for /api/ee/advanced-transforms endpoints and runner-based transform CRUD."
(:require
[clojure.string :as str]
[clojure.test :refer :all]
[metabase-enterprise.transforms-clojure.init]
[metabase-enterprise.transforms-javascript.init]
[metabase-enterprise.transforms-python.init]
[metabase-enterprise.transforms-runner.models.transform-library :as transform-library]
[metabase.permissions.models.permissions-group :as perms-group]
[metabase.test :as mt]
[metabase.transforms.crud :as transforms.crud]
[metabase.transforms.interface :as transforms.i]
[metabase.transforms.test-dataset :as transforms-dataset]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
;;; ------------------------------------------------ CRUD smoke tests ------------------------------------------------
(deftest create-runner-transform-for-all-languages-test
(testing "Creating a transform via the API works for every registered runner language"
(mt/with-premium-features #{:transforms :transforms-python}
(with-redefs [transforms.crud/check-database-feature (constantly nil)]
(doseq [lang (sort (transforms.i/runner-languages))]
(testing (str "language: " (name lang))
(let [response (mt/user-http-request :crowberto :post "transform"
{:name (str "Test " (name lang) " transform")
:source {:type (name lang)
:body "placeholder"
:source-tables {}
:source-database (mt/id)}
:target {:type "table"
:schema nil
:name (str (name lang) "_test_output")
:database (mt/id)}})]
(is (pos-int? (:id response))
(str "Expected a transform id for " (name lang) ", got: " (pr-str response)))
(when-let [id (:id response)]
(mt/user-http-request :crowberto :delete (format "transform/%d" id))))))))))
;;; ------------------------------------------- Library endpoint tests -----------------------------------------------
(deftest get-library-path-test
(testing "GET /api/ee/advanced-transforms/library/:path"
(mt/with-premium-features #{:transforms-python :transforms}
(testing "requires transform permissions"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :get 403 "ee/advanced-transforms/library/common"))))
(mt/with-data-analyst-role! (mt/user->id :lucky)
(mt/with-db-perm-for-group! (perms-group/all-users) (mt/id) :perms/transforms :yes
(testing "returns nil when no library exists"
(t2/delete! :model/TransformLibrary :language "python")
(is (= "Not found." (mt/user-http-request :lucky :get 404 "ee/advanced-transforms/library/common"))))
(testing "returns existing library"
(t2/delete! :model/TransformLibrary :language "python")
(transform-library/update-library-source! "python" "common" "def my_function():\n return 42")
(is (=? {:source "def my_function():\n return 42"
:path "common.py"
:created_at some?
:updated_at some?}
(mt/user-http-request :lucky :get 200 "ee/advanced-transforms/library/common"))))
(testing "rejects invalid paths"
(is (= "Invalid library path. Only 'common' is currently supported."
(:message (mt/user-http-request :lucky :get 400 "ee/advanced-transforms/library/invalid-path"))))))))))
(deftest put-library-path-test
(testing "PUT /api/ee/advanced-transforms/library/:path"
(mt/with-premium-features #{:transforms-python :transforms}
(testing "requires transform permissions"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :put 403 "ee/advanced-transforms/library/common"
{:source "def test(): pass"}))))
(mt/with-data-analyst-role! (mt/user->id :lucky)
(mt/with-db-perm-for-group! (perms-group/all-users) (mt/id) :perms/transforms :yes
(testing "validates request body"
(testing "requires source field"
(is (=? {:errors
{:source "string"}}
(mt/user-http-request :lucky :put 400 "ee/advanced-transforms/library/common"
{}))))
(testing "allows empty string"
(t2/delete! :model/TransformLibrary :language "python")
(is (=? {:source ""
:path "common.py"
:id integer?
:created_at some?
:updated_at some?}
(mt/user-http-request :lucky :put 200 "ee/advanced-transforms/library/common"
{:source ""})))))
(testing "creates new library when none exists"
(t2/delete! :model/TransformLibrary :language "python")
(is (=? {:source "def new_function():\n return 100"
:path "common.py"
:id integer?
:created_at some?
:updated_at some?}
(mt/user-http-request :lucky :put 200 "ee/advanced-transforms/library/common"
{:source "def new_function():\n return 100"})))
(is (= "def new_function():\n return 100"
(t2/select-one-fn :source :model/TransformLibrary :language "python"))))
(testing "updates existing library"
(t2/delete! :model/TransformLibrary :language "python")
(transform-library/update-library-source! "python" "common" "def old_function():\n return 1")
(is (=? {:source "def updated_function():\n return 2"
:path "common.py"
:id integer?
:created_at some?
:updated_at some?}
(mt/user-http-request :lucky :put 200 "ee/advanced-transforms/library/common"
{:source "def updated_function():\n return 2"})))
(is (= "def updated_function():\n return 2"
(t2/select-one-fn :source :model/TransformLibrary :language "python")))
(is (= 1 (t2/count :model/TransformLibrary :language "python")) "Should not create duplicate records"))
(testing "rejects invalid paths"
(is (= "Invalid library path. Only 'common' is currently supported."
(:message (mt/user-http-request :lucky :put 400 "ee/advanced-transforms/library/invalid-path"
{:source "def test(): pass"}))))))))))
(deftest get-javascript-library-path-test
(testing "GET /api/ee/advanced-transforms/library/:path?type=javascript"
(mt/with-premium-features #{:transforms-python :transforms}
(mt/with-data-analyst-role! (mt/user->id :lucky)
(mt/with-db-perm-for-group! (perms-group/all-users) (mt/id) :perms/transforms :yes
(testing "returns nil when no JS library exists"
(t2/delete! :model/TransformLibrary :language "javascript")
(is (= "Not found." (mt/user-http-request :lucky :get 404 "ee/advanced-transforms/library/common"
:type "javascript"))))
(testing "returns existing JS library"
(t2/delete! :model/TransformLibrary :language "javascript")
(transform-library/update-library-source! "javascript" "common" "function myFunc() { return 42; }")
(is (=? {:source "function myFunc() { return 42; }"
:path "common.js"
:created_at some?
:updated_at some?}
(mt/user-http-request :lucky :get 200 "ee/advanced-transforms/library/common"
:type "javascript"))))
(testing "without type param, returns Python library (not JS)"
(t2/delete! :model/TransformLibrary :language "python")
(transform-library/update-library-source! "python" "common" "def py_func(): return 1")
(is (=? {:source "def py_func(): return 1"
:path "common.py"}
(mt/user-http-request :lucky :get 200 "ee/advanced-transforms/library/common")))))))))
(deftest put-javascript-library-path-test
(testing "PUT /api/ee/advanced-transforms/library/:path with type=javascript"
(mt/with-premium-features #{:transforms-python :transforms}
(mt/with-data-analyst-role! (mt/user->id :lucky)
(mt/with-db-perm-for-group! (perms-group/all-users) (mt/id) :perms/transforms :yes
(testing "creates new JS library"
(t2/delete! :model/TransformLibrary :language "javascript")
(is (=? {:source "function newFunc() { return 100; }"
:path "common.js"
:id integer?
:created_at some?
:updated_at some?}
(mt/user-http-request :lucky :put 200 "ee/advanced-transforms/library/common"
{:source "function newFunc() { return 100; }"
:type "javascript"})))
(is (= "function newFunc() { return 100; }"
(t2/select-one-fn :source :model/TransformLibrary :language "javascript"))))
(testing "updates existing JS library"
(is (=? {:source "function updated() { return 2; }"
:path "common.js"}
(mt/user-http-request :lucky :put 200 "ee/advanced-transforms/library/common"
{:source "function updated() { return 2; }"
:type "javascript"})))
(is (= 1 (t2/count :model/TransformLibrary :language "javascript")) "Should not create duplicate"))
(testing "without type param, updates Python library (not JS)"
(t2/delete! :model/TransformLibrary :language "python")
(is (=? {:source "def py_func(): return 1"
:path "common.py"}
(mt/user-http-request :lucky :put 200 "ee/advanced-transforms/library/common"
{:source "def py_func(): return 1"})))
(is (= 1 (t2/count :model/TransformLibrary :language "python")))))))))
;;; ------------------------------------------- Test-run endpoint tests ----------------------------------------------
(deftest test-run-test
(mt/with-premium-features #{:transforms :transforms-python}
(let [program ["import pandas as pd"
"import sys"
"def transform():"
" print(\"out1\")"
" print(\"err1\", file=sys.stderr)"
" print(\"out2\")"
" print(\"err2\", file=sys.stderr)"
" return pd.DataFrame({'x': [42, 43]})"]
body {:source_tables {:test (t2/select-one-pk :model/Table :db_id (mt/id))}, :code (str/join "\n" program)}
{:keys [error logs output]} (mt/user-http-request :crowberto :post 200 "ee/advanced-transforms/test-run" body)]
(is (nil? error))
(is (str/includes? logs "out1\nerr1\nout2\nerr2"))
(is (=? {:cols [{:name "x"}] :rows [{:x 42} {:x 43}]} output)))))
(defn- test-run [& {:keys [program user features source-tables extra-opts]
:or {program ["import pandas as pd" "def transform():" " return pd.DataFrame()"]
user :crowberto
source-tables {:test (t2/select-one-pk :model/Table :db_id (mt/id) :active true)}
features #{:transforms :transforms-python}}}]
(let [body (merge {:source_tables source-tables, :code (str/join "\n" program)} extra-opts)]
(mt/with-premium-features features
(mt/user-http-request-full-response user :post "ee/advanced-transforms/test-run" body))))
(deftest test-run-with-inputs-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
(mt/dataset transforms-dataset/transforms-test
(let [program ["def transform(customers):" " return customers"]
source-tables {"customers" (mt/id :transforms_customers)}
{:keys [status body]} (test-run :program program :source-tables source-tables)]
(is (= 200 status))
(is (nil? (:error body)))
(is (seq (:rows (:output body))))))))
(deftest test-run-empty-results-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
(mt/dataset transforms-dataset/transforms-test
(let [program ["import pandas as pd" "def transform(customers):" " return pd.DataFrame()"]
source-tables {"customers" (mt/id :transforms_customers)}
{:keys [status body]} (test-run :program program :source-tables source-tables)]
(is (= 200 status))
(is (nil? (:error body)))
(is (empty? (:rows (:output body))))))))
(deftest test-run-error-test
(let [program ["import pandas as pd" "def transform():" " totallynotsourcecode"]
{:keys [status body]} (test-run :program program)]
(is (= 200 status))
(is (=? {:message "Python execution failure (exit code 1)"} (:error body)))
(is (str/includes? (:logs body "") "Traceback"))))
(deftest test-run-feature-test
(is (=? {:status 402} (test-run :features #{})))
(testing "transforms alone is not enough"
(is (=? {:status 402} (test-run :features #{:transforms}))))
(is (=? {:status 200} (test-run :features #{:transforms :transforms-python}))))
(deftest test-run-permissions-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
(mt/with-data-analyst-role! (mt/user->id :lucky)
(mt/with-group-for-user [lucky-group :lucky {:name "Lucky Transforms Group"}]
(mt/with-db-perm-for-group! lucky-group (mt/id) :perms/transforms :yes
(is (=? {:status 403} (test-run :user :rasta)))
(is (=? {:status 200} (test-run :user :lucky))))))))
(deftest test-run-input-limit-test
(mt/dataset transforms-dataset/transforms-test
(testing "maximum"
(is (=? {:status 400} (test-run :extra-opts {:per_input_row_limit 10000}))))
(testing "minimum"
(is (=? {:status 400} (test-run :extra-opts {:per_input_row_limit 0}))))
(testing "truncates sources"
(let [program ["def transform(customers):" " return customers"]
source-tables {"customers" (mt/id :transforms_customers)}
response (test-run :program program :source-tables source-tables :extra-opts {:per_input_row_limit 2})]
(is (=? {:body {:output {:rows #(= 2 (count %))}}} response))))))
(deftest test-run-output-limit-test
(testing "maximum"
(is (=? {:status 400} (test-run :extra-opts {:output_row_limit 10000}))))
(testing "minimum"
(is (=? {:status 400} (test-run :extra-opts {:output_row_limit 0}))))
(testing "truncates outputs"
(let [program ["import pandas as pd" "def transform():" " return pd.DataFrame({'x': [1,2,3,4]})"]
response (test-run :program program :extra-opts {:output_row_limit 2})]
(is (=? {:body {:output {:rows #(= 2 (count %))}}} response)))))
(deftest test-run-timeout-test
(let [program ["import pandas as pd"
"import time"
"def transform():"
" time.sleep(30)"
" return pd.DataFrame()"]]
(mt/with-temporary-setting-values [python-runner-test-run-timeout-seconds 1]
(let [response (test-run :program program)]
(is (=? {:body {:error {:message "Python execution timed out"}}} response))))))
(deftest test-run-cols-meta-empty-return-test
(let [program ["import pandas as pd"
"def transform():"
" return pd.DataFrame(columns=['y', 'z', 'x'])"]
response (test-run :program program)]
(is (=? {:output {:cols [{:name "y"} {:name "z"} {:name "x"}]}} (:body response)))))

View File

@@ -0,0 +1,137 @@
(ns metabase-enterprise.transforms-runner.execute-test
"Tests for the shared runner execution infrastructure.
Covers pure functions: source-table-value->dependency, message-log state, table-schema."
(:require
[clojure.test :refer :all]
[clojure.test.check.clojure-test :refer [defspec]]
[clojure.test.check.generators :as gen]
[clojure.test.check.properties :as prop]
[metabase-enterprise.transforms-runner.execute :as runner.execute]))
(set! *warn-on-reflection* true)
;;; ------------------------------------------------ source-table-value->dependency --------------------------------
(deftest source-table-value->dependency-int-test
(testing "integer table ID returns {:table id}"
(is (= {:table 42} (runner.execute/source-table-value->dependency 42)))
(is (= {:table 1} (runner.execute/source-table-value->dependency 1)))))
(deftest source-table-value->dependency-ref-with-table-id-test
(testing "map with :table_id returns {:table table_id}"
(is (= {:table 100}
(runner.execute/source-table-value->dependency
{:database_id 1 :schema "public" :table "foo" :table_id 100})))))
(deftest source-table-value->dependency-ref-without-table-id-test
(testing "map without :table_id returns {:table-ref ...} with only the relevant keys"
(is (= {:table-ref {:database_id 1 :schema "public" :table "foo"}}
(runner.execute/source-table-value->dependency
{:database_id 1 :schema "public" :table "foo"})))
(testing "extra keys are stripped"
(is (= {:table-ref {:database_id 1 :schema nil :table "bar"}}
(runner.execute/source-table-value->dependency
{:database_id 1 :schema nil :table "bar" :extra "ignored"}))))))
(defspec source-table-value->dependency-int-prop 100
(prop/for-all [n gen/nat]
(let [result (runner.execute/source-table-value->dependency n)]
(and (= {:table n} result)
(int? (:table result))))))
(def ^:private gen-source-table-ref
"Generator for source table ref maps (without :table_id)."
(gen/let [db-id gen/nat
schema (gen/one-of [(gen/return nil) gen/string-alphanumeric])
table gen/string-alphanumeric]
{:database_id db-id :schema schema :table table}))
(defspec source-table-value->dependency-ref-no-table-id-prop 100
(prop/for-all [ref gen-source-table-ref]
(let [result (runner.execute/source-table-value->dependency ref)]
(and (contains? result :table-ref)
(= (select-keys ref [:database_id :schema :table])
(:table-ref result))))))
(defspec source-table-value->dependency-ref-with-table-id-prop 100
(prop/for-all [ref gen-source-table-ref
tid gen/nat]
(let [result (runner.execute/source-table-value->dependency (assoc ref :table_id tid))]
(= {:table tid} result))))
;;; ------------------------------------------------ Message Log ---------------------------------------------------
(deftest empty-message-log-test
(let [ml (#'runner.execute/empty-message-log)]
(is (= {:pre [] :runner nil :post []} @ml))))
(deftest log!-before-runner-test
(testing "log! appends to :pre before runner logs arrive"
(let [ml (#'runner.execute/empty-message-log)]
(#'runner.execute/log! ml "first")
(#'runner.execute/log! ml "second")
(is (= ["first" "second"] (:pre @ml)))
(is (nil? (:runner @ml)))
(is (= [] (:post @ml))))))
(deftest log!-after-runner-test
(testing "log! appends to :post after runner logs are set"
(let [ml (#'runner.execute/empty-message-log)]
(#'runner.execute/log! ml "before")
(#'runner.execute/replace-runner-logs! ml [{:message "runner msg"}])
(#'runner.execute/log! ml "after")
(is (= ["before"] (:pre @ml)))
(is (= [{:message "runner msg"}] (:runner @ml)))
(is (= ["after"] (:post @ml))))))
(deftest replace-runner-logs!-test
(testing "replace-runner-logs! overwrites previous runner logs"
(let [ml (#'runner.execute/empty-message-log)]
(#'runner.execute/replace-runner-logs! ml [{:message "old"}])
(#'runner.execute/replace-runner-logs! ml [{:message "new1"} {:message "new2"}])
(is (= [{:message "new1"} {:message "new2"}] (:runner @ml))))))
(deftest message-log->transform-run-message-test
(testing "concatenates pre, runner messages, and post"
(let [ml (#'runner.execute/empty-message-log)]
(#'runner.execute/log! ml "Loading tables")
(#'runner.execute/replace-runner-logs! ml [{:message "Processing..."} {:message "Done"}])
(#'runner.execute/log! ml "Writing output")
(is (= "Loading tables\nProcessing...\nDone\nWriting output"
(#'runner.execute/message-log->transform-run-message ml)))))
(testing "works with empty log"
(let [ml (#'runner.execute/empty-message-log)]
(is (= "" (#'runner.execute/message-log->transform-run-message ml)))))
(testing "works with only pre messages"
(let [ml (#'runner.execute/empty-message-log)]
(#'runner.execute/log! ml "only pre")
(is (= "only pre" (#'runner.execute/message-log->transform-run-message ml))))))
(deftest exceptional-run-message-test
(testing "combines log and exception message"
(let [ml (#'runner.execute/empty-message-log)]
(#'runner.execute/log! ml "Started")
(#'runner.execute/replace-runner-logs! ml [{:message "Running"}])
(let [ex (ex-info "boom" {:transform-message "Python execution failure (exit code 1)"})]
(is (= "Started\nRunning\nPython execution failure (exit code 1)"
(#'runner.execute/exceptional-run-message ml ex))))))
(testing "uses generic message when no :transform-message"
(let [ml (#'runner.execute/empty-message-log)
ex (ex-info "boom" {})]
(is (string? (#'runner.execute/exceptional-run-message ml ex))))))
;;; ------------------------------------------------ table-schema -------------------------------------------------
(deftest table-schema-test
(testing "builds schema from table name and metadata"
(let [metadata {:fields [{:name "id" :base_type "Integer"}
{:name "name" :base_type "Text"}]}
result (#'runner.execute/table-schema "my_table" metadata)]
(is (= :my_table (:name result)))
(is (= 2 (count (:columns result))))
(is (= "id" (-> result :columns first :name)))
(is (true? (-> result :columns first :nullable?)))))
(testing "keyword table name passes through"
(let [result (#'runner.execute/table-schema :already_keyword {:fields []})]
(is (= :already_keyword (:name result)))
(is (= [] (:columns result))))))

View File

@@ -0,0 +1,56 @@
(ns metabase-enterprise.transforms-runner.feature-gating-test
"Tests for feature gating that depend on runner language registration.
Pure predicate tests live in metabase.transforms.feature-gating-test (OSS)."
(:require
[clojure.test :refer :all]
[metabase.test :as mt]
[metabase.transforms.feature-gating :as transforms.gating]
[metabase.transforms.interface :as transforms.i]
[metabase.transforms.util :as transforms.util]))
(set! *warn-on-reflection* true)
(use-fixtures :once (fn [thunk]
(doseq [lang [:python :javascript :clojure]]
(transforms.i/register-runner! lang))
(thunk)))
(def ^:private runner-languages
[:python :javascript :clojure])
(defn- make-transform [source-type]
{:source {:type (name source-type)}
:target {:database 1 :type "table" :name "out" :schema "s"}})
(deftest enabled-source-types-full-features-test
(mt/with-premium-features #{:transforms :transforms-python}
(let [types (transforms.gating/enabled-source-types)]
(testing "includes query types"
(is (contains? types "native"))
(is (contains? types "mbql")))
(testing "includes python"
(is (contains? types "python")))
(testing "includes all runner languages"
(doseq [lang (transforms.gating/runner-language-types)]
(testing (str "includes " lang)
(is (contains? types lang))))))))
(deftest check-feature-enabled-full-features-test
(mt/with-premium-features #{:transforms :transforms-python}
(testing "query transform enabled"
(is (true? (transforms.util/check-feature-enabled (make-transform :query)))))
(testing "all runner languages enabled"
(doseq [lang runner-languages]
(testing (str "language: " lang)
(is (true? (transforms.util/check-feature-enabled (make-transform lang)))))))))
(deftest check-feature-enabled-no-python-feature-test
(mt/with-premium-features #{:transforms}
(testing "query transform still enabled"
(is (true? (transforms.util/check-feature-enabled (make-transform :query)))))
(testing "python disabled without :transforms-python"
(is (false? (transforms.util/check-feature-enabled (make-transform :python)))))
(testing "other runner languages disabled without :transforms-python"
(doseq [lang [:javascript :clojure]]
(testing (str "language: " lang)
(is (false? (transforms.util/check-feature-enabled (make-transform lang)))))))))

View File

@@ -0,0 +1,157 @@
(ns metabase-enterprise.transforms-runner.models.transform-library-test
(:require
[clojure.test :refer :all]
[metabase-enterprise.transforms-runner.models.transform-library :as transform-library]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
(defn- clean-libraries! []
(t2/delete! :model/TransformLibrary))
(deftest update-library-source-python-test
(testing "update-library-source! for python"
(testing "creates new record when none exists"
(clean-libraries!)
(is (=? {:source "def new_func(): return 1"
:path "common.py"
:language "python"
:id integer?
:created_at some?
:updated_at some?}
(transform-library/update-library-source! "python" "common" "def new_func(): return 1")))
(is (= 1 (t2/count :model/TransformLibrary :language "python"))))
(testing "updates existing record"
(is (=? {:source "def updated(): return 2"
:path "common.py"
:language "python"}
(transform-library/update-library-source! "python" "common" "def updated(): return 2")))
(is (= 1 (t2/count :model/TransformLibrary :language "python")) "Should not create duplicate"))
(testing "rejects invalid paths"
(is (thrown-with-msg? clojure.lang.ExceptionInfo
#"Invalid library path"
(transform-library/update-library-source! "python" "invalid-path" "code"))))))
(deftest update-library-source-javascript-test
(testing "update-library-source! for javascript"
(testing "creates new record when none exists"
(clean-libraries!)
(is (=? {:source "function newFunc() { return 1; }"
:path "common.js"
:language "javascript"
:id integer?
:created_at some?
:updated_at some?}
(transform-library/update-library-source! "javascript" "common" "function newFunc() { return 1; }")))
(is (= 1 (t2/count :model/TransformLibrary :language "javascript"))))
(testing "updates existing record"
(is (=? {:source "function updated() { return 2; }"
:path "common.js"
:language "javascript"}
(transform-library/update-library-source! "javascript" "common" "function updated() { return 2; }")))
(is (= 1 (t2/count :model/TransformLibrary :language "javascript")) "Should not create duplicate"))
(testing "rejects invalid paths"
(is (thrown-with-msg? clojure.lang.ExceptionInfo
#"Invalid library path"
(transform-library/update-library-source! "javascript" "invalid-path" "code"))))))
(deftest get-library-by-path-test
(testing "get-library-by-path"
(clean-libraries!)
(transform-library/update-library-source! "python" "common" "py code")
(transform-library/update-library-source! "javascript" "common" "js code")
(testing "returns python library"
(is (=? {:source "py code" :path "common.py" :language "python"}
(transform-library/get-library-by-path "python" "common"))))
(testing "returns javascript library"
(is (=? {:source "js code" :path "common.js" :language "javascript"}
(transform-library/get-library-by-path "javascript" "common"))))
(testing "rejects invalid paths"
(is (thrown-with-msg? clojure.lang.ExceptionInfo
#"Invalid library path"
(transform-library/get-library-by-path "python" "invalid")))
(is (thrown-with-msg? clojure.lang.ExceptionInfo
#"Invalid library path"
(transform-library/get-library-by-path "javascript" "invalid"))))))
(deftest normalize-path-test
(testing "normalize-path adds correct extension per language"
(is (= "common.py" (#'transform-library/normalize-path "python" "common")))
(is (= "common.js" (#'transform-library/normalize-path "javascript" "common")))
(is (= "common.clj" (#'transform-library/normalize-path "clojure" "common"))))
(testing "does not duplicate extension"
(is (= "common.py" (#'transform-library/normalize-path "python" "common.py")))
(is (= "common.js" (#'transform-library/normalize-path "javascript" "common.js"))))
(testing "works with extension when fetching"
(clean-libraries!)
(transform-library/update-library-source! "python" "common.py" "code1")
(is (=? {:source "code1" :path "common.py"}
(transform-library/get-library-by-path "python" "common.py")))
(is (=? {:source "code1" :path "common.py"}
(transform-library/get-library-by-path "python" "common")))))
(deftest language-isolation-test
(testing "libraries for different languages are independent"
(clean-libraries!)
(transform-library/update-library-source! "python" "common" "py-source")
(transform-library/update-library-source! "javascript" "common" "js-source")
(is (= 2 (t2/count :model/TransformLibrary)))
(is (= "py-source" (:source (transform-library/get-library-by-path "python" "common"))))
(is (= "js-source" (:source (transform-library/get-library-by-path "javascript" "common"))))
(testing "updating one language does not affect the other"
(transform-library/update-library-source! "python" "common" "py-updated")
(is (= "py-updated" (:source (transform-library/get-library-by-path "python" "common"))))
(is (= "js-source" (:source (transform-library/get-library-by-path "javascript" "common")))))))
(deftest builtin-entity-id-test
(testing "builtin-entity-id returns expected values for legacy languages"
(is (= "cWWH9qJPvHNB3rP2vLZrK" (transform-library/builtin-entity-id "python")))
(is (= "aHWo_0yPLwKqpQPlFNzCg" (transform-library/builtin-entity-id "javascript"))))
(testing "builtin-entity-id returns deterministic hash for new languages"
(let [clj-id (transform-library/builtin-entity-id "clojure")]
(is (= 21 (count clj-id)))
(is (= clj-id (transform-library/builtin-entity-id "clojure"))
"Same language should produce the same ID")))
(testing "all-builtin-entity-ids returns set of all registered language IDs"
(let [ids (transform-library/all-builtin-entity-ids)]
(is (set? ids))
(is (contains? ids "cWWH9qJPvHNB3rP2vLZrK") "Should include python")
(is (contains? ids "aHWo_0yPLwKqpQPlFNzCg") "Should include javascript"))))
(deftest ensure-builtin-library-test
(testing "creates a library row when none exists"
(clean-libraries!)
(transform-library/ensure-builtin-library! "python")
(let [lib (t2/select-one :model/TransformLibrary :language "python")]
(is (some? lib))
(is (= "common.py" (:path lib)))
(is (= "" (:source lib)))
(is (= "cWWH9qJPvHNB3rP2vLZrK" (:entity_id lib)))))
(testing "is idempotent — second call is a no-op"
(transform-library/ensure-builtin-library! "python")
(is (= 1 (t2/count :model/TransformLibrary :language "python"))))
(testing "does not overwrite existing source"
(transform-library/update-library-source! "python" "common" "user code")
(transform-library/ensure-builtin-library! "python")
(is (= "user code" (:source (t2/select-one :model/TransformLibrary :language "python")))))
(testing "works for javascript with correct extension and entity_id"
(clean-libraries!)
(transform-library/ensure-builtin-library! "javascript")
(let [lib (t2/select-one :model/TransformLibrary :language "javascript")]
(is (= "common.js" (:path lib)))
(is (= "aHWo_0yPLwKqpQPlFNzCg" (:entity_id lib))))))

View File

@@ -0,0 +1,193 @@
(ns metabase-enterprise.transforms-runner.polyglot-contract-test
"Contract equivalence tests: all runner-based languages must implement the
transforms.interface multimethods identically given the same transform shape.
Also covers transforms.util pure functions for all language types."
(:require
[clojure.test :refer :all]
[clojure.test.check.clojure-test :refer [defspec]]
[clojure.test.check.generators :as gen]
[clojure.test.check.properties :as prop]
[metabase-enterprise.transforms-clojure.impl]
[metabase-enterprise.transforms-javascript.impl]
[metabase-enterprise.transforms-python.impl]
[metabase.models.transforms.transform]
[metabase.transforms.interface :as transforms.i]
[metabase.transforms.util :as transforms.util]))
(set! *warn-on-reflection* true)
(def ^:private runner-languages
"All registered runner-based language dispatch values."
[:python :javascript :clojure])
;;; ------------------------------------------------ Generators ----------------------------------------------------
(def ^:private gen-source-table-int
"Generator for integer table IDs."
(gen/fmap inc gen/nat))
(def ^:private gen-source-table-ref
"Generator for source table reference maps."
(gen/let [db-id (gen/fmap inc gen/nat)
schema (gen/one-of [(gen/return nil) gen/string-alphanumeric])
table gen/string-alphanumeric
table-id (gen/one-of [(gen/return nil) (gen/fmap inc gen/nat)])]
(cond-> {:database_id db-id :schema schema :table table}
table-id (assoc :table_id table-id))))
(def ^:private gen-source-table-value
"Generator for either int or ref source table values."
(gen/one-of [gen-source-table-int gen-source-table-ref]))
(def ^:private gen-source-tables
"Generator for source-tables maps (string keys -> source table values)."
(gen/let [entries (gen/vector
(gen/tuple gen/string-alphanumeric gen-source-table-value)
0 5)]
(into {} entries)))
(defn- gen-runner-transform
"Generator for a runner transform with a specific language type."
[lang]
(gen/let [source-db (gen/fmap inc gen/nat)
target-db (gen/fmap inc gen/nat)
source-tables gen-source-tables
body gen/string-alphanumeric]
{:source {:type (name lang)
:source-database source-db
:source-tables source-tables
:body body}
:target {:database target-db
:type "table"
:name "output"
:schema "public"}}))
;;; --------------------------------- Contract: target-db-id is language-independent --------------------------------
(deftest target-db-id-contract-test
(testing "all runner languages extract target-db-id identically"
(let [transform {:source {:type "python" :source-database 1 :source-tables {} :body "x"}
:target {:database 99 :type "table" :name "out" :schema "s"}}]
(doseq [lang runner-languages]
(testing (str "language: " lang)
(is (= 99 (transforms.i/target-db-id
(assoc-in transform [:source :type] (name lang))))))))))
(defspec target-db-id-equivalence-prop 50
(prop/for-all [transform (gen-runner-transform :python)]
(let [expected (transforms.i/target-db-id transform)]
(every? (fn [lang]
(= expected
(transforms.i/target-db-id
(assoc-in transform [:source :type] (name lang)))))
runner-languages))))
;;; --------------------------------- Contract: source-db-id is language-independent --------------------------------
(deftest source-db-id-contract-test
(testing "all runner languages extract source-db-id identically"
(let [transform {:source {:type "python" :source-database 42 :source-tables {} :body "x"}
:target {:database 1 :type "table" :name "out" :schema "s"}}]
(doseq [lang runner-languages]
(testing (str "language: " lang)
(is (= 42 (transforms.i/source-db-id
(assoc-in transform [:source :type] (name lang))))))))))
(defspec source-db-id-equivalence-prop 50
(prop/for-all [transform (gen-runner-transform :python)]
(let [expected (transforms.i/source-db-id transform)]
(every? (fn [lang]
(= expected
(transforms.i/source-db-id
(assoc-in transform [:source :type] (name lang)))))
runner-languages))))
;;; --------------------------------- Contract: table-dependencies is language-independent --------------------------
(deftest table-dependencies-contract-test
(testing "all runner languages compute table-dependencies identically"
(let [transform {:source {:type "python"
:source-database 1
:source-tables {"a" 100
"b" {:database_id 1 :schema "s" :table "t"}
"c" {:database_id 2 :schema nil :table "u" :table_id 200}}
:body "x"}
:target {:database 1 :type "table" :name "out" :schema "s"}}
expected (transforms.i/table-dependencies transform)]
(doseq [lang runner-languages]
(testing (str "language: " lang)
(is (= expected
(transforms.i/table-dependencies
(assoc-in transform [:source :type] (name lang))))))))))
(defspec table-dependencies-equivalence-prop 50
(prop/for-all [transform (gen-runner-transform :python)]
(let [expected (transforms.i/table-dependencies transform)]
(every? (fn [lang]
(= expected
(transforms.i/table-dependencies
(assoc-in transform [:source :type] (name lang)))))
runner-languages))))
;;; --------------------------------- transforms.util pure functions ------------------------------------------------
(deftest transform-type-test
(doseq [lang runner-languages]
(testing (str "transform-type returns " lang " for runner transform")
(is (= lang (transforms.util/transform-type
{:source {:type (name lang)}}))))))
(deftest runner-transform?-test
(doseq [lang runner-languages]
(testing (str lang " is a runner transform")
(is (true? (transforms.util/runner-transform?
{:source {:type (name lang)}})))))
(testing "query is not a runner transform"
(is (false? (transforms.util/runner-transform?
{:source {:type "query"}})))))
(deftest transform-source-database-test
(testing "runner transforms get source-database from :source"
(doseq [lang runner-languages]
(testing (str "language: " lang)
(is (= 42 (transforms.util/transform-source-database
{:source {:type (name lang) :source-database 42}}))))))
(testing "query transforms get database from :query"
(is (= 99 (transforms.util/transform-source-database
{:source {:type "query" :query {:database 99}}})))))
(deftest transform-source-type-test
(testing "runner types return themselves"
(doseq [lang runner-languages]
(testing (str "language: " lang)
(is (= lang (transforms.util/transform-source-type
{:type (name lang)}))))))
(testing "unknown type throws"
(is (thrown-with-msg? clojure.lang.ExceptionInfo
#"Unknown transform source type"
(transforms.util/transform-source-type {:type "cobol"})))))
(deftest required-database-features-test
(testing "all runner transforms require :transforms/python"
(doseq [lang runner-languages]
(testing (str "language: " lang)
(is (= [:transforms/python]
(transforms.util/required-database-features
{:source {:type (name lang)}}))))))
(testing "query transforms require :transforms/table"
(is (= [:transforms/table]
(transforms.util/required-database-features
{:source {:type "query"}})))))
(deftest search-index-query-text-all-runner-languages-test
(testing "maybe-extract-transform-query-text extracts :body for all runner languages, not just :python"
(let [extract-fn @#'metabase.models.transforms.transform/maybe-extract-transform-query-text
body "print('hello world')"]
(doseq [lang runner-languages]
(testing (str "language: " lang)
(let [transform {:source_type (name lang)
:source {:type (name lang)
:body body
:source-tables {}}}]
(is (= body (extract-fn transform))
(str "Expected body extraction for " lang))))))))

View File

@@ -0,0 +1,105 @@
(ns metabase-enterprise.transforms-runner.schema-test
"Schema validation tests for runner-based transform sources.
These require runner language registration, so they live in enterprise."
(:require
[clojure.test :refer :all]
[metabase.transforms.interface :as transforms.i]
[metabase.transforms.schema :as transforms.schema]
[metabase.util.malli.registry :as mr]))
(set! *warn-on-reflection* true)
(use-fixtures :once (fn [thunk]
(doseq [lang [:python :javascript :clojure]]
(transforms.i/register-runner! lang))
(thunk)))
(def ^:private runner-languages
[:python :javascript :clojure])
(def ^:private valid-runner-source
{:source-database 1
:source-tables {"t1" 100}
:body "some code"})
;;; ---------------------------------------- Valid runner sources ---------------------------------------------------
(deftest runner-source-validates-for-all-languages-test
(doseq [lang runner-languages]
(testing (str lang " source validates")
(is (true? (mr/validate ::transforms.schema/transform-source
(assoc valid-runner-source :type lang)))))))
(deftest runner-source-with-table-ref-validates-test
(doseq [lang runner-languages]
(testing (str lang " with table ref source-tables validates")
(is (true? (mr/validate ::transforms.schema/transform-source
(assoc valid-runner-source
:type lang
:source-tables {"t1" {:database_id 1
:schema "public"
:table "foo"}})))))))
(deftest runner-source-with-incremental-strategy-validates-test
(doseq [lang runner-languages]
(testing (str lang " with optional incremental strategy validates")
(is (true? (mr/validate ::transforms.schema/transform-source
(assoc valid-runner-source
:type lang
:source-incremental-strategy
{:type "checkpoint"
:checkpoint {:type "checkpoint"}})))))))
(deftest runner-source-optional-source-database-test
(doseq [lang runner-languages]
(testing (str lang " without source-database validates (it's optional)")
(is (true? (mr/validate ::transforms.schema/transform-source
(-> valid-runner-source
(dissoc :source-database)
(assoc :type lang))))))))
;;; ---------------------------------------- Invalid runner sources ------------------------------------------------
(deftest runner-source-missing-body-invalid-test
(doseq [lang runner-languages]
(testing (str lang " without :body is invalid")
(is (false? (mr/validate ::transforms.schema/transform-source
(-> valid-runner-source
(dissoc :body)
(assoc :type lang))))))))
(deftest runner-source-missing-source-tables-invalid-test
(doseq [lang runner-languages]
(testing (str lang " without :source-tables is invalid")
(is (false? (mr/validate ::transforms.schema/transform-source
(-> valid-runner-source
(dissoc :source-tables)
(assoc :type lang))))))))
(deftest runner-source-wrong-body-type-invalid-test
(doseq [lang runner-languages]
(testing (str lang " with non-string :body is invalid")
(is (false? (mr/validate ::transforms.schema/transform-source
(assoc valid-runner-source :type lang :body 42)))))))
(deftest unknown-source-type-invalid-test
(testing "unknown source type is invalid"
(is (false? (mr/validate ::transforms.schema/transform-source
(assoc valid-runner-source :type :cobol))))))
;;; ---------------------------------------- Query source ----------------------------------------------------------
(deftest query-source-not-confused-with-runner-test
(testing "a runner-shaped source with :type :query does not validate as runner"
(is (false? (mr/validate ::transforms.schema/transform-source
(assoc valid-runner-source :type :query))))))
;;; ---------------------------------------- Shape equivalence across languages ------------------------------------
(deftest all-runner-languages-accept-same-shape-test
(testing "the exact same map (minus :type) validates for every runner language"
(let [base (dissoc valid-runner-source :type)]
(doseq [lang runner-languages]
(testing (str lang)
(is (true? (mr/validate ::transforms.schema/transform-source
(assoc base :type lang)))))))))

View File

@@ -10,29 +10,30 @@ import { idTag, invalidateTags, providePythonLibraryTags } from "./tags";
export const pythonLibraryApi = EnterpriseApi.injectEndpoints({
endpoints: (builder) => ({
getPythonLibrary: builder.query<PythonLibrary, GetPythonLibraryRequest>({
query: ({ path }) => ({
url: `/api/ee/transforms-python/library/${path}`,
query: ({ path, type }) => ({
url: `/api/ee/advanced-transforms/library/${path}`,
method: "GET",
params: { type },
}),
providesTags: (library) =>
library ? providePythonLibraryTags(library) : [],
}),
updatePythonLibrary: builder.mutation<void, UpdatePythonLibraryRequest>({
query: ({ path, source }) => ({
url: `/api/ee/transforms-python/library/${path}`,
query: ({ path, source, type }) => ({
url: `/api/ee/advanced-transforms/library/${path}`,
method: "PUT",
body: { source },
body: { source, type },
}),
invalidatesTags: (_, error, { path }) =>
invalidateTags(error, [idTag("python-transform-library", path)]),
onQueryStarted: async (
{ path, ...patch },
{ path, type, ...patch },
{ dispatch, queryFulfilled },
) => {
const patchResult = dispatch(
pythonLibraryApi.util.updateQueryData(
"getPythonLibrary",
{ path },
{ path, type },
(draft) => {
Object.assign(draft, patch);
},

View File

@@ -11,10 +11,11 @@ export const pythonRunnerApi = EnterpriseApi.injectEndpoints({
TestPythonTransformResponse,
TestPythonTransformRequest
>({
query: ({ code, source_tables }) => ({
url: "/api/ee/transforms-python/test-run",
query: ({ type, code, source_tables }) => ({
url: "/api/ee/advanced-transforms/test-run",
method: "POST",
body: {
type,
code,
source_tables,
},

View File

@@ -307,7 +307,10 @@ function getSourceCode(
function getTransformUrl(transform: SuggestedTransform): string {
return match(transform)
.with({ id: P.number }, ({ id }) => Urls.transformEdit(id))
.with({ source: { type: "python" } }, () => Urls.newPythonTransform())
.with({ source: { type: "python" } }, () => Urls.newAdvancedTransform())
.with({ source: { type: "javascript" } }, () =>
Urls.newAdvancedTransform("javascript"),
)
.with({ source: { type: "query" } }, () => Urls.newNativeTransform())
.exhaustive();
}

View File

@@ -1,35 +1,36 @@
import type { Extension } from "@uiw/react-codemirror";
import cx from "classnames";
import _ from "underscore";
import { useMemo } from "react";
import { CodeEditor } from "metabase/common/components/CodeEditor";
import type { AdvancedTransformType } from "metabase-types/api";
import S from "./PythonEditor.module.css";
import { completion } from "./utils";
import { getCompletionExtensions } from "./utils";
export function PythonEditor({
type,
value,
proposedValue,
onChange,
withPandasCompletions,
className,
readOnly,
extensions: externalExtensions,
...rest
}: {
type: AdvancedTransformType;
value: string;
proposedValue?: string;
onChange?: (value: string) => void;
withPandasCompletions?: boolean;
className?: string;
readOnly?: boolean;
extensions?: Extension[];
}) {
const extensions = _.compact([
...(externalExtensions ?? []),
withPandasCompletions ? completion : undefined,
]);
const extensions = useMemo(
() => [...(externalExtensions ?? []), ...getCompletionExtensions(type)],
[externalExtensions, type],
);
return (
<CodeEditor
@@ -37,7 +38,7 @@ export function PythonEditor({
value={value}
proposedValue={proposedValue}
onChange={onChange}
language="python"
language={type === "javascript" ? "typescript" : type}
extensions={extensions}
readOnly={readOnly}
{...rest}

View File

@@ -4,8 +4,11 @@ import {
autocompletion,
} from "@codemirror/autocomplete";
import { pythonLanguage } from "@codemirror/lang-python";
import type { Extension } from "@uiw/react-codemirror";
import { t } from "ttag";
import type { AdvancedTransformType } from "metabase-types/api";
const pandasDefinitions = (): Record<string, Completion[]> => ({
pd: [
{
@@ -667,9 +670,16 @@ function genericPandasCompletions(context: CompletionContext) {
}
}
export const completion = [
pythonLanguage.data.of({
autocomplete: genericPandasCompletions,
}),
autocompletion(),
];
export function getCompletionExtensions(
type: AdvancedTransformType,
): Extension[] {
if (type === "python") {
return [
pythonLanguage.data.of({
autocomplete: genericPandasCompletions,
}),
autocompletion(),
];
}
return [autocompletion()];
}

View File

@@ -124,7 +124,7 @@ export function PythonDataPicker({
<Box>
<Text fw="bold">{t`Pick tables and alias them`}</Text>
<Text size="sm" c="text-tertiary" mb="sm">
{t`Select tables to use as data sources and provide aliases that can be referenced in your Python script.`}
{t`Select tables to use as data sources and provide aliases that can be referenced in your transform.`}
</Text>
<Stack gap="md">
{tableSelections.map((selection, index) => (

View File

@@ -9,14 +9,16 @@ import { useDispatch } from "metabase/lib/redux";
import * as Urls from "metabase/lib/urls";
import { RunButtonWithTooltip } from "metabase/query_builder/components/RunButtonWithTooltip";
import { Button, Flex, Icon, Stack, Tooltip } from "metabase/ui";
import type { AdvancedTransformType } from "metabase-types/api";
import { SHARED_LIB_IMPORT_PATH } from "../../../constants";
import { SHARED_LIB_IMPORT_PATHS } from "../../../constants";
import { PythonEditor } from "../../PythonEditor";
import { ResizableBoxHandle } from "./ResizableBoxHandle";
import { createPythonImportTokenLocator } from "./utils";
import { createImportTokenLocator } from "./utils";
type PythonEditorBodyProps = {
type: AdvancedTransformType;
disabled?: boolean;
source: string;
proposedSource?: string;
@@ -34,11 +36,12 @@ type PythonEditorBodyProps = {
onRejectProposed?: () => void;
};
const EDITOR_HEIGHT = 400;
const EDITOR_HEIGHT = 330;
const HEADER_HEIGHT = 65 + 50; // Top bar height + transform header height
const PREVIEW_MAX_INITIAL_HEIGHT = 192;
export function PythonEditorBody({
type,
disabled,
source,
proposedSource,
@@ -63,7 +66,7 @@ export function PythonEditorBody({
(e: MouseEvent) => {
const openInNewTab = e.metaKey || e.ctrlKey || e.button === 1;
const href = Urls.transformPythonLibrary({
path: SHARED_LIB_IMPORT_PATH,
path: SHARED_LIB_IMPORT_PATHS[type],
});
if (openInNewTab) {
window.open(href, "_blank", "noopener,noreferrer");
@@ -71,27 +74,27 @@ export function PythonEditorBody({
dispatch(push(href));
}
},
[dispatch],
[dispatch, type],
);
const clickableTokensExtension = useMemo(
() =>
clickableTokens([
{
tokenLocator: createPythonImportTokenLocator("common"),
tokenLocator: createImportTokenLocator(type, "common"),
onClick: (e) => navigateToCommonLibrary(e),
},
]),
[navigateToCommonLibrary],
[navigateToCommonLibrary, type],
);
const editorContent = (
<Flex h="100%" align="end" bg="background-secondary" pos="relative">
<PythonEditor
type={type}
value={source}
proposedValue={proposedSource}
onChange={onChange}
withPandasCompletions
readOnly={!isEditMode || disabled}
extensions={[clickableTokensExtension]}
data-testid="python-editor"
@@ -134,7 +137,7 @@ export function PythonEditorBody({
isDirty={isDirty}
onRun={onRun}
onCancel={onCancel}
getTooltip={() => t`Run Python script`}
getTooltip={() => t`Run transform`}
/>
)}
</Stack>

View File

@@ -41,6 +41,7 @@ function setup({
onCancel={jest.fn()}
onAcceptProposed={onAcceptProposed}
onRejectProposed={onRejectProposed}
type="python"
/>,
);
}

View File

@@ -1,6 +1,8 @@
import type { EditorView } from "@codemirror/view";
import type { SyntaxNodeRef } from "@lezer/common";
import type { AdvancedTransformType } from "metabase-types/api";
export function insertImport(source: string, path: string) {
const lines = source.split("\n");
@@ -35,7 +37,7 @@ export function hasImport(source: string, path: string) {
return regex.test(source);
}
export const createPythonImportTokenLocator =
const createPythonImportTokenLocator =
(moduleName: string) =>
(view: EditorView, node: SyntaxNodeRef): boolean => {
if (node.type.name !== "VariableName") {
@@ -54,3 +56,36 @@ export const createPythonImportTokenLocator =
}
return false;
};
const createJavascriptImportTokenLocator =
(moduleName: string) =>
(view: EditorView, node: SyntaxNodeRef): boolean => {
if (node.type.name !== "VariableDefinition") {
return false;
}
const variableName = view.state.doc.sliceString(node.from, node.to);
if (variableName !== moduleName) {
return false;
}
let cur: SyntaxNodeRef | null = node;
while (cur) {
if (cur.type.name === "ImportDeclaration") {
return true;
}
cur = cur.node.parent;
}
return false;
};
export function createImportTokenLocator(
type: AdvancedTransformType,
moduleName: string,
) {
if (type === "python") {
return createPythonImportTokenLocator(moduleName);
} else if (type === "javascript") {
return createJavascriptImportTokenLocator(moduleName);
}
const _exhaustive: never = type;
return _exhaustive;
}

View File

@@ -161,7 +161,7 @@ function ResultsFooter({
<Text
c="text-primary"
ml="xs"
>{t`An error occurred while executing your Python script.`}</Text>
>{t`An error occurred while executing your transform.`}</Text>
<DismissButton onDismiss={onDismiss} />
</Flex>
);

View File

@@ -2,5 +2,26 @@
height: 100%;
min-height: 0;
overflow: auto;
position: relative;
width: 100%;
}
.typeLabel {
background-color: var(--mb-color-background-primary);
border-color: var(--mb-color-border);
border-radius: var(--mantine-spacing-sm) 0 0 0;
border-style: solid;
border-width: 1px 0 0 1px;
bottom: 0;
color: var(--mb-color-text-secondary);
right: 0;
padding: 0 0.5rem;
position: absolute;
&.editMode {
border-radius: 0 var(--mantine-spacing-sm) 0 0;
border-width: 1px 1px 0 0;
left: 0;
right: auto;
}
}

View File

@@ -1,13 +1,15 @@
import { useHotkeys } from "@mantine/hooks";
import cx from "classnames";
import { useEffect } from "react";
import { usePrevious } from "react-use";
import type { PythonTransformEditorProps } from "metabase/plugins";
import { Flex, Stack } from "metabase/ui";
import type {
DatabaseId,
PythonTransformTableAliases,
Table,
import { Box, Flex, Stack, Text } from "metabase/ui";
import {
ADVANCED_TRANSFORM_TYPES,
type DatabaseId,
type PythonTransformTableAliases,
type Table,
} from "metabase-types/api";
import { isPythonTransformSource } from "../../utils";
@@ -17,6 +19,7 @@ import { PythonEditorBody } from "./PythonEditorBody";
import { PythonEditorResults } from "./PythonEditorResults";
import S from "./PythonTransformEditor.module.css";
import { PythonTransformTopBar } from "./PythonTransformTopBar";
import { TransformTypeSelect } from "./TransformTypeSelect";
import { useTestPythonTransform } from "./hooks";
import { updateTransformSignature } from "./utils";
@@ -64,6 +67,7 @@ export function PythonTransformEditor({
source.body,
sourceTables,
tableInfo,
source.type,
);
const newSource = {
@@ -119,6 +123,7 @@ export function PythonTransformEditor({
};
useHotkeys([["mod+Enter", handleCmdEnter]], []);
const canChangeType = !transform?.id;
return (
<Flex h="100%" w="100%" direction="column">
@@ -141,6 +146,7 @@ export function PythonTransformEditor({
)}
<Stack w="100%" h="100%" gap={0}>
<PythonEditorBody
type={source.type}
disabled={uiOptions?.readOnly}
isRunnable={isPythonTransformSource(source)}
isRunning={isRunning}
@@ -163,6 +169,17 @@ export function PythonTransformEditor({
/>
)}
</Stack>
{!!source?.type && (
<Box className={cx(S.typeLabel, { [S.editMode]: isEditMode })}>
{canChangeType ? (
<TransformTypeSelect value={source.type} />
) : (
<Text fz="sm" c="text-secondary" px="xs">
{ADVANCED_TRANSFORM_TYPES[source.type].displayName}
</Text>
)}
</Box>
)}
</Flex>
</Flex>
);

View File

@@ -0,0 +1,60 @@
import { replace } from "react-router-redux";
import { useDispatch } from "metabase/lib/redux";
import { newAdvancedTransform } from "metabase/lib/urls";
import { Button, Combobox, Group, Icon, Text, useCombobox } from "metabase/ui";
import {
ADVANCED_TRANSFORM_TYPES,
type AdvancedTransformType,
isAdvancedTransformType,
} from "metabase-types/api";
const transformTypes = Object.keys(
ADVANCED_TRANSFORM_TYPES,
) as AdvancedTransformType[];
type AdvancedTransformTypeSelectProps = {
value: AdvancedTransformType;
};
export function TransformTypeSelect({
value,
}: AdvancedTransformTypeSelectProps) {
const dispatch = useDispatch();
const combobox = useCombobox();
const onChange = (type: string) => {
if (isAdvancedTransformType(type)) {
dispatch(replace(newAdvancedTransform(type)));
}
};
return (
<Combobox onOptionSubmit={onChange} store={combobox} position="top">
<Combobox.Target>
<Button
c="text-secondary"
onClick={() => combobox.toggleDropdown()}
pl="xs"
pr={0}
py={0}
h="1.5rem"
variant="subtle"
>
<Group fz="sm" gap="sm" align="center" fw="normal">
{ADVANCED_TRANSFORM_TYPES[value].displayName}
<Icon name="chevrondown" size={8} />
</Group>
</Button>
</Combobox.Target>
<Combobox.Dropdown miw="8rem">
<Combobox.Options>
{transformTypes.map((type) => (
<Combobox.Option value={type} key={type} py="xs">
<Text>{ADVANCED_TRANSFORM_TYPES[type].displayName}</Text>
</Combobox.Option>
))}
</Combobox.Options>
</Combobox.Dropdown>
</Combobox>
);
}

View File

@@ -33,6 +33,7 @@ export function useTestPythonTransform(
}
const request = executePython({
type: source.type,
code: source.body,
source_tables: source["source-tables"],
});
@@ -65,7 +66,7 @@ function getError(error: unknown) {
if (typeof error === "object" && error !== null) {
if ("name" in error && error.name === "AbortError") {
return { message: t`Python script execution was canceled` };
return { message: t`Transform execution was canceled` };
}
}
return {

View File

@@ -1,4 +1,5 @@
import type {
AdvancedTransformType,
PythonTransformTableAliases,
Table,
TableId,
@@ -8,25 +9,26 @@ export function updateTransformSignature(
script: string,
tables: PythonTransformTableAliases,
tableInfo: Table[],
type: AdvancedTransformType,
): string {
return type === "python"
? updatePythonTransformSignature(script, tables, tableInfo)
: updateJavascriptTransformSignature(script, tables, tableInfo);
}
function updatePythonTransformSignature(
script: string,
tables: PythonTransformTableAliases,
tableInfo: Table[],
): string {
const tableAliases = Object.keys(tables);
const transformRegex = /^def\s+transform\s*\([^)]*\)\s*:\s*\n(\s*)/m;
const signatureOneLine = `def transform(${tableAliases.join(", ")}):`;
const newSignature =
signatureOneLine.length > 80 && tableAliases.length > 0
? `def transform(\n${tableAliases.map((alias) => ` ${alias}`).join(",\n")},\n):`
: signatureOneLine;
function getTableName(tableId: TableId) {
const table = tableInfo.find((t) => t.id === tableId);
if (!table) {
return undefined;
}
return [table.db?.name, table.schema, table.name].filter(Boolean).join(".");
}
if (transformRegex.test(script)) {
// Capture the existing indentation after the colon
const match = script.match(transformRegex);
@@ -47,7 +49,7 @@ export function updateTransformSignature(
.map((alias) => {
const padding = " ".repeat(maxAliasLength - alias.length);
const tableId = tables[alias];
const tableName = getTableName(tableId);
const tableName = getTableName(tableInfo, tableId);
return ` ${alias}:${padding} DataFrame containing the data from the "${tableName}" table`;
})
.join("\n");
@@ -119,7 +121,7 @@ ${tableAliases
.map((alias) => {
const padding = " ".repeat(maxAliasLength - alias.length);
const tableId = tables[alias];
const tableName = getTableName(tableId);
const tableName = getTableName(tableInfo, tableId);
if (alias === tableName) {
return ` ${alias}:${padding} DataFrame containing the data from the corresponding table`;
}
@@ -138,3 +140,138 @@ ${tableAliases
return script + functionTemplate;
}
function updateJavascriptTransformSignature(
script: string,
tables: PythonTransformTableAliases,
tableInfo: Table[],
): string {
const tableAliases = Object.keys(tables);
const transformRegex =
/^export\sfunction\s+transform\s*\([^)]*\)\s*{\s*\n(\s*)/m;
const signatureOneLine = `export function transform(${tableAliases.join(", ")}) {`;
const newSignature =
signatureOneLine.length > 80 && tableAliases.length > 0
? `export function transform(\n${tableAliases.map((alias) => ` ${alias}`).join(",\n")},\n) {`
: signatureOneLine;
if (transformRegex.test(script)) {
// Capture the existing indentation after the colon
const match = script.match(transformRegex);
const originalIndent = match ? match[1] : " ";
let updatedScript = script.replace(
transformRegex,
newSignature + "\n" + originalIndent,
);
const argsRegex =
/(\s+\/\*\*[\s\S]*?)\s*Args:\s*\n([\s\S]*?)\s*(\n\s+Returns:|\n\s+""")/;
const argsMatch = updatedScript.match(argsRegex);
if (tableAliases.length > 0) {
const maxAliasLength = Math.max(...tableAliases.map((a) => a.length));
const newArgsSection = tableAliases
.map((alias) => {
const padding = " ".repeat(maxAliasLength - alias.length);
const tableId = tables[alias];
const tableName = getTableName(tableInfo, tableId);
return ` ${alias}:${padding} Object array containing the data from the "${tableName}" table`;
})
.join("\n");
if (argsMatch) {
// Replace existing Args section
updatedScript = updatedScript.replace(
argsRegex,
` $1
Args:
${newArgsSection}
$3`,
);
} else {
// No Args section exists, look for the pattern before Returns and insert Args
// Also remove the instructional text about selecting tables
const insertRegex =
/(\s+\/\*\*[\s\S]*?)\s*Select tables above to add them as function parameters\.\s*(\n\s+Returns:|\n\s+""")/;
const insertMatch = updatedScript.match(insertRegex);
if (insertMatch) {
updatedScript = updatedScript.replace(
insertRegex,
`$1
Args:
${newArgsSection}
$2`,
);
} else {
// Fallback: just look for Returns section without the specific text
// Remove any extra whitespace before Returns and ensure single blank line
const fallbackRegex = /(\s+"""[\s\S]*?)\s*(\n\s+Returns:|\n\s+""")/;
const fallbackMatch = updatedScript.match(fallbackRegex);
if (fallbackMatch) {
updatedScript = updatedScript.replace(
fallbackRegex,
`$1
Args:
${newArgsSection}
$2`,
);
}
}
}
} else if (argsMatch) {
// Remove existing Args section when no tables
updatedScript = updatedScript.replace(argsRegex, "$1$3");
}
return updatedScript;
}
const maxAliasLength =
tableAliases.length > 0
? Math.max(...tableAliases.map((a) => a.length))
: 0;
const functionTemplate = `
${newSignature}
/**
Transform function that processes the input data.
${
tableAliases.length > 0
? `
Args:
${tableAliases
.map((alias) => {
const padding = " ".repeat(maxAliasLength - alias.length);
const tableId = tables[alias];
const tableName = getTableName(tableInfo, tableId);
if (alias === tableName) {
return ` ${alias}:${padding} Array containing the data from the corresponding table`;
}
return ` ${alias}:${padding} Array containing the data from table "${tableName}"`;
})
.join("\n")}`
: ""
}
Returns:
The transformed data object array
*/
return []
}
`;
return script + functionTemplate;
}
function getTableName(tableInfo: Table[], tableId: TableId) {
const table = tableInfo.find((t) => t.id === tableId);
if (!table) {
return undefined;
}
return [table.db?.name, table.schema, table.name].filter(Boolean).join(".");
}

View File

@@ -1 +1,6 @@
export const SHARED_LIB_IMPORT_PATH = "common.py";
import type { AdvancedTransformType } from "metabase-types/api";
export const SHARED_LIB_IMPORT_PATHS: Record<AdvancedTransformType, string> = {
python: "common.py",
javascript: "common.js",
};

View File

@@ -1,8 +1,8 @@
import { PLUGIN_TRANSFORMS_PYTHON } from "metabase/plugins";
import { hasPremiumFeature } from "metabase-enterprise/settings";
import { PythonTransformEditor } from "./components/PythonTransformEditor";
import { SHARED_LIB_IMPORT_PATH } from "./constants";
import { PythonTransformEditor } from "./components/PythonTransformEditor/PythonTransformEditor";
import { SHARED_LIB_IMPORT_PATHS } from "./constants";
import { PythonRunnerSettingsPage } from "./pages/PythonRunnerSettingsPage";
import { getPythonTransformsRoutes, getPythonUpsellRoutes } from "./routes";
import { getPythonSourceValidationResult } from "./utils";
@@ -24,5 +24,5 @@ export function initializePlugin() {
PLUGIN_TRANSFORMS_PYTHON.getPythonTransformsRoutes = getPythonUpsellRoutes;
}
PLUGIN_TRANSFORMS_PYTHON.sharedLibImportPath = SHARED_LIB_IMPORT_PATH;
PLUGIN_TRANSFORMS_PYTHON.sharedLibImportPaths = SHARED_LIB_IMPORT_PATHS;
}

View File

@@ -10,6 +10,7 @@ import * as Urls from "metabase/lib/urls";
import { Button, Group } from "metabase/ui";
type PythonLibraryEditorHeaderProps = {
typeDisplayName: string;
isDirty?: boolean;
isSaving?: boolean;
onSave: () => void;
@@ -17,6 +18,7 @@ type PythonLibraryEditorHeaderProps = {
};
export const PythonLibraryEditorHeader = ({
typeDisplayName,
isDirty,
isSaving,
onSave,
@@ -27,10 +29,14 @@ export const PythonLibraryEditorHeader = ({
breadcrumbs={
<DataStudioBreadcrumbs>
<Link to={Urls.transformList()}>{t`Transforms`}</Link>
{t`Python library`}
{typeDisplayName + " " + t`library`}
</DataStudioBreadcrumbs>
}
title={<PanelHeaderTitle>{t`Python library`}</PanelHeaderTitle>}
title={
<PanelHeaderTitle>
{typeDisplayName + " " + t`library`}
</PanelHeaderTitle>
}
actions={
(isDirty || isSaving) && (
<Group wrap="nowrap">

View File

@@ -2,6 +2,7 @@ import { useLayoutEffect, useState } from "react";
import type { Route } from "react-router";
import { t } from "ttag";
import { skipToken } from "metabase/api";
import { LeaveRouteConfirmModal } from "metabase/common/components/LeaveConfirmModal";
import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
import { PageContainer } from "metabase/data-studio/common/components/PageContainer";
@@ -15,6 +16,10 @@ import {
useUpdatePythonLibraryMutation,
} from "metabase-enterprise/api/python-transform-library";
import { getIsRemoteSyncReadOnly } from "metabase-enterprise/remote_sync/selectors";
import {
ADVANCED_TRANSFORM_TYPES,
type AdvancedTransformType,
} from "metabase-types/api";
import { PythonEditor } from "../../components/PythonEditor";
@@ -26,26 +31,43 @@ type PythonLibraryEditorPageProps = {
route: Route;
};
const EMPTY_LIBRARY_SOURCE = `
# This is your Python library.
# You can add functions and classes here that can be reused in Python transforms.
function getTypeFromPath(path: string): AdvancedTransformType | undefined {
if (path.endsWith(".py")) {
return "python";
}
if (path.endsWith(".js")) {
return "javascript";
}
}
function getEmptyLibrarySource(type?: AdvancedTransformType) {
if (type == null) {
return "";
}
const { displayName, commentString } = ADVANCED_TRANSFORM_TYPES[type];
return `
${commentString} This is your ${displayName} library.
${commentString} You can add code here that can be reused in ${displayName} transforms.
`
.trim()
.concat("\n");
.trim()
.concat("\n");
}
export function PythonLibraryEditorPage({
params,
route,
}: PythonLibraryEditorPageProps) {
const { path } = params;
const [source, setSource] = useState(EMPTY_LIBRARY_SOURCE);
const type = getTypeFromPath(path);
const emptyLibrarySource = getEmptyLibrarySource(type);
const [source, setSource] = useState(emptyLibrarySource);
const isRemoteSyncReadOnly = useSelector(getIsRemoteSyncReadOnly);
const {
data: library,
isLoading,
error,
} = useGetPythonLibraryQuery({ path });
} = useGetPythonLibraryQuery(type != null ? { path, type } : skipToken);
const [updatePythonLibrary, { isLoading: isSaving }] =
useUpdatePythonLibraryMutation();
const { sendSuccessToast, sendErrorToast } = useMetadataToasts();
@@ -56,7 +78,7 @@ export function PythonLibraryEditorPage({
}
if (isResourceNotFoundError(error)) {
setSource(EMPTY_LIBRARY_SOURCE);
setSource(emptyLibrarySource);
return;
}
@@ -66,11 +88,14 @@ export function PythonLibraryEditorPage({
}
async function handleSave() {
if (type == null) {
return;
}
try {
await updatePythonLibrary({ path, source }).unwrap();
sendSuccessToast(t`Python library saved`);
await updatePythonLibrary({ path, source, type }).unwrap();
sendSuccessToast(t`Library saved`);
} catch (error) {
sendErrorToast(t`Python library could not be saved`);
sendErrorToast(t`Library could not be saved`);
}
}
@@ -79,13 +104,13 @@ export function PythonLibraryEditorPage({
if (library?.source) {
setSource(library.source);
} else {
setSource(EMPTY_LIBRARY_SOURCE);
setSource(emptyLibrarySource);
}
}, [library]);
}, [library, emptyLibrarySource]);
const isDirty = source !== (library?.source || EMPTY_LIBRARY_SOURCE);
const isDirty = source !== (library?.source || emptyLibrarySource);
if (isLoading || (error && !isResourceNotFoundError(error))) {
if (isLoading || (error && !isResourceNotFoundError(error)) || type == null) {
return (
<Box p="md">
<LoadingAndErrorWrapper loading={isLoading} error={error} />
@@ -93,11 +118,14 @@ export function PythonLibraryEditorPage({
);
}
const typeDisplayName = ADVANCED_TRANSFORM_TYPES[type].displayName;
return (
<>
<PageContainer>
<Stack>
<PythonLibraryEditorHeader
typeDisplayName={typeDisplayName}
onSave={handleSave}
onRevert={handleRevert}
isDirty={isDirty && !isRemoteSyncReadOnly}
@@ -109,7 +137,7 @@ export function PythonLibraryEditorPage({
className={S.flexStart}
color="warning"
p="0.75rem"
title={t`The Python library is not editable because Remote Sync is in read-only mode.`}
title={t`The library is not editable because Remote Sync is in read-only mode.`}
variant="outline"
w="auto"
/>
@@ -118,9 +146,9 @@ export function PythonLibraryEditorPage({
<Card withBorder p={0}>
<PythonEditor
type={type}
value={source}
onChange={setSource}
withPandasCompletions
className={S.editor}
data-testid="python-editor"
readOnly={isRemoteSyncReadOnly}

View File

@@ -10,25 +10,25 @@ import { useSetting } from "metabase/common/hooks";
export function PythonRunnerSettingsPage() {
const isHosted = useSetting("is-hosted?");
// Python Runner settings are managed by Metabase Cloud for hosted instances
// Advanced Transform Runner settings are managed by Metabase Cloud for hosted instances
if (isHosted) {
return null;
}
return (
<SettingsPageWrapper title={t`Python Runner`}>
<SettingsPageWrapper title={t`Advanced Transform Runner`}>
<SettingsSection title={t`Service Configuration`}>
<AdminSettingInput
name="python-runner-url"
title={t`Python Execution Server URL`}
description={t`URL for the Python execution server that runs transform functions.`}
title={t`Advanced Transform Execution Server URL`}
description={t`URL for the server that runs advanced transform functions.`}
placeholder="http://localhost:5001"
inputType="text"
/>
<AdminSettingInput
name="python-runner-timeout-seconds"
title={t`Python Script Execution Timeout`}
description={t`Timeout in seconds for Python script execution. Defaults to 30 minutes (1800 seconds).`}
title={t`Advanced Transform Execution Timeout`}
description={t`Timeout in seconds for advanced transform execution. Defaults to 30 minutes (1800 seconds).`}
placeholder="1800"
inputType="number"
/>
@@ -37,7 +37,7 @@ export function PythonRunnerSettingsPage() {
<AdminSettingInput
name="python-storage-s-3-endpoint"
title={t`S3 Endpoint`}
description={t`S3 endpoint URL for storing Python execution artifacts.`}
description={t`S3 endpoint URL for storing advanced transform execution artifacts.`}
inputType="text"
/>
<AdminSettingInput
@@ -49,7 +49,7 @@ export function PythonRunnerSettingsPage() {
<AdminSettingInput
name="python-storage-s-3-bucket"
title={t`S3 Bucket`}
description={t`S3 bucket name for storing Python execution artifacts.`}
description={t`S3 bucket name for storing advanced transform execution artifacts.`}
inputType="text"
/>
<AdminSettingInput

View File

@@ -1,7 +1,7 @@
import { Route } from "react-router";
import { Redirect, Route } from "react-router";
import { ModalRoute } from "metabase/hoc/ModalRoute";
import { NewPythonTransformPage } from "metabase/transforms/pages/NewTransformPage";
import { NewAdvancedTransformPage } from "metabase/transforms/pages/NewTransformPage";
import { TransformListPage } from "metabase/transforms/pages/TransformListPage";
import { PythonLibraryEditorPage } from "./pages/PythonLibraryEditorPage";
@@ -11,7 +11,9 @@ export function getPythonTransformsRoutes() {
return (
<>
<Route path="library/:path" component={PythonLibraryEditorPage} />
<Route path="new/python" component={NewPythonTransformPage} />
<Redirect from="new/python" to="new/advanced/python" />
<Redirect from="new/advanced" to="new/advanced/python" />
<Route path="new/advanced/:type" component={NewAdvancedTransformPage} />
</>
);
}
@@ -25,8 +27,9 @@ export function getPythonUpsellRoutes() {
modal={PythonTransformsUpsellModal}
noWrap
/>
<Redirect from="new/python" to="new/advanced/python" />
<ModalRoute
path="new/python"
path="new/advanced/:type"
modal={PythonTransformsUpsellModal}
noWrap
/>

View File

@@ -10,13 +10,13 @@ export function getPythonSourceValidationResult(
source: PythonTransformSourceDraft,
): PythonTransformSourceValidationResult {
if (!source["source-database"]) {
return { isValid: false, errorMessage: t`Select a source a database` };
return { isValid: false, errorMessage: t`Select a source database` };
}
if (source.body.trim() === "") {
return {
isValid: false,
errorMessage: t`The Python script cannot be empty`,
errorMessage: t`The transform cannot be empty`,
};
}
@@ -33,5 +33,5 @@ export function getPythonSourceValidationResult(
export function isPythonTransformSource(
source: PythonTransformSourceDraft,
): source is PythonTransformSource {
return source.type === "python" && source["source-database"] !== undefined;
return source["source-database"] !== undefined;
}

View File

@@ -9,6 +9,7 @@ import type {
ConcreteTableId,
Engine,
RelatedDashboardXRays,
Transform,
TransformId,
VisualizationDisplay,
} from "metabase-types/api";
@@ -292,12 +293,13 @@ export type TransformJobTriggerManualRunEvent = ValidateEvent<{
export type TransformCreateEvent = ValidateEvent<{
event: "transform_create";
event_detail: "query" | "native" | "python" | "saved-question";
event_detail: "query" | "native" | "advanced" | "saved-question";
}>;
export type TransformCreatedEvent = ValidateEvent<{
event: "transform_created";
target_id: number;
event_detail: Transform["source_type"];
}>;
export type TransformRunTagsUpdated = ValidateEvent<{

View File

@@ -23,12 +23,31 @@ export type TransformOwner = Pick<
"id" | "email" | "first_name" | "last_name"
>;
export const ADVANCED_TRANSFORM_TYPES = {
python: {
displayName: "Python",
commentString: "#",
},
javascript: {
displayName: "JavaScript",
commentString: "//",
},
} satisfies Record<string, { displayName: string; commentString: string }>;
export type AdvancedTransformType = keyof typeof ADVANCED_TRANSFORM_TYPES;
export function isAdvancedTransformType(
type: string,
): type is AdvancedTransformType {
return type in ADVANCED_TRANSFORM_TYPES;
}
export type Transform = {
id: TransformId;
name: string;
description: string | null;
source: TransformSource;
source_type: "native" | "python" | "mbql";
source_type: "native" | "mbql" | AdvancedTransformType;
target: TransformTarget;
collection_id: CollectionId | null;
created_at: string;
@@ -70,14 +89,14 @@ export type TransformSourceCheckpointStrategy = {
export type SourceIncrementalStrategy = TransformSourceCheckpointStrategy;
export type PythonTransformSourceDraft = {
type: "python";
type: AdvancedTransformType;
body: string;
"source-database": DatabaseId | undefined;
"source-tables": PythonTransformTableAliases;
};
export type PythonTransformSource = {
type: "python";
type: AdvancedTransformType;
body: string;
"source-database": DatabaseId;
"source-tables": PythonTransformTableAliases;
@@ -96,9 +115,16 @@ export type TransformTargetAppendStrategy = {
type: "append";
};
export type DraftTransformSource =
| Transform["source"]
| QueryTransformSource
| PythonTransformSource
| PythonTransformSourceDraft;
export function isAdvancedTransformSource(
source: DraftTransformSource,
): source is PythonTransformSource | PythonTransformSourceDraft {
return isAdvancedTransformType(source.type);
}
export type DraftTransform = Partial<
Pick<Transform, "id" | "name" | "description" | "target">
> & { source: DraftTransformSource };
@@ -266,6 +292,7 @@ export type ListTransformRunsResponse = {
} & PaginationResponse;
export type TestPythonTransformRequest = {
type: AdvancedTransformType;
code: string;
source_tables: PythonTransformTableAliases;
};
@@ -286,11 +313,13 @@ export type PythonLibrary = {
export type GetPythonLibraryRequest = {
path: string;
type: AdvancedTransformType;
};
export type UpdatePythonLibraryRequest = {
path: string;
source: string;
type: AdvancedTransformType;
};
export type ExtractColumnsFromQueryRequest = {

View File

@@ -1,4 +1,5 @@
import type {
AdvancedTransformType,
CardId,
CollectionId,
DatabaseId,
@@ -45,12 +46,8 @@ export function newNativeTransform() {
return `${TRANSFORMS_ROOT_URL}/new/native`;
}
export function newPythonTransform() {
return `${TRANSFORMS_ROOT_URL}/new/python`;
}
export function pythonTransformsUpsell() {
return `${TRANSFORMS_ROOT_URL}/python-transforms`;
export function newAdvancedTransform(type: AdvancedTransformType = "python") {
return `${TRANSFORMS_ROOT_URL}/new/advanced/${type}`;
}
export function newTransformFromCard(cardId: CardId) {

View File

@@ -8,6 +8,7 @@ import {
} from "metabase/plugins/components/PluginPlaceholder";
import type Question from "metabase-lib/v1/Question";
import type {
AdvancedTransformType,
CheckDependenciesResponse,
GetDependencyGraphRequest,
PythonTransformSourceDraft,
@@ -68,7 +69,7 @@ export type PythonTransformsPlugin = {
PythonRunnerSettingsPage: ComponentType;
getAdminRoutes: () => ReactNode;
getTransformsNavLinks: () => ReactNode;
sharedLibImportPath: string;
sharedLibImportPaths: Record<AdvancedTransformType, string>;
};
type DependenciesPlugin = {
@@ -154,7 +155,7 @@ const getDefaultPluginTransformsPython = (): PythonTransformsPlugin => ({
PythonRunnerSettingsPage: NotFoundPlaceholder,
getAdminRoutes: () => null,
getTransformsNavLinks: () => null,
sharedLibImportPath: "",
sharedLibImportPaths: {} as Record<AdvancedTransformType, string>,
});
export const PLUGIN_TRANSFORMS_PYTHON = getDefaultPluginTransformsPython();

View File

@@ -2,6 +2,7 @@ import { trackSimpleEvent } from "metabase/lib/analytics";
import type {
InspectorCardId,
InspectorLensId,
Transform,
TransformId,
TransformJobId,
} from "metabase-types/api";
@@ -33,7 +34,7 @@ export function trackTransformJobTriggerManualRun({
export function trackTransformCreate({
creationType,
}: {
creationType: "query" | "native" | "python" | "saved-question";
creationType: "query" | "native" | "advanced" | "saved-question";
}) {
trackSimpleEvent({
event: "transform_create",
@@ -41,14 +42,11 @@ export function trackTransformCreate({
});
}
export function trackTransformCreated({
transformId,
}: {
transformId: TransformId;
}) {
export function trackTransformCreated({ transform }: { transform: Transform }) {
trackSimpleEvent({
event: "transform_created",
target_id: transformId,
target_id: transform.id,
event_detail: transform.source_type,
});
}

View File

@@ -10,6 +10,7 @@ import { useDispatch, useSelector } from "metabase/lib/redux";
import * as Urls from "metabase/lib/urls";
import { getShouldShowPythonTransformsUpsell } from "metabase/transforms/selectors";
import { Button, Center, Icon, Loader, Menu, Tooltip } from "metabase/ui";
import type { AdvancedTransformType } from "metabase-types/api";
import { trackTransformCreate } from "../../analytics";
import { CreateTransformCollectionModal } from "../CreateTransformCollectionModal";
@@ -33,14 +34,14 @@ export const CreateTransformMenu = () => {
const { data: databases, isLoading } = useListDatabasesQuery({
include_analytics: true,
});
const shouldShowPythonScriptOption =
const shouldShowAdvancedTransformsOptions =
hasPythonTransformsFeature || shouldShowPythonTransformsUpsell;
const handlePythonClick = () => {
dispatch(push(Urls.newPythonTransform())); // Route will show upsell modal if feature is not enabled
const navigateToNewAdvancedTransform = (type: AdvancedTransformType) => {
dispatch(push(Urls.newAdvancedTransform(type))); // Route will show upsell modal if feature is not enabled
if (hasPythonTransformsFeature) {
trackTransformCreate({ creationType: "python" });
trackTransformCreate({ creationType: "advanced" });
}
};
@@ -63,6 +64,42 @@ export const CreateTransformMenu = () => {
) : (
<>
<Menu.Label>{t`Create your transform with…`}</Menu.Label>
<Menu.Item
leftSection={<Icon name="sql" />}
onClick={() => {
trackTransformCreate({ creationType: "native" });
dispatch(push(Urls.newNativeTransform()));
}}
>
{t`SQL`}
</Menu.Item>
{shouldShowAdvancedTransformsOptions && (
<>
<Menu.Item
leftSection={<Icon name="code_block" />}
rightSection={
!hasPythonTransformsFeature ? (
<UpsellGem size={14} />
) : null
}
onClick={() => navigateToNewAdvancedTransform("python")}
>
{t`Python`}
</Menu.Item>
<Menu.Item
leftSection={<Icon name="code_block" />}
rightSection={
!hasPythonTransformsFeature ? (
<UpsellGem size={14} />
) : null
}
onClick={() => navigateToNewAdvancedTransform("javascript")}
>
{t`JavaScript`}
</Menu.Item>
</>
)}
<Menu.Item
leftSection={<Icon name="notebook" />}
onClick={() => {
@@ -72,27 +109,6 @@ export const CreateTransformMenu = () => {
>
{t`Query builder`}
</Menu.Item>
<Menu.Item
leftSection={<Icon name="sql" />}
onClick={() => {
trackTransformCreate({ creationType: "native" });
dispatch(push(Urls.newNativeTransform()));
}}
>
{t`SQL query`}
</Menu.Item>
{shouldShowPythonScriptOption && (
<Menu.Item
leftSection={<Icon name="code_block" />}
rightSection={
!hasPythonTransformsFeature ? <UpsellGem size={14} /> : null
}
onClick={handlePythonClick}
>
{t`Python script`}
</Menu.Item>
)}
<Menu.Item
leftSection={<Icon name="insight" />}
onClick={() => {

View File

@@ -24,7 +24,11 @@ import {
Tooltip,
} from "metabase/ui";
import type * as Lib from "metabase-lib";
import type { TransformSource } from "metabase-types/api";
import {
type AdvancedTransformType,
type TransformSource,
isAdvancedTransformSource,
} from "metabase-types/api";
import {
MBQLKeysetColumnSelect,
@@ -56,7 +60,7 @@ export const IncrementalTransformSettings = ({
// Check if this is a Python transform with exactly one source table
// Incremental transforms are only supported for single-table Python transforms
const isPythonTransform = source.type === "python";
const isPythonTransform = isAdvancedTransformSource(source);
const isMultiTablePythonTransform =
isPythonTransform && Object.keys(source["source-tables"]).length > 1;
@@ -65,7 +69,7 @@ export const IncrementalTransformSettings = ({
isPythonTransform,
})
.with({ isMbqlQuery: true }, () => "query" as const)
.with({ isPythonTransform: true }, () => "python" as const)
.with({ isPythonTransform: true }, () => source.type)
.otherwise(() => "native" as const);
const { url: incrementalTransformsDocsUrl, showMetabaseLinks } = useDocsUrl(
isPythonTransform
@@ -205,7 +209,7 @@ function TargetStrategyFields({
type SourceStrategyFieldsProps = {
source: TransformSource;
query: Lib.Query | null;
type: "query" | "native" | "python";
type: "query" | "native" | AdvancedTransformType;
readOnly?: boolean;
};
@@ -252,7 +256,7 @@ function SourceStrategyFields({
disabled={readOnly}
/>
)}
{type === "python" && "source-tables" in source && (
{isAdvancedTransformSource(source) && "source-tables" in source && (
<PythonKeysetColumnSelect
name="checkpointFilterUniqueKey"
label={t`Field to check for new values`}

View File

@@ -88,12 +88,8 @@ export function useSourceState({
}, [source, suggestedTransform, metadata]);
const isDirty = useMemo(() => {
return (
transformId == null ||
proposedSource != null ||
!isSameSource(source, initialSource)
);
}, [source, initialSource, proposedSource, transformId]);
return proposedSource != null || !isSameSource(source, initialSource);
}, [source, initialSource, proposedSource]);
const setSourceAndRejectProposed = (source: DraftTransformSource) => {
if (suggestedTransform != null) {

View File

@@ -32,7 +32,9 @@ export const useCreateTransform = (
databaseId,
);
const transform = await createTransformMutation(request).unwrap();
trackTransformCreated({ transformId: transform.id });
trackTransformCreated({
transform,
});
return transform;
};

View File

@@ -22,10 +22,14 @@ import { getMetadata } from "metabase/selectors/metadata";
import { useTransformPermissions } from "metabase/transforms/hooks/use-transform-permissions";
import { Box, Center } from "metabase/ui";
import * as Lib from "metabase-lib";
import type {
Database,
DraftTransformSource,
Transform,
import {
type AdvancedTransformType,
type Database,
type DraftTransformSource,
type PythonTransformSourceDraft,
type Transform,
isAdvancedTransformSource,
isAdvancedTransformType,
} from "metabase-types/api";
import { TransformEditor } from "../../components/TransformEditor";
@@ -38,6 +42,7 @@ import { CreateTransformModal } from "./CreateTransformModal";
import {
getDefaultValues,
getInitialCardSource,
getInitialJavascriptSource,
getInitialNativeSource,
getInitialPythonSource,
getInitialQuerySource,
@@ -156,7 +161,7 @@ function NewTransformPageBody({
overflow: "hidden",
}}
>
{source.type === "python" ? (
{isAdvancedTransformSource(source) ? (
<PLUGIN_TRANSFORMS_PYTHON.TransformEditor
source={source}
proposedSource={
@@ -219,15 +224,38 @@ export function NewNativeTransformPage({ route }: NewNativeTransformPageProps) {
return <NewTransformPage initialSource={initialSource} route={route} />;
}
type NewPythonTransformPageProps = {
type NewAdvancedTransformPageParams = {
type?: string;
};
type NewAdvancedTransformPageProps = {
params: NewAdvancedTransformPageParams;
route: Route;
};
export function NewPythonTransformPage({ route }: NewPythonTransformPageProps) {
const initialSource = useMemo(() => getInitialPythonSource(), []);
return <NewTransformPage initialSource={initialSource} route={route} />;
export function NewAdvancedTransformPage({
params,
route,
}: NewAdvancedTransformPageProps) {
const typeParam = params?.type || "";
const type: AdvancedTransformType = isAdvancedTransformType(typeParam)
? typeParam
: "python";
const initialSource = useMemo(() => sourceFunctionMap[type](), [type]);
return (
<NewTransformPage initialSource={initialSource} route={route} key={type} />
);
}
const sourceFunctionMap: Record<
AdvancedTransformType,
() => PythonTransformSourceDraft
> = {
javascript: getInitialJavascriptSource,
python: getInitialPythonSource,
};
type NewCardTransformPageParams = {
cardId: string;
};

View File

@@ -47,6 +47,31 @@ def transform():
};
}
export function getInitialJavascriptSource(): PythonTransformSourceDraft {
return {
type: "javascript",
"source-database": undefined,
"source-tables": {},
body: `// Write your Javascript transformation script here
import * as common from "./common.js";
export function transform() {
/**
* Your transformation function.
*
* Select tables above to add them as function parameters.
*
* Returns:
* Object array to write to the destination table
*/
// Your transformation logic here
return [{"message": "Hello from Javascript transform!"}];
}`,
};
}
export function getInitialCardSource(card: Card): QueryTransformSource {
return { type: "query", query: card.dataset_query };
}

View File

@@ -145,20 +145,39 @@ export const TransformListPage = ({
const data = buildTreeData(collections, transforms);
// Only show Python library item if there's at least one item in the table
// It will trigger the upsell modal if the feature isn't enabled.
const shouldShowPythonLibraryRow =
const shouldShowCodeLibraryRow =
data.length > 0 &&
(hasPythonTransformsFeature || shouldShowPythonTransformsUpsell);
if (shouldShowPythonLibraryRow) {
if (shouldShowCodeLibraryRow) {
data.push({
id: "library",
name: t`Python library`,
nodeType: "library",
icon: "snippet",
url: Urls.transformPythonLibrary({
path: PLUGIN_TRANSFORMS_PYTHON.sharedLibImportPath,
}),
id: "code-library",
name: t`Code library`,
nodeType: "folder",
icon: "folder",
source_readable: transformsDatabases.length > 0,
children: [
{
id: "python-library",
name: t`Python library`,
nodeType: "library",
icon: "snippet",
url: Urls.transformPythonLibrary({
path: PLUGIN_TRANSFORMS_PYTHON.sharedLibImportPaths["python"],
}),
source_readable: transformsDatabases.length > 0,
},
{
id: "javascript-library",
name: t`JavaScript library`,
nodeType: "library",
icon: "snippet",
url: Urls.transformPythonLibrary({
path: PLUGIN_TRANSFORMS_PYTHON.sharedLibImportPaths["javascript"],
}),
source_readable: transformsDatabases.length > 0,
},
],
});
}
return data;

View File

@@ -7,7 +7,11 @@ import { getMetadata } from "metabase/selectors/metadata";
import { EditDefinitionButton } from "metabase/transforms/components/TransformEditor/EditDefinitionButton";
import { getValidationResult } from "metabase/transforms/utils";
import * as Lib from "metabase-lib";
import type { DraftTransformSource, Transform } from "metabase-types/api";
import {
type DraftTransformSource,
type Transform,
isAdvancedTransformType,
} from "metabase-types/api";
type Props = {
handleCancel: VoidFunction;
@@ -52,7 +56,7 @@ export const TransformPaneHeaderActions = (props: Props) => {
isNative: false,
};
}, [source, metadata]);
const isPythonTransform = source.type === "python";
const isPythonTransform = isAdvancedTransformType(source.type);
if (
!readOnly &&

View File

@@ -25,11 +25,12 @@ import {
import { getInitialUiState } from "metabase/querying/editor/components/QueryEditor";
import { useTransformPermissions } from "metabase/transforms/hooks/use-transform-permissions";
import { Box, Center, Group, Icon } from "metabase/ui";
import type {
Database,
DatasetQuery,
DraftTransformSource,
Transform,
import {
type Database,
type DatasetQuery,
type DraftTransformSource,
type Transform,
isAdvancedTransformSource,
} from "metabase-types/api";
import { useQueryComplexityChecks } from "../../components/QueryComplexityWarning";
@@ -233,7 +234,7 @@ function TransformQueryPageBody({
illustrationElement={<Icon name="key" size={100} />}
/>
</Center>
) : source.type === "python" ? (
) : isAdvancedTransformSource(source) ? (
<PLUGIN_TRANSFORMS_PYTHON.TransformEditor
source={source}
proposedSource={
@@ -322,7 +323,7 @@ export function TransformQueryPageEditor({
onRunTransform,
onRun,
}: TransformQueryPageEditorProps) {
return source.type === "python" ? (
return isAdvancedTransformSource(source) ? (
<PLUGIN_TRANSFORMS_PYTHON.TransformEditor
source={source}
uiOptions={uiOptions}

View File

@@ -6,17 +6,20 @@ import type { OmniPickerCollectionItem } from "metabase/common/components/Picker
import { parseTimestamp } from "metabase/lib/time-dayjs";
import * as Lib from "metabase-lib";
import type Metadata from "metabase-lib/v1/metadata/Metadata";
import type {
CollectionNamespace,
Database,
DatabaseId,
DraftTransformSource,
TemplateTag,
Transform,
TransformRun,
TransformRunMethod,
TransformRunStatus,
TransformSource,
import {
type CollectionNamespace,
type Database,
type DatabaseId,
type DraftTransformSource,
type PythonTransformSourceDraft,
type TemplateTag,
type Transform,
type TransformRun,
type TransformRunMethod,
type TransformRunStatus,
type TransformSource,
isAdvancedTransformSource,
isAdvancedTransformType,
} from "metabase-types/api";
import { CHECKPOINT_TEMPLATE_TAG } from "./constants";
@@ -83,7 +86,7 @@ export function sourceDatabaseId(source: TransformSource): DatabaseId | null {
return source.query.database;
}
if (source.type === "python") {
if (isAdvancedTransformType(source.type)) {
return source["source-database"];
}
@@ -134,8 +137,15 @@ export function isSameSource(
if (source1.type === "query" && source2.type === "query") {
return Lib.areLegacyQueriesEqual(source1.query, source2.query);
}
if (source1.type === "python" && source2.type === "python") {
return _.isEqual(source1, source2);
if (
isAdvancedTransformSource(source1) &&
isAdvancedTransformSource(source2)
) {
// source-database is not set as part of the initialSource, so we omit it from the comparison
return _.isEqual(
_.omit(source1, "source-database"),
_.omit(source2, "source-database"),
);
}
return false;
}
@@ -164,7 +174,10 @@ export function isSourceEmpty(
export function isCompleteSource(
source: DraftTransformSource,
): source is TransformSource {
return source.type !== "python" || source["source-database"] != null;
return (
!isAdvancedTransformType(source.type) ||
(source as PythonTransformSourceDraft)["source-database"] != null
);
}
export type ValidationResult = {

View File

@@ -3574,6 +3574,77 @@ databaseChangeLog:
path: instance_analytics_views/query_log/v2/h2-query_log.sql
relativeToChangelogFile: true
- changeSet:
id: v59.2026-02-24T12:00:00
author: galdre
comment: Rename python_library to transform_library for unified multi-language library support
changes:
- renameTable:
oldTableName: python_library
newTableName: transform_library
- changeSet:
id: v59.2026-02-24T12:00:01
author: galdre
comment: Add language column to transform_library (existing rows default to python)
changes:
- addColumn:
tableName: transform_library
columns:
- column:
name: language
type: varchar(32)
remarks: Language identifier (python, javascript, r, clojure, julia)
defaultValue: python
constraints:
nullable: false
rollback:
- dropColumn:
tableName: transform_library
columnName: language
- changeSet:
id: v59.2026-02-24T12:00:02
author: galdre
comment: Drop old path-only unique constraint (replaced by language+path constraint)
changes:
- dropUniqueConstraint:
tableName: transform_library
constraintName: uk_python_library_path
rollback:
- addUniqueConstraint:
tableName: transform_library
columnNames: path
constraintName: uk_python_library_path
- changeSet:
id: v59.2026-02-24T12:01:00
author: galdre
comment: Add (language, path) unique constraint to transform_library
changes:
- addUniqueConstraint:
tableName: transform_library
columnNames: language, path
constraintName: uk_transform_library_language_path
rollback:
- dropUniqueConstraint:
tableName: transform_library
constraintName: uk_transform_library_language_path
- changeSet:
id: v59.2026-02-24T12:02:00
author: galdre
comment: Insert default common.js entry in transform_library
preConditions:
- onFail: MARK_RAN
- sqlCheck:
expectedResult: 0
sql: SELECT count(*) FROM transform_library WHERE language = 'javascript' AND path = 'common.js'
changes:
- sql:
sql: INSERT INTO transform_library (path, source, entity_id, language) VALUES ('common.js', '', 'aHWo_0yPLwKqpQPlFNzCg', 'javascript')
rollback:
# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
########################################################################################################################

View File

@@ -165,7 +165,8 @@
:TransformJob []
[:TransformJob :job_tags] []
:TransformTag []
:PythonLibrary [:path
:TransformLibrary [:language
:path
:source
:entity_id
:created_at

View File

@@ -74,7 +74,7 @@
:model/PermissionsRevision metabase.permissions.models.permissions-revision
:model/PersistedInfo metabase.model-persistence.models.persisted-info
:model/PremiumFeaturesCache metabase.premium-features.models.premium-features-cache
:model/PythonLibrary metabase-enterprise.transforms-python.models.python-library
:model/TransformLibrary metabase-enterprise.transforms-runner.models.transform-library
:model/Pulse metabase.pulse.models.pulse
:model/PulseCard metabase.pulse.models.pulse-card
:model/PulseChannel metabase.pulse.models.pulse-channel

Some files were not shown because too many files have changed in this diff Show More