Compare commits

...

206 Commits

Author SHA1 Message Date
Chris Truter
e481c02476 Crud and editing working 2025-12-10 04:00:04 +02:00
Chris Truter
06dd052d57 Add default mock values for WorkspaceTransform 2025-12-10 02:36:23 +02:00
Chris Truter
a85d113127 Simplifying 2025-12-10 01:16:37 +02:00
Chris Truter
3c5de95c17 Enforce workspace name uniqueness 2025-12-10 01:07:33 +02:00
Chris Truter
98716ba649 Cleaning up 2025-12-10 01:06:37 +02:00
Chris Truter
b457d5ebaa Don't be evil 2025-12-09 23:06:39 +02:00
Chris Truter
32b4d0ae3c WIP 2025-12-09 23:05:24 +02:00
Chris Truter
bd66d4c246 Some cleaning 2025-12-09 23:05:24 +02:00
Chris Truter
9ce4259dff WIP 2025-12-09 23:05:24 +02:00
Chris Truter
6e8eadd2a0 wip 2025-12-09 23:05:24 +02:00
Chris Truter
ba279bfc29 wip 2025-12-09 23:05:24 +02:00
Chris Truter
49a00dea09 Strip isolated schemas out of target picker 2025-12-09 22:35:50 +02:00
Chris Truter
ce44815e8e Implement basic workspace execution, for simple graphs 2025-12-09 17:53:59 +02:00
Chris Truter
a4c37283fc Fix types of stub /execute data 2025-12-09 17:41:43 +02:00
Kamil Mielnik
d191b6c0a1 Assert merging workspace 2025-12-09 22:28:37 +07:00
Uladzimir Havenchyk
0cfd5936c5 add bulk transfrom run 2025-12-09 18:27:40 +03:00
Uladzimir Havenchyk
573ba81c1e move workspace modal 2025-12-09 17:34:31 +03:00
Uladzimir Havenchyk
de0616bd90 simplify change target modal logic 2025-12-09 16:50:50 +03:00
Kamil Mielnik
8f7fa76f87 Add transform checkout test 2025-12-09 20:04:33 +07:00
Kamil Mielnik
7f9886044a Fix closing tabs 2025-12-09 19:31:10 +07:00
Kamil Mielnik
becf8db5df Test renaming workspaces 2025-12-09 19:07:51 +07:00
Kamil Mielnik
c78784afef Test archiving 2025-12-09 18:59:39 +07:00
Kamil Mielnik
ac2c41b83f Add 1st e2e test for workspaces + helpers boilerplate + data-testid 2025-12-09 18:44:07 +07:00
Oleksandr Solovyov
c38c69059e fix results of POST /ee/workspace/:id/transform not appearing in contents 2025-12-09 12:37:18 +02:00
Oleksandr Solovyov
f83890077a copy transforms apis inside workspaces 2025-12-08 18:29:18 +02:00
Kamil Mielnik
a5922d9323 Add tooltip with error message 2025-12-08 21:49:59 +07:00
Uladzimir Havenchyk
866dbe4be9 metabot: add db into context and use conversation snapshot per workspace 2025-12-08 16:33:56 +03:00
Kamil Mielnik
bd3c618303 Avoid showing status of other workspaces 2025-12-08 20:21:04 +07:00
Kamil Mielnik
ffbe648b7b Add setup log 2025-12-08 20:15:49 +07:00
Ngoc Khuat
e080d9e1ff fix not savign the database details 2025-12-08 16:36:35 +07:00
Oleksandr Solovyov
af88617937 async api for workspace creation (#66579) 2025-12-08 11:02:03 +02:00
Ngoc Khuat
7bff4573e3 # is unwanted 2025-12-08 15:11:54 +07:00
Uladzimir Havenchyk
098370174a metabot in workspaces - alpha version 2025-12-05 23:13:46 +03:00
Kamil Mielnik
14facfdbb7 Move change target button 2025-12-05 20:37:25 +07:00
Kamil Mielnik
ddc9c7c6c3 Fix clicking Metabot tab (first time) opens Setup tab
- Fixes GDGT-1355
2025-12-05 20:24:16 +07:00
Kamil Mielnik
63f4dcbf7c Fix metabot overflow
- Fixes GDGT-1354
2025-12-05 20:14:36 +07:00
Kamil Mielnik
7f35c17f3e Remove workspace_id attribute from Transform
- in CodeTab the usage was redundant
  - GET /api/ee/transform will not return transforms that belong to a workspace
- in EditTransformMenu the usage was redundant
  - you can't open a transform that belongs to a workspace in this page
- in TransformTab the usage can be replaced by looking up workspace contents
- Closes GDGT-1349
2025-12-05 19:27:56 +07:00
lbrdnk
4f98f13971 Hack to make transforms exec work until there is ws specific api 2025-12-05 12:49:22 +01:00
Ngoc Khuat
651918f3e8 remove duplicated schema database_id 2025-12-05 17:41:05 +07:00
Stas Gavrylov
59c652d537 Add sorting for tabs 2025-12-05 11:34:02 +01:00
Ngoc Khuat
bc3687c4a5 use admin connection for h2 test-data 2025-12-05 16:03:37 +07:00
Ngoc Khuat
05bb1daa42 fix a few transforms api and workspace api tests 2025-12-05 15:52:38 +07:00
Ngoc Khuat
6b6d55cd22 fix serdes tests 2025-12-05 15:31:51 +07:00
Stas Gavrylov
1361b9bda0 Merge branch 'workspaces-merge-tabs' into workspaces-2025-11 2025-12-05 09:26:38 +01:00
Ngoc Khuat
6644f846e8 there are too many migrations on fk_collection_workspace_id, anyway this attempts to fix the cross branch migration CI 2025-12-05 15:20:47 +07:00
Stas Gavrylov
98b5096eda Refactor tabs logic to keep a single array, allowing better UX 2025-12-05 09:19:04 +01:00
Uladzimir Havenchyk
0cc951e1a4 replace target input with change target modal 2025-12-04 23:58:02 +03:00
Ngoc Khuat
24910610c4 fix dag cte for for h2, h2 is stricter and requires column declaration
for h2
2025-12-04 23:32:30 +07:00
Stas Gavrylov
12fd210a41 Squashed commit of the following:
commit 303e05ec59
Author: Stas Gavrylov <stasgavrylov@gmail.com>
Date:   Thu Dec 4 17:25:58 2025 +0100

    merge main branch

commit a67169c5e5
Author: Stas Gavrylov <stasgavrylov@gmail.com>
Date:   Thu Dec 4 16:15:35 2025 +0100

    Cleanup. Add tables refetching on transform run

commit 14e7b758f4
Author: lbrdnk <lbrdnk@users.noreply.github.com>
Date:   Thu Dec 4 14:51:28 2025 +0100

    input tables adj

commit a0feade8ae
Author: lbrdnk <lbrdnk@users.noreply.github.com>
Date:   Thu Dec 4 14:46:11 2025 +0100

    /tables no graph, alt impl

commit 3509629d84
Author: Stas Gavrylov <stasgavrylov@gmail.com>
Date:   Thu Dec 4 11:45:56 2025 +0100

    remove questionWithMetadata

commit bccfd459c5
Author: Stas Gavrylov <stasgavrylov@gmail.com>
Date:   Thu Dec 4 11:39:21 2025 +0100

    wip4

commit f733db75b1
Author: Stas Gavrylov <stasgavrylov@gmail.com>
Date:   Wed Dec 3 16:59:42 2025 +0100

    wip3

commit 4bc044c577
Author: Stas Gavrylov <stasgavrylov@gmail.com>
Date:   Wed Dec 3 16:37:26 2025 +0100

    wip2

commit 2ca514359c
Author: Stas Gavrylov <stasgavrylov@gmail.com>
Date:   Wed Dec 3 16:22:29 2025 +0100

    wip

commit 6a29d09d17
Author: Stas Gavrylov <stasgavrylov@gmail.com>
Date:   Wed Dec 3 13:20:52 2025 +0100

    wip

commit 15c9599bfa
Author: Stas Gavrylov <stasgavrylov@gmail.com>
Date:   Wed Dec 3 11:45:01 2025 +0100

    Refactor edits detection logic

commit d1ac164c32
Author: Stas Gavrylov <stasgavrylov@gmail.com>
Date:   Tue Dec 2 16:11:02 2025 +0100

    Add Data tab with a list of input and output tables
2025-12-04 17:26:55 +01:00
Kamil Mielnik
7484249328 Fix - Changing transform output table always shows an error
- Fixes GDGT-1344
- Simplifies useTransformValidation
2025-12-04 22:00:23 +07:00
Oleksandr Solovyov
d034244735 expose upstream transform details to fe 2025-12-04 15:58:51 +02:00
Ngoc Khuat
3b837f7593 Isolation v2 (#66374) 2025-12-04 20:49:19 +07:00
Ngoc Khuat
36419de6e7 swapped pool supports mutli pools of the same db (#66498) 2025-12-04 15:06:43 +02:00
Kamil Mielnik
b488c1b082 Fix state management issue (changing output table resets source tables and code inputs)
- Fixes GDGT-1328
2025-12-04 19:40:19 +07:00
Kamil Mielnik
869cfcc024 Improve types 2025-12-04 19:39:55 +07:00
Kamil Mielnik
81a7572e0a Fix endpoint usage 2025-12-04 19:11:52 +07:00
Kamil Mielnik
2ff73cd864 Update validate-target endpoint mapping 2025-12-04 04:07:37 -08:00
Uladzimir Havenchyk
3ebdecc2d2 add change target modal to workspace transform 2025-12-04 08:19:44 +03:00
Oleksandr Solovyov
f0bfd6898f transform filtering (#66411) 2025-12-03 17:30:19 +02:00
Oleksandr Solovyov
1b6528c9f1 don't import unexported stuff 2025-12-03 17:28:35 +02:00
Oleksandr Solovyov
f8f2e235f6 very sorry 2025-12-03 17:15:32 +02:00
Oleksandr Solovyov
9086aa10e9 more useful validate-target 2025-12-03 17:14:19 +02:00
Uladzimir Havenchyk
cfa8c389b2 add transform edit menu 2025-12-02 23:45:47 +03:00
Uladzimir Havenchyk
c670809704 remove unused files 2025-12-02 23:45:47 +03:00
lbrdnk
1f78793830 Add remove add remove test 2025-12-02 19:50:41 +01:00
lbrdnk
4e342e8481 Tables docstring 2025-12-02 19:50:41 +01:00
lbrdnk
b61455bb53 /tables, empty handling 2025-12-02 19:50:41 +01:00
lbrdnk
18c743403e /tables endpoint tests 2025-12-02 19:50:40 +01:00
Stas Gavrylov
b1fd062915 Add ability to rename workspace 2025-12-02 16:55:14 +01:00
Chris Truter
15b446ab04 Remove inline config that's breaking cljfmt 2025-12-02 17:39:23 +02:00
Oleksandr Solovyov
0f61a79a6b make merge work if there were no remapped tables (#66467) 2025-12-02 17:30:51 +02:00
lbrdnk
9ebbf353f6 unblock fe ignoring brokne graph 2025-12-02 16:12:26 +01:00
Stas Gavrylov
be25ff92a1 Fix docs 2025-12-02 15:38:18 +01:00
Kamil Mielnik
11ef81fdee Autofocus 2025-12-02 21:33:04 +07:00
Kamil Mielnik
ce08daf26f Use form in new workspace modal 2025-12-02 21:32:29 +07:00
Kamil Mielnik
e2b5e9dd9d Empty state 2025-12-02 21:29:52 +07:00
lbrdnk
49bbcf7467 Avoid calling delete with nil ids 2025-12-02 15:20:25 +01:00
Kamil Mielnik
757cca20db Remove transform from workspace
- Closes GDGT-1319
2025-12-02 21:19:02 +07:00
Oleksandr Solovyov
b196d10ad3 fix error reporting so that tests are passing 2025-12-02 15:52:24 +02:00
Kamil Mielnik
3e92c2d2ad Disable "Merge" button when there are no transforms in the workspace
- Closes GDGT-1326
2025-12-02 20:30:45 +07:00
lbrdnk
f1d495dfde whitespace 2025-12-02 14:30:31 +01:00
lbrdnk
a14b9c1b3b Add support for transform deletion 2025-12-02 14:28:33 +01:00
Kamil Mielnik
6cffef70f5 Fix alignment 2025-12-02 20:15:51 +07:00
Kamil Mielnik
a5a8fa1f1e Fix crash 2025-12-02 20:14:47 +07:00
Kamil Mielnik
6c7102aa37 Fix app-wide transforms filtering 2025-12-02 20:14:47 +07:00
Oleksandr Solovyov
809e5b6724 api for workspace name change and test fix 2025-12-02 15:11:57 +02:00
Stas Gavrylov
5f75951717 Remove tabs centered positioning 2025-12-02 14:10:23 +01:00
Stas Gavrylov
8ff78faef0 Add scrollIntoView for tabs 2025-12-02 14:09:24 +01:00
Stas Gavrylov
2bd68fc928 Add output table name input validation 2025-12-02 14:04:27 +01:00
Kamil Mielnik
288f07c709 Disable transform editor when transform is not a part of the workspace 2025-12-02 20:00:53 +07:00
Ngoc Khuat
1f199c1dfb Workspaces swapping db details (#66324) 2025-12-02 19:50:40 +07:00
Stas Gavrylov
51efac3d49 Fix workspace transforms origin 2025-12-02 13:35:48 +01:00
Kamil Mielnik
a9ccb8def6 Extract CheckOutTransformButton & SaveTransformButton 2025-12-02 19:31:31 +07:00
Stas Gavrylov
b2dacc7a92 Add transform creation within current workspace 2025-12-02 13:22:33 +01:00
Stas Gavrylov
d769903e13 Refactor table name validation to use Yup schema extension 2025-12-02 13:22:32 +01:00
Stas Gavrylov
167af6a806 Extend CreateTransformModal with external schemas and async table name validation. Integrate to WorkspacePage 2025-12-02 13:22:32 +01:00
Oleksandr Solovyov
e2ad85b627 upstream_id is optional 2025-12-02 14:21:44 +02:00
Chris Truter
c1c1926751 Fix some tests 2025-12-02 10:05:05 +02:00
Chris Truter
40dbb2dd5e WIP 2025-12-02 09:57:13 +02:00
Chris Truter
f206ac8d9b Disallow transforms with card dependencies in workspaces 2025-12-02 09:57:11 +02:00
lbrdnk
c93f52c810 Workspaces GET /tables endpoint (#66421)
* /tables endpoint

* Update keys

* Adjust test

* Update test

* remarks

* Avoid using :outputs or :transforms from the graph structure
2025-12-01 22:28:41 +01:00
lbrdnk
fac1eda9fa Workspaces dependent python transforms id adjustments (#66399)
* Python transitive transforms

* wording
2025-12-01 22:27:12 +01:00
Uladzimir Havenchyk
5baa9d4709 allow merging workspaces when all transforms saved 2025-12-01 22:30:45 +03:00
Uladzimir Havenchyk
78272586f8 redirect to the transform list after merge of a workspace 2025-12-01 21:47:19 +03:00
Ngoc Khuat
86926f69d3 return global transforms only by default and add exclude_workspace_id filter 2025-12-01 22:58:47 +07:00
Kamil Mielnik
0ee7e2fa48 Target table name input 2025-12-01 22:12:15 +07:00
Ngoc Khuat
c3f7c0e3f8 hydrate full transform contents 2025-12-01 21:00:11 +07:00
Chris Truter
1774564beb API to check whether transform target is OK 2025-12-01 15:46:44 +02:00
Chris Truter
557ce7c3c6 Fix linter warnings 2025-11-30 14:54:11 +02:00
Chris Truter
a60fa0c2cb Fix whitespace 2025-11-30 14:50:06 +02:00
Chris Truter
39e4f68e31 Add linter (overkill?) and fix for bad claude migration yaml 2025-11-30 14:43:29 +02:00
Uladzimir Havenchyk
072c06e2bd add a skeleton for merge functionality 2025-11-29 01:15:32 +03:00
Chris Truter
e781690132 Fix assertion to actually check something 2025-11-28 16:49:44 +02:00
Kamil Mielnik
68b19966fc Run transform button 2025-11-28 21:24:41 +07:00
Kamil Mielnik
8b5db401e8 Save button 2025-11-28 21:21:29 +07:00
Oleksandr Solovyov
946fc08c43 complain for wrong args to api 2025-11-28 15:20:10 +02:00
Stas Gavrylov
f5cbb8cc48 Update edited transforms display logic 2025-11-28 13:17:00 +01:00
Oleksandr Solovyov
44d6075e8c do not ask for workspace transform to provide db/schema for target 2025-11-28 14:12:23 +02:00
Chris Truter
fa2ae8e884 Fix broken SQL error 2025-11-28 14:09:51 +02:00
Stas Gavrylov
36e3b6ec3a Update WS transforms styling 2025-11-28 12:43:56 +01:00
Oleksandr Solovyov
ba9a841466 fix graph queries 2025-11-28 13:32:36 +02:00
Oleksandr Solovyov
5ebc738f29 less obtuse names 2025-11-28 13:18:22 +02:00
Stas Gavrylov
045e9a224f Refactor Setup tab 2025-11-28 12:00:34 +01:00
Stas Gavrylov
a6e22daac1 Add onChange for TransformEditor. Refactoring 2025-11-28 11:48:08 +01:00
Kamil Mielnik
6bdd48ffe3 Check if transform is in the workspace 2025-11-28 17:37:16 +07:00
Kamil Mielnik
5bb0f1f560 Style 2025-11-28 17:36:54 +07:00
Kamil Mielnik
2d263d19e3 Save button adds transform into the workspace 2025-11-28 17:30:12 +07:00
Kamil Mielnik
5e98f44b80 Add updateWorkspaceContents API mapping 2025-11-28 17:24:46 +07:00
Kamil Mielnik
da6555f5cf Stub handlers 2025-11-28 17:24:46 +07:00
Kamil Mielnik
937f3f7b03 Add TransformTab 2025-11-28 17:24:46 +07:00
Stas Gavrylov
71d065f0a9 Fix Python sidebar 2025-11-28 11:23:39 +01:00
Oleksandr Solovyov
b30071a69a adding entities and fresh transforms to workspace (#66364) 2025-11-28 12:14:40 +02:00
Stas Gavrylov
407fc2a5ab Update Code tab styling 2025-11-28 11:13:16 +01:00
Stas Gavrylov
de113f72ce Merge branch 'workspaces-master' into workspaces-2025-11 2025-11-28 11:09:08 +01:00
Stas Gavrylov
6199e0056f Merge master 2025-11-28 10:51:57 +01:00
lbrdnk
fd6b176446 Dependecy support for python transforms (#66239)
* Python transform dependencies support

* Upstream deps test

* Pass the transform
2025-11-28 10:16:24 +01:00
lbrdnk
42d1202c18 Support for multiple interdependent transforms in workspaces (#66348)
* Interdependent native Xs not executed test

WIP

Fix X mirroring as per huddle

Test: use atom for cleanup

transform-node->data

Use new mirroring and remap code

typo

typo

Make isolation naming functions public

Fix adding workspace namespace, not tested yet

Fix source-for-dup-transform

Dirty interedependent native transforms test

Instead of writing over graph structure, wrap it in context

Cleanup

Cleanup test, not final

Remove redundant test

Consolidate schema set into single update

Remove juxt in favor of select-fn->fn

nits

* Fix stuff after merging feat branch

graph :dependencies now hold transforms not tables

this change adjusts the rest of mirroring to work with that

* Better test cleanup

* hack to avoid failing when provisioning ws with no entities to checkout
2025-11-28 08:31:36 +01:00
Uladzimir Havenchyk
d1a4656cb8 move archive action to workspace item 2025-11-27 23:13:17 +03:00
Uladzimir Havenchyk
cc934ea766 fix layout 2025-11-27 20:32:00 +03:00
Stas Gavrylov
8bf117b047 Fix tabs header wrap 2025-11-27 17:31:55 +01:00
Uladzimir Havenchyk
caea0b31cf move routes out of transforms 2025-11-27 19:25:15 +03:00
Stas Gavrylov
1d6dd0d16d Add tabs icons 2025-11-27 16:51:41 +01:00
Stas Gavrylov
dca47362f8 Disable DB selection in Python editor 2025-11-27 16:23:30 +01:00
Stas Gavrylov
2f8b666797 Add transforms filtering by DB type and display of source DB name 2025-11-27 16:09:33 +01:00
Stas Gavrylov
f3aef4f4ad Some types refactoring. Make transforms names Ellipsified 2025-11-27 15:52:20 +01:00
Stas Gavrylov
5ac6249eb3 Filter transforms. Add tabs layout improvements 2025-11-27 15:35:15 +01:00
Stas Gavrylov
8acbee8000 tabs improvements 2025-11-27 15:21:41 +01:00
Kamil Mielnik
bfd2ad8f0f Better Metabot isolation 2025-11-27 21:13:49 +07:00
Stas Gavrylov
a982db6bf5 Fix for resetting active transform 2025-11-27 15:13:39 +01:00
Stas Gavrylov
3e6d266a33 Add tabs for transforms 2025-11-27 15:11:37 +01:00
Stas Gavrylov
e197cf97af Minor tabs styling updates 2025-11-27 14:47:41 +01:00
Kamil Mielnik
97d89cc855 Make tabs controlled, open transform tab when opening a transform 2025-11-27 20:44:05 +07:00
Kamil Mielnik
955cc0b4be Render Metabot tab only when plugin is enabled 2025-11-27 20:39:28 +07:00
Kamil Mielnik
1b4424393d Change data studio sidebar width 2025-11-27 20:37:29 +07:00
Kamil Mielnik
3337769411 Add Metabot tab 2025-11-27 20:27:26 +07:00
Stas Gavrylov
0b07779984 Initial version of transforms editor integrated to workspaces 2025-11-27 13:06:04 +01:00
Chris Truter
2e54d68fb2 Put mirrored transforms in the WS collection (#66358)
In anticipation of transforms moving into collection in master, let's get our code ready for that.
2025-11-27 14:01:47 +02:00
Chris Truter
e67c3e625d Defend against table name collissions 2025-11-27 13:32:19 +02:00
Chris Truter
325fe07ab1 Actually database is on the target also 2025-11-27 13:20:24 +02:00
Chris Truter
dc53fec17e Fixes to dag + workaround for deps gotcha 2025-11-27 13:04:17 +02:00
Chris Truter
17c3ed6635 Preconditions and checksum overrides for updated migrations 2025-11-27 13:04:17 +02:00
Chris Truter
f661f0c21e Fix module test 2025-11-27 13:04:17 +02:00
Chris Truter
89d1394fb7 Fix unused linter for stub API 2025-11-27 13:04:17 +02:00
Stas Gavrylov
0cac006357 Fix transforms list rendering 2025-11-27 11:07:46 +01:00
Chris Truter
c9a5ea4dd1 Better multi tx test 2025-11-27 10:30:20 +02:00
Chris Truter
79f7d83c4d Add a less trivial path-induced-subgraph test 2025-11-27 09:39:52 +02:00
Uladzimir Havenchyk
56a1ac2862 add archive method 2025-11-26 23:55:39 +03:00
Uladzimir Havenchyk
8d0acd1436 basic ui 2025-11-26 18:48:18 +03:00
Uladzimir Havenchyk
b6628af72f fix merge conflict 2025-11-26 12:37:34 +03:00
Uladzimir Havenchyk
0169970a8b Merge remote-tracking branch 'origin/workspaces-master' into workspaces-2025-11 2025-11-26 12:32:40 +03:00
Chris Truter
9cc27dfa59 Mode cosmetics 2025-11-25 17:15:55 +02:00
Oleksandr Solovyov
9221d59b7e wrong default 2025-11-25 16:56:40 +02:00
Oleksandr Solovyov
22047bb890 claude fixes tests (#66244) 2025-11-25 16:55:45 +02:00
Stas Gavrylov
772216505c Merge branch 'workspaces-list-transforms' into workspaces-2025-11 2025-11-25 12:57:50 +01:00
Stas Gavrylov
3d7541f3c7 Display list of all transforms and workspace transforms 2025-11-25 12:57:31 +01:00
Oleksandr Solovyov
5a59eef925 oops, do not hydrate for the list 2025-11-25 13:52:13 +02:00
Chris Truter
dc05f59514 Stub Workspaces API (#66124)
Co-authored-by: Oleksandr Solovyov <o@solovyov.net>
2025-11-25 13:47:10 +02:00
Stas Gavrylov
f64ab4ebab Adjust workspace page layout. Extract route 2025-11-25 12:45:17 +01:00
Kamil Mielnik
19497a7470 Fix redirect after creating a workspace from a transform (use imperative code instead of a React effect) 2025-11-25 16:54:08 +07:00
Kamil Mielnik
823690bc76 Fix skip tokens 2025-11-25 16:49:24 +07:00
Kamil Mielnik
82930d7f02 Fix workspaces API types and RTK query tags 2025-11-25 16:49:15 +07:00
Chris Truter
fa9b90beb1 Fix cross rollback test, hopefully 2025-11-21 18:38:19 +02:00
Chris Truter
d4d0c64400 Basic Merge functionality (#66137) 2025-11-21 18:22:44 +02:00
Chris Truter
cb70117c05 Fixes for Workspace merge (#66136) 2025-11-21 18:20:59 +02:00
Oleksandr Solovyov
3d9c45e126 Basic workspace merging (#66118) 2025-11-21 18:18:23 +02:00
Chris Truter
516df0c70e Bug fix for output table remapping 2025-11-21 11:05:51 +02:00
Chris Truter
4cd2aa7c3e Add list and show page for workspaces (#66112) 2025-11-21 10:44:18 +02:00
Chris Truter
e6943a7389 Max UX more fancy (links between mirrors) (#66101) 2025-11-21 10:43:33 +02:00
Chris Truter
a43c90f357 More idempotent migration fixes 2025-11-21 00:09:30 +02:00
Chris Truter
91a42b4fac Add constraint that each downstream entity has a unique mapping 2025-11-21 00:09:20 +02:00
Chris Truter
34689d9a05 Ignore checksums, for luck 2025-11-20 18:49:03 +02:00
Chris Truter
528b264f87 Fix migration lints, hopefully without breaking pr env 2025-11-20 18:47:01 +02:00
Chris Truter
0580e2dc34 Fix migration ordering 2025-11-20 18:41:06 +02:00
Chris Truter
91c985eca9 Adding missing indexes and cascade behavior 2025-11-20 18:06:43 +02:00
Chris Truter
3bb34fa1c9 Fix small lints and sanity tests
Fix deadcode lint
Fix serialization tests
Fix revision description test (workspace_id column doesn't change)
2025-11-20 18:06:15 +02:00
Chris Truter
f539b334dd Support multiple transforms (mostly) (#66079) 2025-11-20 17:56:43 +02:00
Chris Truter
9cfe59b1a2 Whoops, need plurals 2025-11-20 15:26:51 +02:00
Chris Truter
fb59bf9325 Graph Analysis for Workspace dependencies (#66069) 2025-11-20 15:15:48 +02:00
Chris Truter
2b2ff2dae5 Use join tables to track workspace entity mappings (#66062)
First class joins are kinda nice:

1.  Detection of relationships and navigation back-and-forth in the FE becomes easy. 
2. Database constraints might catch bugs.
3. Clean-up of obsolete mappings is trivial.
4. Debugging and testing is potentially more convenient too.

Maintenance on them will be low, we just need to handle creating them, and 

I haven't covered `metabase_field` _yet_, as that would be more complex. Waiting to confirm they're really necessary, and if so, whether having their mapping in JSON is sufficient.
2025-11-20 12:53:15 +02:00
Chris Truter
0dd7d78c07 Crude toposorted dag calculation 2025-11-20 10:25:51 +02:00
lbrdnk
33daf1d822 Mirror single transform (#65982)
* Mirror single transform

Creates dwh namespace
Creates mirror target table
Creates mirror transform artifact
with adjusted
target
name

Abusing toucan models
sending those in entity-info

Most of the code incrementally
updates entity-info
with checkout progress
and we should continue with that
with more reasonable way
(i.e. not toucan models but some
subset of relevant keys)

Testing code at the end of
copying ns

Namespaces will change
into different subsystems probably
for now this is poc

* Assert toplevel, docstring

* Adjust namespaces and cleanup

* drop old file

* Tweak for rich comment, incomplete

* Vibe UX

* Tighten up workspace demo-0 code (#66044)

* Update FE to track API changes. Fix some small, subtle FE and BE issues.

* Critical error

---------

Co-authored-by: Chris Truter <chris@metabase.com>
Co-authored-by: Chris Truter <crisptrutski@users.noreply.github.com>
2025-11-20 08:57:21 +01:00
Oleksandr Solovyov
9faad636cb add namespace to workspace collections 2025-11-19 18:47:16 +02:00
dan sutton
b201dca6cf empty for CI 2025-11-19 09:20:17 -06:00
Chris Truter
f1982e56e3 Fix contents API shape 2025-11-19 12:13:13 +02:00
Chris Truter
439c3f3be5 Add API endpoint to return the contents of a workspace 2025-11-19 12:11:01 +02:00
Oleksandr Solovyov
37449925fa migration and basic model file (#65956)
* migration and basic model file

* basic api
2025-11-19 12:04:35 +02:00
233 changed files with 11029 additions and 580 deletions

View File

@@ -1325,6 +1325,7 @@
metabase.sync.core
metabase.sync.init
metabase.sync.schedules
metabase.sync.sync-metadata.tables
metabase.sync.task.sync-databases
metabase.sync.util}
:uses #{analyze
@@ -2277,6 +2278,7 @@
app-db
config
driver
events
lib
lib-be
query-processor
@@ -2291,6 +2293,23 @@
models
premium-features
upload
util}}
enterprise/workspaces
{:team "Metabot"
:api #{metabase-enterprise.workspaces.api
metabase-enterprise.workspaces.core
metabase-enterprise.workspaces.init}
:uses #{api
api-keys
driver
enterprise/transforms
events
models
queries
request
sync
system
util}}}
;; namespaces matching these patterns (with `re-find`) are excluded from the module linter. Since regex literals

View File

@@ -93,6 +93,7 @@
metabase-enterprise.test/with-gtaps! [[:inner 0]]
metabase-enterprise.test/with-gtaps-for-user! [[:inner 0]]
metabase-enterprise.test/with-user-attributes [[:block 2]]
metabase-enterprise.workspaces.models.workspace-log/track! [[:block 2]]
metabase.actions.actions-test/with-actions-test-data-and-actions-permissively-enabled [[:block 0]]
metabase.actions.test-util/with-actions [[:block 1]]
metabase.actions.test-util/with-actions-disabled [[:block 0]]

View File

@@ -61,6 +61,7 @@
(eval . (put-clojure-indent 'u/prog1 1))
(eval . (put-clojure-indent 'u/select-keys-when '(:form)))
(eval . (put-clojure-indent 'with-meta '(:form)))
(eval . (put-clojure-indent 'ws.log/track! 2))
;; these ones have to be done with `define-clojure-indent' for now because of upstream bug
;; https://github.com/clojure-emacs/clojure-mode/issues/600 once that's resolved we should use `put-clojure-indent'
;; instead. Please don't add new entries unless they don't work with `put-clojure-indent'

View File

@@ -144,6 +144,18 @@
change-log
#(pos? (compare % "v49.00-000"))))
(defn require-primary-key-exists-has-table-name
"Ensures that all primaryKeyExists preconditions specify a tableName."
[change-log]
(doseq [{:keys [changeSet]} change-log
:when changeSet
:let [s (pr-str (:preConditions changeSet))]
:when (and (str/includes? s ":primaryKeyExists")
(not (str/includes? s ":tableName")))]
(throw (validation-error
(format "Migration '%s' has primaryKeyExists precondition without tableName" (:id changeSet))
{:id (:id changeSet)}))))
(s/def ::changeSet
(s/spec :change-set.strict/change-set))
@@ -163,6 +175,7 @@
(require-no-bare-blob-or-text-types change-log)
(require-no-bare-boolean-types change-log)
(require-no-datetime-type change-log)
(require-primary-key-exists-has-table-name change-log)
(let [{:keys [changeSet]
:as all} (group-by only-key change-log)]
(when-not (set/subset? (set (keys all)) #{:property :objectQuotingStrategy :changeSet})

View File

@@ -75,7 +75,11 @@ export function entityPickerModalItem(level, name) {
}
export function entityPickerModalTab(name) {
return cy.findAllByRole("tab").filter(`:contains(${name})`);
if (typeof name === "string") {
return cy.findAllByRole("tab").filter(`:contains(${name})`);
} else {
return cy.findAllByRole("tab").first().parent().findByText(name);
}
}
// displays at least these tabs:

View File

@@ -0,0 +1,83 @@
import type { WorkspaceId } from "metabase-types/api";
export const Workspaces = {
visit: (id: WorkspaceId) => {
return cy.visit(`/data-studio/workspace/${id}`);
},
visitDataStudio() {
return cy.visit("/data-studio");
},
visitTransformListPage() {
return cy.visit("/data-studio/transforms");
},
getWorkspacesPage() {
return cy.findByTestId("workspaces-page");
},
getWorkspacePage() {
return cy.findByTestId("workspace-page");
},
getWorkspaceContent() {
return cy.findByTestId("workspace-content");
},
getWorkspaceSidebar() {
return cy.findByTestId("workspace-sidebar");
},
getWorkspacesSection() {
return cy.findByTestId("workspaces-section");
},
getWorkspaceItem(name: string | RegExp) {
return Workspaces.getWorkspacesSection().findByRole("button", { name });
},
getWorkspaceItemActions(name: string | RegExp) {
return Workspaces.getWorkspaceItem(name).findByLabelText("More actions");
},
getWorkspaceNameInput() {
return cy.findByPlaceholderText("Workspace name");
},
getMergeWorkspaceButton() {
return cy.findByRole("button", { name: /Merge/ });
},
getTransformTargetButton() {
return cy.findByRole("button", { name: /Change target/ });
},
getTransformRunButton() {
return cy.findByRole("button", { name: /Run/ });
},
getTransformSaveButton() {
return cy.findByRole("button", { name: /Save/ });
},
getWorkspaceTransforms() {
return cy.findByTestId("workspace-transforms");
},
getMainlandTransforms() {
return cy.findByTestId("mainland-transforms");
},
getNewWorkspaceButton() {
return cy.findByRole("button", { name: /New workspace/ });
},
getNewWorkspaceNameInput() {
return cy.findByPlaceholderText("Enter workspace name");
},
getNewWorkspaceDatabaseInput() {
return cy.findByPlaceholderText("Select a database");
},
};

View File

@@ -13,9 +13,9 @@ export * from "./e2e-comment-helpers";
export * from "./e2e-custom-column-helpers";
export * from "./e2e-dashboard-helpers";
export * from "./e2e-dashboard-visualizer-helpers";
export * from "./e2e-data-studio-helpers";
export * from "./e2e-database-metadata-helpers";
export * from "./e2e-datamodel-helpers";
export * from "./e2e-data-studio-helpers";
export * from "./e2e-dependency-graph-helpers";
export * from "./e2e-detail-view-helpers";
export * from "./e2e-dimension-list-helpers";
@@ -59,4 +59,5 @@ export * from "./e2e-upload-helpers";
export * from "./e2e-users-helpers";
export * from "./e2e-visual-tests-helpers";
export * from "./e2e-viz-settings-helpers";
export * from "./e2e-workspaces-helpers";
export * from "./embedding-sdk-helpers/constants";

View File

@@ -0,0 +1,349 @@
import dedent from "ts-dedent";
import { SAMPLE_DB_ID, WRITABLE_DB_ID } from "e2e/support/cypress_data";
import { SAMPLE_DATABASE } from "e2e/support/cypress_sample_database";
import type {
CardType,
PythonTransformTableAliases,
TransformTagId,
} from "metabase-types/api";
const { H } = cy;
const { DataStudio, Workspaces } = H;
const { Transforms } = DataStudio;
const { ORDERS_ID } = SAMPLE_DATABASE;
const DB_NAME = "Writable Postgres12";
const SOURCE_TABLE = "Animals";
const TARGET_TABLE = "transform_table";
const TARGET_TABLE_2 = "transform_table_2";
const TARGET_SCHEMA = "Schema A";
const TARGET_SCHEMA_2 = "Schema B";
const CUSTOM_SCHEMA = "custom_schema";
describe("scenarios > data studio > workspaces", () => {
beforeEach(() => {
H.restore("postgres-writable");
H.resetTestTable({ type: "postgres", table: "many_schemas" });
H.resetSnowplow();
cy.signInAsAdmin();
H.activateToken("bleeding-edge");
H.resyncDatabase({ dbId: WRITABLE_DB_ID, tableName: SOURCE_TABLE });
// cy.intercept("PUT", "/api/field/*").as("updateField");
// cy.intercept("POST", "/api/ee/transform").as("createTransform");
// cy.intercept("PUT", "/api/ee/transform/*").as("updateTransform");
// cy.intercept("DELETE", "/api/ee/transform/*").as("deleteTransform");
// cy.intercept("DELETE", "/api/ee/transform/*/table").as(
// "deleteTransformTable",
// );
// cy.intercept("POST", "/api/ee/transform-tag").as("createTag");
// cy.intercept("PUT", "/api/ee/transform-tag/*").as("updateTag");
// cy.intercept("DELETE", "/api/ee/transform-tag/*").as("deleteTag");
});
// afterEach(() => {
// H.expectNoBadSnowplowEvents();
// });
it("should be able to create, navigate, archive, and rename workspaces", () => {
Workspaces.visitDataStudio();
Workspaces.getWorkspacesSection()
.findByText("No workspaces yet")
.should("be.visible");
createWorkspace("Workspace A");
cy.location("pathname").should("match", /data-studio\/workspaces\/\d+/);
cy.log("shows workspace name");
Workspaces.getWorkspaceNameInput().should("have.value", "Workspace A");
Workspaces.getMergeWorkspaceButton().should("be.disabled");
Workspaces.getWorkspaceContent().within(() => {
cy.log("starts on setup tab, and has only 2 tabs");
H.tabsShouldBe("Setup", ["Setup", "Agent Chat"]);
cy.log("shows workspace db");
cy.findByText("Writable Postgres12").should("be.visible");
cy.log("shows workspace setup logs");
cy.findByText("Provisioning database isolation").should("be.visible");
cy.findByText("Setting up the workspace").should("be.visible");
cy.findByText("Workspace ready!").should("be.visible");
});
Workspaces.getWorkspaceSidebar().within(() => {
cy.log("starts on Code tab, and has only 2 tabs");
H.tabsShouldBe("Code", ["Code", "Data"]);
cy.log("shows transforms list");
cy.findByText("Workspace is empty").should("be.visible");
});
createWorkspace("Workspace B");
Workspaces.getWorkspaceNameInput().should("have.value", "Workspace B");
Workspaces.getWorkspacesSection().within(() => {
cy.findByText("Workspace B").should("be.visible");
cy.findByText("Workspace A").should("be.visible").click();
});
Workspaces.getWorkspaceNameInput().should("have.value", "Workspace A");
cy.log("can archive a workspace");
Workspaces.getWorkspaceItemActions(/Workspace A/).click();
H.popover().findByText("Archive").click();
H.undoToast().should("have.text", "Workspace archived successfully");
Workspaces.getWorkspaceItem(/Workspace A/).should("not.exist");
cy.location("pathname").should("eq", "/data-studio/workspaces");
Workspaces.getWorkspacesPage().within(() => {
cy.findByText("Workspaces").should("be.visible");
cy.findByText("Workspace A").should("not.exist");
cy.log("can navigate from workspaces list to a workspace");
cy.findByText("Workspace B").should("be.visible").click();
});
cy.location("pathname").should("match", /data-studio\/workspaces\/\d+/);
Workspaces.getWorkspaceNameInput().should("have.value", "Workspace B");
Workspaces.getWorkspaceNameInput().clear().type("Renamed workspace").blur();
// H.undoToast().should("have.text", "Workspace renamed"); // TODO: uncomment when implemented
Workspaces.getWorkspaceItem(/Renamed workspace/).should("be.visible");
// TODO: workspace deletion?
});
it("should be able to check out exisitng transforms into a new workspace from the transform page", () => {
const sourceTable = `${TARGET_SCHEMA}.${SOURCE_TABLE}`;
createTransforms({ visit: true });
cy.findByRole("button", { name: /Edit transform/ }).click();
H.popover().within(() => {
cy.findByText("No workspaces yet").should("be.visible");
cy.findByText("New workspace").should("be.visible").click();
});
Workspaces.getNewWorkspaceNameInput().should("have.value", "New workspace");
Workspaces.getNewWorkspaceDatabaseInput().should(
"have.value",
"Writable Postgres12",
);
H.modal().findByText("Create").click();
cy.location("pathname").should("match", /data-studio\/workspaces\/\d+/);
Workspaces.getWorkspaceNameInput().should("have.value", "New workspace");
Workspaces.getWorkspacePage()
.findByText("Workspace is empty")
.should("not.exist");
Workspaces.getMainlandTransforms()
.findByText("Python transform")
.should("be.visible");
Workspaces.getMainlandTransforms()
.findByText("MBQL transform")
.should("not.exist");
Workspaces.getWorkspaceTransforms()
.findByText("SQL transform")
.should("be.visible")
.click();
Workspaces.getWorkspaceContent().within(() => {
H.tabsShouldBe("SQL transform", ["Setup", "Agent Chat", "SQL transform"]);
});
Workspaces.getMergeWorkspaceButton().should("be.enabled");
Workspaces.getTransformTargetButton().should("be.enabled");
Workspaces.getTransformRunButton().should("be.enabled");
Workspaces.getTransformSaveButton().should("be.disabled");
H.NativeEditor.type(" LIMIT 2");
Workspaces.getMergeWorkspaceButton().should("be.disabled");
Workspaces.getTransformTargetButton().should("be.disabled");
Workspaces.getTransformRunButton().should("be.disabled");
Workspaces.getTransformSaveButton().should("be.enabled").click();
Workspaces.getMergeWorkspaceButton().should("be.enabled");
Workspaces.getTransformTargetButton().should("be.enabled");
Workspaces.getTransformSaveButton().should("be.disabled");
Workspaces.getTransformRunButton().should("be.enabled").click();
Workspaces.getWorkspaceSidebar().within(() => {
cy.findByRole("tab", { name: "Data" }).click();
cy.findByLabelText(sourceTable).should("be.visible").click();
});
Workspaces.getWorkspaceContent().within(() => {
H.tabsShouldBe(sourceTable, [
"Setup",
"Agent Chat",
"SQL transform",
sourceTable,
]);
H.assertTableData({
columns: ["Name", "Score"],
firstRows: [
["Duck", "10"],
["Horse", "20"],
["Cow", "30"],
],
});
});
Workspaces.getWorkspaceSidebar().within(() => {
cy.findByRole("tab", { name: "Data" }).click();
cy.findByLabelText(/mb__isolation_/)
.should("be.visible")
.click();
});
Workspaces.getWorkspaceContent().within(() => {
H.tabsShouldBe(/mb__isolation_/, [
"Setup",
"Agent Chat",
"SQL transform",
sourceTable,
/mb__isolation_/,
]);
H.assertTableData({
columns: ["Name", "Score"],
firstRows: [
["Duck", "10"],
["Horse", "20"],
],
});
cy.findByText("Cow").should("not.exist");
cy.findByText("30").should("not.exist");
});
Workspaces.getMergeWorkspaceButton().click();
cy.findByRole("link", { name: /SQL transform/ }).click();
H.NativeEditor.value().should(
"eq",
'SELECT * FROM "Schema A"."Animals" LIMIT 2',
);
Transforms.runTab().click();
runTransformAndWaitForSuccess();
Transforms.targetTab().click();
getTableLink().should("contain.text", "transform_table").click();
H.assertTableData({
columns: ["Name", "Score"],
firstRows: [
["Duck", "10"],
["Horse", "20"],
],
});
H.tableInteractiveBody().findByText("Cow").should("not.exist");
H.tableInteractiveBody().findByText("30").should("not.exist");
});
// it("should be able to check out existing transform into a new workspace from the workspace page", () => {
// createTransforms({ visit: true });
// });
});
function createWorkspace(name: string) {
Workspaces.getNewWorkspaceButton().click();
H.modal().findByText("Create new workspace").should("be.visible");
Workspaces.getNewWorkspaceNameInput().clear().type(name);
Workspaces.getNewWorkspaceDatabaseInput().click();
H.popover().within(() => {
// cy.findByText("Internal Metabase Database").should("not.exist"); // TODO: uncomment once it works
// cy.findByText("Sample Database").should("not.exist"); // TODO: uncomment once it works
cy.findByText("Writable Postgres12").should("be.visible").click();
});
H.modal().findByText("Create").click();
}
function createTransforms({ visit }: { visit?: boolean } = {}) {
createMbqlTransform({
targetTable: TARGET_TABLE,
visitTransform: true,
});
H.getTableId({ name: "Animals", databaseId: WRITABLE_DB_ID }).then((id) => {
createPythonTransform({
body: dedent`
import pandas as pd
def transform(foo):
return pd.DataFrame([{"foo": 42 }])
`,
sourceTables: { foo: id },
});
});
createSqlTransform({
sourceQuery: `SELECT * FROM "${TARGET_SCHEMA}"."${SOURCE_TABLE}"`,
visitTransform: visit,
});
}
function createMbqlTransform(
opts: {
sourceTable?: string;
targetTable?: string;
targetSchema?: string | null;
tagIds?: TransformTagId[];
name?: string;
databaseId?: number;
visitTransform?: boolean;
} = {},
) {
return H.createMbqlTransform({
sourceTable: SOURCE_TABLE,
targetTable: TARGET_TABLE,
targetSchema: TARGET_SCHEMA,
name: "MBQL transform",
...opts,
});
}
function createSqlTransform(opts: {
sourceQuery: string;
targetTable?: string;
targetSchema?: string;
tagIds?: TransformTagId[];
visitTransform?: boolean;
}) {
return H.createSqlTransform({
targetTable: TARGET_TABLE,
targetSchema: TARGET_SCHEMA,
...opts,
});
}
function createPythonTransform(opts: {
body: string;
sourceTables: PythonTransformTableAliases;
targetTable?: string;
targetSchema?: string;
tagIds?: TransformTagId[];
visitTransform?: boolean;
}) {
return H.createPythonTransform({
targetTable: TARGET_TABLE,
targetSchema: TARGET_SCHEMA,
...opts,
});
}
function getTableLink({ isActive = true }: { isActive?: boolean } = {}) {
return cy
.findByTestId("table-link")
.should("have.attr", "aria-disabled", String(!isActive));
}
function runTransformAndWaitForSuccess() {
getRunButton().click();
getRunButton().should("have.text", "Ran successfully");
}
function getRunButton(options: { timeout?: number } = {}) {
return cy.findAllByTestId("run-button").eq(0, options);
}

View File

@@ -42,6 +42,7 @@
[metabase-enterprise.transforms-python.api]
[metabase-enterprise.transforms.api]
[metabase-enterprise.upload-management.api]
[metabase-enterprise.workspaces.api]
[metabase.api.macros :as api.macros]
[metabase.api.routes.common :as api.routes.common]
[metabase.api.util.handlers :as handlers]
@@ -139,7 +140,8 @@
"/transform" (premium-handler metabase-enterprise.transforms.api/routes :transforms)
"/transform-job" (premium-handler metabase-enterprise.transforms.api/transform-job-routes :transforms)
"/transform-tag" (premium-handler metabase-enterprise.transforms.api/transform-tag-routes :transforms)
"/upload-management" (premium-handler metabase-enterprise.upload-management.api/routes :upload-management)})
"/upload-management" (premium-handler metabase-enterprise.upload-management.api/routes :upload-management)
"/workspace" metabase-enterprise.workspaces.api/routes})
;;; ↑↑↑ KEEP THIS SORTED OR ELSE ↑↑↑
(def ^:private routes-map

View File

@@ -24,4 +24,5 @@
[metabase-enterprise.stale.init]
[metabase-enterprise.support-access-grants.init]
[metabase-enterprise.transforms-python.init]
[metabase-enterprise.transforms.init]))
[metabase-enterprise.transforms.init]
[metabase-enterprise.workspaces.init]))

View File

@@ -41,6 +41,12 @@
query-deps
param-card-ids)))
(mu/defn upstream-deps:python-transform :- ::deps.schema/upstream-deps
"Given a Toucan `:model/Transform`, return its upstream dependencies as a map from the kind to a set of IDs."
[{{tables :source-tables} :source :as _py-transform}
:- [:map [:source-tables {:optional true} [:map-of :string :int]]]]
{:table (set (vals tables))})
(mu/defn upstream-deps:transform :- ::deps.schema/upstream-deps
"Given a Transform (in Toucan form), return its upstream dependencies."
[{{:keys [query], source-type :type} :source :as transform} :-
@@ -49,9 +55,15 @@
[:query
[:map [:query ::lib.schema/query]]]
[:python
[:map]]]]]]
(if (= (keyword source-type) :query)
[:map [:source-tables {:optional true} [:map-of :string :int]]]]]]]]
(cond
(= (keyword source-type) :query)
(upstream-deps:query query)
(= (keyword source-type) :python)
(upstream-deps:python-transform transform)
:else
(do (log/warnf "Don't know how to analyze the deps of Transform %d with source type '%s'" (:id transform) source-type)
{})))

View File

@@ -115,4 +115,7 @@
"User"
"UserKeyValue"
"UserParameterValue"
"ViewLog"])
"ViewLog"
"Workspace"
"WorkspaceMappingTable"
"WorkspaceMappingTransform"])

View File

@@ -4,6 +4,7 @@
[metabase-enterprise.transforms.api.transform-job]
[metabase-enterprise.transforms.api.transform-tag]
[metabase-enterprise.transforms.canceling :as transforms.canceling]
[metabase-enterprise.transforms.execute :as transforms.execute]
[metabase-enterprise.transforms.interface :as transforms.i]
[metabase-enterprise.transforms.models.transform :as transform.model]
[metabase-enterprise.transforms.models.transform-run :as transform-run]
@@ -79,14 +80,32 @@
[402 (deferred-tru "Premium features required for this transform type are not enabled.")]))
(defn get-transforms
"Get a list of transforms."
[& {:keys [last_run_start_time last_run_statuses tag_ids]}]
"Get a list of transforms.
By default, only returns global transforms (workspace_id=null).
When `exclude_workspace_id` is provided, additionally excludes global transforms
that have been mirrored into the specified workspace (i.e., they have a
workspace_mapping_transform entry pointing to a transform in that workspace)."
[& {:keys [last_run_start_time last_run_statuses tag_ids exclude_workspace_id database_id type]}]
(api/check-superuser)
(let [transforms (t2/select :model/Transform {:order-by [[:id :asc]]})]
(let [transforms (if exclude_workspace_id
(t2/select :model/Transform
{:left-join [[:workspace_mapping_transform :wmt]
[:and
[:= :transform.id :wmt.upstream_id]
[:= :wmt.workspace_id exclude_workspace_id]]]
:where [:and
[:= :transform.workspace_id nil]
[:= :wmt.upstream_id nil]]
:order-by [[:id :asc]]})
(t2/select :model/Transform
{:where [:= :workspace_id nil]
:order-by [[:id :asc]]}))]
(into []
(comp (transforms.util/->date-field-filter-xf [:last_run :start_time] last_run_start_time)
(transforms.util/->status-filter-xf [:last_run :status] last_run_statuses)
(transforms.util/->tag-filter-xf [:tag_ids] tag_ids)
(transforms.util/->database-id-filter-xf database_id)
(transforms.util/->type-filter-xf type)
(map #(update % :last_run transforms.util/localize-run-timestamps)))
(t2/hydrate transforms :last_run :transform_tag_ids :creator))))
@@ -94,13 +113,18 @@
;; of the REST API
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-query-params-use-kebab-case]}
(api.macros/defendpoint :get "/"
"Get a list of transforms."
"Get a list of global transforms (workspace_id=null).
When `exclude_workspace_id` is provided, additionally excludes global transforms
that have been mirrored into the specified workspace."
[_route-params
query-params :-
[:map
[:last_run_start_time {:optional true} [:maybe ms/NonBlankString]]
[:last_run_statuses {:optional true} [:maybe (ms/QueryVectorOf [:enum "started" "succeeded" "failed" "timeout"])]]
[:tag_ids {:optional true} [:maybe (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]]]]
[:tag_ids {:optional true} [:maybe (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]]
[:exclude_workspace_id {:optional true} [:maybe ms/PositiveInt]]
[:database_id {:optional true} [:maybe ms/PositiveInt]]
[:type {:optional true} [:maybe (ms/QueryVectorOf [:enum "query" "native" "python"])]]]]
(get-transforms query-params))
(api.macros/defendpoint :post "/"
@@ -186,19 +210,10 @@
:limit (request/limit)))
(update :data #(map transforms.util/localize-run-timestamps %))))
(api.macros/defendpoint :put "/:id"
"Update a transform."
[{:keys [id]} :- [:map
[:id ms/PositiveInt]]
_query-params
body :- [:map
[:name {:optional true} :string]
[:description {:optional true} [:maybe :string]]
[:source {:optional true} ::transforms.schema/transform-source]
[:target {:optional true} ::transforms.schema/transform-target]
[:run_trigger {:optional true} ::run-trigger]
[:tag_ids {:optional true} [:sequential ms/PositiveInt]]]]
(api/check-superuser)
(defn update-transform!
"Update a transform. Validates features, database support, cycles, and target conflicts.
Returns the updated transform with hydrated associations."
[id body]
(let [transform (t2/with-transaction [_]
;; Cycle detection should occur within the transaction to avoid race
(let [old (t2/select-one :model/Transform id)
@@ -223,17 +238,36 @@
(events/publish-event! :event/transform-update {:object transform :user-id api/*current-user-id*})
transform))
(api.macros/defendpoint :put "/:id"
"Update a transform."
[{:keys [id]} :- [:map
[:id ms/PositiveInt]]
_query-params
body :- [:map
[:name {:optional true} :string]
[:description {:optional true} [:maybe :string]]
[:source {:optional true} ::transforms.schema/transform-source]
[:target {:optional true} ::transforms.schema/transform-target]
[:run_trigger {:optional true} ::run-trigger]
[:tag_ids {:optional true} [:sequential ms/PositiveInt]]]]
(api/check-superuser)
(update-transform! id body))
(defn delete-transform!
"Delete a transform and publish the delete event."
[transform]
(t2/delete! :model/Transform (:id transform))
(events/publish-event! :event/transform-delete
{:object transform
:user-id api/*current-user-id*})
nil)
(api.macros/defendpoint :delete "/:id"
"Delete a transform."
[{:keys [id]} :- [:map
[:id ms/PositiveInt]]]
(api/check-superuser)
(let [transform (api/check-404 (t2/select-one :model/Transform id))]
(t2/delete! :model/Transform id)
(events/publish-event! :event/transform-delete
{:object transform
:user-id api/*current-user-id*}))
nil)
(delete-transform! (api/check-404 (t2/select-one :model/Transform id))))
(api.macros/defendpoint :delete "/:id/table"
"Delete a transform's output table."
@@ -255,17 +289,15 @@
(transforms.canceling/cancel-run! (:id run))))
nil)
(api.macros/defendpoint :post "/:id/run"
"Run a transform."
[{:keys [id]} :- [:map
[:id ms/PositiveInt]]]
(api/check-superuser)
(let [transform (api/check-404 (t2/select-one :model/Transform id))
_ (check-feature-enabled! transform)
start-promise (promise)]
(defn run-transform!
"Run a transform. Returns a 202 response with run_id.
The transform must already be fetched and validated."
[transform]
(check-feature-enabled! transform)
(let [start-promise (promise)]
(u.jvm/in-virtual-thread*
(transforms.i/execute! transform {:start-promise start-promise
:run-method :manual}))
(transforms.execute/execute! transform {:start-promise start-promise
:run-method :manual}))
(when (instance? Throwable @start-promise)
(throw @start-promise))
(let [result @start-promise
@@ -275,6 +307,13 @@
:run_id run-id})
(assoc :status 202)))))
(api.macros/defendpoint :post "/:id/run"
"Run a transform."
[{:keys [id]} :- [:map
[:id ms/PositiveInt]]]
(api/check-superuser)
(run-transform! (api/check-404 (t2/select-one :model/Transform id))))
(defn- extract-columns-from-query
"Attempts to extract column names from an MBQL query without executing it.

View File

@@ -1,6 +1,7 @@
(ns metabase-enterprise.transforms.core
"API namespace for the `metabase-enterprise.transform` module."
(:require
[metabase-enterprise.transforms.models.transform]
[metabase-enterprise.transforms.models.transform-run]
[metabase-enterprise.transforms.settings]
[metabase-enterprise.transforms.util]
@@ -14,4 +15,6 @@
python-transform?
query-transform?]
[metabase-enterprise.transforms.models.transform-run
timeout-run!])
timeout-run!]
[metabase-enterprise.transforms.models.transform
update-transform-tags!])

View File

@@ -1,81 +1,22 @@
(ns metabase-enterprise.transforms.execute
(:require
[metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
[metabase-enterprise.transforms.util :as transforms.util]
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
[metabase.events.core :as events]
[metabase.lib.schema.common :as schema.common]
[metabase.query-processor.compile :as qp.compile]
[metabase.util.log :as log]
[metabase.util.malli.registry :as mr]
[toucan2.core :as t2]))
[metabase-enterprise.transforms.interface :as transforms.i]))
(set! *warn-on-reflection* true)
(mr/def ::transform-details
[:map
[:transform-type [:enum {:decode/normalize schema.common/normalize-keyword} :table :table-incremental]]
[:conn-spec :any]
[:query ::qp.compile/compiled]
[:output-table [:keyword {:decode/normalize schema.common/normalize-keyword}]]])
(defn execute!
"Execute a transform.
(mr/def ::transform-opts
[:map
[:overwrite? :boolean]])
(defn- transform-opts [{:keys [transform-type]}]
(case transform-type
:table {:overwrite? true}
;; once we have more than just append, dispatch on :target-incremental-strategy
:table-incremental {}))
(defn run-mbql-transform!
"Run `transform` and sync its target table.
This is executing synchronously, but supports being kicked off in the background
by delivering the `start-promise` just before the start when the beginning of the execution has been booked
in the database."
([transform] (run-mbql-transform! transform nil))
([{:keys [id source target] :as transform} {:keys [run-method start-promise]}]
(try
(let [db (get-in source [:query :database])
{driver :engine :as database} (t2/select-one :model/Database db)
transform-details {:db-id db
:database database
:transform-id id
:transform-type (keyword (:type target))
:conn-spec (driver/connection-spec driver database)
:query (transforms.util/compile-source transform)
:output-schema (:schema target)
:output-table (transforms.util/qualified-table-name driver target)}
opts (transform-opts transform-details)
features (transforms.util/required-database-features transform)]
(when (transforms.util/db-routing-enabled? database)
(throw (ex-info "Transforms are not supported on databases with DB routing enabled."
{:driver driver, :database database})))
(when-not (every? (fn [feature] (driver.u/supports? (:engine database) feature database)) features)
(throw (ex-info "The database does not support the requested transform target type."
{:driver driver, :database database, :features features})))
;; mark the execution as started and notify any observers
(let [{run-id :id} (transforms.util/try-start-unless-already-running id run-method)]
(when start-promise
(deliver start-promise [:started run-id]))
(log/info "Executing transform" id "with target" (pr-str target))
(transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]]
(transforms.util/run-cancelable-transform!
run-id driver transform-details
(fn [_cancel-chan] (driver/run-transform! driver transform-details opts))))
(transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]
(transforms.util/sync-target! target database)
;; This event must be published only after the sync is complete - the new table needs to be in AppDB.
(events/publish-event! :event/transform-run-complete {:object transform-details}))))
(catch Throwable t
(log/error t "Error executing transform")
(when start-promise
;; if the start-promise has been delivered, this is a no-op,
;; but we assume nobody would catch the exception anyway
(deliver start-promise t))
(throw t)))))
For workspace transforms, execution must go through the workspace-specific endpoint.
Calling this function directly with a workspace transform will throw a 404 error."
[transform {:keys [start-promise] :as opts}]
;; This is used on FE for workspace transform execution until proper API is provided in workspaces module!
;; Hence temp enabling that.
;; TODO: Cleanup when moving to woskapce transform execution api.
(if false #_(:workspace_id transform)
(deliver start-promise
(ex-info "Workspace transforms must be executed through the workspace API"
{:status-code 404
:transform-id (:id transform)
:workspace-id (:workspace_id transform)}))
(transforms.i/execute! transform opts)))

View File

@@ -223,12 +223,14 @@
[_model-name opts]
{:copy [:name :description :entity_id]
:skip [:dependency_analysis_version :source_type]
:transform {:created_at (serdes/date)
:creator_id (serdes/fk :model/User)
:source {:export #(update % :query serdes/export-mbql)
:import #(update % :query serdes/import-mbql)}
:target {:export serdes/export-mbql :import serdes/import-mbql}
:tags (serdes/nested :model/TransformTransformTag :transform_id opts)}})
:transform {:created_at (serdes/date)
:creator_id (serdes/fk :model/User)
:workspace_id (serdes/fk :model/Workspace)
:collection_id (serdes/fk :model/Collection)
:source {:export #(update % :query serdes/export-mbql)
:import #(update % :query serdes/import-mbql)}
:target {:export serdes/export-mbql :import serdes/import-mbql}
:tags (serdes/nested :model/TransformTransformTag :transform_id opts)}})
(defmethod serdes/dependencies "Transform"
[{:keys [source tags]}]

View File

@@ -1,7 +1,15 @@
(ns metabase-enterprise.transforms.query-impl
(:require
[metabase-enterprise.transforms.execute :as transforms.execute]
[metabase-enterprise.transforms.interface :as transforms.i]))
[metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
[metabase-enterprise.transforms.interface :as transforms.i]
[metabase-enterprise.transforms.util :as transforms.util]
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
[metabase.events.core :as events]
[metabase.lib.schema.common :as schema.common]
[metabase.util.log :as log]
[metabase.util.malli.registry :as mr]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -13,5 +21,71 @@
[transform]
(-> transform :source :query :database))
(mr/def ::transform-details
[:map
[:transform-type [:enum {:decode/normalize schema.common/normalize-keyword} :table]]
[:conn-spec :any]
[:query :string]
[:output-table [:keyword {:decode/normalize schema.common/normalize-keyword}]]])
(mr/def ::transform-opts
[:map
[:overwrite? :boolean]])
(defn- transform-opts [{:keys [transform-type]}]
(case transform-type
:table {:overwrite? true}
;; once we have more than just append, dispatch on :target-incremental-strategy
:table-incremental {}))
(defn run-mbql-transform!
"Run `transform` and sync its target table.
This is executing synchronously, but supports being kicked off in the background
by delivering the `start-promise` just before the start when the beginning of the execution has been booked
in the database."
([transform] (run-mbql-transform! transform nil))
([{:keys [id source target] :as transform} {:keys [run-method start-promise]}]
(try
(let [db (get-in source [:query :database])
{driver :engine :as database} (t2/select-one :model/Database db)
transform-details {:db-id db
:database database
:transform-id id
:transform-type (keyword (:type target))
:conn-spec (driver/connection-spec driver database)
:query (transforms.util/compile-source transform)
:output-schema (:schema target)
:output-table (transforms.util/qualified-table-name driver target)}
opts (transform-opts transform-details)
features (transforms.util/required-database-features transform)]
(when (transforms.util/db-routing-enabled? database)
(throw (ex-info "Transforms are not supported on databases with DB routing enabled."
{:driver driver, :database database})))
(when-not (every? (fn [feature] (driver.u/supports? (:engine database) feature database)) features)
(throw (ex-info "The database does not support the requested transform target type."
{:driver driver, :database database, :features features})))
;; mark the execution as started and notify any observers
(let [{run-id :id} (transforms.util/try-start-unless-already-running id run-method)]
(when start-promise
(deliver start-promise [:started run-id]))
(log/info "Executing transform" id "with target" (pr-str target))
(transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]]
(transforms.util/run-cancelable-transform!
run-id driver transform-details
(fn [_cancel-chan] (driver/run-transform! driver transform-details opts))))
(transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]
(transforms.util/sync-target! target database)
;; This event must be published only after the sync is complete - the new table needs to be in AppDB.
(events/publish-event! :event/transform-run-complete {:object transform-details}))))
(catch Throwable t
(log/error t "Error executing transform")
(when start-promise
;; if the start-promise has been delivered, this is a no-op,
;; but we assume nobody would catch the exception anyway
(deliver start-promise t))
(throw t)))))
(defmethod transforms.i/execute! :query [transform opts]
(transforms.execute/run-mbql-transform! transform opts))
(run-mbql-transform! transform opts))

View File

@@ -45,14 +45,14 @@
[:map
[:database {:optional true} :int]
[:type [:= "table"]]
[:schema {:optional true} [:or ms/NonBlankString :nil]]
[:schema {:optional true} [:maybe ms/NonBlankString]]
[:name :string]])
(mr/def ::table-incremental-target
[:map
[:database {:optional true} :int]
[:type [:= "table-incremental"]]
[:schema {:optional true} [:or ms/NonBlankString :nil]]
[:schema {:optional true} [:maybe ms/NonBlankString]]
[:name :string]
[:target-incremental-strategy ::target-incremental-strategy]])

View File

@@ -436,33 +436,49 @@
(defn- matching-timestamp?
[job field-path {:keys [start end]}]
(when-let [field-instant (->instant (get-in job field-path))]
(let [start-instant (some-> start u.date/parse ->instant)
end-instant (some-> end u.date/parse ->instant)]
(and (or (nil? start)
(not (.isBefore field-instant start-instant)))
(or (nil? end)
(.isAfter end-instant field-instant))))))
(let [parse #(-> % u.date/parse ->instant)]
;; logic here is to find when it's not matching and invert this
(not (or (and start (.isBefore field-instant (parse start)))
(and end (.isAfter field-instant (parse end))))))))
(defn ->date-field-filter-xf
"Returns an xform for a date filter."
[field-path filter-value]
(let [range (some-> filter-value (params.dates/date-string->range {:inclusive-end? false}))]
(if range
(filter #(matching-timestamp? % field-path range))
identity)))
(if-let [range (some-> filter-value (params.dates/date-string->range {:inclusive-end? false}))]
(filter #(matching-timestamp? % field-path range))
identity))
(defn ->status-filter-xf
"Returns an xform for a transform run status filter."
[field-path statuses]
(let [statuses (->> statuses (map keyword) set not-empty)]
(if statuses
(filter #(statuses (get-in % field-path)))
identity)))
(if-let [statuses (->> statuses (map keyword) set not-empty)]
(filter #(statuses (get-in % field-path)))
identity))
(defn ->tag-filter-xf
"Returns an xform for a transform tag filter."
[field-path tag-ids]
(let [tag-ids (-> tag-ids set not-empty)]
(if tag-ids
(filter #(some tag-ids (get-in % field-path)))
identity)))
(if-let [tag-ids (-> tag-ids set not-empty)]
(filter #(some tag-ids (get-in % field-path)))
identity))
(defn ->type-filter-xf
"Returns an xform for a transform source type filter."
;; TODO: Ngoc - 2025-12-05 We have transform.source_type column so we can just use it to query
[types]
(if-let [types (->> types (map keyword) set not-empty)]
(filter (fn [transform]
(contains? types (:source_type transform))))
identity))
(defn ->database-id-filter-xf
"Returns an xform for a transform database ID filter.
Matches transforms where the source database OR target database matches the given database-id."
[database-id]
(if database-id
(filter (fn [transform]
(let [python-source-db-id (transforms.i/source-db-id transform)
target-db-id (transforms.i/target-db-id transform)]
(or (= database-id python-source-db-id)
(= database-id target-db-id)))))
identity))

View File

@@ -12,6 +12,7 @@
[metabase.app-db.core :as app-db]
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
[metabase.events.core :as events]
[metabase.util :as u]
[metabase.util.format :as u.format]
[metabase.util.i18n :as i18n]
@@ -360,6 +361,7 @@
(log/info "Executing Python transform" transform-id "with target" (pr-str target))
(let [start-ms (u/start-timer)
transform-details {:db-id (:id db)
:transform-id transform-id
:transform-type (keyword (:type target))
:conn-spec (driver/connection-spec driver db)
:output-schema (:schema target)
@@ -372,7 +374,8 @@
result (transforms.instrumentation/with-stage-timing [run-id [:computation :python-execution]]
(transforms.util/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn))]
(transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]
(transforms.util/sync-target! target db))
(transforms.util/sync-target! target db)
(events/publish-event! :event/transform-run-complete {:object transform-details}))
{:run_id run-id
:result result}))
(catch Throwable t

View File

@@ -0,0 +1,552 @@
(ns metabase-enterprise.workspaces.api
"`/api/ee/workspace/` routes"
(:require
[clojure.string :as str]
[honey.sql.helpers :as sql.helpers]
[metabase-enterprise.transforms.api :as transforms.api]
[metabase-enterprise.transforms.util :as transforms.util]
[metabase-enterprise.workspaces.common :as ws.common]
[metabase-enterprise.workspaces.models.workspace-log]
[metabase-enterprise.workspaces.types :as ws.t]
[metabase.api.common :as api]
[metabase.api.macros :as api.macros]
[metabase.api.routes.common :refer [+auth]]
[metabase.driver.util :as driver.u]
[metabase.queries.schema :as queries.schema]
[metabase.request.core :as request]
[metabase.util :as u]
[metabase.util.i18n :refer [deferred-tru]]
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]
[toucan2.core :as t2]))
(def ^:private log-limit "Maximum number of recent workspace log items to show" 20)
(mr/def ::appdb-or-ref-id [:or ::ws.t/appdb-id ::ws.t/ref-id])
;; Workspace status
(mr/def ::graph-status [:enum :stale :calculating :ready])
(mr/def ::db-status [:enum :uninitialized :provisioning :granting :cleaning :ready])
;; Proposed simplified FE statuses
(mr/def ::display-status [:enum :pending :ready] #_[:enum :uninitialized :database-not-read :graph-not-ready :ready])
(defn- display-status [graph-status db-status]
(cond
(= :uninitialized db-status) :uninitialized
(not (#{:ready :cleaning} db-status)) :database-not-ready
(not= :ready graph-status) :graph-not-ready
:else :ready))
(def ^:private Workspace
[:map
[:id ::ws.t/appdb-id]
[:name :string]
[:collection_id ::ws.t/appdb-id]
[:database_id ::ws.t/appdb-id]
[:status ::display-status]
[:created_at ms/TemporalInstant]
[:updated_at ms/TemporalInstant]
[:archived_at [:maybe :any]]])
;; Transform-related schemas (adapted from transforms/api.clj)
(mr/def ::transform-source
[:multi {:dispatch (comp keyword :type)}
[:query
[:map
[:type [:= "query"]]
[:query ::queries.schema/query]]]
[:python
[:map {:closed true}
[:source-database {:optional true} :int]
[:source-tables [:map-of [:string {:min 1}] :int]]
[:type [:= "python"]]
[:body :string]]]])
(mr/def ::transform-target
[:map {:closed true}
[:database {:optional true} ::ws.t/appdb-id]
[:type [:enum "table"]]
[:schema {:optional true} [:maybe [:string {:min 1}]]]
[:name [:string {:min 1}]]])
(mr/def ::run-trigger
[:enum "none" "global-schedule"])
(defn- check-transforms-enabled!
[db-id]
(let [database (api/check-400 (t2/select-one :model/Database db-id)
(deferred-tru "The target database cannot be found."))]
(api/check (transforms.util/check-feature-enabled nil)
[402 (deferred-tru "Premium features required for transforms are not enabled.")])
(api/check-400 (not (:is_sample database))
(deferred-tru "Cannot run transforms on the sample database."))
(api/check-400 (not (:is_audit database))
(deferred-tru "Cannot run transforms on audit databases."))
(api/check-400 (driver.u/supports? (:engine database) :transforms/table database)
(deferred-tru "The database does not support the requested transform target type."))
(api/check-400 (not (transforms.util/db-routing-enabled? database))
(deferred-tru "Transforms are not supported on databases with DB routing enabled."))))
(defn- ws->response [ws]
(select-keys ws [:id :name :collection_id :database_id :status :created_at :updated_at :archived_at]))
;;; routes
(def ^:private WorkspaceListing
[:map {:closed true}
[:id ::ws.t/appdb-id]
[:name :string]
[:archived :boolean]])
(api.macros/defendpoint :get "/" :- [:map {:closed true}
[:items [:sequential WorkspaceListing]]
[:limit [:maybe :int]]
[:offset [:maybe :int]]]
"Get a list of all workspaces"
[_route-params
_query-params]
{:items (t2/select [:model/Workspace :id :name [[:not= nil :archived_at] :archived]]
(cond-> {:order-by [[:created_at :desc]]}
(request/limit) (sql.helpers/limit (request/limit))
(request/offset) (sql.helpers/offset (request/offset))))
:limit (request/limit)
:offset (request/offset)})
(mr/def ::input-table
[:map
;; Future-proof with cross-db Python transforms
[:db_id ::ws.t/appdb-id]
[:schema :string]
[:table :string]
[:table_id [:maybe ::ws.t/appdb-id]]])
(mr/def ::output-table
[:map
;; Future-proof with multi-db Workspaces, plus necessary to resolve table references when id is null.
[:db_id ::ws.t/appdb-id]
[:global [:map
[:transform_id [:maybe ::ws.t/appdb-id]]
[:schema :string]
[:table :string]
[:table_id [:maybe ::ws.t/appdb-id]]]]
[:isolated [:map
[:transform_id ::ws.t/ref-id]
[:schema :string]
[:table :string]
[:table_id [:maybe ::ws.t/appdb-id]]]]])
(api.macros/defendpoint :get "/:id/table"
:- [:map {:closed true}
[:inputs [:sequential ::input-table]]
[:outputs [:sequential ::output-table]]]
"Get workspace tables"
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params]
;; This typically needs the big 'ol graph, to know about the enclosed transforms (for their outputs)
;; Easy optimization for the <= 1 entities case and just return the stuff directly though :lightbulb
(let [{:keys [database_id schema]} (api/check-404 (t2/select-one [:model/Workspace :database_id :schema] id))]
;; TODO fix N+1
{:inputs []
:outputs (for [{:keys [ref_id target]} (t2/select [:model/WorkspaceTransform :ref_id :target] :workspace_id id)]
;; transform_id is not guaranteed to be the global_id, topology may have changed
{:db_id database_id
:global {:transform_id nil
:schema (:schema target)
:table (:name target)
:table_id (t2/select-one-pk :model/Table
:db_id database_id
:schema (:schema target)
:name (:name target))}
:isolated {:transform_id ref_id
:schema schema
:table (str (:schema target) "__" (:name target))
:table_id (t2/select-one-pk :model/Table
:db_id database_id
:schema schema
:name (str (:schema target) "__" (:name target)))}})}))
(api.macros/defendpoint :get "/:id" :- Workspace
"Get a single workspace by ID"
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params]
(-> (t2/select-one :model/Workspace :id id) api/check-404 ws->response))
(api.macros/defendpoint :get "/:id/log"
:- [:map
[:workspace_id ms/PositiveInt]
[:status [:enum :pending :ready]]
[:updated_at :any]
[:last_completed_at [:maybe :any]]
[:logs [:sequential [:map
[:id ms/PositiveInt]
[:task :keyword]
[:started_at :any]
[:updated_at :any]
[:completed_at [:maybe :any]]
[:status [:maybe :keyword]]
[:message [:maybe :string]]]]]]
"Get workspace creation status and recent log entries for polling during async setup"
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params]
(let [workspace (api/check-404 (t2/select-one :model/Workspace :id id))
logs (t2/select [:model/WorkspaceLog
:id :task :started_at :completed_at :status :message
:updated_at]
:workspace_id id
{:order-by [[:started_at :desc]]
:limit log-limit})]
{:workspace_id id
:status (:status workspace)
:logs logs
:updated_at (->> (map :updated_at logs) sort reverse first)
:last_completed_at (->> (seq (keep :completed_at logs)) sort reverse first)}))
(def ^:private CreateWorkspace
[:map
[:name [:string {:min 1}]]
[:database_id ::ws.t/appdb-id]])
(api.macros/defendpoint :post "/" :- Workspace
"Create a new workspace
Potential payload:
{:name \"a\" :database_id 2}}"
[_route-params
_query-params
{:keys [database_id] :as body} :- CreateWorkspace]
(when database_id
(check-transforms-enabled! database_id))
(ws->response (ws.common/create-workspace! api/*current-user-id* body)))
(api.macros/defendpoint :put "/:id" :- Workspace
"Update simple workspace properties, like name."
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params
{:keys [name]} :- [:map {:closed true} [:name [:string {:min 1}]]]]
(u/prog1 (api/check-404 (t2/select-one :model/Workspace :id id))
(api/check-400 (nil? (:archived_at <>)) "Cannot update an archived workspace"))
(t2/update! :model/Workspace id {:name name})
(-> (t2/select-one :model/Workspace :id id)
ws->response))
(api.macros/defendpoint :post "/:id/archive" :- Workspace
"Archive a workspace. Deletes the isolated schema and tables, but preserves mirrored entities."
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params
_body-params]
(let [ws (api/check-404 (t2/select-one :model/Workspace :id id))]
(api/check-400 (nil? (:archived_at ws)) "You cannot archive an archived workspace")
;; TODO tear down the isolated database resources, and delete the graph
(t2/update! :model/Workspace id {:archived_at [:now]})
(-> (t2/select-one :model/Workspace :id id)
ws->response)))
(api.macros/defendpoint :post "/:id/unarchive" :- Workspace
"Restore an archived workspace. Recreates the isolated schema and tables."
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params
_body-params]
(let [ws (api/check-404 (t2/select-one :model/Workspace :id id))]
(api/check-400 (some? (:archived_at ws)) "You cannot unarchive a workspace that is not archived")
;; TODO re-provision the isolated database resources, and recompute the graph
(t2/update! :model/Workspace id {:archived_at nil})
(-> (t2/select-one :model/Workspace :id id)
ws->response)))
(api.macros/defendpoint :delete "/:id" :- [:map [:ok [:= true]]]
"Delete a workspace and all its contents, including mirrored entities."
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params]
(api/check-404 (t2/select-one :model/Workspace :id id))
;; TODO implement it https://linear.app/metabase/issue/BOT-574/delete-workspaceid
{:ok true})
(api.macros/defendpoint :post "/:id/run"
:- [:map
[:succeeded [:sequential ::ws.t/ref-id]]
[:failed [:sequential ::ws.t/ref-id]]
[:not_run [:sequential ::ws.t/ref-id]]]
"Execute all transforms in the workspace in dependency order.
Returns which transforms succeeded, failed, and were not run."
[{:keys [id]} :- [:map [:id ::ws.t/appdb-id]]
_query-params
_body-params]
(u/prog1 (t2/select-one :model/Workspace :id id)
(api/check-404 <>)
(api/check-400 (nil? (:archived_at <>)) "Cannot execute archived workspace"))
;; get topo-sorted enclosed transforms, run them in order
;; to keep things simple, stop execution as soon as there is any failure
;; in future we can continue running anything as long as its dependencies succeeded
;; TODO not only is the order dodgy, we're not actually running them.
{:succeeded (or (t2/select-fn-vec :ref_id [:model/WorkspaceTransform :ref_id] :workspace_id id) [])
:failed []
:not_run []})
(mr/def ::graph-node-type [:enum :table :output-table :transform :workspace-transform])
(mr/def ::graph-node
[:map
[:id ::appdb-or-ref-id]
[:type [:enum :input-table :output-table :workspace-transform]]
[:dependents_count [:map-of ::graph-node-type ms/PositiveInt]]
[:data :map]])
(mr/def ::graph-edge
[:map
[:from_entity_id ::appdb-or-ref-id]
[:from_entity_type :string]
[:to_entity_id ::appdb-or-ref-id]
[:to_entity_type :string]])
(def ^:private GraphResult
[:map
[:nodes [:sequential ::graph-node]]
[:edges [:sequential ::graph-edge]]])
(api.macros/defendpoint :get "/:id/graph" :- GraphResult
"Display the dependency graph between the Changeset and the (potentially external) entities that they depend on."
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params]
(api/check-404 (t2/select-one :model/Workspace :id id))
;; TODO decide on whether to show output tables, or to rather show dependencies directly between transforms.
{:nodes [{:id 1, :type "table", :data {:name "Bob"}, :dependants_count {:workspace-transform 1}}
{:id "2", :type "workspace-transform", :data {:name "MyTrans"}, :dependants_count {:output-table 1}}
{:id 3, :type "output-table", :data {:external {:table_id 2, :name "Clarence"}, :internal {:name "_-sdre4rcc@"}}, :dependants_count {}}]
;; I can't remember which way this is supposed to point - it might be meant to point *backwards* rather.
:edges [{:from_entity_type "table"
:from_entity_id 1
:to_entity_type "workspace-transform"
:to_entity_id "3"}
{:from_entity_type "workspace-transform"
:from_entity_id "3"
:to_entity_type "output-table"
:to_entity_id 3}]})
(def ^:private db+schema+table (juxt :database :schema :name))
(defn- internal-target-conflict?
"Check whether the given table is the target of another transform within the workspace. Ignores global transforms."
[ws-id target & [tx-id]]
(contains?
(t2/select-fn-set (comp db+schema+table :target)
[:model/WorkspaceTransform :target]
:ref_id [:not= tx-id]
:workspace_id ws-id)
(db+schema+table target)))
(api.macros/defendpoint :post "/:id/transform/validate/target"
"Validate the target table for a workspace transform"
[{:keys [id]} :- [:map [:id ::ws.t/appdb-id]]
{:keys [transform-id]} :- [:map [:transform-id {:optional true} ::ws.t/ref-id]]
{:keys [db_id target]} :- [:map
[:db_id {:optional true} ::ws.t/appdb-id]
[:target [:map
[:database {:optional true} ::ws.t/appdb-id]
[:type :string]
[:schema :string]
[:name :string]]]]]
(let [workspace (api/check-404 (t2/select-one [:model/Workspace :database_id] id))
target (update target :database #(or % db_id))
tx-id (when transform-id (parse-long transform-id))
ws-db-id (:database_id workspace)]
(cond
(not= "table" (:type target))
{:status 403 :body (deferred-tru "Unsupported target type")}
(and db_id ws-db-id (not= db_id ws-db-id))
{:status 403 :body (deferred-tru "Must target the workspace database")}
(not (or db_id ws-db-id))
{:status 403 :body (deferred-tru "Must target a database")}
(str/starts-with? (:schema target) "mb__isolation_")
{:status 403 :body (deferred-tru "Must not target an isolated workspace schema")}
;; Within a workspace, we defer blocking on conflicts outside the workspace
#_{:status 403 :body (deferred-tru "A table with that name already exists.")}
;; TODO consider deferring this validation until merge also.
(internal-target-conflict? id target tx-id)
{:status 403 :body (deferred-tru "Another transform in this workspace already targets that table.")}
:else
{:status 200 :body "OK"})))
(api.macros/defendpoint :post "/:id/transform"
"Add another transform to the Changeset. This could be a fork of an existing global transform, or something new."
[{:keys [id]} :- [:map [:id ::ws.t/appdb-id]]
_query-params
body :- [:map #_{:closed true}
[:name :string]
[:description {:optional true} [:maybe :string]]
[:source ::transform-source]
;; Not sure why this schema is giving trouble
#_[:target ::transform-target]]]
(api/check (transforms.util/check-feature-enabled body)
[402 (deferred-tru "Premium features required for this transform type are not enabled.")])
(let [workspace (u/prog1 (api/check-404 (t2/select-one :model/Workspace :id id))
(api/check-400 (nil? (:archived_at <>)) "Cannot create transforms in an archived workspace"))
;; TODO why 400 here and 403 in the validation route? T_T
_ (api/check-400 (not (internal-target-conflict? id (:target body)))
(deferred-tru "Another transform in this workspace already targets that table."))
global-id (:global_id body (:id body))
body (-> body (dissoc :global_id) (update :target assoc :database_id (:database_id workspace)))]
(ws.common/add-to-changeset! api/*current-user-id* workspace :transform global-id body)))
(defn- malli-map-keys [schema]
(map first (rest schema)))
;; TODO Confirm precisely which fields are needed by the FE
(def ^:private WorkspaceTransformListing
"Schema for a transform in a workspace"
[:map {:closed true}
[:ref_id ::ws.t/ref-id]
[:name :string]
[:source_type [:maybe :keyword]]
;[:creator_id ::ws.t/appdb-id]
;[:last_run :map]
; See https://metaboat.slack.com/archives/C099RKNLP6U/p1765205882655869?thread_ts=1765205222.888209&cid=C099RKNLP6U
#_[:target_stale :boolean]])
(defn- map-source-type [ws-tx]
(-> ws-tx
(assoc :source_type (keyword (:type (:source ws-tx))))
(dissoc :source)))
(api.macros/defendpoint :get "/:id/transform" :- [:map [:transforms [:sequential WorkspaceTransformListing]]]
"Get all transforms in a workspace."
[{:keys [id]} :- [:map [:id ::ws.t/appdb-id]]]
(api/check-404 (t2/select-one :model/Workspace :id id))
{:transforms (map map-source-type (t2/select [:model/WorkspaceTransform :ref_id :name :source] :workspace_id id))})
(def ^:private WorkspaceTransform
[:map
[:ref_id ::ws.t/ref-id]
[:name :string]
[:description [:maybe :string]]
[:source :map]
[:target :map]
[:workspace_id ::ws.t/appdb-id]
;[:creator_id ::ws.t/appdb-id]
[:created_at :any]
[:updated_at :any]])
(defn- fetch-ws-transform [ws-id tx-id]
;; TODO We still need to do some hydration, e.g. of the target table (both internal and external)
(-> (into [:model/WorkspaceTransform] (malli-map-keys WorkspaceTransform))
(t2/select-one :ref_id tx-id :workspace_id ws-id)
(api/check-404)))
(api.macros/defendpoint :get "/:id/transform/:tx-id" :- WorkspaceTransform
"Get a specific transform in a workspace."
[{:keys [id tx-id]} :- [:map [:id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]
(fetch-ws-transform id tx-id))
(api.macros/defendpoint :put "/:id/transform/:tx-id" :- WorkspaceTransform
"Update a transform in a workspace."
[{:keys [id tx-id]} :- [:map [:id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]
_query-params
body :- [:map
[:name {:optional true} :string]
[:description {:optional true} [:maybe :string]]
[:source {:optional true} ::transform-source]
[:target {:optional true} ::transform-target]]]
(api/check-404 (t2/select-one :model/WorkspaceTransform :ref_id tx-id :workspace_id id))
(t2/update! :model/WorkspaceTransform tx-id body)
(fetch-ws-transform id tx-id))
(api.macros/defendpoint :post "/:id/transform/:tx-id/archive" :- :nil
"Mark the given transform to be archived when the workspace is merged.
For provisional transforms we will skip even creating it in the first place."
[{:keys [id tx-id]} :- [:map [:id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]
(api/check-404 (pos? (t2/update! :model/WorkspaceTransform {:ref_id tx-id :workspace id} {:archived_at [:now]})))
nil)
(api.macros/defendpoint :post "/:id/transform/:tx-id/unarchive" :- :nil
"Unmark the given transform for archival. This will recall the last definition it had within the workspace."
[{:keys [id tx-id]} :- [:map [:id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]
(api/check-404 (pos? (t2/update! :model/WorkspaceTransform {:ref_id tx-id :workspace_id id} {:archived_at nil})))
nil)
(api.macros/defendpoint :delete "/:id/transform/:tx-id" :- :nil
"Discard a transform from the changeset.
Equivalent to resetting a checked-out transform to its global definition, or deleting a provisional transform."
[{:keys [id tx-id]} :- [:map [:id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]
(api/check-404 (pos? (t2/delete! :model/WorkspaceTransform :ref_id tx-id :workspace_id id)))
nil)
(api.macros/defendpoint :post "/:id/transform/:tx-id/run"
:- [:map [:message :string] [:run_id {:optional true} [:maybe :int]]]
"Run a transform in a workspace."
[{:keys [id tx-id]} :- [:map [:id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]
(let [transform (api/check-404 (t2/select-one :model/WorkspaceTransform :ref_id tx-id :workspace_id id))]
;; We want to run *this* transform, not its global (which might not exist)
;; ... in order to get this working quickly we'll need to create a temporary global one (scream)
(when-let [global-id (:global_id transform)]
(transforms.api/run-transform! (t2/select :model/Transform global-id)))))
(api.macros/defendpoint :get "/checkout"
:- [:map
[:transforms [:sequential
[:map
[:id ::ws.t/ref-id]
[:name :string]
[:workspace [:map
[:id ms/PositiveInt]
[:name :string]]]]]]]
"Get all downstream transforms for a transform that is not in a workspace.
Returns the transforms that were mirrored from this upstream transform, with workspace info."
[_route-params
{:keys [transform-id]} :- [:map {:closed true} [:id ::ws.t/appdb-id]]]
(let [transforms (t2/select [:model/WorkspaceTransform :ref_id :name :workspace_id] :global_id transform-id)
workspace-ids (map :workspace_id transforms)
workspaces-by-id (when (seq transforms)
(t2/select-fn->fn :id identity [:model/Workspace :id :name] :id [:in workspace-ids]))]
{:transforms (for [transform transforms]
(assoc transform :workspace (get workspaces-by-id (:workspace_id transform))))}))
(api.macros/defendpoint :post "/:id/merge"
:- [:or
[:map
[:errors [:maybe [:sequential [:map [:id ::ws.t/ref-id] [:name :string] [:error :string]]]]]
[:workspace [:map [:id ::ws.t/appdb-id] [:name :string]]]
[:archived_at [:maybe :any]]]
;; error message from check-404 or check-400
:string]
"This will:
1. Update original transforms with workspace versions
2. Archive the workspace and clean up isolated resources
Returns a report of merged entities, and any errors."
[{:keys [id]} :- [:map [:id ::ws.t/appdb-id]]
_query-params
_body-params]
(let [ws (u/prog1 (t2/select-one :model/Workspace :id id)
(api/check-404 <>)
(api/check-400 (nil? (:archived_at <>)) "Cannot merge an archived workspace"))
;; TODO implement individual entity merging
;; return new global ids for provisional transforms
{:keys [merged
errors]} {:merged {:transforms (vec (t2/select-fn-vec :global_id [:model/WorkspaceTransform :global_id] :workspace_id id))}
:errors []}]
(u/prog1
{:merged merged
:errors errors
:workspace {:id id, :name (:name ws)}
:archived_at (when-not (seq errors)
;; TODO call a ws.common method, which can handle the clean-up too
(t2/update! :model/Workspace :id id {:archived_at [:now]})
(t2/select-one-fn :archived_at [:model/Workspace :archived_at] :id id))}
(when-not (seq errors)
;; Most of the APIs and the FE are not respecting when a Workspace is archived yet.
(t2/delete! :model/Workspace id)))))
(def ^{:arglists '([request respond raise])} routes
"`/api/ee/workspace/` routes."
(api.macros/ns-handler *ns* api/+check-superuser +auth))

View File

@@ -0,0 +1,223 @@
(ns metabase-enterprise.workspaces.common
(:require
[clojure.string :as str]
[metabase-enterprise.workspaces.dag :as ws.dag]
[metabase-enterprise.workspaces.driver.common :as driver.common]
[metabase-enterprise.workspaces.isolation :as ws.isolation]
[metabase-enterprise.workspaces.models.workspace-log :as ws.log]
[metabase-enterprise.workspaces.util :as ws.u]
[metabase.api-keys.core :as api-key]
[metabase.api.common :as api]
[metabase.util :as u]
[metabase.util.quick-task :as quick-task]
[toucan2.core :as t2]))
;; should be encapsulated in our dag namespace, or dependency module
(defn check-no-card-dependencies!
"Check that transforms don't depend on cards. Throws 400 if they do."
[transform-ids]
(when-let [card-ids (seq (ws.dag/unsupported-dependency? transform-ids))]
(api/check-400 false
(format "Cannot add transforms that depend on saved questions (cards). Found dependencies on card IDs: %s"
(pr-str (vec card-ids))))))
(defn- extract-suffix-number
"Extract the numeric suffix from a workspace name like 'Foo (3)', or nil if no valid suffix."
[ws-name base-name]
(when (= (str/replace ws-name #"\s*\(\d+\)$" "") base-name)
(when-let [[_ n] (re-find #"\((\d+)\)$" ws-name)]
(parse-long n))))
(defn- generate-unique-workspace-name
"Generate a unique workspace name by appending (N) if the name already exists.
Strategy:
1. If the base name doesn't exist, use it as-is
2. If it exists, strip any existing (N) suffix and find the highest N among similar names
3. Return the base name with (N+1) appended
Note: This function has a potential race condition between checking existence and insertion.
Callers should handle unique constraint violations by retrying."
[base-name]
(when (str/blank? base-name)
(throw (ex-info "Workspace name cannot be empty" {:status-code 400})))
(if-not (t2/exists? :model/Workspace :name base-name)
base-name
(let [stripped-name (str/replace base-name #"\s*\(\d+\)$" "")
existing (t2/select-fn-set :name :model/Workspace
:name [:like (str stripped-name " (%)")])
numbers (keep #(extract-suffix-number % stripped-name) existing)
next-num (inc (apply max 0 numbers))]
(str stripped-name " (" next-num ")"))))
;; TODO: Generate new metabase user for the workspace
;; TODO: Should we move this to model as per the diagram?
(defn- create-workspace-container!
"Create the workspace and its related collection, user, and api key."
[creator-id db-id workspace-name status]
;; TODO (Chris 2025-11-19) Unsure API key name is unique, and remove this (insecure) workaround.
(let [api-key (let [key-name (format "API key for Workspace %s" workspace-name)]
(or (t2/select-one :model/ApiKey :name key-name)
(api-key/create-api-key-with-new-user! {:key-name key-name})))
;; TODO (Chris 2025-11-19) Associate the api-key user with the workspace as well.
ws (t2/insert-returning-instance! :model/Workspace
{:name workspace-name
:creator_id creator-id
:database_id db-id
:api_key_id (:id api-key)
:execution_user (:user_id api-key)
:status status})
coll (t2/insert-returning-instance! :model/Collection
{:name (format "Collection for Workspace %s" workspace-name)
:namespace "workspace"
:workspace_id (:id ws)})
schema (driver.common/isolation-namespace-name ws)
ws (assoc ws
:collection_id (:id coll)
:schema schema)]
;; Set the backlink from the workspace to the collection inside it and set the schema.
(t2/update! :model/Workspace (:id ws) {:collection_id (:id coll)
:schema schema})
;; TODO (Sanya 2025-11-18) - not sure how to transfer this api key to agent
#_(log/infof "Generated API key for workspace: %s" (u.secret/expose (:unmasked_key api-key)))
ws))
(defn- unique-constraint-violation?
"Check if an exception is due to a unique constraint violation."
[e]
(let [msg (str (ex-message e) (some-> (ex-cause e) ex-message))]
(or (str/includes? msg "unique")
(str/includes? msg "duplicate")
(str/includes? msg "UNIQUE"))))
(defn- run-workspace-setup!
"Background job: runs isolation, mirroring, grants. Updates status to :ready when done."
[{ws-id :id :as workspace} database]
(ws.log/track! ws-id :workspace-setup
(let [{:keys [database_details]} (ws.log/track! ws-id :database-isolation
(-> (ws.isolation/ensure-database-isolation! workspace database)
;; it actually returns just those, this is more like a doc than behavior
(select-keys [:schema :database_details])
(u/prog1 (t2/update! :model/Workspace ws-id <>))))
;; TODO analyze graph. in the
{:keys [inputs]} {}]
(when-let [table-ids (seq (keep #(when (= :table (:type %)) (:id %)) inputs))]
(ws.log/track! ws-id :grant-read-access
(let [user (:user database_details)
input-tables (t2/select :model/Table :id [:in table-ids])]
(ws.isolation/grant-read-access-to-tables! database user input-tables)))))
(t2/update! :model/Workspace ws-id {:status :ready})))
(defn- create-workspace-with-unique-name!
"Create a workspace with status=updating, then kick off async setup."
[creator-id database ws-name max-retries]
(loop [attempt 1]
(let [unique-name (generate-unique-workspace-name ws-name)
{:keys [retry
workspace]} (try
{:workspace (create-workspace-container! creator-id (:id database) unique-name :pending)}
(catch Exception e
(if (and (< attempt max-retries) (unique-constraint-violation? e))
{:retry true}
(throw e))))]
(if retry
(recur (inc attempt))
(do
(quick-task/submit-task! #(run-workspace-setup! workspace database))
workspace)))))
;; TODO internal: test!
(defn create-workspace!
"Create workspace"
[creator-id {ws-name-maybe :name
db-id :database_id}]
(let [ws-name (or ws-name-maybe (str (random-uuid)))
database (t2/select-one :model/Database :id db-id)]
(create-workspace-with-unique-name! creator-id database ws-name 5)))
(defn add-to-changeset!
"Add the given "
[_creator-id workspace entity-type global-id body]
(ws.u/assert-transform! entity-type)
(u/prog1
(t2/with-transaction [_]
(let [workspace-id (:id workspace)
workspace-db-id (:database_id workspace)
body (assoc-in body [:target :database] workspace-db-id)]
(t2/insert-returning-instance!
:model/WorkspaceTransform
(assoc (select-keys body [:name :description :source :target])
;; TODO add this to workspace_transform, or implicitly use the id of the user that does the merge?
;:creator_id creator-id
:global_id global-id
:workspace_id workspace-id))))))
(defn- mirror-table-to-delete-where
[database-id targets]
(let [s+t (mapv (fn [[schema name]]
[:and
[:= [:inline schema] :schema]
[:= [:inline name] :name]])
targets)]
(into [:and
[:= [:inline database-id] :db_id]
(into [:or] s+t)])))
(defn remove-entities!
"Remove multiple entities from the workspace."
[workspace entities]
;; count does not have to be 1
(assert (= 1 (count entities))
"Single kv")
(assert (= :transforms (-> entities keys first))
"For transforms")
(assert (every? pos-int? (:transforms entities))
"With seq of ids")
;; drop transform, drop its target table, no checking of dependencies
(let [mirror-transforms-ids (set (:transforms entities))
mirror-transforms-data (t2/select :model/Transform :id [:in mirror-transforms-ids])
database (t2/select-one :model/Database :id (:database_id workspace))
targets (mapv (fn [{:keys [target]}]
(assert (= "table" (:type target)))
[(:schema target) (:name target)])
mirror-transforms-data)
_ (assert (< 0 (count targets)))
tables-where-clause (mirror-table-to-delete-where (:database_id workspace) targets)
tables-data (t2/select :model/Table {:where tables-where-clause})
tables-ids (into #{} (map :id) tables-data)]
(assert (every? pos-int? tables-ids))
(when (seq targets)
(ws.isolation/drop-isolated-tables! database targets))
(when (seq tables-ids)
(t2/delete! :model/Table :id [:in tables-ids]))
(when (seq mirror-transforms-ids)
(t2/delete! :model/Transform :id [:in mirror-transforms-ids]))))
#_:clj-kondo/ignore
(comment
(defn- clean-up-ws!* [& [ws-id]]
;; Pass nil to clean up all workspaces
(let [ws-clause (or ws-id [:not= nil])]
(t2/delete! :model/Collection :workspace_id ws-clause)
(t2/delete! :model/Transform :workspace_id ws-clause)
(doseq [[db_id schema] (t2/select-fn-set (juxt :db_id :schema) :model/Table :workspace_id [:not= nil])]
(t2/delete! :model/Table :schema schema)
(let [db (t2/select-one :model/Database db_id)
driver (metabase.driver.util/database->driver db)
make-spec (requiring-resolve 'metabase.driver.sql-jdbc.connection/connection-details->spec)
jdbc-spec (make-spec driver (:details db))]
(clojure.java.jdbc/execute! jdbc-spec [(str "DROP SCHEMA \"" schema "\" CASCADE")])))
(t2/delete! :model/Workspace :id ws-clause)))
(def upstream-id (t2/select-one-pk :model/Transform :workspace_id nil {:order-by [:id]}))
(def upstream-ids (t2/select-pks-vec :model/Transform :workspace_id nil {:order-by [:id]}))
;; Ensure output table exists
(#'metabase-enterprise.transforms.execute/run-mbql-transform! (t2/select-one :model/Transform upstream-id))
(let [admin-id (t2/select-one-pk :model/User :is_superuser true {:order-by [:id]})]
(binding [api/*current-user-id* admin-id]
(create-workspace! admin-id {:name "Workplace Workspace", :upstream {:transforms upstream-ids #_[upstream-id]}})))
(clean-up-ws!*))

View File

@@ -0,0 +1,8 @@
(ns metabase-enterprise.workspaces.core
(:require
[metabase-enterprise.workspaces.isolation]
[potemkin :as p]))
(p/import-vars
[metabase-enterprise.workspaces.isolation
with-workspace-isolation])

View File

@@ -0,0 +1,103 @@
;; TODO consider renaming this to dataflow
(ns metabase-enterprise.workspaces.dag
"Used to do graph computations with respect to dataflow within a workspace.
This workspace should encapsulate all coupling to the [[metabase-enterprise.dependencies]] module, including through
database artifacts.
It is purely concerned with calculation, any side effects wanted as a result of the analysis should happen outside.")
#_(def ^:private empty-subgraph
{:working-set []
:inputs []
:outputs []
:entities []
:dependencies {}})
;; Is this working with appdb-ids or ref-ids?
(defn unsupported-dependency?
"Return a map of the entities which have unsupported dependencies, or nil if there are none."
[{transform-ids :transforms :as _entity-map}]
;; TODO filter out those that don't have a card dependency
(when (seq transform-ids)
{:transforms transform-ids}))
;;;; Public API
(defn- path-induced-subgraph
"Given a map of entity types to IDs, compute the path-induced subgraph.
`entities-by-type` is a map like {:transform [1 2 3]}.
Returns a map with:
- :changeset - the entities being edited within the workspace, in topological order
{type, ref-id, parent-id}
- :inputs - tables that the subgraph directly depends on, that are not themselves produced by the subgraph.
ordered lexically by db+schema+table, to be stable across metabase instances, and visibly sorted.
- :outputs - outputs of both the changeset transforms, and the external transforms enclosed by the changeset.
both the external names, and the internal (isolated) names. sorted by the external names.
point back to their relevant transform.
- :entities - the full list of entities, both in the changeset, and those that are enclosed. topo-sorted.
- :dependencies - association list for the subgraph, with keys and values both in topological order"
[working-set]
{:working-set (or working-set [{:type "transform", :ref_id "2", :global_id 1}])
:inputs [{:db_id 1, :schema "public", :table "orders", :table_id 1}]
;; or global + isolated?
:outputs [{:global {:transform_id nil
:schema "public"
:table "customers"
:table_id nil}
:isolated {:transform_id "1"
:schema "isolated__blah"
:table "public__customers"
:table_id 2}}
{:db_id 1
:global {:transform_id 1
:schema "public"
:table "mega_orders"
:table_id 1}
;; integer id - enclosed
:isolated {:transform_id nil
:schema "isolated__blah"
:table "public__mega_orders"
:table_id 3}}
{:id 1
:db_id 1
:global {:transform_id nil
:schema "public"
:table "augmented_orders"
:table_id 4}
;; string id - changeset
:isolated {:transform_id "2"
:schema "isolated__blah"
:table "public__augmented_orders"
:table-id nil}}]
;; should we include tables in the :dependencies graph? i kinda don't like it
;; 1. bigger and uglier for test validation
;; 2. easier to expand than to contract (and not sure which we want to show in ui)
;; 3. don't need the tables for execute.
;; 4. don't need the dependencies list for data tab.
:enclosed [{:type "transform", :id 2}]
;; If writing to JSON, we need to serialize to an association list because of rich keys.
:dependencies {{:type "external-transform", :id 1} [{:type "workspace-transform", :id "1"}]
{:type "workspace-transform", :id "2"} [{:type "input-table", :id 1}
{:type "global-transform" :id 1}]}})
;; source-table ---------> checked-out-transform
;; \____external-transform__/
;;
;; should external-transform be enclosed? answer: no, because it won't become stale from changes in the workspace
;;
;; source-table ----> model ---> checked-out-transform
;;
;; this time we *need* the model to be serialized, because we need to read it in a fresh instance.
;; it doesn't need to be remapped or "run" independently though, so does it make sense to call it enclosed?
;;
;; enclosed: between changeset only? or including the model in the example above?
;; ancestors: as far back as you can go
;;

View File

@@ -0,0 +1,37 @@
(ns metabase-enterprise.workspaces.driver.common
"Common utilities for workspace isolation drivers."
(:require
[clojure.string :as str]
[metabase.system.core :as system]))
;;;; Naming
;; re-using https://github.com/metabase/metabase/pull/61887/commits/c92e4a9cc451c61a13fef19ed9d6107873b17f07
;; (original ws isolation code)
(defn- instance-uuid-slug
"Create a slug from the site UUID, taking the first character of each section."
[site-uuid-string]
(apply str (map first (str/split site-uuid-string #"-"))))
(defn isolation-namespace-name
"Generate namespace/database name for workspace isolation following mb__isolation_<slug>_<workspace-id> pattern.
Uses 'namespace' as the generic term that maps to 'schema' in Postgres, 'database' in ClickHouse, etc."
[workspace]
(assert (some? (:id workspace)) "Workspace must have an :id")
(let [instance-slug (instance-uuid-slug (str (system/site-uuid)))
clean-workspace-id (str/replace (str (:id workspace)) #"[^a-zA-Z0-9]" "_")]
(format "mb__isolation_%s_%s" instance-slug clean-workspace-id)))
(defn isolated-table-name
"Generate name for a table mirroring transform target table in the isolated database namespace."
[{:keys [schema name] :as _source-table}]
;; TODO: This naming scheme is not guaranteed to give a unique name, even if it's likely
;; to always be unique in practice. Consider adding a hash suffix or conflict detection
;; before merging to master.
(format "%s__%s" schema name))
(defn isolation-user-name
"Generate username for workspace isolation."
[workspace]
(let [instance-slug (instance-uuid-slug (str (system/site-uuid)))]
(format "mb_isolation_%s_%s" instance-slug (:id workspace))))

View File

@@ -0,0 +1,56 @@
(ns metabase-enterprise.workspaces.driver.h2
"H2-specific implementations for workspace isolation.
H2 differences from PostgreSQL:
- H2 doesn't support CREATE USER with PASSWORD syntax like Postgres
- H2 uses CREATE USER userName PASSWORD 'password'
- H2 GRANT syntax: GRANT { SELECT | ALL } ON { SCHEMA schemaName | tableName } TO userName
- H2 supports CREATE TABLE ... AS SELECT syntax for table duplication"
(:require
[clojure.java.jdbc :as jdbc]
[metabase-enterprise.workspaces.driver.common :as driver.common]
[metabase-enterprise.workspaces.isolation :as isolation]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.util :as driver.u]))
(defmethod isolation/grant-read-access-to-tables! :h2
[database username tables]
(let [driver (driver.u/database->driver database)
jdbc-spec (sql-jdbc.conn/connection-details->spec driver (:details database))
schemas (distinct (map :schema tables))]
;; H2 uses GRANT SELECT ON SCHEMA schemaName TO userName
(doseq [schema schemas]
(jdbc/execute! jdbc-spec [(format "GRANT SELECT ON SCHEMA \"%s\" TO \"%s\"" schema username)]))
;; Also grant on individual tables for more fine-grained access
(doseq [table tables]
(jdbc/execute! jdbc-spec [(format "GRANT SELECT ON \"%s\".\"%s\" TO \"%s\""
(:schema table) (:name table) username)]))))
(defmethod isolation/init-workspace-database-isolation! :h2
[database workspace]
(let [driver (driver.u/database->driver database)
schema-name (driver.common/isolation-namespace-name workspace)
;; H2 uses simpler user creation syntax
read-user {:user (driver.common/isolation-user-name workspace)
:password (str (random-uuid))}
jdbc-spec (sql-jdbc.conn/connection-details->spec driver (:details database))]
(doseq [sql [(format "CREATE SCHEMA IF NOT EXISTS \"%s\"" schema-name)
;; H2 syntax: CREATE USER userName PASSWORD 'password'
(format "CREATE USER IF NOT EXISTS \"%s\" PASSWORD '%s'"
(:user read-user) (:password read-user))
;; Grant access on the isolation schema
(format "GRANT ALL ON SCHEMA \"%s\" TO \"%s\"" schema-name (:user read-user))]]
(jdbc/execute! jdbc-spec [sql]))
{:schema schema-name
:database_details read-user}))
(defmethod isolation/drop-isolated-tables! :h2
[database s+t-tuples]
(let [driver (driver.u/database->driver database)
jdbc-spec (sql-jdbc.conn/connection-details->spec driver (:details database))]
(run!
(fn [[schema-name table-name]]
(jdbc/execute! jdbc-spec
[(format "DROP TABLE IF EXISTS \"%s\".\"%s\""
schema-name table-name)]))
s+t-tuples)))

View File

@@ -0,0 +1,6 @@
(ns metabase-enterprise.workspaces.driver.init
"Load all workspace driver implementations.
Require this namespace to ensure all driver methods are registered."
(:require
[metabase-enterprise.workspaces.driver.h2]
[metabase-enterprise.workspaces.driver.postgres]))

View File

@@ -0,0 +1,46 @@
(ns metabase-enterprise.workspaces.driver.postgres
"Postgres-specific implementations for workspace isolation."
(:require
[clojure.java.jdbc :as jdbc]
[metabase-enterprise.workspaces.driver.common :as driver.common]
[metabase-enterprise.workspaces.isolation :as isolation]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.util :as driver.u]))
(defmethod isolation/grant-read-access-to-tables! :postgres
[database username tables]
(let [driver (driver.u/database->driver database)
jdbc-spec (sql-jdbc.conn/connection-details->spec driver (:details database))
schemas (distinct (map :schema tables))]
(doseq [schema schemas]
(jdbc/execute! jdbc-spec [(format "GRANT USAGE ON SCHEMA %s TO %s" schema username)]))
(doseq [table tables]
(jdbc/execute! jdbc-spec [(format "GRANT SELECT ON TABLE %s.%s TO %s" (:schema table) (:name table) username)]))))
(defmethod isolation/init-workspace-database-isolation! :postgres
[database workspace]
(let [driver (driver.u/database->driver database)
schema-name (driver.common/isolation-namespace-name workspace)
read-user {:user (driver.common/isolation-user-name workspace)
:password (str (random-uuid))}
jdbc-spec (sql-jdbc.conn/connection-details->spec driver (:details database))]
(doseq [sql [(format "CREATE SCHEMA %s" schema-name)
(format "CREATE USER %s WITH PASSWORD '%s'" (:user read-user) (:password read-user))
;; grant all access on the destination schema
(format "GRANT USAGE ON SCHEMA %s TO %s" schema-name (:user read-user))
;; need to be able insert and dropping, rename tables from this chema
(format "GRANT ALL PRIVILEGES ON SCHEMA %s TO %s" schema-name (:user read-user))]]
(jdbc/execute! jdbc-spec [sql]))
{:schema schema-name
:database_details read-user}))
(defmethod isolation/drop-isolated-tables! :postgres
[database s+t-tuples]
(let [driver (driver.u/database->driver database)
jdbc-spec (sql-jdbc.conn/connection-details->spec driver (:details database))]
(run!
(fn [[schema-name table-name]]
(jdbc/execute! jdbc-spec
[(format "DROP TABLE IF EXISTS \"%s\".\"%s\""
schema-name table-name)]))
s+t-tuples)))

View File

@@ -0,0 +1,7 @@
(ns metabase-enterprise.workspaces.init
(:require
[metabase-enterprise.workspaces.driver.h2]
[metabase-enterprise.workspaces.driver.postgres]
[metabase-enterprise.workspaces.models.workspace]
[metabase-enterprise.workspaces.models.workspace-mapping]
[metabase-enterprise.workspaces.models.workspace-transform]))

View File

@@ -0,0 +1,60 @@
(ns metabase-enterprise.workspaces.isolation
(:require
[metabase-enterprise.workspaces.driver.common :as driver.common]
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
[metabase.util :as u]
[toucan2.core :as t2]))
;;;; Driver multimethods
;; Implementations are in metabase-enterprise.workspaces.driver.{postgres,h2}
(defn dispatch-on-engine
"Take engine from database `db` and dispatch on that."
[database & _args]
(driver.u/database->driver database))
(defmulti grant-read-access-to-tables!
"Grant read access to these tables."
{:added "0.59.0" :arglists '([database username tables])}
#'dispatch-on-engine
:hierarchy #'driver/hierarchy)
(defmulti init-workspace-database-isolation!
"Create database isolation for a workspace. Return the database details."
{:added "0.59.0" :arglists '([database workspace])}
#'dispatch-on-engine
:hierarchy #'driver/hierarchy)
(defmulti duplicate-output-table!
"Create an isolated copy of the given output tables, for a workspace transform to write to.
TODO: Consider removing this method once we have 'remap-on-execute' semantics, where
transforms write directly to the isolated location without needing to duplicate existing tables."
{:added "0.59.0" :arglists '([database workspace output])}
#'dispatch-on-engine
:hierarchy #'driver/hierarchy)
(defmulti drop-isolated-tables!
"Drop isolated tables"
{:added "0.59.0" :arglists '([database s+t-tuples])}
#'dispatch-on-engine)
;;;; Public API
(defn ensure-database-isolation!
"Wrapper around the driver method, to make migrations easier in future."
[workspace database]
(init-workspace-database-isolation! database workspace))
(defn do-with-workspace-isolation
"Impl of* with-workspace-isolation*."
[workspace thunk]
(driver/with-swapped-connection-details (:database_id workspace)
(:database_details workspace)
(thunk)))
(defmacro with-workspace-isolation
"Execute body with necessary isolation."
[workspace & body]
`(do-with-workspace-isolation ~workspace (fn [] ~@body)))

View File

@@ -0,0 +1,49 @@
(ns metabase-enterprise.workspaces.models.workspace
(:require
[metabase.models.interface :as mi]
[metabase.util.json :as json]
[methodical.core :as methodical]
[toucan2.core :as t2]))
(def ^:private type->grouping "How models are grouped in lists"
;; do not forget to update `entities` if you add stuff here
{:transform :transforms})
(methodical/defmethod t2/table-name :model/Workspace [_model] :workspace)
(t2/deftransforms :model/Workspace
{:graph mi/transform-json
:database_details mi/transform-encrypted-json
:status mi/transform-keyword})
(doto :model/Workspace
(derive :metabase/model)
(derive :hook/timestamped?))
(defn- entities [entity-type ids]
(->> (case entity-type
:transform
(t2/select [:model/Transform
:id :name :description :source :target :source_type :creator_id :collection_id
:created_at :updated_at :transform.workspace_id :wmt.upstream_id
[:upstream.target :upstream]]
:transform.workspace_id [:in ids]
{:left-join [[:workspace_mapping_transform :wmt] [:= :wmt.downstream_id :transform.id]
[:transform :upstream] [:= :upstream.id :wmt.upstream_id]]
:order-by [:id]}))
(mapv #(-> %
(update :upstream json/decode+kw)
(assoc :type entity-type)))))
(defn- ws-contents [ids]
(fn []
(->> (mapcat #(entities % ids) (keys type->grouping))
(reduce (fn [acc e]
(update-in acc [(:workspace_id e) (type->grouping (:type e))] (fnil conj []) e))
{}))))
(methodical/defmethod t2/batched-hydrate [:model/Workspace :contents]
"Batch hydrate `Workspace` contents"
[_model k wses]
(let [ids (mapv :id wses)]
(mi/instances-with-hydrated-data wses k (ws-contents ids) :id {:default {}})))

View File

@@ -0,0 +1,55 @@
(ns metabase-enterprise.workspaces.models.workspace-log
(:require
[java-time.api :as t]
[metabase.models.interface :as mi]
[methodical.core :as methodical]
[toucan2.core :as t2]))
(methodical/defmethod t2/table-name :model/WorkspaceLog [_model] :workspace_log)
(doto :model/WorkspaceLog
(derive :metabase/model)
(derive :hook/timestamped?))
(t2/deftransforms :model/WorkspaceLog
{:task mi/transform-keyword
:status mi/transform-keyword})
(defn start!
"Create a log entry for a workspace setup task. Returns the created log."
[workspace-id task]
(t2/insert-returning-instance! :model/WorkspaceLog
{:workspace_id workspace-id
:task task
:started_at (t/offset-date-time)
:status :started}))
(defn- complete!
[status log-id & [message]]
(assert (#{:success :failure} status) "It can only be either success or failure")
(let [data (cond-> {:completed_at (t/offset-date-time)
:status status}
message (assoc :message message))]
(t2/update! :model/WorkspaceLog log-id data)))
(defn success!
"Task was completed successfully"
[log-id & [message]]
(complete! :success log-id message))
(defn failure!
"Task has failed"
[log-id & [message]]
(complete! :failure log-id message))
(defmacro track!
"Wrap body with automatic logging: starts a log entry, marks success on completion, failure on exception."
[workspace-id task & body]
`(let [log# (start! ~workspace-id ~task)]
(try
(let [result# (do ~@body)]
(success! (:id log#))
result#)
(catch Exception e#
(failure! (:id log#) (ex-message e#))
(throw e#)))))

View File

@@ -0,0 +1,18 @@
(ns metabase-enterprise.workspaces.models.workspace-mapping
(:require
[methodical.core :as methodical]
[toucan2.core :as t2]))
(methodical/defmethod t2/table-name :model/WorkspaceMappingTable [_model] :workspace_mapping_table)
(methodical/defmethod t2/primary-keys :model/WorkspaceMappingTable [_model] [:upstream_id :downstream_id])
(doto :model/WorkspaceMappingTable
(derive :metabase/model))
(methodical/defmethod t2/table-name :model/WorkspaceMappingTransform [_model] :workspace_mapping_transform)
(methodical/defmethod t2/primary-keys :model/WorkspaceMappingTransform [_model] [:upstream_id :downstream_id])
(doto :model/WorkspaceMappingTransform
(derive :metabase/model))

View File

@@ -0,0 +1,86 @@
(ns metabase-enterprise.workspaces.models.workspace-transform
"Model for WorkspaceTransform - holds the changeset of transforms being created
and edited within a workspace."
(:require
[clojure.string :as str]
[metabase.models.interface :as mi]
[methodical.core :as methodical]
[toucan2.core :as t2]))
;;; -------------------------------------------------- Cute Names --------------------------------------------------
(def ^:private adjectives
"Adjectives for generating cute ref IDs, inspired by Docker's naming scheme."
["admiring" "adoring" "affectionate" "agitated" "amazing" "angry" "awesome"
"beautiful" "blissful" "bold" "boring" "brave" "busy" "charming" "clever"
"cool" "compassionate" "competent" "condescending" "confident" "cranky"
"crazy" "dazzling" "determined" "distracted" "dreamy" "eager" "ecstatic"
"elastic" "elated" "elegant" "eloquent" "epic" "exciting" "fervent"
"festive" "flamboyant" "focused" "friendly" "frosty" "funny" "gallant"
"gifted" "goofy" "gracious" "great" "happy" "hardcore" "heuristic" "hopeful"
"hungry" "infallible" "inspiring" "intelligent" "interesting" "jolly"
"jovial" "keen" "kind" "laughing" "loving" "lucid" "magical" "modest"
"musing" "mystifying" "naughty" "nervous" "nice" "nifty" "nostalgic"
"objective" "optimistic" "peaceful" "pedantic" "pensive" "practical"
"priceless" "quirky" "quizzical" "recursing" "relaxed" "reverent" "romantic"
"sad" "serene" "sharp" "silly" "sleepy" "stoic" "strange" "stupefied"
"suspicious" "sweet" "tender" "thirsty" "trusting" "unruffled" "upbeat"
"vibrant" "vigilant" "vigorous" "wizardly" "wonderful" "xenodochial" "youthful"
"zealous" "zen"])
(def ^:private animals
"Animals for generating cute ref IDs."
["albatross" "alligator" "alpaca" "ant" "anteater" "antelope" "armadillo"
"baboon" "badger" "bat" "bear" "beaver" "bee" "bison" "boar" "buffalo"
"butterfly" "camel" "capybara" "caribou" "cat" "caterpillar" "cheetah"
"chicken" "chimpanzee" "chinchilla" "cobra" "condor" "cormorant" "coyote"
"crab" "crane" "crocodile" "crow" "deer" "dingo" "dog" "dolphin" "donkey"
"dove" "dragonfly" "duck" "eagle" "echidna" "eel" "elephant" "elk" "emu"
"falcon" "ferret" "finch" "firefly" "flamingo" "fox" "frog" "gazelle"
"gerbil" "giraffe" "goat" "goose" "gopher" "gorilla" "grasshopper" "grouse"
"hamster" "hare" "hawk" "hedgehog" "heron" "hippopotamus" "hornet" "horse"
"hummingbird" "hyena" "ibex" "iguana" "impala" "jackal" "jaguar" "jellyfish"
"kangaroo" "kingfisher" "kiwi" "koala" "komodo" "kookaburra" "lemur"
"leopard" "lion" "lizard" "llama" "lobster" "lynx" "magpie" "mallard"
"manatee" "mandrill" "marmot" "meerkat" "mink" "mole" "mongoose" "monkey"
"moose" "moth" "mouse" "mule" "narwhal" "newt" "nightingale" "ocelot"
"octopus" "okapi" "opossum" "orangutan" "orca" "oryx" "osprey" "ostrich"
"otter" "owl" "ox" "panda" "panther" "parrot" "partridge" "peacock"
"pelican" "penguin" "pheasant" "pig" "pigeon" "platypus" "pony" "porcupine"
"porpoise" "puma" "quail" "rabbit" "raccoon" "ram" "raven" "reindeer"
"rhinoceros" "salamander" "salmon" "sardine" "scorpion" "seahorse" "seal"
"shark" "sheep" "shrew" "skunk" "sloth" "snail" "snake" "sparrow" "spider"
"squid" "squirrel" "starfish" "stingray" "stork" "swallow" "swan" "tapir"
"tiger" "toad" "toucan" "turkey" "turtle" "viper" "vulture" "wallaby"
"walrus" "warthog" "wasp" "weasel" "whale" "wolf" "wolverine" "wombat"
"woodpecker" "yak" "zebra"])
(defn generate-ref-id
"Generate a cute adjective-animal style ref ID with a random suffix.
Format: adjective-animal-xxxx (where xxxx is a random 4-char hex suffix)"
[]
(let [adj (rand-nth adjectives)
animal (rand-nth animals)
suffix (format "%04x" (rand-int 65536))]
(str adj "-" animal "-" suffix)))
;;; -------------------------------------------------- Model Setup --------------------------------------------------
(methodical/defmethod t2/table-name :model/WorkspaceTransform [_model] :workspace_transform)
(methodical/defmethod t2/primary-keys :model/WorkspaceTransform [_model] [:ref_id])
(doto :model/WorkspaceTransform
(derive :metabase/model)
(derive :hook/timestamped?)
(derive :hook/entity-id))
(t2/deftransforms :model/WorkspaceTransform
{:ref_id {:in identity :out str/trim}
:source mi/transform-json
:target mi/transform-json})
(t2/define-before-insert :model/WorkspaceTransform
[instance]
(cond-> instance
(not (:ref_id instance)) (assoc :ref_id (generate-ref-id))))

View File

@@ -0,0 +1,23 @@
(ns metabase-enterprise.workspaces.types
(:require
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]))
(mr/def ::appdb-id ms/PositiveInt)
(mr/def ::ref-id [:string {:min 1}])
(def entity-types
"The kinds of entities we can store within a Workspace."
[{:name "Transform"
:key :transform
:group :transforms
:model :model/Transform}])
(mr/def ::entity-type (into [:enum] (map :key) entity-types))
(mr/def ::entity-grouping (into [:enum] (map :group) entity-types))
;; Map like {:transforms [1 2 3]}
(mr/def ::entity-map
[:map-of ::entity-grouping [:sequential ms/PositiveInt]])

View File

@@ -0,0 +1,36 @@
(ns metabase-enterprise.workspaces.util)
(defn assert-transform!
"Test whether we support the given entity type within workspaces yet.
Named for the only case we support currently, to make call sites assumptions more obvious."
[entity-type]
(when (not= "transform" (name entity-type))
(throw (ex-info "Entity type not supported"
{:status-code 400
:entity-type entity-type}))))
(defn- toposort-visit [node child->parents visited result]
(cond
(visited node) [visited result]
:else (let [parents (child->parents node [])
[visited' result'] (reduce (fn [[v r] p]
(toposort-visit p child->parents v r))
[(conj visited node) result]
parents)]
[visited' (conj result' node)])))
(defn toposort-dfs
"Perform a topological sort using depth-first search.
Takes a map from child nodes to their parent nodes (dependencies).
Returns nodes in topological order (dependencies before dependents)."
[child->parents]
;; TODO (Chris 2025-11-20): Detect cycles and throw an error. (In practice inputs will never be cyclic, but still.)
(let [all-nodes (set (keys child->parents))]
(loop [visited #{}
result []
remaining all-nodes]
(if (empty? remaining)
result
(let [node (first remaining)
[visited' result'] (toposort-visit node child->parents visited result)]
(recur visited' result' (disj remaining node)))))))

View File

@@ -169,6 +169,20 @@
:table #{checkins-id venues-id}}
(calculation/upstream-deps:transform transform)))))))
(deftest ^:parallel upstream-deps-python-transform-test
(mt/with-premium-features #{:transforms}
(let [products-id (mt/id :products)
orders-id (mt/id :orders)]
(mt/with-temp [:model/Transform transform {:name "Test Transform"
:source {:type :python
:source-tables {"PRODUCTS" products-id
"ORDERS" orders-id}
:body "..."}
:target {:schema "PUBLIC"
:name "test_output"}}]
(is (= {:table #{products-id orders-id}}
(calculation/upstream-deps:transform transform)))))))
(deftest ^:parallel upstream-deps-card-native-with-parameter-source-test
(let [mp (mt/metadata-provider)
products-id (mt/id :products)

View File

@@ -96,6 +96,8 @@
:model/TransformJobRun
:model/TransformJobTransformTag
:model/TransformTransformTag
:model/WorkspaceMappingTable
:model/WorkspaceMappingTransform
:model/PythonLibrary
:model/Undo
:model/User
@@ -106,7 +108,8 @@
:model/ConnectionImpersonation
:model/CloudMigration
:model/Comment
:model/CommentReaction})
:model/CommentReaction
:model/Workspace})
(deftest ^:parallel comprehensive-entity-id-test
(let [entity-id-models (->> (v2.entity-ids/toucan-models)

View File

@@ -1,6 +1,7 @@
(ns ^:mb/driver-tests metabase-enterprise.transforms.api-test
"Tests for /api/transform endpoints."
(:require
[clojure.string :as str]
[clojure.test :refer :all]
[medley.core :as m]
[metabase-enterprise.transforms.query-test-util :as query-test-util]
@@ -269,6 +270,229 @@
(is (=? [{:id t2-id}]
(mt/user-http-request :crowberto :get 200 "ee/transform" :tag_ids [tag2-id])))))))))
(deftest filter-transforms-by-workspace-test
(testing "GET /api/ee/transform filters out workspace transforms by default"
(mt/with-premium-features #{:transforms :workspaces}
(mt/with-temp [:model/Transform {global-id :id} {}
:model/Workspace {ws-id :id} {:name "Test Workspace"}
:model/Transform {ws-transform-id :id} {:workspace_id ws-id}]
(testing "by default, only global transforms (workspace_id=null) are returned"
(let [transforms (mt/user-http-request :crowberto :get 200 "ee/transform")
ids (set (map :id transforms))]
(is (contains? ids global-id))
(is (not (contains? ids ws-transform-id)))))))))
(deftest exclude-workspace-id-filter-test
(testing "GET /api/ee/transform with exclude_workspace_id excludes mirrored transforms"
(mt/with-premium-features #{:transforms :workspaces}
(mt/with-temp [;; Global transforms (workspace_id = null)
:model/Transform {global1-id :id} {:name "Global Transform 1"}
:model/Transform {global2-id :id} {:name "Global Transform 2"}
:model/Transform {global3-id :id} {:name "Global Transform 3"}
;; Workspace
:model/Workspace {ws-id :id} {:name "Test Workspace"}
;; Workspace transforms (mirrored from global1 and global2)
:model/Transform {ws-xf1-id :id} {:name "Workspace Transform 1"
:workspace_id ws-id}
:model/Transform {ws-xf2-id :id} {:name "Workspace Transform 2"
:workspace_id ws-id}
;; Mappings: global1 -> ws-xf1, global2 -> ws-xf2
:model/WorkspaceMappingTransform _ {:upstream_id global1-id
:downstream_id ws-xf1-id
:workspace_id ws-id}
:model/WorkspaceMappingTransform _ {:upstream_id global2-id
:downstream_id ws-xf2-id
:workspace_id ws-id}]
(testing "without exclude_workspace_id, returns all global transforms"
(let [transforms (mt/user-http-request :crowberto :get 200 "ee/transform")
ids (set (map :id transforms))]
(is (contains? ids global1-id))
(is (contains? ids global2-id))
(is (contains? ids global3-id))
;; workspace transforms should NOT be returned (they have workspace_id set)
(is (not (contains? ids ws-xf1-id)))
(is (not (contains? ids ws-xf2-id)))))
(testing "with exclude_workspace_id, excludes global transforms mirrored into that workspace"
(let [transforms (mt/user-http-request :crowberto :get 200 "ee/transform"
:exclude_workspace_id ws-id)
ids (set (map :id transforms))]
;; global1 and global2 are mirrored into the workspace, so they should be excluded
(is (not (contains? ids global1-id)))
(is (not (contains? ids global2-id)))
;; global3 is NOT mirrored, so it should be returned
(is (contains? ids global3-id))
;; workspace transforms should still NOT be returned
(is (not (contains? ids ws-xf1-id)))
(is (not (contains? ids ws-xf2-id))))))))
(testing "exclude_workspace_id only excludes transforms mirrored into that specific workspace"
(mt/with-premium-features #{:transforms :workspaces}
(mt/with-temp [;; Global transforms
:model/Transform {global1-id :id} {:name "Global Transform 1"}
:model/Transform {global2-id :id} {:name "Global Transform 2"}
;; Two workspaces
:model/Workspace {ws1-id :id} {:name "Workspace 1"}
:model/Workspace {ws2-id :id} {:name "Workspace 2"}
;; global1 mirrored only into ws1
:model/Transform {ws1-xf-id :id} {:name "WS1 Transform"
:workspace_id ws1-id}
:model/WorkspaceMappingTransform _ {:upstream_id global1-id
:downstream_id ws1-xf-id
:workspace_id ws1-id}
;; global2 mirrored only into ws2
:model/Transform {ws2-xf-id :id} {:name "WS2 Transform"
:workspace_id ws2-id}
:model/WorkspaceMappingTransform _ {:upstream_id global2-id
:downstream_id ws2-xf-id
:workspace_id ws2-id}]
(testing "exclude_workspace_id=ws1 excludes only global1"
(let [transforms (mt/user-http-request :crowberto :get 200 "ee/transform"
:exclude_workspace_id ws1-id)
ids (set (map :id transforms))]
(is (not (contains? ids global1-id)))
(is (contains? ids global2-id))))
(testing "exclude_workspace_id=ws2 excludes only global2"
(let [transforms (mt/user-http-request :crowberto :get 200 "ee/transform"
:exclude_workspace_id ws2-id)
ids (set (map :id transforms))]
(is (contains? ids global1-id))
(is (not (contains? ids global2-id)))))))))
(deftest filter-transforms-by-type-test
(testing "should be able to filter transforms by source type"
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/with-premium-features #{:transforms :transforms-python}
(mt/with-temp [:model/Transform {native-transform-id :id}
{:name "Native Transform"
:source {:type "query"
:query {:database (mt/id)
:type "native"
:native {:query "SELECT 1"
:template-tags {}}}}
:target {:type "table"
:name (str "test_native_" (u/generate-nano-id))}}
:model/Transform {python-transform-id :id}
{:name "Python Transform"
:source {:type "python"
:body "print('hello')"
:source-tables {}}
:target {:type "table"
:name (str "test_python_" (u/generate-nano-id))
:database (mt/id)}}]
(testing "filter by native type"
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :type ["native"])]
(is (some #(= native-transform-id (:id %)) results))
(is (not (some #(= python-transform-id (:id %)) results)))))
(testing "filter by python type"
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :type ["python"])]
(is (some #(= python-transform-id (:id %)) results))
(is (not (some #(= native-transform-id (:id %)) results)))))
(testing "filter by both types returns all"
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :type ["native" "python"])]
(is (some #(= native-transform-id (:id %)) results))
(is (some #(= python-transform-id (:id %)) results)))))))))
(deftest filter-transforms-by-database-id-test
(testing "should be able to filter transforms by database_id"
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/with-premium-features #{:transforms}
(mt/with-temp [:model/Database {other-db-id :id} {:engine :h2 :details {}}
:model/Transform {transform1-id :id}
{:name "Transform for main DB"
:source {:type "query"
:query {:database (mt/id)
:type "native"
:native {:query "SELECT 1"
:template-tags {}}}}
:target {:type "table"
:name (str "test_main_" (u/generate-nano-id))}}
:model/Transform {transform2-id :id}
{:name "Transform for other DB"
:source {:type "query"
:query {:database other-db-id
:type "native"
:native {:query "SELECT 2"
:template-tags {}}}}
:target {:type "table"
:name (str "test_other_" (u/generate-nano-id))}}]
(testing "filter by main database id"
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :database_id (mt/id))]
(is (some #(= transform1-id (:id %)) results))
(is (not (some #(= transform2-id (:id %)) results)))))
(testing "filter by other database id"
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :database_id other-db-id)]
(is (some #(= transform2-id (:id %)) results))
(is (not (some #(= transform1-id (:id %)) results)))))
(testing "filter by non-existent database returns empty for our transforms"
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :database_id 999999)]
(is (not (some #(= transform1-id (:id %)) results)))
(is (not (some #(= transform2-id (:id %)) results))))))))))
(deftest filter-transforms-by-database-id-target-test
(testing "should match transforms where target database matches"
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/with-premium-features #{:transforms :transforms-python}
(mt/with-temp [:model/Database {target-db-id :id} {:engine :h2 :details {}}
:model/Transform {transform-id :id}
{:name "Python Transform with target DB"
:source {:type "python"
:body "print('hello')"
:source-tables {}}
:target {:type "table"
:name (str "test_target_" (u/generate-nano-id))
:database target-db-id}}]
(testing "filter by target database id"
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :database_id target-db-id)]
(is (some #(= transform-id (:id %)) results)))))))))
(deftest filter-transforms-combined-test
(testing "should be able to combine database_id and type filters"
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/with-premium-features #{:transforms :transforms-python}
(mt/with-temp [:model/Database {other-db-id :id} {:engine :h2 :details {}}
:model/Transform {query-main-id :id}
{:name "Query on main DB"
:source {:type "query"
:query {:database (mt/id)
:type "native"
:native {:query "SELECT 1"
:template-tags {}}}}
:target {:type "table"
:name (str "test_qm_" (u/generate-nano-id))}}
:model/Transform {python-main-id :id}
{:name "Python on main DB"
:source {:type "python"
:body "print('hello')"
:source-tables {}}
:target {:type "table"
:name (str "test_pm_" (u/generate-nano-id))
:database (mt/id)}}
:model/Transform {query-other-id :id}
{:name "Query on other DB"
:source {:type "query"
:query {:database other-db-id
:type "native"
:native {:query "SELECT 2"
:template-tags {}}}}
:target {:type "table"
:name (str "test_qo_" (u/generate-nano-id))}}]
(testing "filter by main database and query type"
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform"
:database_id (mt/id)
:type ["query"])]
(is (some #(= query-main-id (:id %)) results))
(is (not (some #(= python-main-id (:id %)) results)))
(is (not (some #(= query-other-id (:id %)) results)))))
(testing "filter by main database and python type"
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform"
:database_id (mt/id)
:type ["python"])]
(is (some #(= python-main-id (:id %)) results))
(is (not (some #(= query-main-id (:id %)) results)))
(is (not (some #(= query-other-id (:id %)) results))))))))))
(deftest get-transforms-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/with-premium-features #{:transforms}

View File

@@ -108,3 +108,81 @@
(catch Exception _e
;; Ignore cleanup errors
nil)))))))))
;;; ------------------------------------------------------------
;;; Filter xf tests
;;; ------------------------------------------------------------
(deftest ^:parallel type-filter-xf-test
(testing "->type-filter-xf filters transforms by source type"
;; Note: :query :type "query" = MBQL, :query :type "native" = native SQL
(let [mbql-transform {:id 1 :source {:type "query" :query {:type "query"}}}
native-transform {:id 2 :source {:type "query" :query {:type "native"}}}
python-transform {:id 3 :source {:type "python"}}
transforms [mbql-transform native-transform python-transform]]
(are [x y] (= x (into [] (transforms.util/->type-filter-xf y) transforms))
transforms nil
transforms []
[mbql-transform] ["query"]
[native-transform] ["native"]
[python-transform] ["python"]
[mbql-transform native-transform] ["query" "native"]
transforms ["query" "native" "python"]
[] ["whatever"]))))
(deftest ^:parallel database-id-filter-xf-test
(testing "->database-id-filter-xf filters transforms by database ID"
(let [db1-query-x {:id 1
:name "Query on DB1"
:source {:type "query" :query {:database 10}}
:target {:type "table" :name "t1" :database 1}}
db2-target-x {:id 2
:name "Python targeting DB4"
:source {:type "python" :body "" :source-tables {}}
:target {:type "table" :name "t4" :database 2}}
transforms [db1-query-x db2-target-x]]
(are [x y] (= x (into [] (transforms.util/->database-id-filter-xf y) transforms))
transforms nil
[db1-query-x] 1
[] 10 ; source does not match
[db2-target-x] 2
[] 999))))
(deftest ^:parallel matching-timestamp?-test
(testing "matching-timestamp? checks if a timestamp falls within a date range [start, end)"
(let [matching-timestamp? #'transforms.util/matching-timestamp?
field-path [:start_time]
range-jan-feb {:start "2024-01-01T00:00:00Z" :end "2024-02-01T00:00:00Z"}
range-start-only {:start "2024-01-01T00:00:00Z" :end nil}
range-end-only {:start nil :end "2024-02-01T00:00:00Z"}]
(testing "with both start and end bounds"
(are [expected timestamp]
(= expected (matching-timestamp? {:start_time timestamp} field-path range-jan-feb))
nil nil ; missing field returns nil
true "2024-01-15T12:00:00Z" ; timestamp in middle of range
false "2023-12-15T12:00:00Z" ; timestamp before range
false "2024-02-15T12:00:00Z" ; timestamp after range
true "2024-01-01T00:00:00Z" ; start boundary is inclusive
true "2024-02-01T00:00:00Z")) ; end boundary is inclusive too 🤷
(testing "with only start bound"
(are [expected timestamp]
(= expected (matching-timestamp? {:start_time timestamp} field-path range-start-only))
true "2024-01-15T12:00:00Z" ; timestamp after start
true "2024-02-15T12:00:00Z" ; any timestamp after start
false "2023-12-15T12:00:00Z")) ; timestamp before start
(testing "with only end bound"
(are [expected timestamp]
(= expected (matching-timestamp? {:start_time timestamp} field-path range-end-only))
true "2024-01-15T12:00:00Z" ; timestamp before end
true "2023-12-15T12:00:00Z" ; any timestamp before end
false "2024-02-15T12:00:00Z")) ; timestamp after end
(testing "returns nil when field value is missing"
(are [job] (nil? (matching-timestamp? job field-path range-jan-feb))
{}
{:other "value"})))))

View File

@@ -0,0 +1,645 @@
(ns ^:mb/driver-tests metabase-enterprise.workspaces.api-test
(:require
[clojure.string :as str]
[clojure.test :refer :all]
[metabase-enterprise.transforms.interface :as transforms.i]
[metabase-enterprise.transforms.test-dataset :as transforms-dataset]
[metabase-enterprise.transforms.test-util :refer [with-transform-cleanup!]]
[metabase-enterprise.workspaces.isolation :as ws.isolation]
[metabase.lib.core :as lib]
[metabase.search.test-util :as search.tu]
[metabase.test :as mt]
[metabase.test.util :as tu]
[toucan2.core :as t2])
(:import (java.time OffsetDateTime)))
(set! *warn-on-reflection* true)
(use-fixtures :once (fn [tests]
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/with-premium-features [:workspaces :dependencies :transforms]
(search.tu/with-index-disabled
(tests))))))
(use-fixtures :each (fn [tests]
(mt/with-model-cleanup [:model/Collection
:model/Transform
:model/Workspace
:model/WorkspaceTransform
:model/WorkspaceMappingTransform]
(tests))))
(defn ws-url [id & [path]]
(str "ee/workspace/" id path))
(defn- ws-ready
"Poll until workspace status becomes :ready or timeout"
[ws-or-id]
(let [ws-id (cond-> ws-or-id
(map? ws-or-id) :id)]
(try
(tu/poll-until 300 (or (t2/select-one :model/Workspace :id ws-id :status :ready)
(Thread/sleep 10)))
(catch Exception e
(if (:timeout-ms (ex-data e))
(throw (ex-info "Workspace is not ready yet" {:logs (t2/select [:model/WorkspaceLog :task :status :message]
:workspace_id ws-id
{:order-by [[:started_at :desc]]})}))
(throw e))))))
(deftest workspace-endpoints-require-superuser-test
(mt/with-temp [:model/Workspace workspace {:name "Private Workspace"}]
(testing "GET /api/ee/workspace requires superuser"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :get 403 "ee/workspace"))))
(testing "GET /api/ee/workspace/:id requires superuser"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :get 403 (ws-url (:id workspace) "")))))
(testing "POST /api/ee/workspace requires superuser"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :post 403 "ee/workspace"
{:name "Unauthorized Workspace"})))))
(mt/with-temp [:model/Workspace workspace {:name "Put Workspace"}]
(testing "PUT /api/ee/workspace/:id requires superuser"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :put 403 (ws-url (:id workspace) "")
{:name "Updated"})))))
(mt/with-temp [:model/Workspace workspace {:name "Delete Workspace"}]
(testing "DELETE /api/ee/workspace/:id requires superuser"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :delete 403 (ws-url (:id workspace) ""))))))
(mt/with-temp [:model/Workspace workspace {:name "Promote Workspace"}]
(testing "POST /api/ee/workspace/:id/promote requires superuser"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :post 403 (ws-url (:id workspace) "/merge")))))))
(deftest workspace-crud-flow-test
(let [workspace-name (str "Workspace " (random-uuid))
created (mt/user-http-request :crowberto :post 200 "ee/workspace"
{:name workspace-name
:database_id (mt/id)})
workspace-id (:id created)
collection-id (:collection_id created)]
(is (=? {:id int?
:collection_id int?
:name workspace-name}
created))
(testing "the collection exists, and we have a back reference"
(is (t2/exists? :model/Workspace :id workspace-id :collection_id collection-id))
(is (t2/exists? :model/Collection :id collection-id :workspace_id workspace-id)))
(testing "workspace appears in list response"
(let [{:keys [items]} (mt/user-http-request :crowberto :get 200 "ee/workspace")]
(is (some #(= workspace-id (:id %)) items))))
(testing "workspace can be fetched individually"
(let [response (mt/user-http-request :crowberto :get 200 (ws-url workspace-id ""))]
(is (= workspace-id (:id response)))))
(testing "workspace can be archived"
(let [updated (mt/user-http-request :crowberto :post 200 (str "ee/workspace/" workspace-id "/archive"))]
(is (some? (:archived_at updated)))))
(testing "workspace can be unarchived"
(let [updated (mt/user-http-request :crowberto :post 200 (str "ee/workspace/" workspace-id "/unarchive"))]
(is (nil? (:archived_at updated)))))))
(deftest ^:parallel promote-workspace-test
(testing "POST /api/ee/workspace/:id/promote requires superuser"
(mt/with-temp [:model/Workspace workspace {:name "Promote Test"}]
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :post 403 (ws-url (:id workspace) "/merge"))))))
(testing "Cannot merge an already archived workspace"
(mt/with-temp [:model/Workspace workspace {:name "Archived Workspace"
:archived_at (OffsetDateTime/now)}]
(is (= "Cannot merge an archived workspace"
(mt/user-http-request :crowberto :post 400 (ws-url (:id workspace) "/merge")))))))
(deftest merge-workspace-with-transform-test
(testing "POST /api/ee/workspace/:id/merge promotes transforms and archives workspace"
(mt/with-temp [:model/Table _table {:schema "public" :name "merge_test_table"}
:model/Transform x1 {:name "Upstream Transform"
:description "Original description"
:target {:type "table"
:database 1
:schema "public"
:name "merge_test_table"}}]
(let [{ws-id :id} (ws-ready (mt/user-http-request :crowberto :post 200 "ee/workspace"
{:name "Merge test"
:database_id (mt/id)}))]
(mt/user-http-request :crowberto :post 200 (ws-url ws-id "/transform")
(merge {:global_id (:id x1)}
(select-keys x1 [:name :description :source :target])))
(testing "We've got our workspace with transform to merge"
(is (int? ws-id))
;; (sanya) TODO: maybe switch to using transform APIs once we get our own
(let [x2-id (t2/select-one-fn :downstream_id :model/WorkspaceMappingTransform :upstream_id (:id x1))]
(t2/update! :model/Transform :id x2-id {:description "Modified in workspace"})))
(testing "returns merged transforms"
(is (=? {:merged {:transforms [(:id x1)]}
:errors []
:workspace {:id ws-id :name "Merge test"}}
(mt/user-http-request :crowberto :post 200 (ws-url ws-id "/merge")))))
;; TODO re-implement merge
#_(testing "original transform was updated with workspace version"
(is (= "Modified in workspace"
(t2/select-one-fn :description :model/Transform :id (:id x1)))))
(testing "workspace was deleted after successful merge"
(is (nil? (t2/select-one :model/Workspace :id ws-id))))))))
(deftest create-workspace-transform-permissions-test
(testing "POST /api/ee/workspace/:id/transform requires superuser"
(mt/with-temp [:model/Workspace workspace {:name "Transform Test"}]
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :post 403 (ws-url (:id workspace) "/transform")
{:name "Should Fail"
:source {:type "query"
:query {}}
:target {:type "table"
:name "should_fail"}}))))))
(deftest create-workspace-transform-archived-test
(testing "Cannot create transform in archived workspace"
(mt/with-temp [:model/Workspace workspace {:name "Archived"
:archived_at (OffsetDateTime/now)}]
(is (= "Cannot create transforms in an archived workspace"
(mt/user-http-request :crowberto :post 400 (ws-url (:id workspace) "/transform")
{:name "Should Fail"
:source {:type "query"
:query (mt/mbql-query venues)}
:target {:type "table"
:name "should_fail"}}))))))
(deftest add-transforms-to-workspace-test
(testing "Add transforms to workspace via POST /transform"
(with-transform-cleanup! [orig-name "ws_tables_test"]
(mt/with-temp [:model/Transform {x1-id :id :as x1} {:target {:type "table"
:database (mt/id)
:schema "public"
:name orig-name}}]
(let [{ws-id :id} (ws-ready (mt/user-http-request :crowberto :post 200 "ee/workspace"
{:name "Add Transforms Test"
:database_id (mt/id)}))]
(is (int? ws-id))
(testing "Can check out a global transform into workspace"
(is (=? {:ref_id string?
:global_id x1-id}
(mt/user-http-request :crowberto :post 200 (ws-url ws-id "/transform")
(merge {:global_id x1-id}
(select-keys x1 [:name :description :source :target]))))))
(testing "Can create a new provisional transform"
(is (=? {:ref_id string?
:global_id nil?
:name "New Transform"}
(mt/user-http-request :crowberto :post 200 (ws-url ws-id "/transform")
{:name "New Transform"
:source {:type "query"
:query (mt/mbql-query venues)}
:target {:type "table"
:name "new_transform_output"}}))))
(testing "Cannot add transforms to archived workspace"
(t2/update! :model/Workspace ws-id {:archived_at (OffsetDateTime/now)})
(is (= "Cannot create transforms in an archived workspace"
(mt/user-http-request :crowberto :post 400 (ws-url ws-id "/transform")
{:name "Should Fail"
:source {:type "query"
:query (mt/mbql-query venues)}
:target {:type "table"
:name "should_fail"}})))))))))
(deftest add-entities-requires-superuser-test
(testing "POST /api/ee/workspace/:id/add requires superuser"
(mt/with-temp [:model/Workspace workspace {:name "Permission Test"}]
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :post 403 (ws-url (:id workspace) "/transform")
{:name "blah", :source {}, :target {}}))))))
(deftest create-workspace-transform-test
(mt/dataset transforms-dataset/transforms-test
(let [{ws-id :id} (ws-ready (mt/user-http-request :crowberto :post 200 "ee/workspace"
{:name "Test Workspace"
:database_id (mt/id)}))]
(with-transform-cleanup! [table-name "workspace_transform_test"]
(is (=? {:ref_id string?
:workspace_id ws-id
;:creator_id (mt/user->id :crowberto)
:target {:database (mt/id)}}
(mt/user-http-request :crowberto :post 200 (ws-url ws-id "/transform")
{:name "Workspace Transform"
:source {:type "query"
:query (mt/mbql-query transforms_products)}
:target {:type "table"
:name table-name}})))
(is (=? {:id ws-id, :status "ready"}
(mt/user-http-request :crowberto :get 200 (ws-url ws-id))))
(is (=? {:transforms [{:ref_id string?, :name "Workspace Transform", :source_type "query"}]}
(mt/user-http-request :crowberto :get 200 (ws-url ws-id "/transform"))))))))
(deftest tables-endpoint-empty-ws-test
(let [user-id (mt/user->id :crowberto)
ws (mt/user-http-request :crowberto :post 200 "ee/workspace/"
{:name "My test ws"
:creator_id user-id
:database_id (mt/id)})]
(is (= {:inputs []
:outputs []}
(mt/user-http-request :crowberto :get 200 (ws-url (:id ws) "/table"))))))
(deftest tables-endpoint-transform-not-run-test
(let [mp (mt/metadata-provider)
query (lib/native-query mp "select * from orders limit 10;")
orig-schema "public"]
(with-transform-cleanup! [orig-name "ws_tables_test"]
(mt/with-temp [:model/Transform x1 {:source_type "native"
:name "My X1"
:source {:type "query"
:query query}
:target {:type "table"
:database (mt/id)
:schema "public"
:name orig-name}}]
(let [;; create the workspace
{ws-id :id} (mt/user-http-request :crowberto :post 200 "ee/workspace"
{:name "Test Workspace"
:database_id (mt/id)})
;; add the transform
ref_id (:ref_id (mt/user-http-request :crowberto :post 200 (ws-url ws-id "/transform") x1))
;; get the tables
tables-result (mt/user-http-request :crowberto :get 200 (ws-url ws-id "/table"))]
(testing "/tables returns expected results"
(is (=? {:inputs [#_{:id (mt/id :orders) :schema orig-schema :table "orders"}]
:outputs [{:db_id (mt/id)
:global {:schema "public", :table orig-name}
:isolated {:transform_id ref_id}}]}
tables-result))))))))
(deftest tables-endpoint-test
(let [mp (mt/metadata-provider)
query (lib/native-query mp "select * from orders limit 10;")
orig-schema "public"]
(with-transform-cleanup! [orig-name "ws_tables_test"]
(mt/with-temp [:model/Transform x1 {:source_type "native"
:name "My X1"
:source {:type "query"
:query query}
:target {:type "table"
:database (mt/id)
:schema "public"
:name orig-name}}]
;; TODO: add a provisional transform, and make sure one of the output tables is created
;; create the global table
(transforms.i/execute! x1 {:run-method :manual})
(let [;; create the workspace
workspace (ws-ready (mt/user-http-request :crowberto :post 200 "ee/workspace"
{:name "Test Workspace"
:database_id (mt/id)}))
;; add the transform
ref-id (:ref_id (mt/user-http-request :crowberto :post 200 (ws-url (:id workspace) "/transform") x1))
;; get the tables
tables-result (mt/user-http-request :crowberto :get 200 (ws-url (:id workspace) "/table"))
mirror-transform (t2/select-one :model/WorkspaceTransform :workspace_id (:id workspace))
mirror-table (t2/select-one :model/Table
:schema (-> mirror-transform :target :schema)
:name (-> mirror-transform :target :name))]
(testing "/table returns expected results"
;; TODO: implement inputs (requires query analysis)
(is (=? {:inputs [#_{:id (mt/id :orders) :schema orig-schema :table "orders"}]
:outputs
[{:db_id (mt/id)
:global {:schema orig-schema :table orig-name, :table_id (:id mirror-table)}
:isolated {:transform_id ref-id}}]}
tables-result))))))))
;;;; Card dependency rejection tests
(defn- query-with-source-card
"Create a pMBQL query that uses a card as its source."
[card-id]
{:lib/type :mbql/query
:database (mt/id)
:stages [{:lib/type :mbql.stage/mbql
:source-card card-id}]})
(defn- create-transform-with-card-source!
"Create a transform whose source query depends on a card.
The after-insert hook triggers dependency calculation automatically."
[card]
(t2/insert-returning-instance! :model/Transform
{:name "Transform depending on card"
:source {:type :query
:query (query-with-source-card (:id card))}
:target {:type "table"
:database (mt/id)
:schema "public"
:name "card_dep_output"}}))
(deftest add-transform-rejects-card-dependencies-test
(testing "Cannot add transforms that depend on cards"
(mt/with-temp [:model/Card card {:name "Test Card"
:database_id (mt/id)
:dataset_query (mt/mbql-query venues)}]
(let [tx (create-transform-with-card-source! card)
ws (mt/user-http-request :crowberto :post 200 "ee/workspace"
{:name "Card Dep Workspace"
:database_id (mt/id)})
;; TODO add validation
_response (mt/user-http-request :crowberto :post 200 #_403 (ws-url (:id ws) "/transform") tx)]
#_(is (re-find #"Cannot add transforms that depend on saved questions" ws))))))
(deftest add-transform-rejects-transitive-card-dependencies-test
(testing "Cannot add transforms that transitively depend on cards"
(mt/with-temp [:model/Card card {:name "Base Card"
:database_id (mt/id)
:dataset_query (mt/mbql-query venues)}]
;; tx1 depends on card
(let [tx1 (create-transform-with-card-source! card)
;; tx2 depends on tx1 (via a manually created dependency - simulating transform chain)
tx2 (t2/insert-returning-instance! :model/Transform
{:name "Transform 2 - depends on tx1"
:source {:type :query
:query {:database (mt/id)
:type :native
:native {:query "SELECT 1"}}}
:target {:type "table"
:database (mt/id)
:schema "public"
:name "tx2_output"}})]
;; Create dependency: tx2 depends on tx1
(t2/insert! :model/Dependency
{:from_entity_type "transform"
:from_entity_id (:id tx2)
:to_entity_type "transform"
:to_entity_id (:id tx1)})
;; Try to add tx2 (which transitively depends on card via tx1) to workspace
(let [ws (mt/user-http-request :crowberto :post 200 "ee/workspace"
{:name "Transitive Card Dep Workspace"
:database_id (mt/id)})
;; TODO add validation
_response (mt/user-http-request :crowberto :post 200 #_403 (ws-url (:id ws) "/transform") tx2)]
#_(is (re-find #"Cannot add transforms that depend on saved questions" response)))))))
(deftest rename-workspace-test
(testing "POST /api/ee/workspace/:id/name updates the workspace name"
(mt/with-temp [:model/Collection {coll-id :id} {}
:model/Workspace workspace {:name "Original Name"
:database_id (mt/id)
:collection_id coll-id}]
(let [response (mt/user-http-request :crowberto :put 200 (ws-url (:id workspace))
{:name "Updated Name"})]
(is (= "Updated Name"
(:name response)
(t2/select-one-fn :name :model/Workspace :id (:id workspace)))))))
(testing "Requires superuser"
(mt/with-temp [:model/Workspace workspace {:name "Permission Test"}]
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :put 403 (ws-url (:id workspace))
{:name "Should Fail"})))))
(testing "Cannot rename an archived workspace"
(mt/with-temp [:model/Workspace workspace {:name "Archived"
:archived_at (OffsetDateTime/now)}]
(is (= "Cannot update an archived workspace"
(mt/user-http-request :crowberto :put 400 (ws-url (:id workspace))
{:name "Should Fail"}))))))
(deftest add-transform-rejects-card-dependencies-to-existing-workspace-test
(testing "Cannot add transforms with card dependencies to existing workspace"
(mt/with-temp [:model/Card card {:name "Test Card"
:database_id (mt/id)
:dataset_query (mt/mbql-query venues)}]
(let [tx (create-transform-with-card-source! card)
workspace-id (:id (mt/user-http-request :crowberto :post 200 "ee/workspace"
{:name "Empty Workspace"
:database_id (mt/id)}))
;; TODO add validation
_response (mt/user-http-request :crowberto :post 200 (ws-url workspace-id "/transform") tx)]
#_(is (re-find #"Cannot add transforms that depend on saved questions" response))))))
(deftest validate-target-test
(let [table (t2/select-one :model/Table :active true)]
(mt/with-temp [:model/Workspace {ws-id :id} {:name "test"}
:model/WorkspaceTransform _x1 {:workspace_id ws-id
:target {:database (:db_id table)
:type "table"
:schema (:schema table)
:name (str "q_" (:name table))}}]
(testing "Unique"
(is (= "OK"
(mt/with-log-level [metabase.driver.sql-jdbc.sync.describe-table :fatal]
(mt/user-http-request :crowberto :post 200 (ws-url ws-id "/transform/validate/target")
{:db_id (mt/id)
:target {:type "table"
:schema "public"
:name (str/replace (str (random-uuid)) "-" "_")}})))))
;; We've decided to defer this error until merge.
;; Also, this logic is going to become more relaxed, where we're allowed to take over a "dormant" table.
#_(testing "Conflict outside of workspace"
(is (= "A table with that name already exists."
(mt/user-http-request :crowberto :post 403 (ws-url ws-id "/transform/validate/target")
{:db_id (:db_id table)
:target {:type "table"
:schema (:schema table)
:name (:name table)}}))))
(testing "Conflict inside of workspace"
(let [table (t2/select-one :model/Table :active true)]
(is (= "Another transform in this workspace already targets that table."
(mt/with-log-level [metabase.driver.sql-jdbc.sync.describe-table :fatal]
(mt/user-http-request :crowberto :post 403 (ws-url ws-id "/transform/validate/target")
{:db_id (:db_id table)
:target {:type "table"
:schema (:schema table)
:name (str "q_" (:name table))}})))))))))
;;;; Async workspace creation tests
(deftest create-workspace-returns-updating-status-test
(testing "Creating workspace returns status :pending immediately"
(is (=? {:status "pending"}
(mt/user-http-request :crowberto :post 200 "ee/workspace"
{:name "async-test" :database_id (mt/id)})))))
(deftest workspace-status-becomes-ready-test
(testing "Workspace status transitions to :ready after setup completes"
(let [ws (ws-ready (mt/user-http-request :crowberto :post 200 "ee/workspace"
{:name "async-ready-test" :database_id (mt/id)}))]
(is (= :ready (:status ws))))))
(deftest workspace-log-endpoint-test
(testing "GET /api/ee/workspace/:id/log returns status and log entries"
(let [{ws-id :id} (ws-ready (mt/user-http-request :crowberto :post 200 "ee/workspace"
{:name "log-test" :database_id (mt/id)}))]
(is (=? {:workspace_id ws-id
:status "ready"
:updated_at some?
:last_completed_at some?
:logs [{:task "database-isolation"
:status "success"}
{:task "workspace-setup"}]}
(mt/user-http-request :crowberto :get 200 (ws-url ws-id "/log")))))))
(deftest workspace-log-endpoint-404-test
(testing "GET /api/ee/workspace/:id/log returns 404 for non-existent workspace"
(is (= "Not found."
(mt/user-http-request :crowberto :get 404 "ee/workspace/999999/log")))))
(deftest workspace-log-entries-created-test
(testing "WorkspaceLog entries are created during setup"
(let [{ws-id :id} (ws-ready (mt/user-http-request :crowberto :post 200 "ee/workspace"
{:name "log-entries-test" :database_id (mt/id)}))]
(is (=? [{:task :database-isolation
:status :success}
{:task :workspace-setup
:status :success}]
(t2/select :model/WorkspaceLog :workspace_id ws-id {:order-by [[:started_at :desc]]}))))))
(deftest workspace-setup-failure-logs-error-test
(testing "Failed workspace setup logs error message"
(with-redefs [ws.isolation/ensure-database-isolation!
(fn [& _] (throw (ex-info "Test isolation error" {})))]
(let [{ws-id :id} (mt/user-http-request :crowberto :post 200 "ee/workspace"
{:name "fail-test" :database_id (mt/id)})
_ (try (ws-ready ws-id) (catch Exception _))]
(Thread/sleep 500)
(is (=? [{:task :database-isolation
:status :failure
:message "Test isolation error"}
{:task :workspace-setup
:status :failure}]
(t2/select :model/WorkspaceLog :workspace_id ws-id {:order-by [[:started_at :desc]]})))))))
;;; ---------------------------------------- Workspace Transform CRUD Tests ----------------------------------------
(deftest get-workspace-transforms-test
(testing "GET /api/ee/workspace/:id/transform"
(mt/with-temp [:model/Workspace workspace {:name "List Transforms Test"}
:model/WorkspaceTransform tx1 {:name "Transform 1"
:workspace_id (:id workspace)}
:model/WorkspaceTransform tx2 {:name "Transform 2"
:workspace_id (:id workspace)}
:model/Transform _tx3 {:name "Global Transform"}]
(testing "returns transforms in workspace"
(is (=? {:transforms [{:ref_id (:ref_id tx1)}
{:ref_id (:ref_id tx2)}]}
(mt/user-http-request :crowberto :get 200 (ws-url (:id workspace) "/transform")))))
(testing "requires superuser"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :get 403 (ws-url (:id workspace) "/transform"))))))
(testing "returns empty list when no transforms"
(mt/with-temp [:model/Workspace workspace {:name "Empty Workspace"}]
(is (= {:transforms []}
(mt/user-http-request :crowberto :get 200 (ws-url (:id workspace) "/transform"))))))
(testing "returns 404 for non-existent workspace"
(is (= "Not found."
(mt/user-http-request :crowberto :get 404 "ee/workspace/999999/transform"))))))
(deftest get-workspace-transform-by-id-test
(testing "GET /api/ee/workspace/:id/transform/:txid"
(mt/with-temp [:model/Workspace workspace1 {:name "Workspace 1"}
:model/Workspace workspace2 {:name "Workspace 2"}
:model/WorkspaceTransform transform {:name "My Transform"
:description "Test description"
:workspace_id (:id workspace1)}]
(testing "returns specific transform"
(is (=? {:ref_id (:ref_id transform)
:name "My Transform"
:description "Test description"}
(mt/user-http-request :crowberto :get 200
(ws-url (:id workspace1) (str "/transform/" (:ref_id transform)))))))
(testing "returns 404 if transform not in workspace"
(is (= "Not found."
(mt/user-http-request :crowberto :get 404
(ws-url (:id workspace2) (str "/transform/" (:ref_id transform)))))))
(testing "requires superuser"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :get 403
(ws-url (:id workspace1) (str "/transform/" (:ref_id transform))))))))))
(deftest update-workspace-transform-test
(testing "PUT /api/ee/workspace/:id/transform/:txid"
(mt/with-temp [:model/Workspace workspace1 {:name "Workspace 1"}
:model/Workspace workspace2 {:name "Workspace 2"}
:model/WorkspaceTransform transform {:name "Original Name"
:description "Original description"
:workspace_id (:id workspace1)}]
(testing "updates transform"
(is (=? {:ref_id (:ref_id transform)
:name "Updated Name"
:description "Updated description"}
(mt/user-http-request :crowberto :put 200
(ws-url (:id workspace1) (str "/transform/" (:ref_id transform)))
{:name "Updated Name"
:description "Updated description"})))
(is (= "Updated Name" (t2/select-one-fn :name :model/WorkspaceTransform :ref_id (:ref_id transform)))))
(testing "returns 404 if transform not in workspace"
(is (= "Not found."
(mt/user-http-request :crowberto :put 404
(ws-url (:id workspace2) (str "/transform/" (:ref_id transform)))
{:name "Should Fail"}))))
(testing "requires superuser"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :put 403
(ws-url (:id workspace1) (str "/transform/" (:ref_id transform)))
{:name "Should Fail"})))))))
(deftest delete-workspace-transform-test
(testing "DELETE /api/ee/workspace/:id/transform/:txid"
(mt/with-temp [:model/Workspace workspace1 {:name "Workspace 1"}
:model/Workspace workspace2 {:name "Workspace 2"}
:model/WorkspaceTransform transform1 {:name "Transform in WS1"
:workspace_id (:id workspace1)}
:model/WorkspaceTransform transform2 {:name "To Delete"
:workspace_id (:id workspace1)}]
(testing "returns 404 if transform not in workspace"
(is (= "Not found."
(mt/user-http-request :crowberto :delete 404
(ws-url (:id workspace2) (str "/transform/" (:ref_id transform1)))))))
(testing "requires superuser"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :delete 403
(ws-url (:id workspace1) (str "/transform/" (:ref_id transform1)))))))
(testing "deletes transform"
(is (nil? (mt/user-http-request :crowberto :delete 204
(ws-url (:id workspace1) (str "/transform/" (:ref_id transform2))))))
(is (some? (t2/select-one :model/WorkspaceTransform :ref_id (:ref_id transform1))))
(is (nil? (t2/select-one :model/WorkspaceTransform :ref_id (:ref_id transform2))))))))
(deftest run-workspace-transform-test
(testing "POST /api/ee/workspace/:id/transform/:txid/run"
(mt/with-temp [:model/Workspace workspace1 {:name "Workspace 1"}
:model/Workspace workspace2 {:name "Workspace 2"}
:model/WorkspaceTransform transform {:name "Transform in WS1"
:workspace_id (:id workspace1)}]
(testing "returns 404 if transform not in workspace"
(is (= "Not found."
(mt/user-http-request :crowberto :post 404
(ws-url (:id workspace2) (str "/transform/" (:ref_id transform) "/run"))))))
(testing "requires superuser"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :post 403
(ws-url (:id workspace1) (str "/transform/" (:ref_id transform) "/run")))))))))
(deftest run-workspace-test
(testing "POST /api/ee/workspace/:id/execute"
(mt/with-temp [:model/Workspace workspace1 {:name "Workspace 1"}
:model/Workspace workspace2 {:name "Workspace 2"}
:model/WorkspaceTransform transform {:name "Transform in WS1"
:workspace_id (:id workspace1)}]
(testing "returns empty when no transforms"
(is (= {:succeeded []
:failed []
:not_run []}
(mt/user-http-request :crowberto :post 200 (ws-url (:id workspace2) "/run")))))
(testing "executes transforms in workspace"
(is (= {:succeeded [(:ref_id transform)]
:failed []
:not_run []}
(mt/user-http-request :crowberto :post 200 (ws-url (:id workspace1) "/run"))))))))

View File

@@ -0,0 +1,422 @@
(ns ^:mb/driver-tests metabase-enterprise.workspaces.common-test
(:require
[clojure.java.jdbc :as jdbc]
[clojure.test :refer [deftest testing is]]
[medley.core :as m]
[metabase-enterprise.transforms.interface :as transforms.i]
[metabase-enterprise.transforms.test-util :as transforms.tu]
[metabase-enterprise.workspaces.common :as ws.common]
[metabase.driver :as driver]
[metabase.driver.sql-jdbc.connection :as driver.conn]
[metabase.lib.core :as lib]
[metabase.test :as mt]
[toucan2.core :as t2]))
(defn- setup!
[schema]
(let [spec (driver.conn/connection-details->spec :postgres (:details (mt/db)))]
(jdbc/execute! spec (format "DROP SCHEMA IF EXISTS %s CASCADE" schema))
(jdbc/execute! spec (format "CREATE SCHEMA %s" schema))))
(defn teardown!
[ws-id]
(let [[ws-graph ws-schema] (t2/select-one-fn (juxt :graph :schema) :model/Workspace :id ws-id)
spec (driver.conn/connection-details->spec :postgres (:details (mt/db)))
output-ids (-> ws-graph :outputs (->> (map :id)))
dup-tables-ids (into #{}
(map #(-> % :mapping :id))
(-> ws-graph :outputs))]
(when (seq dup-tables-ids)
(t2/delete! :model/Table :id [:in dup-tables-ids]))
(when (seq output-ids)
(t2/delete! :model/Table :id [:in output-ids]))
(when (some? ws-id)
(t2/delete! :model/Workspace :id ws-id))
(when (string? (not-empty ws-schema))
(jdbc/execute! spec (format "DROP SCHEMA IF EXISTS %s CASCADE" ws-schema)))))
(deftest create-workspace-independent-native-transforms-test
(mt/test-driver
:postgres
(let [mp (mt/metadata-provider)
native-base (lib/native-query mp "select * from ORDERS LIMIT 10")
test-schema "test_schema"
new-workspace (atom nil)]
(try
(setup! test-schema)
(mt/with-temp
[:model/Transform x1 {:name "x1"
:source_type :native
:source {:type :query
:query native-base}
:target {:type :table
:database (mt/id)
:schema test-schema
:name "t1"}}
:model/Transform x2 {:name "x2"
:source_type :native
:source {:type :query
:query (assoc-in native-base [:stages 0 :native]
(str "SELECT *\n"
"FROM PRODUCTS\n"
"LIMIT 10\n"))}
:target {:type :table
:database (mt/id)
:schema test-schema
:name "t2"}}
:model/Transform x3 {:name "x3"
:source_type :native
:source {:type :query
:query (assoc-in native-base [:stages 0 :native]
(str "SELECT *\n"
"FROM PEOPLE\n"
"LIMIT 10\n"))}
:target {:type :table
:database (mt/id)
:schema test-schema
:name "t3"}}]
(transforms.i/execute! x1 {:run-method :manual})
(transforms.i/execute! x2 {:run-method :manual})
(transforms.i/execute! x3 {:run-method :manual})
(let [result (mt/with-current-user (mt/user->id :crowberto)
(ws.common/create-workspace! (mt/user->id :crowberto)
{:name "test_ws"
:database_id (mt/id)
:upstream {:transforms (mapv :id [x1 x2 x3])}}))
checkouts (-> result :graph :check-outs)
transforms (-> result :graph :transforms)]
(reset! new-workspace result)
(testing "Name is correct"
(is (= "test_ws" (:name result))))
(testing "Checkouts are as expected"
(is (= [{:type :transform :id (:id x1)}
{:type :transform :id (:id x2)}
{:type :transform :id (:id x3)}]
(sort-by :id checkouts))))
(testing "Transforms look as expected"
(is (= #{{:id (:id x1), :type :transform, :mapping {#_#_:id pos-int?, :name "x1"}}
{:id (:id x2), :type :transform, :mapping {#_#_:id pos-int?, :name "x2"}}
{:id (:id x3), :type :transform, :mapping {#_#_:id pos-int?, :name "x3"}}}
(into #{} (map #(m/update-existing % :mapping dissoc :id)) transforms))))))
;; For development, subject to change
(catch Throwable t
(def ttt t)
(throw t))
(finally
(teardown! (:id @new-workspace)))))))
(deftest mirroring-multiple-interdependent-native-transforms-test
(mt/test-driver
:postgres
(let [mp (mt/metadata-provider)
native-base (lib/native-query mp "select * from ORDERS LIMIT 10")
test-schema "test_schema"
new-workspace (atom nil)]
(try
(setup! test-schema)
(testing "Interdependent native transforms"
(mt/with-temp
[:model/Transform x1 {:name "x1"
:source_type :native
:source {:type :query
:query native-base}
:target {:type :table
:database (mt/id)
:schema test-schema
:name "t1"}}
:model/Transform x2 {:name "x2"
:source_type :native
:source {:type :query
:query (assoc-in native-base [:stages 0 :native]
(str "SELECT *\n"
(format "FROM \"%s\".\"%s\"\n" test-schema "t1")
"LIMIT 10\n"))}
:target {:type :table
:database (mt/id)
:schema test-schema
:name "t2"}}
:model/Transform x3 {:name "x3"
:source_type :native
:source {:type :query
:query (assoc-in native-base [:stages 0 :native]
(str "SELECT *\n"
(format "FROM \"%s\".\"%s\"\n" test-schema "t2")
"LIMIT 10\n"))}
:target {:type :table
:database (mt/id)
:schema test-schema
:name "t3"}}]
(transforms.i/execute! x1 {:run-method :manual})
(transforms.i/execute! x2 {:run-method :manual})
(transforms.i/execute! x3 {:run-method :manual})
(let [result-ws (mt/with-current-user (mt/user->id :crowberto)
(ws.common/create-workspace! (mt/user->id :crowberto)
{:name "test_ws"
:database_id (mt/id)
:upstream {:transforms (mapv :id [x1 x3])}}))
checkouts (-> result-ws :graph :check-outs)
transforms (-> result-ws :graph :transforms)]
(reset! new-workspace result-ws)
(testing "Workspace has expected name"
(is (= "test_ws" (:name result-ws))))
(testing "Checkouts contain only the explicitly checked out transforms"
(is (= #{{:type :transform :id (:id x1)}
{:type :transform :id (:id x3)}}
(set checkouts))))
(testing "Workspace has correct mappings in graph"
(is (= #{{:id (:id x1), :type :transform, :mapping {#_#_:id pos-int?, :name "x1"}}
{:id (:id x2), :type :transform, :mapping {#_#_:id pos-int?, :name "x2"}}
{:id (:id x3), :type :transform, :mapping {#_#_:id pos-int?, :name "x3"}}}
(into #{} (map #(m/update-existing % :mapping dissoc :id)) transforms))))
(let [orig-id->mirror-id (into {}
(map (fn [{:keys [id mapping]}]
[id (:id mapping)]))
transforms)
mirror-id->mirror (t2/select-fn->fn :id identity :model/Transform :id
[:in (vals orig-id->mirror-id)])
x1-mirror (-> (:id x1) orig-id->mirror-id mirror-id->mirror)
x2-mirror (-> (:id x2) orig-id->mirror-id mirror-id->mirror)
x3-mirror (-> (:id x3) orig-id->mirror-id mirror-id->mirror)]
(testing "X1 was properly mirrored"
(testing "name"
(is (= "x1" (:name x1-mirror))))
(testing "target adjusted"
(is (=? {:type "table"
:database (mt/id)
:schema (:schema result-ws)
;; table name = orig schema + __ + orig target name atm
:name (str test-schema "__" (-> x1 :target :name))}
(:target x1-mirror))))
(testing "source not modifed"
(is (=? (:source x1)
(:source x1-mirror)))))
(testing "X2 was properly mirrored"
(testing "name"
(is (= (:name x2) (:name x2-mirror))))
(testing "target"
(is (=? {:type "table"
:database (mt/id)
:schema (:schema result-ws)
:name (str test-schema "__" (-> x2 :target :name))}
(:target x2-mirror))))
(testing "source query referencing target of x1 mirror"
;; TODO (lbrdnk 2025-11-27): Quotes are removed, figure out whether what has
;; some unwanted consequences.
(is (=? {:query {:stages [{:native (format "SELECT *\nFROM %s.%s\nLIMIT 10\n"
(-> x1-mirror :target :schema)
(-> x1-mirror :target :name))}]}}
(:source x2-mirror)))))
(testing "X3 was properly mirrored"
(testing "name"
(is (= (:name x3) (:name x3-mirror))))
(testing "target"
(is (=? {:type "table"
:database (mt/id)
:schema (:schema result-ws)
:name (str test-schema "__" (-> x3 :target :name))}
(:target x3-mirror))))
(testing "source query referencing target of x2 mirror"
(is (=? {:query {:stages [{:native (format "SELECT *\nFROM %s.%s\nLIMIT 10\n"
(-> x2-mirror :target :schema)
(-> x2-mirror :target :name))}]}}
(:source x3-mirror)))))))))
;; For development, subject to change
(catch Throwable t
(def ttt t)
(throw t))
(finally
(teardown! (:id @new-workspace)))))))
(deftest python-transforms-checkout-test
(mt/test-driver
:postgres
(let [target-schema "public"]
(try (transforms.tu/with-transform-cleanup! [t1 "t1"
t2 "t2"]
(mt/with-temp
[:model/Transform x1 {:name "initial"
:source {:type "python"
:source-database (mt/id)
:source-tables {"checkins" (mt/id :checkins)
"venues" (mt/id :venues)}
:body (str "import pandas as pd\n"
"\n"
"def transform(checkins, venues):\n"
" return checkins.set_index('venue_id').join(venues.set_index('id'), lsuffix='_venues')\n")}
:target {:type "table"
:database (mt/id)
:schema target-schema
:name t1}}]
(transforms.i/execute! x1 {:run-method :manual})
(let [t1-table (t2/select-one :model/Table :name t1)]
(mt/with-temp [:model/Transform x2 {:name "dependent"
:source {:type "python"
:source-database (mt/id)
:source-tables {"t1" (:id t1-table)}
:body (str "import pandas as pd\n"
"\n"
"def transform(t1):\n"
" return t1")}
:target {:type "table"
:database (mt/id)
:schema target-schema
:name t2}}]
(transforms.i/execute! x2 {:run-method :manual})
(let [user-id (mt/user->id :crowberto)
result-workspace (mt/with-test-user :crowberto
(ws.common/create-workspace! user-id
{:name "my test workspace x"
:database_id (mt/id)
:upstream {:transforms [(:id x1) (:id x2)]}}))
mirror-initial (t2/select-one :model/Transform :name "initial" :workspace_id (:id result-workspace))
mirror-dependent (t2/select-one :model/Transform :name "dependent" :workspace_id (:id result-workspace))
t1-mirror (t2/select-one :model/Table
:schema (-> mirror-initial :target :schema)
:name (-> mirror-initial :target :name))]
(testing "targets"
(is (=? {:name (str "public__" (-> x1 :target :name))
:schema (:schema result-workspace)}
(:target mirror-initial)))
(is (=? {:name (str "public__" (-> x2 :target :name))
:schema (:schema result-workspace)}
(:target mirror-dependent))))
(testing "sources"
(is (=? {:source-tables {"venues" (mt/id :venues)
"checkins" (mt/id :checkins)}}
(:source mirror-initial)))
(is (=? {:source-tables {"t1" (:id t1-mirror)}}
(:source mirror-dependent)))))))))
(catch Throwable t
(def ttt t)
(throw t))))))
(deftest remove-transforms-test
(mt/test-driver
:postgres
(let [mp (mt/metadata-provider)
q1 (lib/native-query mp "select * from orders limit 10;")
q2 (lib/native-query mp "select * from products limit 10;")
test-schema "my_test_schema"
ws-id (atom nil)]
(try
(setup! test-schema)
(transforms.tu/with-transform-cleanup! [t1 "t1"
t2 "t2"]
(mt/with-temp
[:model/Transform x1 {:source_type :native
:source {:type :query
:query q1}
:target {:type :table
:database (mt/id)
:schema test-schema
:name t1}}
:model/Transform x2 {:source_type :native
:source {:type :query
:query q2}
:target {:type :table
:database (mt/id)
:schema test-schema
:name t2}}]
;; Create the target tables
(transforms.i/execute! x1 {:run-method :manual})
(transforms.i/execute! x2 {:run-method :manual})
;; Add into a workspace
(let [creator-id (mt/user->id :crowberto)
ws (mt/with-test-user :crowberto
(ws.common/create-workspace! creator-id {:name "my test ws xxx"
:database_id (mt/id)
:upstream {:transforms [(:id x1) (:id x2)]}}))
_ (reset! ws-id (:id ws))
mirrored-transforms-ids (t2/select-fn-vec :downstream_id :model/WorkspaceMappingTransform :workspace_id (:id ws))
ws-db (t2/select-one :model/Database :id (:database_id ws))
mirrored-tables-ids (t2/select-fn-vec :downstream_id :model/WorkspaceMappingTable :workspace_id (:id ws))
mirrored-tables (t2/select :model/Table :id [:in mirrored-tables-ids])]
(testing "Sanity"
(testing "Mirrored transforms created"
(is (= 2 (count mirrored-transforms-ids))))
(testing "Mirrored tables created in isolated schema"
(run!
(fn [{:keys [schema name]}]
(is (true? (driver/table-exists? :postgres ws-db {:schema schema :name name}))))
mirrored-tables))
(testing "Mirrored tables created in appdb"
(is (= 2 (count mirrored-tables-ids)))))
(testing "Remove transforms from a workspace"
(mt/with-test-user :crowberto
;; ACT!
(ws.common/remove-entities! ws {:transforms mirrored-transforms-ids}))
(testing "Mirrored transforms deleted"
(is (empty? (t2/select :model/Transform :id [:in mirrored-transforms-ids]))))
(testing "Mirrored tables dropped from the isolated schema"
(run!
(fn [{:keys [schema name]}]
;; driver/table-exists? -- _prints_ exception on non-existent relation, while having correct
;; return value.
;; one way or another -- this test will be rewritten completely. Then we will handle that.
(is (false? (driver/table-exists? :postgres ws-db {:schema schema :name name}))))
mirrored-tables))
(testing "Mirrored tables deleted from the appdb"
(is (empty? (t2/select :model/Table :id [:in mirrored-tables-ids]))))
(testing "Mirrored entites are not present in the mapping tables"
(is (empty? (t2/select :model/WorkspaceMappingTable :workspace_id (:id ws))))
(is (empty? (t2/select :model/WorkspaceMappingTransform :workspace_id (:id ws)))))))))
(catch Throwable t
(def ttt t)
(throw t))
(finally
(teardown! @ws-id))))))
(deftest add-one-by-one-and-remove-test
(mt/test-driver
:postgres
(let [mp (mt/metadata-provider)
q1 (lib/native-query mp "select * from orders limit 10;")
test-schema "my_test_schema"
ws-id (atom nil)]
(try
(setup! test-schema)
(transforms.tu/with-transform-cleanup! [t1 "t1"]
(mt/with-temp
[:model/Transform x1 {:source_type :native
:source {:type :query
:query q1}
:target {:type :table
:database (mt/id)
:schema test-schema
:name t1}}]
;; Create the target tables
(transforms.i/execute! x1 {:run-method :manual})
;; Add into a workspace
(let [creator-id (mt/user->id :crowberto)
;; create empty workspace
ws (mt/user-http-request :crowberto :post 200
"/ee/workspace"
{:name "my test ws xxx"
:creator_id creator-id
:database_id (mt/id)})
_ (reset! ws-id (:id ws))]
(let [add-res (mt/user-http-request :crowberto :post 200
(str "ee/workspace/" (:id ws) "/contents")
{:add {:transforms [(:id x1)]}})]
(testing "Successfully added"
(is (some? (-> add-res :contents :transforms (nth 0)))))
(let [rem-res (mt/user-http-request :crowberto :post 200
(str "ee/workspace/" (:id ws) "/contents")
{:remove {:transforms [(-> add-res :contents :transforms (nth 0) :id)]}})]
(testing "successfully removed"
(is (empty? (-> rem-res :contents :transforms))))))
(let [add-res (mt/user-http-request :crowberto :post 200
(str "ee/workspace/" (:id ws) "/contents")
{:add {:transforms [(:id x1)]}})]
(testing "Successfully added"
(is (some? (-> add-res :contents :transforms (nth 0)))))
(let [rem-res (mt/user-http-request :crowberto :post 200
(str "ee/workspace/" (:id ws) "/contents")
{:remove {:transforms [(-> add-res :contents :transforms (nth 0) :id)]}})]
(testing "successfully removed"
(is (empty? (-> rem-res :contents :transforms)))))))))
(catch Throwable t
(def ttt t)
(throw t))
(finally
(teardown! @ws-id))))))

View File

@@ -0,0 +1,115 @@
(ns metabase-enterprise.workspaces.dag-abstract
"Abstract graph solver using shorthand notation for testing.
Shorthand uses keywords like :x1, :t2, :m3 where:
- :x<n> is a transform
- :t<n> is a table (source/generated)
- :m<n> is a model (card)"
(:require
[clojure.set :as set]
[flatland.ordered.map :as ordered-map]
[metabase-enterprise.workspaces.dag :as ws.dag]
[metabase-enterprise.workspaces.util :as ws.u]
[metabase.util :as u]))
;;;; Shorthand notation helpers
(defn- related [t id]
(keyword (str (name t) (subs (name id) 1))))
(def ^:private x->t (partial related :t))
(defn is?
"Check if id is of type t (e.g., (is? :x :x1) => true)."
[t id]
(= (name t) (str (first (name id)))))
(defn- lexico [id]
(let [n (name id)]
[(parse-long (subs n 1))
({\x 1 \t 2 \m 3} (first n))]))
(defn expand-shorthand
"As shorthand, we can use transforms in the place of their induced tables.
This function expands the graph to insert the induced tables, and replace the corresponding references."
[{:keys [dependencies] :as graph}]
(assoc graph :dependencies
(let [xs (set (filter #(is? :x %) (concat (keys dependencies) (mapcat val dependencies))))]
(into (sorted-map-by #(compare (lexico %1) (lexico %2)))
(merge (zipmap (map x->t xs) (map vector xs))
(into {} (filter (comp #(is? :t %) key)) dependencies)
(update-vals
(into {} (remove (comp #(is? :t %) key)) dependencies)
(fn [deps] (->> deps
(map (fn [id]
(if (is? :x id)
(x->t id)
id)))
(sort-by lexico)
(vec)))))))))
;;;; Graph traversal helpers
(defn- children [dependencies]
(reduce
(fn [acc [child parents]]
(reduce (fn [acc p] (update acc p u/conjv child)) acc parents))
(sorted-map-by #(compare (lexico %1) (lexico %2)))
dependencies))
(defn- ->ts [ids]
(into #{}
(keep (fn [id]
(cond (is? :t id) id
(is? :x id) (x->t id))))
ids))
;;;; Toposort
(defn- toposort-map [child->parents]
(let [child->parents (if (map? child->parents) child->parents (into {} child->parents))]
(into (ordered-map/ordered-map)
(keep (fn [n] (when-let [deps (get child->parents n)] [n deps])))
(ws.u/toposort-dfs child->parents))))
;;;; Abstract path-induced subgraph solver
(def ^:private max-iterations 100)
(defn path-induced-subgraph
"Abstract solver for path-induced subgraph using shorthand notation."
[{:keys [check-outs dependencies] :as _graph}]
(let [->parents dependencies
->children (children dependencies)]
(loop [iteration 1
below check-outs
above check-outs
seen-down? check-outs
seen-up? check-outs
to-descend check-outs
to-ascend check-outs]
(if (and (empty? to-descend) (empty? to-ascend))
(let [nodes (set/intersection above below)
entities (remove #(is? :t %) nodes)
tables (sort-by lexico (filter #(is? :t %) nodes))]
{:check-outs (sort-by lexico check-outs)
:inputs (sort-by lexico (remove nodes (->ts (mapcat ->parents check-outs))))
:tables tables
:transforms (sort-by lexico (filter #(is? :x %) nodes))
:entities (sort-by lexico entities)
:dependencies (apply dissoc
(toposort-map
(for [n nodes]
[n (vec (filter nodes (get dependencies n)))]))
tables)})
(if (> iteration max-iterations)
{:error "max iterations exceeded"}
(let [next-down (into #{} (comp (mapcat ->children) (remove seen-down?)) to-descend)
next-up (into #{} (comp (mapcat ->parents) (remove seen-up?)) to-ascend)]
(recur (inc iteration)
(into below next-down)
(into above next-up)
(into seen-down? next-down)
(into seen-up? next-up)
next-down
next-up)))))))

View File

@@ -0,0 +1,202 @@
(ns metabase-enterprise.workspaces.dag-test
(:require
[clojure.string :as str]
[clojure.test :refer :all]
[flatland.ordered.map :as ordered-map]
[metabase-enterprise.workspaces.dag :as ws.dag]
[metabase-enterprise.workspaces.dag-abstract :as dag-abstract]
[metabase.test :as mt]
[metabase.util :as u]
[toucan2.core :as t2]))
(use-fixtures
:once
(fn [f]
(mt/with-premium-features [:dependencies :transforms :workspaces]
(mt/with-model-cleanup [:model/Dependency :model/Transform :model/Table]
(f)))))
;;;; Example graphs for testing
(def ^:private example
{:check-outs #{:x3 :m6 :m10 :m13}
:dependencies {:x3 [:x1 :t2]
:x4 [:x3]
:m6 [:x4 :t5]
:m10 [:t9]
:x11 [:m10]
:m12 [:x11]
:m13 [:x11 :m12]}})
;;;; Test data helpers
(defn- transform? [kw] (= \x (first (name kw))))
(defn- table? [kw] (= \t (first (name kw))))
(defn- kw->id [kw] (parse-long (subs (name kw) 1)))
(defn- create-test-graph!
"Create test transforms and tables from shorthand notation, returning id mappings.
Shorthand uses keywords like :x1, :t2, :m3 where:
- :x<n> is a transform
- :t<n> is a table (source/generated)
- :m<n> is a model (card)
Returns a map from shorthand id to real database id, e.g. {:x1 123, :t2 456}
NOTE: Caller should wrap in mt/with-model-cleanup for proper cleanup."
[{:keys [check-outs dependencies]}]
(let [schema (str/replace (str (random-uuid)) "-" "_")
all-ids (set (concat (keys dependencies)
(mapcat val dependencies)
check-outs))
transforms (filter transform? all-ids)
tables (filter table? all-ids)
table-ids (u/for-map [t tables]
[t (t2/insert-returning-pk! :model/Table
{:db_id (mt/id)
:schema schema
:name (str "test_table_" (kw->id t))
:active true})])
;; Create transforms that reference their input tables
tx-ids (u/for-map [tx transforms]
(let [parent-tables (->> (get dependencies tx []) (filter table?) (map table-ids))
id (t2/insert-returning-pk! :model/Transform
{:name (str "Test " (name tx))
:source {:type :query
:query (if (seq parent-tables)
{:database (mt/id)
:type :query
:query {:source-table (first parent-tables)
:joins (for [pt (rest parent-tables)]
{:source-table pt
:condition [:= 1 1]})}}
{:database (mt/id)
:type :native
:native {:query "SELECT 1"}})}
:target {:type "table"
:database (mt/id)
:schema schema
:name (str "test_table_" (kw->id tx))}})]
[tx id]))]
;; TODO This is a workaround for the dependency between transforms and their output only being inserted on run.
;; We will need to do something about this when we mirror as well - ideally the deps module would "just work"
(doseq [[tx-kw tx-id] tx-ids]
(t2/insert! :model/Dependency
{:from_entity_type "table"
:from_entity_id (table-ids (keyword (str "t" (kw->id tx-kw))))
:to_entity_type "transform"
:to_entity_id tx-id}))
(merge tx-ids table-ids)))
(defn- translate-result
"Translate result from real IDs back to shorthand notation for easier comparison."
[result id-map]
(let [reverse-map (u/for-map [[k v] id-map] [v k])
translate (fn [{:keys [id]}] (get reverse-map id))]
(-> (reduce (fn [m k] (update m k #(set (map translate %))))
result
[:check-outs :inputs :outputs :transforms])
(update :dependencies
(fn [deps]
(into {}
(map (fn [[k v]]
[(translate k) (set (map translate v))]))
deps))))))
;;;; Tests
(deftest path-induced-subgraph-shorthand-test
(testing "graph built from shorthand matches abstract solver"
(let [shorthand {:check-outs #{:x1}
:dependencies {:x1 [:t1]}}
id-map (create-test-graph! shorthand)
result nil #_(ws.dag/path-induced-subgraph {:transforms [(id-map :x1)]})
translated (translate-result result id-map)]
#_(is (= {:check-outs #{:x1}
:transforms #{:x1}
:inputs #{:t1}}
translated)))))
(deftest path-induced-subgraph-larger-test
(testing "graph built from shorthand matches abstract solver"
(let [shorthand {:check-outs #{:x2, :x4}
:dependencies {:x1 [:t0]
:x2 [:x1, :t10]
:x3 [:x2, :t8]
:x4 [:x3]
:x5 [:x2, :x4, :t9]}}
id-map (create-test-graph! (dag-abstract/expand-shorthand shorthand))
result nil #_(ws.dag/path-induced-subgraph {:transforms (mapv id-map (:check-outs shorthand))})
translated (translate-result result id-map)]
#_(is (=? {:check-outs #{:x2, :x4}
:transforms #{:x2, :x3, :x4}
:inputs #{:t0, :t1, :t8, :t10}
:outputs #{:t2, :t3, :t4}
;; Dependencies outside the subgraph are not listed
:dependencies {:x2 #{}
:x3 #{:x2}
:x4 #{:x3}}}
translated)))))
(deftest expand-solver-test
(testing "expand-shorthand inserts interstitial nodes for transform output tables"
(is (= {:check-outs #{:x3 :m6 :m10 :m13}
:dependencies {:t1 [:x1]
:x3 [:t1 :t2]
:t3 [:x3]
:x4 [:t3]
:t4 [:x4]
:m6 [:t4 :t5]
:m10 [:t9]
:x11 [:m10]
:t11 [:x11]
:m12 [:t11]
:m13 [:t11 :m12]}}
(dag-abstract/expand-shorthand example)))))
(deftest abstract-path-induced-subgraph-test
(testing "path-induced-subgraph computes correct result for example graph"
(is (= {:check-outs [:x3 :m6 :m10 :m13]
:inputs [:t1 :t2 :t5 :t9]
:tables [:t3 :t4 :t11]
:transforms [:x3 :x4 :x11]
:entities [:x3 :x4 :m6 :m10 :x11 :m12 :m13]
:dependencies (ordered-map/ordered-map
:m10 []
:x11 [:m10]
:m12 [:t11]
:m13 [:t11 :m12]
:x3 []
:x4 [:t3]
:m6 [:t4])}
(dag-abstract/path-induced-subgraph (dag-abstract/expand-shorthand example))))))
;;;; Card dependency detection tests
(deftest unsupported-dependency-no-transforms-test
(testing "empty input returns nil"
(is (nil? (ws.dag/unsupported-dependency? {})))
(is (nil? (ws.dag/unsupported-dependency? {:transforms []})))))
(deftest unsupported-dependency-mbql-card-test
#_(testing "transform with card dependencies are unsupported"
(let [tx-with-no-dependencies 1
;; These tests transforms must seem pretty redundant - but it's intentional: we may relax them one case at
;; at time in the future.
;; ----------------------
;; We don't allow MBQL card dependencies, as we do not support re-mapping their references on execution.
;; This is only a problem if the card depends on a table that is shadowed in the isolated schema, but we
;; want to keep the semantics simple.
tx-with-direct-mbql-card-dependency 2
;; Even a transitive dependency would need to be remapped.
tx-with-indirect-mbql-card-dependency 3
;; In fact, we don't allow *any* card dependency for two reasons:
;; 1. We don't yet support reference re-mapping across entity references.
;; 2. It's an anti-pattern to build transforms on models, the relationship is intended to be the other way
;; around.
tx-with-sql-card-dependency 4]
(is (= nil (ws.dag/unsupported-dependency? {:transforms [1]})))
(is (= {:transforms [2 3 4]} (ws.dag/unsupported-dependency? {:transforms [1 2 3 4]}))))))

View File

@@ -0,0 +1,20 @@
(ns ^:mb/driver-tests metabase-enterprise.workspaces.drivers-test
(:require
[clojure.test :refer :all]
[metabase.driver :as driver]
[metabase.test :as mt]))
(deftest drivers-support-workspaces-has-to-support-db-swapping-test
(mt/test-drivers (mt/normal-drivers-with-feature :workspace)
(let [db-swapped? (fn []
(try
(mt/run-mbql-query venues {:limit 1})
false
(catch Exception _e
true)))]
(testing "db is swapped if being executed inside a macros"
(driver/with-swapped-connection-details (mt/id) {:user "unicorn"}
(is (db-swapped?))))
(testing "sanity check that it's not swapped outside of the macros"
(is (not (db-swapped?)))))))

View File

@@ -0,0 +1,56 @@
(ns ^:mb/driver-tests metabase-enterprise.workspaces.e2e-test
(:require
[clojure.string :as str]
[clojure.test :refer :all]
[metabase-enterprise.transforms.interface :as transforms.i]
[metabase-enterprise.transforms.test-util :as transforms.tu]
[metabase-enterprise.workspaces.common :as ws.common]
[metabase-enterprise.workspaces.core :as workspaces]
[metabase.driver :as driver]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.query-processor.preprocess :as qp.preprocess]
^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]
[metabase.test :as mt]
[toucan2.core :as t2]))
(defn- execute-workspace-transform!
"Execute a transform within workspace isolation context. For testing purposes."
[workspace transform opts]
(workspaces/with-workspace-isolation workspace
(transforms.i/execute! transform opts)))
(defn- mbql->native [query]
(qp.store/with-metadata-provider (mt/id)
(sql.qp/mbql->native driver/*driver* (qp.preprocess/preprocess query))))
(deftest isolation-e2e-test
(mt/test-drivers #_(mt/normal-drivers-with-feature :workspace) #{:postgres #_:h2}
(transforms.tu/with-transform-cleanup! [output-table-name (str "test_table_1_" (mt/random-name))]
(mt/with-model-cleanup [:model/Transform :model/Workspace]
(mt/with-temp [:model/Transform {transform-id :id} {:name "Transform 1"
:source {:type "query"
:query (mt/native-query (mbql->native (mt/mbql-query orders {:limit 1})))}
:target {:type "table"
:name output-table-name}}]
(let [workspace (mt/with-current-user (mt/user->id :crowberto)
(ws.common/create-workspace! (mt/user->id :crowberto)
{:name (mt/random-name)
:database_id (mt/id)
:upstream {:transforms [transform-id]}}))
isolated-transform-id (t2/select-one-fn :downstream_id :model/WorkspaceMappingTransform :workspace_id (:id workspace))
isolated-transform (t2/select-one :model/Transform isolated-transform-id)
executed-transform (execute-workspace-transform! workspace isolated-transform {:run-method :manual})
output-table (-> executed-transform :object :output-table)]
(testing "execute the transform with the original query is fine and the output is in an isolated schema"
(transforms.tu/wait-for-table (name output-table) 1000)
(is (str/starts-with? (namespace output-table) "mb__isolation"))
(is (= 1 (count (transforms.tu/table-rows (name output-table))))))
(testing "changing the query without granting access will fail"
(t2/update! :model/Transform isolated-transform-id {:source {:type "query"
;; TODO we should use native query here
:query (mt/mbql-query venues {:limit 1})}})
(is (thrown-with-msg?
Exception
#"ERROR: permission denied for table.*"
(execute-workspace-transform! workspace (t2/select-one :model/Transform isolated-transform-id) {:run-method :manual}))))))))))

View File

@@ -5,12 +5,27 @@ import { useMetabotAgent } from "metabase-enterprise/metabot/hooks";
import { trackQueryFixClicked } from "../../analytics";
export function FixSqlQueryButton() {
type FixSqlQueryButtonProps = {
rawSql?: string | null;
errorMessage?: string | null;
};
export function FixSqlQueryButton({
rawSql,
errorMessage,
}: FixSqlQueryButtonProps) {
const { submitInput } = useMetabotAgent();
const handleClick = () => {
trackQueryFixClicked();
submitInput("Fix this SQL query");
const promptParts = ["Fix this SQL query."];
if (errorMessage) {
promptParts.push(`The database returned this error: ${errorMessage}`);
}
if (rawSql) {
promptParts.push(`SQL:\n${rawSql}`);
}
submitInput(promptParts.join("\n\n"));
};
return <Button onClick={handleClick}>{t`Have Metabot fix it`}</Button>;

View File

@@ -21,3 +21,4 @@ export * from "./transform";
export * from "./transform-job";
export * from "./transform-tag";
export * from "./upload-management";
export * from "./workspace";

View File

@@ -25,6 +25,9 @@ import type {
TransformJob,
TransformRun,
TransformTag,
Workspace,
WorkspaceContentItem,
WorkspaceContents,
} from "metabase-types/api";
export const ENTERPRISE_TAG_TYPES = [
@@ -49,6 +52,7 @@ export const ENTERPRISE_TAG_TYPES = [
"remote-sync-branches",
"remote-sync-current-task",
"python-transform-library",
"workspace",
"support-access-grant",
"support-access-grant-current",
"library-collection",
@@ -82,6 +86,32 @@ export function invalidateTags(
return !error ? tags : [];
}
export function provideWorkspacesTags(
workspaces: Workspace[],
): TagDescription<EnterpriseTagType>[] {
return [listTag("workspace"), ...workspaces.flatMap(provideWorkspaceTags)];
}
export function provideWorkspaceTags(
workspace: Workspace,
): TagDescription<EnterpriseTagType>[] {
return [idTag("workspace", workspace.id)];
}
export function provideWorkspaceContentsTags(
workspaceContents: WorkspaceContents,
): TagDescription<EnterpriseTagType>[] {
return provideWorkspaceContentItemsTags(
workspaceContents.contents.transforms,
);
}
export function provideWorkspaceContentItemsTags(
items: WorkspaceContentItem[],
): TagDescription<EnterpriseTagType>[] {
return items.flatMap((transform) => idTag("transform", transform.id));
}
export function provideTransformTags(
transform: Transform,
): TagDescription<EnterpriseTagType>[] {

View File

@@ -0,0 +1,208 @@
import type {
CreateWorkspaceRequest,
CreateWorkspaceTransformRequest,
CreateWorkspaceTransformResponse,
TransformDownstreamMapping,
TransformId,
TransformUpstreamMapping,
ValidateTableNameRequest,
ValidateTableNameResponse,
Workspace,
WorkspaceContents,
WorkspaceExecuteRequest,
WorkspaceExecuteResponse,
WorkspaceId,
WorkspaceListResponse,
WorkspaceLogResponse,
WorkspaceMergeResponse,
WorkspaceTablesResponse,
WorkspaceUpdateContentsRequest,
} from "metabase-types/api";
import { EnterpriseApi } from "./api";
import {
idTag,
invalidateTags,
listTag,
provideWorkspaceContentItemsTags,
provideWorkspaceContentsTags,
provideWorkspaceTags,
provideWorkspacesTags,
tag,
} from "./tags";
export const workspaceApi = EnterpriseApi.injectEndpoints({
endpoints: (builder) => ({
getWorkspaces: builder.query<WorkspaceListResponse, void>({
query: () => ({
method: "GET",
url: "/api/ee/workspace",
}),
providesTags: (response) =>
response ? provideWorkspacesTags(response.items) : [],
}),
getWorkspace: builder.query<Workspace, WorkspaceId>({
query: (id) => ({
method: "GET",
url: `/api/ee/workspace/${id}`,
}),
providesTags: (workspace) =>
workspace ? provideWorkspaceTags(workspace) : [],
}),
createWorkspace: builder.mutation<Workspace, CreateWorkspaceRequest>({
query: (body) => ({
method: "POST",
url: "/api/ee/workspace",
body,
}),
invalidatesTags: (_, error) =>
invalidateTags(error, [listTag("workspace"), listTag("transform")]),
}),
getWorkspaceContents: builder.query<WorkspaceContents, WorkspaceId>({
query: (id) => ({
method: "GET",
url: `/api/ee/workspace/${id}`,
}),
transformResponse: (workspace: any) => {
return {
contents: workspace?.contents ?? { transforms: [] },
} as WorkspaceContents;
},
providesTags: (workspaceContents) =>
workspaceContents
? provideWorkspaceContentsTags(workspaceContents)
: [],
}),
updateWorkspaceContents: builder.mutation<
WorkspaceContents,
WorkspaceUpdateContentsRequest
>({
query: ({ id, ...body }) => ({
method: "POST",
url: `/api/ee/workspace/${id}/contents`,
body,
}),
invalidatesTags: (_, error, { id }) =>
invalidateTags(error, [idTag("workspace", id), tag("transform")]),
}),
createWorkspaceTransform: builder.mutation<
CreateWorkspaceTransformResponse,
{ id: WorkspaceId } & CreateWorkspaceTransformRequest
>({
query: ({ id, ...body }) => ({
method: "POST",
url: `/api/ee/workspace/${id}/transform`,
body,
}),
invalidatesTags: (_, error, { id }) =>
invalidateTags(error, [idTag("workspace", id), tag("transform")]),
}),
getTransformUpstreamMapping: builder.query<
TransformUpstreamMapping,
TransformId
>({
query: (id) => ({
method: "GET",
url: `/api/ee/workspace/mapping/transform/${id}/upstream`,
}),
providesTags: (mapping) =>
mapping?.transform
? provideWorkspaceContentItemsTags([mapping.transform])
: [],
}),
getTransformDownstreamMapping: builder.query<
TransformDownstreamMapping,
TransformId
>({
query: (id) => ({
method: "GET",
url: `/api/ee/workspace/mapping/transform/${id}/downstream`,
}),
providesTags: (mapping) =>
mapping ? provideWorkspaceContentItemsTags(mapping.transforms) : [],
}),
mergeWorkspace: builder.mutation<WorkspaceMergeResponse, WorkspaceId>({
query: (id) => ({
method: "POST",
url: `/api/ee/workspace/${id}/merge`,
}),
invalidatesTags: (_, error) =>
invalidateTags(error, [tag("workspace"), tag("transform")]),
}),
archiveWorkspace: builder.mutation<Workspace, WorkspaceId>({
query: (id) => ({
method: "POST",
url: `/api/ee/workspace/${id}/archive`,
}),
invalidatesTags: (_, error) =>
invalidateTags(error, [tag("workspace"), tag("transform")]),
}),
validateTableName: builder.mutation<
ValidateTableNameResponse,
ValidateTableNameRequest
>({
query: ({ id, ...body }) => ({
method: "POST",
url: `/api/ee/workspace/${id}/validate-target`,
body,
}),
}),
updateWorkspaceName: builder.mutation<
Workspace,
{ id: WorkspaceId; name: string }
>({
query: ({ id, ...body }) => ({
method: "POST",
url: `/api/ee/workspace/${id}/name`,
body,
}),
invalidatesTags: (_, error, { id }) =>
invalidateTags(error, [idTag("workspace", id)]),
}),
getWorkspaceTables: builder.query<WorkspaceTablesResponse, WorkspaceId>({
query: (id) => ({
method: "GET",
url: `/api/ee/workspace/${id}/tables`,
}),
providesTags: (_, __, id) => [idTag("workspace", id)],
}),
getWorkspaceLog: builder.query<WorkspaceLogResponse, WorkspaceId>({
query: (id) => ({
method: "GET",
url: `/api/ee/workspace/${id}/log`,
}),
providesTags: (_, _error, id) => [idTag("workspace", id)],
}),
executeWorkspace: builder.mutation<
WorkspaceExecuteResponse,
WorkspaceExecuteRequest
>({
query: ({ id, stale_only }) => ({
method: "POST",
url: `/api/ee/workspace/${id}/execute`,
params: stale_only ? { stale_only: 1 } : undefined,
}),
invalidatesTags: (_, error, { id }) =>
invalidateTags(error, [idTag("workspace", id), tag("transform")]),
}),
}),
});
export const {
useGetWorkspacesQuery,
useGetWorkspaceQuery,
useCreateWorkspaceMutation,
useGetWorkspaceContentsQuery,
useUpdateWorkspaceContentsMutation,
useLazyGetWorkspaceContentsQuery,
useCreateWorkspaceTransformMutation,
useGetTransformUpstreamMappingQuery,
useGetTransformDownstreamMappingQuery,
useMergeWorkspaceMutation,
useArchiveWorkspaceMutation,
useValidateTableNameMutation,
useUpdateWorkspaceNameMutation,
useGetWorkspaceTablesQuery,
useGetWorkspaceLogQuery,
useExecuteWorkspaceMutation,
} = workspaceApi;

View File

@@ -1,5 +1,13 @@
.nav {
border-right: 1px solid var(--mb-color-border);
transition: width 200ms ease;
width: auto;
min-width: 250px;
}
.navCollapsed {
width: 60px;
min-width: 60px;
}
.tab {
@@ -14,9 +22,99 @@
.tab,
.toggle {
color: var(--mb-color-text-primary);
font-weight: 600;
&:hover,
&.selected {
color: var(--mb-color-brand);
}
}
.workspacesSection {
border-top: 1px solid var(--mb-color-border);
padding-top: 0.75rem;
}
.workspacesSectionHeader {
background-color: var(--mb-color-bg-secondary);
color: var(--mb-color-text-primary);
font-weight: 600;
transition: background-color 150ms ease;
&:hover {
background-color: var(--mb-color-brand-light);
color: var(--mb-color-brand);
}
&.selected {
background-color: var(--mb-color-brand-light);
color: var(--mb-color-brand);
}
}
.workspaceItem {
background-color: var(--mb-color-bg-white);
color: var(--mb-color-text-primary);
cursor: pointer;
border: 1px solid var(--mb-color-border);
transition:
background-color 150ms ease,
border-color 150ms ease,
color 150ms ease;
&:hover {
background-color: var(--mb-color-bg-light);
border-color: var(--mb-color-border-dark);
}
&:hover .workspaceMenuButton {
opacity: 1;
}
&.selected {
background-color: var(--mb-color-brand-lighter);
border-color: var(--mb-color-brand);
color: var(--mb-color-brand);
}
&.selected .workspaceMenuButton {
opacity: 1;
}
&:disabled {
opacity: 0.5;
cursor: not-allowed;
}
}
.workspaceMenuButton {
color: var(--mb-color-text-secondary);
opacity: 0;
transition:
opacity 150ms ease,
background-color 150ms ease,
color 150ms ease;
flex-shrink: 0;
&:hover {
background-color: var(--mb-color-bg-medium);
color: var(--mb-color-text-primary);
}
}
.newWorkspaceButton {
background-color: transparent;
color: var(--mb-color-text-primary);
cursor: pointer;
border: none;
transition: color 150ms ease;
&:hover {
color: var(--mb-color-brand);
}
&:disabled {
opacity: 0.5;
cursor: not-allowed;
}
}

View File

@@ -1,10 +1,14 @@
import cx from "classnames";
import { type ReactNode, useMemo, useState } from "react";
import dayjs from "dayjs";
import { type ReactNode, useCallback, useMemo, useState } from "react";
import { push } from "react-router-redux";
import { t } from "ttag";
import { useListDatabasesQuery } from "metabase/api";
import { ForwardRefLink } from "metabase/common/components/Link";
import { useSelector } from "metabase/lib/redux";
import { useDispatch, useSelector } from "metabase/lib/redux";
import * as Urls from "metabase/lib/urls";
import { useMetadataToasts } from "metabase/metadata/hooks";
import {
PLUGIN_DEPENDENCIES,
PLUGIN_FEATURE_LEVEL_PERMISSIONS,
@@ -12,15 +16,27 @@ import {
} from "metabase/plugins";
import { getLocation } from "metabase/selectors/routing";
import {
ActionIcon,
Box,
FixedSizeIcon,
Flex,
Icon,
type IconName,
Menu,
Skeleton,
Stack,
Text,
Tooltip,
UnstyledButton,
} from "metabase/ui";
import {
useArchiveWorkspaceMutation,
useCreateWorkspaceMutation,
useGetWorkspacesQuery,
} from "metabase-enterprise/api";
import { DataStudioContext } from "metabase-enterprise/data-studio/common/contexts/DataStudioContext";
import { CreateWorkspaceModal } from "metabase-enterprise/data-studio/workspaces/components/CreateWorkspaceModal/CreateWorkspaceModal";
import type { Database, WorkspaceId } from "metabase-types/api";
import S from "./DataStudioLayout.module.css";
@@ -31,6 +47,7 @@ type DataStudioLayoutProps = {
export function DataStudioLayout({ children }: DataStudioLayoutProps) {
const [isSidebarOpened, setIsSidebarOpened] = useState(false);
const [isSidebarAvailable, setIsSidebarAvailable] = useState(false);
const [isNavExpanded, setIsNavExpanded] = useState(true);
const contextValue = useMemo(
() => ({
isSidebarOpened,
@@ -48,6 +65,8 @@ export function DataStudioLayout({ children }: DataStudioLayoutProps) {
isSidebarOpened={isSidebarOpened}
isSidebarAvailable={isSidebarAvailable}
onSidebarToggle={setIsSidebarOpened}
isNavExpanded={isNavExpanded}
onNavToggle={setIsNavExpanded}
/>
<Box h="100%" flex={1} miw={0}>
{children}
@@ -61,12 +80,16 @@ type DataStudioNavProps = {
isSidebarOpened: boolean;
isSidebarAvailable: boolean;
onSidebarToggle: (isOpened: boolean) => void;
isNavExpanded: boolean;
onNavToggle: (isExpanded: boolean) => void;
};
function DataStudioNav({
isSidebarOpened,
isSidebarAvailable,
onSidebarToggle,
isNavExpanded,
onNavToggle,
}: DataStudioNavProps) {
const { pathname } = useSelector(getLocation);
const canAccessDataModel = useSelector(
@@ -76,42 +99,62 @@ function DataStudioNav({
PLUGIN_TRANSFORMS.canAccessTransforms,
);
const isDataTab = pathname.startsWith(Urls.dataStudioData());
const isTransformsTab = pathname.startsWith(Urls.transformList());
const isWorkspacePage = pathname.startsWith(Urls.dataStudioWorkspaceList());
const isTransformsTab =
pathname.startsWith(Urls.transformList()) && !isWorkspacePage;
const isModelingTab = pathname.startsWith(Urls.dataStudioModeling());
const isDependenciesTab = pathname.startsWith(Urls.dependencyGraph());
return (
<Stack className={S.nav} h="100%" p="0.75rem" justify="space-between">
<Stack
className={cx(S.nav, { [S.navCollapsed]: !isNavExpanded })}
h="100%"
p="0.75rem"
justify="space-between"
>
<Stack gap="0.75rem">
{canAccessDataModel && (
<DataStudioTab
label={t`Data`}
icon="database"
to={Urls.dataStudioData()}
isSelected={isDataTab}
/>
)}
{canAccessTransforms && (
<DataStudioTab
label={t`Transforms`}
icon="transform"
to={Urls.transformList()}
isSelected={isTransformsTab}
/>
)}
<DataStudioTab
label={t`Modeling`}
icon="model"
to={Urls.dataStudioModeling()}
isSelected={isModelingTab}
<DataStudioNavToggle
isNavExpanded={isNavExpanded}
onNavToggle={onNavToggle}
/>
{PLUGIN_DEPENDENCIES.isEnabled && (
<Stack gap="0.75rem">
{canAccessDataModel && (
<DataStudioTab
label={t`Data`}
icon="database"
to={Urls.dataStudioData()}
isSelected={isDataTab}
isExpanded={isNavExpanded}
/>
)}
{canAccessTransforms && (
<DataStudioTab
label={t`Transforms`}
icon="transform"
to={Urls.transformList()}
isSelected={isTransformsTab}
isExpanded={isNavExpanded}
/>
)}
<DataStudioTab
label={t`Dependency graph`}
icon="schema"
to={Urls.dependencyGraph()}
isSelected={isDependenciesTab}
label={t`Modeling`}
icon="model"
to={Urls.dataStudioModeling()}
isSelected={isModelingTab}
isExpanded={isNavExpanded}
/>
{PLUGIN_DEPENDENCIES.isEnabled && (
<DataStudioTab
label={t`Dependency graph`}
icon="schema"
to={Urls.dependencyGraph()}
isSelected={isDependenciesTab}
isExpanded={isNavExpanded}
/>
)}
</Stack>
{canAccessTransforms && (
<WorkspacesSection isExpanded={isNavExpanded} />
)}
</Stack>
{isSidebarAvailable && (
@@ -129,23 +172,241 @@ type DataStudioTabProps = {
icon: IconName;
to: string;
isSelected: boolean;
isExpanded: boolean;
};
const TOOLTIP_OPEN_DELAY = 1000;
function DataStudioTab({ label, icon, to, isSelected }: DataStudioTabProps) {
function DataStudioTab({
label,
icon,
to,
isSelected,
isExpanded,
}: DataStudioTabProps) {
const content = (
<Box
className={cx(S.tab, { [S.selected]: isSelected })}
component={ForwardRefLink}
to={to}
p="0.75rem"
bdrs="md"
>
<Flex
align="center"
gap="sm"
justify={isExpanded ? "flex-start" : "center"}
>
<FixedSizeIcon name={icon} display="block" />
{isExpanded && (
<Text size="sm" fw={600}>
{label}
</Text>
)}
</Flex>
</Box>
);
if (!isExpanded) {
return (
<Tooltip label={label} position="right" openDelay={TOOLTIP_OPEN_DELAY}>
{content}
</Tooltip>
);
}
return content;
}
type WorkspacesSectionProps = {
isExpanded: boolean;
};
function WorkspacesSection({
isExpanded: isNavExpanded,
}: WorkspacesSectionProps) {
const dispatch = useDispatch();
const [isWorkspacesExpanded, setIsWorkspacesExpanded] = useState(true);
const [isCreateModalOpen, setIsCreateModalOpen] = useState(false);
const { pathname } = useSelector(getLocation);
const { data: workspacesData, isLoading } = useGetWorkspacesQuery();
const { data: databaseData, isLoading: isLoadingDatabases } =
useListDatabasesQuery({ include_analytics: true });
const [createWorkspace, { isLoading: isCreating }] =
useCreateWorkspaceMutation();
const workspaces = useMemo(
() =>
[...(workspacesData?.items ?? [])].sort(
(a, b) =>
new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime(),
),
[workspacesData],
);
const databaseOptions = useMemo(
() =>
(databaseData?.data ?? []).map((db: Database) => ({
value: String(db.id),
label: db.name,
})),
[databaseData],
);
const handleOpenWorkspace = useCallback(
(workspaceId: number) => {
dispatch(push(Urls.dataStudioWorkspace(workspaceId)));
},
[dispatch],
);
const handleOpenCreateModal = useCallback(() => {
setIsCreateModalOpen(true);
}, []);
const handleCloseCreateModal = useCallback(() => {
setIsCreateModalOpen(false);
}, []);
const { sendErrorToast, sendSuccessToast } = useMetadataToasts();
const handleCreateWorkspace = useCallback(
async ({ name, databaseId }: { name: string; databaseId: string }) => {
try {
const workspace = await createWorkspace({
name,
database_id: Number(databaseId),
upstream: {},
}).unwrap();
handleCloseCreateModal();
handleOpenWorkspace(workspace.id);
} catch (error) {
sendErrorToast(t`Failed to create workspace`);
}
},
[
createWorkspace,
handleCloseCreateModal,
handleOpenWorkspace,
sendErrorToast,
],
);
const [archiveWorkspace] = useArchiveWorkspaceMutation();
const handleWorkspaceArchive = async (id: WorkspaceId) => {
try {
await archiveWorkspace(id).unwrap();
sendSuccessToast(t`Workspace archived successfully`);
dispatch(push(Urls.dataStudioWorkspaceList()));
} catch (error) {
sendErrorToast(t`Failed to archive workspace`);
}
};
const isWorkspaceListPage = pathname === Urls.dataStudioWorkspaceList();
if (!isNavExpanded) {
// In collapsed mode, show only an icon button with tooltip
return (
<Tooltip
label={t`Workspaces`}
position="right"
openDelay={TOOLTIP_OPEN_DELAY}
>
<UnstyledButton
className={cx(S.tab, { [S.selected]: isWorkspaceListPage })}
component={ForwardRefLink}
to={Urls.dataStudioWorkspaceList()}
p="0.75rem"
bdrs="md"
>
<Flex align="center" justify="center">
<FixedSizeIcon name="git_branch" display="block" />
</Flex>
</UnstyledButton>
</Tooltip>
);
}
return (
<Tooltip label={label} position="right" openDelay={TOOLTIP_OPEN_DELAY}>
<Box
className={cx(S.tab, { [S.selected]: isSelected })}
component={ForwardRefLink}
to={to}
<Stack
data-testid="workspaces-section"
gap="0.5rem"
className={S.workspacesSection}
>
<UnstyledButton
className={cx(S.workspacesSectionHeader, {
[S.selected]: isWorkspaceListPage,
})}
onClick={() => setIsWorkspacesExpanded(!isWorkspacesExpanded)}
p="0.75rem"
bdrs="md"
>
<FixedSizeIcon name={icon} display="block" />
</Box>
</Tooltip>
<Flex align="center" justify="space-between" w="100%">
<Text size="sm" fw={600}>
{t`Workspaces`}
</Text>
<Icon
name={isWorkspacesExpanded ? "chevrondown" : "chevronright"}
size={16}
/>
</Flex>
</UnstyledButton>
{isWorkspacesExpanded && (
<>
<UnstyledButton
className={S.newWorkspaceButton}
onClick={handleOpenCreateModal}
disabled={isLoadingDatabases}
p="0.5rem"
bdrs="md"
>
<Flex align="center" gap="xs">
<Icon name="add" size={16} />
<Text size="sm" fw={500}>
{t`New workspace`}
</Text>
</Flex>
</UnstyledButton>
<Stack gap="0.75rem" style={{ overflowY: "auto", maxHeight: "50vh" }}>
{isLoading ? (
<>
<Skeleton height={80} radius="md" />
<Skeleton height={80} radius="md" />
</>
) : workspaces.length === 0 ? (
<Text c="text-secondary" size="xs" px="0.5rem">
{t`No workspaces yet`}
</Text>
) : (
workspaces.map((workspace) => {
const isSelected =
pathname === Urls.dataStudioWorkspace(workspace.id);
return (
<WorkspaceItem
key={workspace.id}
workspace={workspace}
isSelected={isSelected}
onOpen={handleOpenWorkspace}
archiveWorkspace={handleWorkspaceArchive}
/>
);
})
)}
</Stack>
</>
)}
<CreateWorkspaceModal
opened={isCreateModalOpen}
onClose={handleCloseCreateModal}
onSubmit={handleCreateWorkspace}
databaseOptions={databaseOptions}
isSubmitting={isCreating}
/>
</Stack>
);
}
@@ -161,6 +422,7 @@ function DataStudioSidebarToggle({
return (
<Tooltip
label={isSidebarOpened ? t`Close sidebar` : t`Open sidebar`}
position="right"
openDelay={TOOLTIP_OPEN_DELAY}
>
<UnstyledButton
@@ -176,3 +438,101 @@ function DataStudioSidebarToggle({
</Tooltip>
);
}
type DataStudioNavToggleProps = {
isNavExpanded: boolean;
onNavToggle: (isExpanded: boolean) => void;
};
function DataStudioNavToggle({
isNavExpanded,
onNavToggle,
}: DataStudioNavToggleProps) {
return (
<Tooltip
label={isNavExpanded ? t`Collapse navigation` : t`Expand navigation`}
position="right"
openDelay={TOOLTIP_OPEN_DELAY}
>
<UnstyledButton
className={S.toggle}
p="0.75rem"
bdrs="md"
onClick={() => onNavToggle(!isNavExpanded)}
>
<FixedSizeIcon name={isNavExpanded ? "chevronleft" : "chevronright"} />
</UnstyledButton>
</Tooltip>
);
}
interface WorkspaceItemProps {
workspace: { id: number; name: string; updated_at: string };
isSelected: boolean;
onOpen: (workspaceId: WorkspaceId) => void;
archiveWorkspace: (workspaceId: WorkspaceId) => Promise<void>;
}
function WorkspaceItem({
workspace,
isSelected,
onOpen,
archiveWorkspace,
}: WorkspaceItemProps) {
const timeAgo = dayjs(workspace.updated_at).fromNow();
const handleArchive = () => {
archiveWorkspace(workspace.id);
};
const handleDelete = () => {
archiveWorkspace(workspace.id);
};
return (
<UnstyledButton
className={cx(S.workspaceItem, { [S.selected]: isSelected })}
onClick={() => onOpen(workspace.id)}
p="0.75rem"
bdrs="md"
>
<Flex align="flex-start" justify="space-between" gap="xs">
<Stack gap="xs" style={{ flex: 1, minWidth: 0 }}>
<Text size="sm" fw={600} truncate>
{workspace.name}
</Text>
<Text size="xs" c="text-secondary" truncate>
{t`Updated ${timeAgo}`}
</Text>
</Stack>
<Menu position="right" withinPortal>
<Menu.Target>
<ActionIcon
className={S.workspaceMenuButton}
aria-label={t`More actions`}
onClick={(e: React.MouseEvent) => e.stopPropagation()}
size="sm"
variant="subtle"
>
<Icon name="ellipsis" size={16} />
</ActionIcon>
</Menu.Target>
<Menu.Dropdown onClick={(e: React.MouseEvent) => e.stopPropagation()}>
<Menu.Item
leftSection={<Icon name="archive" />}
onClick={handleArchive}
>
{t`Archive`}
</Menu.Item>
<Menu.Divider />
<Menu.Item
leftSection={<Icon name="trash" />}
onClick={handleDelete}
>
{t`Move to trash`}
</Menu.Item>
</Menu.Dropdown>
</Menu>
</Flex>
</UnstyledButton>
);
}

View File

@@ -0,0 +1,22 @@
import { t } from "ttag";
import { usePageTitle } from "metabase/hooks/use-page-title";
import { SectionLayout, SectionTitle } from "../../components/SectionLayout";
type WorkspacesSectionLayoutProps = {
location: Location;
children?: React.ReactNode;
};
export function WorkspacesSectionLayout({
children,
}: WorkspacesSectionLayoutProps) {
usePageTitle(t`Workspaces`);
return (
<SectionLayout title={<SectionTitle title={t`Workspaces`} />}>
{children}
</SectionLayout>
);
}

View File

@@ -0,0 +1 @@
export * from "./WorkspacesSectionLayout";

View File

@@ -15,12 +15,14 @@ import { DataStudioLayout } from "./app/pages/DataStudioLayout";
import { DependenciesSectionLayout } from "./app/pages/DependenciesSectionLayout";
import { ModelingSectionLayout } from "./app/pages/ModelingSectionLayout";
import { TransformsSectionLayout } from "./app/pages/TransformsSectionLayout";
import { WorkspacesSectionLayout } from "./app/pages/WorkspacesSectionLayout";
import { getDataStudioMetadataRoutes } from "./data-model/routes";
import { getDataStudioGlossaryRoutes } from "./glossary/routes";
import { getDataStudioMetricRoutes } from "./metrics/routes";
import { getDataStudioModelingRoutes } from "./modeling/routes";
import { getDataStudioModelRoutes } from "./models/routes";
import { getDataStudioSnippetRoutes } from "./snippets/routes";
import { getDataStudioWorkspaceRoutes } from "./workspaces/routes";
export function getDataStudioRoutes(
store: Store<State>,
@@ -55,6 +57,9 @@ export function getDataStudioRoutes(
{getDataStudioSnippetRoutes()}
{getDataStudioGlossaryRoutes()}
</Route>
<Route path="workspaces" component={WorkspacesSectionLayout}>
{getDataStudioWorkspaceRoutes()}
</Route>
{PLUGIN_DEPENDENCIES.isEnabled && (
<Route path="dependencies" component={DependenciesSectionLayout}>
{PLUGIN_DEPENDENCIES.getDataStudioDependencyRoutes()}

View File

@@ -0,0 +1,105 @@
import { type FormEvent, useEffect, useMemo, useState } from "react";
import { t } from "ttag";
import { Button, Flex, Modal, Select, Stack, TextInput } from "metabase/ui";
type DatabaseOption = {
value: string;
label: string;
};
export type CreateWorkspaceModalProps = {
opened: boolean;
onClose: () => void;
onSubmit: (workspace: { name: string; databaseId: string }) => void;
databaseOptions: DatabaseOption[];
isSubmitting?: boolean;
defaultName?: string;
defaultDatabaseId?: string | null;
};
export function CreateWorkspaceModal({
opened,
onClose,
onSubmit,
databaseOptions,
isSubmitting = false,
defaultName,
defaultDatabaseId = null,
}: CreateWorkspaceModalProps) {
const initialName = useMemo(
() => defaultName ?? t`New workspace`,
[defaultName],
);
const [workspaceName, setWorkspaceName] = useState(initialName);
const [selectedDatabaseId, setSelectedDatabaseId] = useState<string | null>(
defaultDatabaseId,
);
useEffect(() => {
if (!opened) {
return;
}
setWorkspaceName(initialName);
setSelectedDatabaseId(defaultDatabaseId);
}, [opened, initialName, defaultDatabaseId]);
const handleSubmit = (event: FormEvent) => {
event.preventDefault();
if (!selectedDatabaseId || !workspaceName.trim() || isSubmitting) {
return;
}
onSubmit({
name: workspaceName.trim(),
databaseId: selectedDatabaseId,
});
};
return (
<Modal
opened={opened}
onClose={onClose}
title={t`Create new workspace`}
withinPortal
>
<form onSubmit={handleSubmit}>
<Stack gap="md">
<TextInput
autoFocus
label={t`Workspace name`}
placeholder={t`Enter workspace name`}
value={workspaceName}
onChange={(e) => setWorkspaceName(e.target.value)}
required
/>
<Select
label={t`Database`}
placeholder={t`Select a database`}
data={databaseOptions}
value={selectedDatabaseId}
onChange={setSelectedDatabaseId}
required
searchable
/>
<Flex gap="sm" justify="flex-end">
<Button variant="subtle" onClick={onClose}>
{t`Cancel`}
</Button>
<Button
variant="filled"
disabled={
!selectedDatabaseId || !workspaceName.trim() || isSubmitting
}
type="submit"
loading={isSubmitting}
>
{t`Create`}
</Button>
</Flex>
</Stack>
</form>
</Modal>
);
}

View File

@@ -0,0 +1,59 @@
import { useCallback } from "react";
import { t } from "ttag";
import { useMetadataToasts } from "metabase/metadata/hooks";
import { Button, Icon, Menu } from "metabase/ui";
import { useExecuteWorkspaceMutation } from "metabase-enterprise/api";
import type { WorkspaceId } from "metabase-types/api";
type RunWorkspaceMenuProps = {
workspaceId: WorkspaceId;
disabled?: boolean;
};
export function RunWorkspaceMenu({
workspaceId,
disabled = false,
}: RunWorkspaceMenuProps) {
const { sendErrorToast } = useMetadataToasts();
const [executeWorkspace, { isLoading: isExecuting }] =
useExecuteWorkspaceMutation();
const handleExecuteWorkspace = useCallback(
async ({ staleOnly } = { staleOnly: false }) => {
try {
await executeWorkspace({
id: workspaceId,
stale_only: staleOnly,
}).unwrap();
} catch (error) {
sendErrorToast(t`Failed to run transforms`);
}
},
[workspaceId, executeWorkspace, sendErrorToast],
);
return (
<Menu position="bottom-end">
<Menu.Target>
<Button
variant="default"
rightSection={<Icon name="chevrondown" />}
loading={isExecuting}
disabled={disabled}
size="xs"
>
{t`Run transforms`}
</Button>
</Menu.Target>
<Menu.Dropdown>
<Menu.Item onClick={() => handleExecuteWorkspace()}>
{t`Run all transforms`}
</Menu.Item>
<Menu.Item onClick={() => handleExecuteWorkspace({ staleOnly: true })}>
{t`Run stale transforms`}
</Menu.Item>
</Menu.Dropdown>
</Menu>
);
}

View File

@@ -0,0 +1,45 @@
import { t } from "ttag";
import { ForwardRefLink } from "metabase/common/components/Link";
import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
import * as Urls from "metabase/lib/urls";
import { Anchor, Box, Card, Group, Stack, Text, Title } from "metabase/ui";
import { useGetWorkspacesQuery } from "metabase-enterprise/api";
export function WorkspaceListPage() {
const { data: workspacesData, error, isLoading } = useGetWorkspacesQuery();
if (error || isLoading) {
return <LoadingAndErrorWrapper error={error} loading={isLoading} />;
}
const workspaces = workspacesData?.items ?? [];
return (
<Box data-testid="workspaces-page" p="lg">
<Title order={2} mb="lg">{t`Workspaces`}</Title>
{workspaces.length === 0 ? (
<Text c="text-secondary">{t`No workspaces yet`}</Text>
) : (
<Stack gap="md">
{workspaces.map((workspace) => (
<Card key={workspace.id} p="md" shadow="none" withBorder>
<Group justify="space-between">
<Anchor
component={ForwardRefLink}
to={Urls.dataStudioWorkspace(workspace.id)}
fw={500}
>
{workspace.name}
</Anchor>
<Text c="text-secondary" size="sm">
{t`Created ${new Date(workspace.created_at).toLocaleDateString()}`}
</Text>
</Group>
</Card>
))}
</Stack>
)}
</Box>
);
}

View File

@@ -0,0 +1,210 @@
import { useCallback, useMemo, useState } from "react";
import { t } from "ttag";
import * as Yup from "yup";
import { skipToken, useListDatabaseSchemasQuery } from "metabase/api";
import { getErrorMessage } from "metabase/api/utils";
import { ActionIcon, Icon, Menu } from "metabase/ui";
import {
useCreateWorkspaceTransformMutation,
useValidateTableNameMutation,
} from "metabase-enterprise/api";
import {
CreateTransformModal,
type NewTransformValues,
} from "metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal";
import {
getInitialNativeSource,
getInitialPythonSource,
} from "metabase-enterprise/transforms/pages/NewTransformPage/utils";
import type {
CreateWorkspaceTransformRequest,
DatabaseId,
Transform,
TransformSource,
TransformTarget,
WorkspaceId,
} from "metabase-types/api";
type TransformType = "sql" | "python";
type AddTransformMenuProps = {
databaseId: number;
workspaceId: number;
onCreate: (transform: Transform) => void;
};
export const AddTransformMenu = ({
databaseId,
workspaceId,
onCreate,
}: AddTransformMenuProps) => {
const [modalType, setModalType] = useState<TransformType | null>(null);
const [createWorkspaceTransform] = useCreateWorkspaceTransformMutation();
const { data: fetchedSchemas = [] } = useListDatabaseSchemasQuery(
databaseId ? { id: databaseId, include_hidden: false } : skipToken,
);
const allowedSchemas = useMemo(
() =>
fetchedSchemas.filter((schema) => !schema.startsWith("mb__isolation")),
[fetchedSchemas],
);
const getSource = useCallback(
(type: TransformType): TransformSource => {
if (type === "sql") {
const source = getInitialNativeSource();
return {
...source,
query: { ...source.query, database: databaseId },
};
}
return {
...getInitialPythonSource(),
"source-database": databaseId,
};
},
[databaseId],
);
const handleClose = () => setModalType(null);
const handleSubmit = useCallback(
async (values: NewTransformValues): Promise<Transform> => {
const source = getSource(modalType!);
const request: CreateWorkspaceTransformRequest & { id: WorkspaceId } =
values.incremental
? {
id: workspaceId,
name: values.name,
description: null,
source,
target: {
type: "table-incremental" as const,
name: values.targetName,
schema: values.targetSchema,
database: databaseId,
"target-incremental-strategy": {
type: "append" as const,
},
},
}
: {
id: workspaceId,
name: values.name,
description: null,
source,
target: {
type: "table" as const,
name: values.targetName,
schema: values.targetSchema,
database: databaseId,
},
};
const transform = await createWorkspaceTransform(request).unwrap();
onCreate(transform);
handleClose();
return transform;
},
[
databaseId,
workspaceId,
modalType,
createWorkspaceTransform,
onCreate,
getSource,
],
);
const validationSchemaExtension = useTransformValidation({
databaseId,
workspaceId,
});
return (
<>
<Menu position="bottom-end">
<Menu.Target>
<ActionIcon size="2rem" p="0" ml="auto" aria-label={t`Add transform`}>
<Icon name="add" size={16} aria-hidden />
</ActionIcon>
</Menu.Target>
<Menu.Dropdown>
<Menu.Item
leftSection={<Icon name="sql" />}
onClick={() => setModalType("sql")}
>
{t`SQL Transform`}
</Menu.Item>
<Menu.Item
leftSection={<Icon name="code_block" />}
onClick={() => setModalType("python")}
>
{t`Python Transform`}
</Menu.Item>
</Menu.Dropdown>
</Menu>
{modalType && (
<CreateTransformModal
source={getSource(modalType)}
defaultValues={{ name: t`New transform` }}
onClose={handleClose}
schemas={allowedSchemas}
showIncrementalSettings={false}
validationSchemaExtension={validationSchemaExtension}
handleSubmit={handleSubmit}
/>
)}
</>
);
};
export const useTransformValidation = ({
databaseId,
target,
workspaceId,
}: {
databaseId: DatabaseId;
target?: TransformTarget;
workspaceId: WorkspaceId;
}) => {
const [validateTableName] = useValidateTableNameMutation();
const yupSchema = useMemo(
() => ({
targetName: Yup.string()
.required("Target table name is required")
.test(async (value, context) => {
if (!value) {
return context.createError({
message: "Target table name is required",
});
}
const schema = context.parent.targetSchema;
if (target && target.name === value && target.schema === schema) {
return true;
}
try {
const message = await validateTableName({
id: workspaceId,
db_id: databaseId,
target: { type: "table", name: value, schema },
}).unwrap();
return message === "OK" ? true : context.createError({ message });
} catch (error) {
const message = getErrorMessage(error);
return context.createError({ message });
}
}),
}),
[databaseId, target, workspaceId, validateTableName],
);
return yupSchema;
};

View File

@@ -0,0 +1,86 @@
import { t } from "ttag";
import { useDispatch, useSelector } from "metabase/lib/redux";
import { Button } from "metabase/ui";
import {
useLazyGetTransformQuery,
useUpdateWorkspaceContentsMutation,
} from "metabase-enterprise/api";
import {
addSuggestedTransform,
getMetabotSuggestedTransform,
} from "metabase-enterprise/metabot/state";
import type { Transform, TransformId, WorkspaceId } from "metabase-types/api";
import { useWorkspace } from "./WorkspaceProvider";
interface Props {
transform: Transform;
workspaceId: WorkspaceId;
onOpenTransform: (transformId: TransformId) => void;
}
export const CheckOutTransformButton = ({
transform,
workspaceId,
onOpenTransform,
}: Props) => {
const dispatch = useDispatch();
const suggestedTransform = useSelector((state) =>
getMetabotSuggestedTransform(state, transform.id),
);
const [getTransform] = useLazyGetTransformQuery();
const [updateWorkspaceContents] = useUpdateWorkspaceContentsMutation();
const {
addOpenedTransform,
removeOpenedTransform,
setActiveTransform,
removeEditedTransform,
} = useWorkspace();
const handleClick = async () => {
const response = await updateWorkspaceContents({
id: workspaceId,
add: {
transforms: [transform.id],
},
});
const newTransform = response.data?.contents.transforms.find(
(t) => t.upstream_id === transform.id,
);
const newTransformId = newTransform?.id;
if (newTransformId != null) {
// TODO: remove when backend adds contents hydration to previous request
const newTransform = await getTransform(newTransformId).unwrap();
if (newTransform) {
if (suggestedTransform) {
dispatch(
addSuggestedTransform({
...suggestedTransform,
id: newTransform.id,
active: true,
}),
);
}
removeEditedTransform(transform.id);
addOpenedTransform(newTransform);
removeOpenedTransform(transform.id);
setActiveTransform(newTransform);
onOpenTransform(newTransform.id);
}
}
};
return (
<Button
size="sm"
variant="filled"
onClick={handleClick}
>{t`Check out`}</Button>
);
};

View File

@@ -0,0 +1,101 @@
import { useMemo } from "react";
import { t } from "ttag";
import EmptyState from "metabase/common/components/EmptyState";
import { Stack, Text } from "metabase/ui";
import type { Transform, WorkspaceId } from "metabase-types/api";
import { useWorkspace } from "../WorkspaceProvider";
import { TransformListItem } from "./TransformListItem";
import { TransformListItemMenu } from "./TransformListItemMenu";
type CodeTabProps = {
activeTransformId?: number;
transforms: Transform[];
workspaceId: WorkspaceId;
workspaceTransforms: Transform[];
onTransformClick: (transform: Transform) => void;
};
export const CodeTab = ({
activeTransformId,
transforms,
workspaceId,
workspaceTransforms,
onTransformClick,
}: CodeTabProps) => {
const { editedTransforms, hasTransformEdits } = useWorkspace();
const workspaceTransformsUpstreamIds = useMemo(
() => new Set(workspaceTransforms.map((t) => t.upstream_id)),
[workspaceTransforms],
);
const availableTransforms = useMemo(() => {
return transforms.filter((transform) => {
return !workspaceTransformsUpstreamIds.has(transform.id);
});
}, [workspaceTransformsUpstreamIds, transforms]);
const handleTransformClick = (transform: Transform) => {
const edited = editedTransforms.get(transform.id);
const transformToOpen = edited
? ({ ...transform, ...edited } as Transform)
: transform;
onTransformClick(transformToOpen);
};
return (
<Stack h="100%" gap={0}>
<Stack
data-testid="workspace-transforms"
gap="xs"
pb="sm"
style={{
borderBottom: "1px solid var(--mb-color-border)",
}}
>
<Stack gap={0}>
<Text fw={600}>{t`Workspace transforms`}</Text>
{workspaceTransforms.map((transform) => {
const isEdited = hasTransformEdits(transform);
return (
<TransformListItem
key={transform.id}
name={transform.name}
icon="pivot_table"
fw={600}
isActive={activeTransformId === transform.id}
isEdited={isEdited}
menu={
<TransformListItemMenu
transform={transform}
workspaceId={workspaceId}
/>
}
onClick={() => handleTransformClick(transform)}
/>
);
})}
</Stack>
{workspaceTransforms.length === 0 && (
<EmptyState message={t`Workspace is empty`} />
)}
</Stack>
<Stack data-testid="mainland-transforms" py="sm" gap="xs">
{availableTransforms.map((transform) => (
<TransformListItem
key={transform.id}
name={transform.name}
isActive={activeTransformId === transform.id}
isEdited={editedTransforms.has(transform.id)}
onClick={() => handleTransformClick(transform)}
/>
))}
</Stack>
</Stack>
);
};

View File

@@ -0,0 +1,26 @@
.root {
display: flex;
align-items: center;
gap: var(--mantine-spacing-sm);
padding: var(--mantine-spacing-xs) var(--mantine-spacing-sm);
border-radius: var(--mantine-radius-sm);
cursor: pointer;
flex-wrap: nowrap;
&:hover .menu {
visibility: visible;
}
&:hover {
background-color: var(--mb-color-bg-light);
}
}
.name {
flex: 1;
min-width: 0;
}
.menu {
visibility: hidden;
}

View File

@@ -0,0 +1,40 @@
import type { ReactNode } from "react";
import { Box, type BoxProps, Icon, type IconName, Text } from "metabase/ui";
import { StatusDot } from "../components/StatusDot/StatusDot";
import S from "./TransformListItem.module.css";
type TransformListItemProps = {
name: string;
icon?: IconName;
isActive?: boolean;
isEdited?: boolean;
menu?: ReactNode;
onClick?: () => void;
} & BoxProps;
export const TransformListItem = ({
name,
icon = "database",
isActive,
isEdited,
menu,
onClick,
...props
}: TransformListItemProps) => {
return (
<Box className={S.root} onClick={onClick} {...props}>
<Icon name={icon} size={14} c="brand" />
<Text className={S.name} c={isActive ? "brand" : "text-dark"} truncate>
{name}
</Text>
{isEdited && <StatusDot status="changed" />}
<Box className={S.menu} flex="0 0 auto">
{menu}
</Box>
</Box>
);
};

View File

@@ -0,0 +1,56 @@
import { t } from "ttag";
import { useMetadataToasts } from "metabase/metadata/hooks";
import { ActionIcon, Icon, Menu } from "metabase/ui";
import { useUpdateWorkspaceContentsMutation } from "metabase-enterprise/api";
import type { Transform, WorkspaceId } from "metabase-types/api";
import { useWorkspace } from "../WorkspaceProvider";
interface Props {
transform: Transform;
workspaceId: WorkspaceId;
}
export function TransformListItemMenu({ transform, workspaceId }: Props) {
const [updateWorkspaceContents] = useUpdateWorkspaceContentsMutation();
const { removeEditedTransform, removeOpenedTransform } = useWorkspace();
const { sendErrorToast, sendSuccessToast } = useMetadataToasts();
const handleRemove = async () => {
const response = await updateWorkspaceContents({
id: workspaceId,
remove: {
transforms: [transform.id],
},
});
if (response.error) {
sendErrorToast(t`Failed to remove transform from the workspace`);
} else {
sendSuccessToast(t`Transform removed from the workspace`);
removeEditedTransform(transform.id);
removeOpenedTransform(transform.id);
}
};
return (
<Menu position="bottom-end" withinPortal>
<Menu.Target>
<ActionIcon
onClick={(event) => event.stopPropagation()}
size="sm"
variant="subtle"
>
<Icon name="ellipsis" />
</ActionIcon>
</Menu.Target>
<Menu.Dropdown onClick={(event) => event.stopPropagation()}>
<Menu.Item leftSection={<Icon name="trash" />} onClick={handleRemove}>
{t`Remove`}
</Menu.Item>
</Menu.Dropdown>
</Menu>
);
}

View File

@@ -0,0 +1,55 @@
import { t } from "ttag";
import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
import CS from "metabase/css/core/index.css";
import { Box, Stack, Text } from "metabase/ui";
import Visualization from "metabase/visualizations/components/Visualization";
import type { DatabaseId, TableId } from "metabase-types/api";
import { useTablePreview } from "./useTablePreview";
interface DataTabProps {
databaseId: DatabaseId | null;
tableId: TableId | null;
}
export function DataTab({ databaseId, tableId }: DataTabProps) {
const { rawSeries, isFetching, error } = useTablePreview({
databaseId,
tableId,
});
if (!databaseId || !tableId) {
return (
<Stack h="100%" align="center" justify="center">
<Text c="text-medium">{t`Select a table to view its data`}</Text>
</Stack>
);
}
if (isFetching) {
return <LoadingAndErrorWrapper loading />;
}
if (error) {
return (
<Stack h="100%" align="center" justify="center">
<Text c="error">{t`Error loading data`}</Text>
</Stack>
);
}
if (!rawSeries) {
return (
<Stack h="100%" align="center" justify="center">
<Text c="text-medium">{t`No data available`}</Text>
</Stack>
);
}
return (
<Box h="100%" className={CS.relative}>
<Visualization queryBuilderMode="dataset" rawSeries={rawSeries} />
</Box>
);
}

View File

@@ -0,0 +1,92 @@
import { t } from "ttag";
import { Stack, Text } from "metabase/ui";
import type {
TableId,
Transform,
WorkspaceTablesResponse,
WorkspaceTransformItem,
} from "metabase-types/api";
import { type OpenTable, useWorkspace } from "../WorkspaceProvider";
import { TableListItem } from "./TableListItem";
type DataTabSidebarProps = {
tables: WorkspaceTablesResponse;
workspaceTransforms: WorkspaceTransformItem[];
dbTransforms: Transform[];
selectedTableId?: TableId | null;
onTransformClick?: (transform: WorkspaceTransformItem) => void;
onTableSelect?: (table: OpenTable) => void;
};
export const DataTabSidebar = ({
tables,
workspaceTransforms,
dbTransforms,
selectedTableId,
onTransformClick,
onTableSelect,
}: DataTabSidebarProps) => {
const { hasTransformEdits } = useWorkspace();
return (
<Stack h="100%" gap="sm">
<Stack
gap="xs"
pb="sm"
style={{ borderBottom: "1px solid var(--mb-color-border)" }}
>
<Text fw={600}>{t`Data active in this workspace`}</Text>
<Stack gap={0}>
{tables.outputs.map((table, index: number) => {
const workspaceTransform = table.workspace
? workspaceTransforms.find(
(t) => t.id === table.workspace?.["transform-id"],
)
: undefined;
const originalTransform = workspaceTransform
? dbTransforms.find(
(t) => t.id === workspaceTransform.upstream_id,
)
: undefined;
const hasChanges = originalTransform
? hasTransformEdits(originalTransform)
: false;
const tableId = table.workspace?.["table-id"];
return (
<TableListItem
key={`output-${index}`}
name={table.workspace?.table || ""}
schema={table.workspace?.schema}
icon="pivot_table"
type="output"
hasChanges={hasChanges}
transform={workspaceTransform}
tableId={tableId}
isSelected={tableId === selectedTableId}
onTransformClick={onTransformClick}
onTableClick={onTableSelect}
/>
);
})}
</Stack>
<Stack gap={0}>
{tables.inputs.map((table, index) => (
<TableListItem
key={`input-${index}`}
name={table.table || ""}
schema={table.schema}
icon="table"
type="input"
tableId={table.id}
onTransformClick={onTransformClick}
onTableClick={onTableSelect}
/>
))}
</Stack>
</Stack>
</Stack>
);
};

View File

@@ -0,0 +1,34 @@
.root {
display: flex;
align-items: center;
gap: var(--mantine-spacing-sm);
padding: var(--mantine-spacing-xs) var(--mantine-spacing-sm);
border-radius: var(--mantine-radius-sm);
flex-wrap: nowrap;
&:hover .menu {
visibility: visible;
}
&:hover {
background-color: var(--mb-color-bg-light);
}
}
.clickable {
cursor: pointer;
}
.selected {
background-color: var(--mb-color-bg-medium);
}
.name {
flex: 1;
min-width: 0;
}
.actionIcon {
&:hover {
background-color: var(--mb-color-bg-medium) !important;
}
}

View File

@@ -0,0 +1,94 @@
import cx from "classnames";
import { t } from "ttag";
import {
ActionIcon,
Box,
Icon,
type IconName,
Text,
Tooltip,
} from "metabase/ui";
import type { TableId, WorkspaceTransformItem } from "metabase-types/api";
import type { OpenTable } from "../WorkspaceProvider";
import { StatusDot } from "../components/StatusDot/StatusDot";
import S from "./TableListItem.module.css";
type TableListItemProps = {
name: string;
schema?: string | null;
icon?: IconName;
type: "input" | "output";
hasChanges?: boolean;
transform?: WorkspaceTransformItem;
tableId?: TableId;
isSelected?: boolean;
onTransformClick?: (transform: WorkspaceTransformItem) => void;
onTableClick?: (table: OpenTable) => void;
};
export const TableListItem = ({
name,
schema,
icon = "table",
type,
hasChanges = false,
transform,
tableId,
isSelected = false,
onTransformClick,
onTableClick,
}: TableListItemProps) => {
const displayName = schema ? `${schema}.${name}` : name;
const handleTransformClick = (e: React.MouseEvent) => {
e.stopPropagation();
if (transform && onTransformClick) {
onTransformClick(transform);
}
};
const handleTableClick = () => {
if (tableId && onTableClick) {
onTableClick({ tableId, name, schema });
}
};
const isClickable = tableId && onTableClick;
return (
<Box
className={cx(S.root, {
[S.selected]: isSelected,
[S.clickable]: isClickable,
})}
aria-label={displayName}
onClick={isClickable ? handleTableClick : undefined}
>
<Icon
name={icon}
size={14}
c={type === "input" ? "saturated-green" : "saturated-yellow"}
/>
<Text className={S.name} c="text-dark" truncate>
{displayName}
</Text>
{hasChanges && <StatusDot status="changed" />}
{type === "output" && transform && (
<Tooltip label={t`Open transform`} position="top">
<ActionIcon
className={S.actionIcon}
ml="auto"
size="sm"
variant="subtle"
onClick={handleTransformClick}
>
<Icon name="code_block" size={14} c="text-medium" />
</ActionIcon>
</Tooltip>
)}
</Box>
);
};

View File

@@ -0,0 +1,4 @@
export { DataTab } from "./DataTab";
export { DataTabSidebar } from "./DataTabSidebar";
export { useTablePreview as useTableQuestion } from "./useTablePreview";
export type { OpenTable } from "../WorkspaceProvider";

View File

@@ -0,0 +1,101 @@
import { useMemo } from "react";
import _ from "underscore";
import {
skipToken,
useGetAdhocQueryQuery,
useGetTableQueryMetadataQuery,
} from "metabase/api";
import { getErrorMessage } from "metabase/api/utils";
import { useSelector } from "metabase/lib/redux";
import { getTableMetadataQuery } from "metabase/metadata/pages/shared";
import { getMetadata } from "metabase/selectors/metadata";
import * as Lib from "metabase-lib";
import type { DatabaseId, RawSeries, TableId } from "metabase-types/api";
import { createMockCard } from "metabase-types/api/mocks";
interface UseTableQuestionProps {
databaseId: DatabaseId | null;
tableId: TableId | null;
}
interface UseTablePreviewResult {
rawSeries?: RawSeries;
isFetching: boolean;
isLoading: boolean;
isError: boolean;
error?: string;
}
export function useTablePreview({
databaseId,
tableId,
}: UseTableQuestionProps): UseTablePreviewResult {
const metadata = useSelector(getMetadata);
// Need to fetch this to make sure metadata is populated with proper table information on
// initial page load.
useGetTableQueryMetadataQuery(
tableId && tableId in metadata.tables
? skipToken
: getTableMetadataQuery(tableId),
);
const metadataProvider =
Object.keys(metadata.tables).length > 0
? Lib.metadataProvider(databaseId, metadata)
: null;
const table =
metadataProvider && tableId
? Lib.tableOrCardMetadata(metadataProvider, tableId)
: null;
const query =
table && metadataProvider
? Lib.queryFromTableOrCardMetadata(metadataProvider, table)
: null;
const { data, refetch, ...rest } = useGetAdhocQueryQuery(
query == null
? skipToken
: {
...Lib.toJsQuery(query),
ignore_error: true,
},
);
const base = {
...rest,
error: rest.error ? getErrorMessage(rest.error) : undefined,
rawSeries: undefined,
};
const rawSeries = useMemo(
() =>
query
? [
{
card: createMockCard({
dataset_query: Lib.toJsQuery(query),
display: "table",
visualization_settings: {},
}),
data: {
...data?.data,
rows: _.uniq(data?.data?.rows ?? []),
},
},
]
: ([] as RawSeries),
[data?.data, query],
);
if (data?.status === "failed") {
return { ...base, isError: true, error: getErrorMessage(data) };
}
if (query == null || !data?.data || data.data.cols.length === 0) {
return base;
}
return { ...base, rawSeries };
}

View File

@@ -0,0 +1,29 @@
import { PLUGIN_METABOT } from "metabase/plugins";
import { useRegisterMetabotTransformContext } from "metabase-enterprise/transforms/hooks/use-register-transform-metabot-context";
import type { DraftTransformSource, Transform } from "metabase-types/api";
type MetabotTabProps = {
transform?: Transform;
source?: DraftTransformSource;
};
const MetabotContextRegistration = ({ transform, source }: MetabotTabProps) => {
useRegisterMetabotTransformContext(transform, source);
return null;
};
export const MetabotTab = ({ transform, source }: MetabotTabProps) => {
const MetabotProvider = PLUGIN_METABOT.getMetabotProvider();
const MetabotChat = PLUGIN_METABOT.MetabotChat;
const metabotConfig = {
// Show top-level mention menu for transforms and databases.
suggestionModels: ["transform", "database"],
};
return (
<MetabotProvider>
<MetabotContextRegistration transform={transform} source={source} />
<MetabotChat config={metabotConfig} />
</MetabotProvider>
);
};

View File

@@ -0,0 +1,56 @@
import { t } from "ttag";
import { Button } from "metabase/ui";
import { useUpdateTransformMutation } from "metabase-enterprise/api";
import { isSameSource } from "metabase-enterprise/transforms/utils";
import type { DatabaseId, Transform } from "metabase-types/api";
import { type EditedTransform, useWorkspace } from "./WorkspaceProvider";
interface Props {
databaseId: DatabaseId;
editedTransform: EditedTransform;
transform: Transform;
}
export const SaveTransformButton = ({
databaseId,
editedTransform,
transform,
}: Props) => {
const [updateTransform] = useUpdateTransformMutation();
const { updateTransformState } = useWorkspace();
const hasSourceChanged = !isSameSource(
editedTransform.source,
transform.source,
);
const hasTargetNameChanged =
transform.target.name !== editedTransform.target.name;
const hasChanges = hasSourceChanged || hasTargetNameChanged;
const handleClick = async () => {
const updated = await updateTransform({
id: transform.id,
source: editedTransform.source,
name: editedTransform.name,
target: {
type: "table",
name: editedTransform.target.name,
schema: transform.target.schema,
database: databaseId,
},
}).unwrap();
updateTransformState(updated);
};
return (
<Button
disabled={!hasChanges}
size="sm"
variant="filled"
onClick={handleClick}
>{t`Save`}</Button>
);
};

View File

@@ -0,0 +1,105 @@
import { useMemo } from "react";
import { match } from "ts-pattern";
import { t } from "ttag";
import LoadingAndGenericErrorWrapper from "metabase/common/components/LoadingAndGenericErrorWrapper";
import { Group, Icon, Loader, Stack, Tooltip } from "metabase/ui";
import { useGetWorkspaceLogQuery } from "metabase-enterprise/api";
import type {
WorkspaceId,
WorkspaceLogStatus,
WorkspaceTask,
} from "metabase-types/api";
interface SetupTabProps {
workspaceId: WorkspaceId;
}
const LOGS_POLLING_INTERVAL = 1000;
export const SetupLog = ({ workspaceId }: SetupTabProps) => {
const { data, error, isLoading } = useGetWorkspaceLogQuery(workspaceId, {
pollingInterval: LOGS_POLLING_INTERVAL,
refetchOnMountOrArgChange: true,
});
const logs = useMemo(() => data?.logs ?? [], [data]);
if (error || isLoading) {
return <LoadingAndGenericErrorWrapper error={error} loading={isLoading} />;
}
return (
<Stack
bd="1px solid var(--mb-color-border)"
bg="background-info"
bdrs="md"
gap="sm"
p="md"
>
{logs.length === 0 && data?.status === "pending" && <Loader size="xs" />}
{logs.map((log) => {
return (
<Group gap="xs" key={log.id}>
<Tooltip disabled={!log.message} label={log.message}>
<Group>
<LogIcon status={log.status} />
</Group>
</Tooltip>
<span>
{translateTask(log.task)}
{log.status === "started" && "…"}
</span>
</Group>
);
})}
{data?.status === "ready" && (
<Group gap="xs">
<LogIcon status="success" />
{t`Workspace ready!`}
</Group>
)}
</Stack>
);
};
function LogIcon({ status }: { status: WorkspaceLogStatus | null }) {
return match(status)
.with("failure", () => {
return <Icon c="warning" name="warning_triangle_filled" />;
})
.with("started", () => {
return <Loader size="xs" />;
})
.with("success", () => {
return <Icon c="success" name="check" />;
})
.with(null, () => {
return <Icon name="chevronright" />;
})
.exhaustive();
}
function translateTask(task: WorkspaceTask | string) {
if (task === "workspace-setup") {
return t`Setting up the workspace`;
}
if (task === "database-isolation") {
return t`Provisioning database isolation`;
}
if (task === "mirror-entities") {
return t`Mirroring entities`;
}
if (task === "grant-read-access") {
return t`Granting permissions`;
}
return task;
}

View File

@@ -0,0 +1,45 @@
import { t } from "ttag";
import { Stack, Text, Title } from "metabase/ui";
import type { WorkspaceId } from "metabase-types/api";
import { SetupLog } from "./SetupLog";
interface SetupTabProps {
databaseName?: string;
workspaceId: WorkspaceId;
}
export const SetupTab = ({ databaseName, workspaceId }: SetupTabProps) => {
return (
<Stack gap="lg">
<Stack gap="xs">
<Title order={3}>{t`Workspace Setup`}</Title>
<Text c="text-medium">
{t`Configure your data warehouse connection for this workspace`}
</Text>
</Stack>
<Stack gap="xs">
<Text fw="bold">{t`Data warehouse`}</Text>
<Text>
<code>{databaseName}</code>
</Text>
<Text c="text-light" size="sm">
{t`Data warehouse selection is locked after editing or timeout`}
</Text>
</Stack>
<Stack gap="xs">
<Text fw="bold">{t`Setup log`}</Text>
<SetupLog
key={workspaceId} // avoid showing status of other workspaces (forces RTK query hook remount)
workspaceId={workspaceId}
/>
</Stack>
</Stack>
);
};

View File

@@ -0,0 +1,65 @@
import { useMemo, useState } from "react";
import _ from "underscore";
import {
TransformQueryPageEditor,
type TransformQueryPageEditorUiState,
} from "metabase-enterprise/transforms/pages/TransformQueryPage/TransformQueryPage";
import type { DraftTransformSource } from "metabase-types/api";
type TransformEditorProps = {
disabled: boolean;
source: DraftTransformSource;
onChange: (source: DraftTransformSource) => void;
proposedSource?: DraftTransformSource;
onAcceptProposed?: () => void;
onRejectProposed?: () => void;
};
export function TransformEditor({
disabled,
source,
onChange,
proposedSource,
onAcceptProposed,
onRejectProposed,
}: TransformEditorProps) {
const [uiState, setUiState] = useState(getInitialUiStateForTransform);
const uiOptions = useMemo(
() => ({
canChangeDatabase: false,
readOnly: disabled,
}),
[disabled],
);
const handleSourceChange = (source: DraftTransformSource) => {
onChange?.(source);
};
return (
<TransformQueryPageEditor
source={source}
proposedSource={proposedSource}
isDirty={false}
uiState={uiState}
setUiState={setUiState}
uiOptions={uiOptions}
databases={[]}
setSourceAndRejectProposed={handleSourceChange}
acceptProposed={onAcceptProposed ?? _.noop}
rejectProposed={onRejectProposed ?? _.noop}
/>
);
}
function getInitialUiStateForTransform() {
return {
lastRunResult: null,
lastRunQuery: null,
selectionRange: [],
modalSnippet: null,
sidebarType: null,
modalType: null,
} as TransformQueryPageEditorUiState;
}

View File

@@ -0,0 +1,237 @@
import { useDisclosure } from "@mantine/hooks";
import { useCallback } from "react";
import { t } from "ttag";
import { useDispatch, useSelector } from "metabase/lib/redux";
import { useMetadataToasts } from "metabase/metadata/hooks";
import { getMetadata } from "metabase/selectors/metadata";
import { Box, Button, Group, Icon, Stack, rem } from "metabase/ui";
import { useRunTransformMutation, workspaceApi } from "metabase-enterprise/api";
import {
deactivateSuggestedTransform,
getMetabotSuggestedTransform,
} from "metabase-enterprise/metabot/state";
import { UpdateTargetModal } from "metabase-enterprise/transforms/pages/TransformTargetPage/TargetSection/UpdateTargetModal";
import {
isSameSource,
isTransformRunning,
} from "metabase-enterprise/transforms/utils";
import * as Lib from "metabase-lib";
import Question from "metabase-lib/v1/Question";
import type {
DatabaseId,
DraftTransformSource,
Transform,
TransformId,
WorkspaceId,
} from "metabase-types/api";
import { CheckOutTransformButton } from "./CheckOutTransformButton";
import { SaveTransformButton } from "./SaveTransformButton";
import { TransformEditor } from "./TransformEditor";
import { type EditedTransform, useWorkspace } from "./WorkspaceProvider";
interface Props {
databaseId: DatabaseId;
editedTransform: EditedTransform;
transform: Transform;
workspaceId: WorkspaceId;
workspaceTransforms: Transform[];
onChange: (patch: Partial<EditedTransform>) => void;
onOpenTransform: (transformId: TransformId) => void;
}
export const TransformTab = ({
databaseId,
editedTransform,
transform,
workspaceId,
workspaceTransforms,
onChange,
onOpenTransform,
}: Props) => {
const { updateTransformState } = useWorkspace();
const { sendSuccessToast, sendErrorToast } = useMetadataToasts();
const [
isChangeTargetModalOpen,
{ open: openChangeTargetModal, close: closeChangeTargetModal },
] = useDisclosure();
const dispatch = useDispatch();
const suggestedTransform = useSelector((state) =>
getMetabotSuggestedTransform(state, transform.id),
);
const metadata = useSelector(getMetadata);
const normalizeSource = useCallback(
(source: DraftTransformSource) => {
if (source.type !== "query") {
return source;
}
const question = Question.create({
dataset_query: source.query,
metadata,
});
const query = question.query();
const { isNative } = Lib.queryDisplayInfo(query);
const normalizedQuery = isNative
? Lib.withNativeQuery(query, Lib.rawNativeQuery(query))
: query;
return {
type: "query",
query: question.setQuery(normalizedQuery).datasetQuery(),
};
},
[metadata],
);
const proposedSource =
suggestedTransform?.source &&
!isSameSource(suggestedTransform.source, editedTransform.source)
? normalizeSource(suggestedTransform.source)
: undefined;
const hasSourceChanged = !isSameSource(
editedTransform.source,
transform.source,
);
const hasChanges = hasSourceChanged;
const isSaved = workspaceTransforms.some((t) => t.id === transform.id);
const [runTransform] = useRunTransformMutation();
const handleRun = async () => {
try {
await runTransform(transform.id).unwrap();
// Invalidate the workspace tables cache since transform execution
// may affect the list of workspace tables.
if (isSaved) {
dispatch(
workspaceApi.util.invalidateTags([
{ type: "workspace", id: workspaceId },
]),
);
}
} catch (error) {
console.error("Failed to run transform", error);
}
};
const handleSourceChange = (source: DraftTransformSource) => {
onChange({ source });
};
const handleAcceptProposed = useCallback(() => {
if (proposedSource == null) {
return;
}
if (!isSaved) {
sendErrorToast(
t`Add this transform to the workspace before applying Metabot changes.`,
);
return;
}
onChange({ source: proposedSource });
dispatch(deactivateSuggestedTransform(suggestedTransform?.id));
}, [
proposedSource,
onChange,
dispatch,
suggestedTransform?.id,
isSaved,
sendErrorToast,
]);
const handleRejectProposed = useCallback(() => {
if (suggestedTransform) {
dispatch(deactivateSuggestedTransform(suggestedTransform.id));
}
}, [dispatch, suggestedTransform]);
const handleTargetUpdate = useCallback(
(updatedTransform: Transform) => {
updateTransformState(updatedTransform);
sendSuccessToast(t`Transform target updated`);
closeChangeTargetModal();
},
[closeChangeTargetModal, updateTransformState, sendSuccessToast],
);
const isRunning = isTransformRunning(transform);
return (
<Stack gap={0} h="100%">
<Group
flex="0 0 auto"
justify="space-between"
mih={rem(73)} // avoid CLS when showing/hiding output table input
p="md"
style={{ borderBottom: "1px solid var(--mb-color-border)" }}
>
<Group>
{isSaved && (
<Button
leftSection={<Icon name="pencil_lines" />}
size="sm"
disabled={isRunning || hasChanges}
onClick={openChangeTargetModal}
>{t`Change target`}</Button>
)}
</Group>
<Group>
{isSaved && (
<Button
disabled={hasChanges}
leftSection={<Icon name="play" />}
size="sm"
onClick={handleRun}
>{t`Run`}</Button>
)}
{isSaved && (
<SaveTransformButton
databaseId={databaseId}
editedTransform={editedTransform}
transform={transform}
/>
)}
{!isSaved && (
<CheckOutTransformButton
transform={transform}
workspaceId={workspaceId}
onOpenTransform={onOpenTransform}
/>
)}
</Group>
</Group>
{editedTransform && (
<Box flex="1">
<TransformEditor
disabled={!isSaved}
source={editedTransform.source}
proposedSource={proposedSource}
onAcceptProposed={handleAcceptProposed}
onRejectProposed={handleRejectProposed}
onChange={handleSourceChange}
/>
</Box>
)}
{isChangeTargetModalOpen && (
<UpdateTargetModal
transform={transform}
onUpdate={handleTargetUpdate}
onClose={closeChangeTargetModal}
/>
)}
</Stack>
);
};

View File

@@ -0,0 +1,20 @@
.tabsPanel {
display: flex;
gap: 0.5rem;
flex-wrap: nowrap;
overflow: auto;
scrollbar-width: none;
-ms-overflow-style: none;
&::-webkit-scrollbar {
display: none;
}
&::before {
display: none;
}
& button {
border-radius: 0;
}
}

View File

@@ -0,0 +1,690 @@
import type { DragEndEvent } from "@dnd-kit/core";
import { DndContext, PointerSensor, useSensor } from "@dnd-kit/core";
import {
restrictToHorizontalAxis,
restrictToParentElement,
} from "@dnd-kit/modifiers";
import {
SortableContext,
arrayMove,
horizontalListSortingStrategy,
} from "@dnd-kit/sortable";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { push, replace } from "react-router-redux";
import { t } from "ttag";
import { useListDatabasesQuery } from "metabase/api";
import { Sortable } from "metabase/common/components/Sortable";
import { useDispatch, useSelector } from "metabase/lib/redux";
import { checkNotNull } from "metabase/lib/types";
import * as Urls from "metabase/lib/urls";
import { useRegisterMetabotContextProvider } from "metabase/metabot";
import { useMetadataToasts } from "metabase/metadata/hooks";
import { PLUGIN_METABOT } from "metabase/plugins";
import {
ActionIcon,
Box,
Button,
Flex,
Group,
Icon,
Stack,
Tabs,
Text,
} from "metabase/ui";
import {
useGetWorkspaceQuery,
useGetWorkspaceTablesQuery,
useListTransformsQuery,
useMergeWorkspaceMutation,
useUpdateWorkspaceNameMutation,
} from "metabase-enterprise/api";
import { PaneHeaderInput } from "metabase-enterprise/data-studio/common/components/PaneHeader";
import { RunWorkspaceMenu } from "metabase-enterprise/data-studio/workspaces/components/RunWorkspaceMenu/RunWorkspaceMenu";
import { useMetabotAgent } from "metabase-enterprise/metabot/hooks/use-metabot-agent";
import { useMetabotReactions } from "metabase-enterprise/metabot/hooks/use-metabot-reactions";
import {
type MetabotState,
metabotActions,
} from "metabase-enterprise/metabot/state/reducer";
import { getMetabot } from "metabase-enterprise/metabot/state/selectors";
import { NAME_MAX_LENGTH } from "metabase-enterprise/transforms/constants";
import type { DraftTransformSource, Transform } from "metabase-types/api";
import { AddTransformMenu } from "./AddTransformMenu";
import { CodeTab } from "./CodeTab/CodeTab";
import { DataTab, DataTabSidebar } from "./DataTab";
import { MetabotTab } from "./MetabotTab";
import { SetupTab } from "./SetupTab";
import { TransformTab } from "./TransformTab";
import styles from "./WorkspacePage.module.css";
import {
type EditedTransform,
type OpenTable,
WorkspaceProvider,
type WorkspaceTab,
useWorkspace,
} from "./WorkspaceProvider";
type WorkspacePageProps = {
params: {
workspaceId: string;
};
};
type MetabotConversationSnapshot = Pick<
MetabotState,
| "messages"
| "history"
| "state"
| "reactions"
| "activeToolCalls"
| "errorMessages"
| "conversationId"
>;
function WorkspacePageContent({ params }: WorkspacePageProps) {
const id = Number(params.workspaceId);
const dispatch = useDispatch();
const metabotState = useSelector(getMetabot as any) as MetabotState;
const { sendErrorToast } = useMetadataToasts();
const isMetabotAvailable = PLUGIN_METABOT.isEnabled();
const [tab, setTab] = useState<string>("setup");
const pointerSensor = useSensor(PointerSensor, {
activationConstraint: { distance: 10 },
});
const metabotStateRef = useRef<MetabotState>(metabotState);
useEffect(() => {
metabotStateRef.current = metabotState;
}, [metabotState]);
const metabotSnapshots = useRef<Map<number, MetabotConversationSnapshot>>(
new Map(),
);
const { data: databases = { data: [] } } = useListDatabasesQuery({});
const { data: allTransforms = [] } = useListTransformsQuery({});
const { data: workspace, isLoading: isLoadingWorkspace } =
useGetWorkspaceQuery(id);
useRegisterMetabotContextProvider(async () => {
if (!workspace?.database_id) {
return;
}
return { default_database_id: workspace.database_id };
}, [workspace?.database_id]);
const { data: workspaceTables = { inputs: [], outputs: [] } } =
useGetWorkspaceTablesQuery(id);
const { navigateToPath, setNavigateToPath } = useMetabotReactions();
const {
resetConversation: resetMetabotConversation,
visible: isMetabotVisible,
} = useMetabotAgent();
const [mergeWorkspace, { isLoading: isMerging }] =
useMergeWorkspaceMutation();
const [updateWorkspaceName] = useUpdateWorkspaceNameMutation();
const sourceDb = databases?.data.find(
(db) => db.id === workspace?.database_id,
);
const dbTransforms = useMemo(
() =>
allTransforms.filter((t) => {
// TODO: @uladzimirdev add guards
if (t.source_type === "python") {
return (
"source-database" in t.source &&
t.source["source-database"] === sourceDb?.id
);
}
if (t.source_type === "native") {
return (
"query" in t.source && t.source.query.database === sourceDb?.id
);
}
return false;
}),
[allTransforms, sourceDb],
);
const {
openedTabs,
activeTab,
activeTransform,
activeEditedTransform,
activeTable,
setActiveTransform,
setActiveTab,
addOpenedTab,
removeOpenedTab,
setOpenedTabs,
addOpenedTransform,
patchEditedTransform,
hasUnsavedChanges,
} = useWorkspace();
const [metabotContextTransform, setMetabotContextTransform] = useState<
Transform | undefined
>();
const [metabotContextSource, setMetabotContextSource] = useState<
DraftTransformSource | undefined
>();
const workspaceTransforms = useMemo(
() => workspace?.contents?.transforms ?? [],
[workspace],
);
useEffect(() => {
if (activeTab) {
setTab(activeTab.id);
}
}, [id, activeTab, setTab]);
const tabsListRef = useRef<HTMLDivElement>(null);
useEffect(() => {
if (tabsListRef.current && tab) {
const activeTabElement = tabsListRef.current.querySelector(
`[data-active="true"]`,
) as HTMLElement;
if (activeTabElement) {
activeTabElement.scrollIntoView({
behavior: "smooth",
block: "nearest",
});
}
}
}, [tab]);
useEffect(() => {
if (
metabotContextTransform &&
!openedTabs.some(
(tab) =>
tab.type === "transform" &&
tab.transform.id === metabotContextTransform.id,
)
) {
setMetabotContextTransform(undefined);
setMetabotContextSource(undefined);
}
}, [openedTabs, metabotContextTransform]);
useEffect(() => {
const snapshots = metabotSnapshots.current;
const snapshot = snapshots.get(id);
if (snapshot) {
dispatch(metabotActions.setConversationSnapshot(snapshot));
} else {
resetMetabotConversation();
}
return () => {
snapshots.set(id, {
messages: metabotStateRef.current.messages,
history: metabotStateRef.current.history,
state: metabotStateRef.current.state,
reactions: metabotStateRef.current.reactions,
activeToolCalls: metabotStateRef.current.activeToolCalls,
errorMessages: metabotStateRef.current.errorMessages,
conversationId: metabotStateRef.current.conversationId,
});
resetMetabotConversation();
};
}, [id, dispatch, resetMetabotConversation]);
useEffect(() => {
if (isMetabotAvailable && isMetabotVisible) {
setTab("metabot");
setActiveTab(undefined);
}
}, [isMetabotAvailable, isMetabotVisible, setActiveTab]);
useEffect(() => {
if (!navigateToPath) {
return;
}
const transformIdFromPath = (() => {
const match = navigateToPath.match(/\/transform\/(\d+)/);
const extracted = Urls.extractEntityId(navigateToPath);
const idString = match?.[1] ?? (extracted ? String(extracted) : null);
const parsed = idString ? Number(idString) : NaN;
return Number.isFinite(parsed) ? parsed : undefined;
})();
if (transformIdFromPath != null) {
const targetTransform =
workspaceTransforms.find((t) => t.id === transformIdFromPath) ||
allTransforms.find((t) => t.id === transformIdFromPath);
if (targetTransform) {
addOpenedTransform(targetTransform);
setActiveTransform(targetTransform);
setNavigateToPath(null);
return;
}
}
dispatch(push(navigateToPath));
setNavigateToPath(null);
}, [
navigateToPath,
workspaceTransforms,
allTransforms,
addOpenedTransform,
setActiveTransform,
setNavigateToPath,
dispatch,
]);
const handleTransformChange = useCallback(
(patch: Partial<EditedTransform>) => {
patchEditedTransform(checkNotNull(activeTransform).id, patch);
},
[activeTransform, patchEditedTransform],
);
const handleTabClose = useCallback(
(event: React.MouseEvent, tab: WorkspaceTab, index: number) => {
event.stopPropagation();
const isActive =
(tab.type === "transform" &&
activeTransform?.id === tab.transform.id) ||
(tab.type === "table" && activeTable?.tableId === tab.table.tableId);
const remaining = openedTabs.filter((item) => item.id !== tab.id);
removeOpenedTab(tab.id);
if (!isActive) {
return;
}
// Prefer previous tab, otherwise use next tab (which is now at the same index after filtering)
const fallback = remaining[index - 1] ?? remaining[index] ?? undefined;
if (fallback) {
setActiveTab(fallback);
setTab(fallback.id);
} else {
setActiveTab(undefined);
setTab("setup");
}
},
[
activeTransform,
activeTable,
removeOpenedTab,
setActiveTab,
setTab,
openedTabs,
],
);
const handleMergeWorkspace = useCallback(async () => {
try {
const response = await mergeWorkspace(id).unwrap();
if (response.errors && response.errors.length > 0) {
sendErrorToast(
t`Failed to merge workspace: ${response.errors.map((e) => e.error).join(", ")}`,
);
return;
}
dispatch(replace(Urls.transformList()));
} catch (error) {
sendErrorToast(t`Failed to merge workspace`);
}
}, [id, mergeWorkspace, sendErrorToast, dispatch]);
const handleWorkspaceNameChange = useCallback(
async (newName: string) => {
if (!workspace || newName.trim() === workspace.name.trim()) {
return;
}
try {
await updateWorkspaceName({ id, name: newName.trim() }).unwrap();
} catch (error) {
sendErrorToast(t`Failed to update workspace name`);
}
},
[workspace, id, updateWorkspaceName, sendErrorToast],
);
const handleTableSelect = useCallback(
(table: OpenTable) => {
const tableTab: WorkspaceTab = {
id: `table-${table.tableId}`,
name: table.schema ? `${table.schema}.${table.name}` : table.name,
type: "table",
table,
};
addOpenedTab(tableTab);
},
[addOpenedTab],
);
const handleTabDragEnd = useCallback(
(event: DragEndEvent) => {
const activeId = event.active.id;
const overId = event.over?.id;
if (typeof activeId === "string" && typeof overId === "string") {
const activeIndex = openedTabs.findIndex(({ id }) => id === activeId);
const overIndex = openedTabs.findIndex(({ id }) => id === overId);
const reorderedTabs = arrayMove(openedTabs, activeIndex, overIndex);
setOpenedTabs(reorderedTabs);
// Activate the dragged tab after reordering
const draggedTab = reorderedTabs.find((tab) => tab.id === activeId);
if (draggedTab) {
setActiveTab(draggedTab);
}
}
},
[openedTabs, setOpenedTabs, setActiveTab],
);
if (isLoadingWorkspace) {
return (
<Box p="lg">
<Text>{t`Loading...`}</Text>
</Box>
);
}
if (!workspace) {
return (
<Box p="lg">
<Text c="text-dark">{t`Workspace not found`}</Text>
</Box>
);
}
return (
<Stack data-testid="workspace-page" h="100%" gap={0}>
<Group
px="md"
py="sm"
style={{ borderBottom: "1px solid var(--mb-color-border)" }}
justify="space-between"
>
<Flex gap="xs" align="center">
<Icon name="git_branch" size="1rem" aria-hidden />
<PaneHeaderInput
initialValue={workspace.name}
placeholder={t`Workspace name`}
maxLength={NAME_MAX_LENGTH}
onChange={handleWorkspaceNameChange}
/>
</Flex>
<Flex gap="sm">
<RunWorkspaceMenu
workspaceId={id}
disabled={workspaceTransforms.length === 0}
/>
<Button
variant="filled"
onClick={handleMergeWorkspace}
loading={isMerging}
disabled={hasUnsavedChanges() || workspaceTransforms.length === 0}
size="xs"
>
{t`Merge`}
</Button>
</Flex>
</Group>
<Group
align="flex-start"
gap={0}
flex="1 1 auto"
wrap="nowrap"
style={{ overflow: "hidden" }}
>
<Box
data-testid="workspace-content"
w="70%"
h="100%"
style={{
borderRight: "1px solid var(--mb-color-border)",
}}
pos="relative"
>
<Tabs
defaultValue="setup"
display="flex"
h="100%"
style={{ flexDirection: "column" }}
value={tab}
onChange={(tab) => {
if (tab === "metabot") {
if (activeTransform) {
setMetabotContextTransform(activeTransform);
setMetabotContextSource(
activeEditedTransform?.source ?? activeTransform.source,
);
} else {
setMetabotContextTransform(undefined);
setMetabotContextSource(undefined);
}
}
if (tab) {
setTab(tab);
}
if (
tab === "setup" ||
(tab === "metabot" && (activeTransform || activeTable))
) {
setActiveTab(undefined);
}
}}
>
<Flex
wrap="nowrap"
flex="0 0 auto"
px="md"
style={{ borderBottom: "1px solid var(--mb-color-border)" }}
>
<DndContext
onDragEnd={handleTabDragEnd}
modifiers={[restrictToHorizontalAxis, restrictToParentElement]}
sensors={[pointerSensor]}
>
<SortableContext
items={openedTabs}
strategy={horizontalListSortingStrategy}
>
<Tabs.List ref={tabsListRef} className={styles.tabsPanel}>
<Tabs.Tab value="setup">
<Group gap="xs" wrap="nowrap">
<Icon name="database" aria-hidden />
{t`Setup`}
</Group>
</Tabs.Tab>
{isMetabotAvailable && (
<Tabs.Tab value="metabot">
<Group gap="xs" wrap="nowrap">
<Icon name="message_circle" aria-hidden />
{t`Agent Chat`}
</Group>
</Tabs.Tab>
)}
{openedTabs.map((tab, index) => (
<Sortable
id={tab.id}
as="div"
key={tab.id}
draggingStyle={{ opacity: 0.5 }}
>
<Tabs.Tab
draggable
key={tab.id}
value={tab.id}
onClick={() => {
setActiveTab(tab);
}}
>
<Group gap="xs" wrap="nowrap">
<Icon
name={
tab.type === "transform"
? "pivot_table"
: "table"
}
aria-hidden
/>
{tab.name}
<ActionIcon
size="1rem"
p="0"
ml="xs"
onClick={(event) =>
handleTabClose(event, tab, index)
}
>
<Icon name="close" size={10} aria-hidden />
</ActionIcon>
</Group>
</Tabs.Tab>
</Sortable>
))}
</Tabs.List>
</SortableContext>
</DndContext>
</Flex>
<Box
flex={1}
mih={0}
style={{
overflow: tab === "metabot" ? "auto" : undefined,
}}
>
<Tabs.Panel value="setup" h="100%" p="md">
<SetupTab
databaseName={sourceDb?.name}
workspaceId={workspace.id}
/>
</Tabs.Panel>
{isMetabotAvailable && (
<Tabs.Panel
value="metabot"
h="100%"
mah="100%"
pos="relative"
style={{ overflow: "auto" }}
>
<MetabotTab
transform={metabotContextTransform}
source={metabotContextSource}
/>
</Tabs.Panel>
)}
<Tabs.Panel value={`table-${activeTable?.tableId}`} h="100%">
{!activeTable ? null : (
<DataTab
databaseId={workspace?.database_id ?? null}
tableId={activeTable.tableId}
/>
)}
</Tabs.Panel>
<Tabs.Panel value={`transform-${activeTransform?.id}`} h="100%">
{openedTabs.length === 0 ||
!activeTransform ||
!activeEditedTransform ? (
<Text c="text-medium">
{t`Select a transform on the right.`}
</Text>
) : (
<TransformTab
databaseId={checkNotNull(workspace.database_id)}
transform={activeTransform}
editedTransform={activeEditedTransform}
workspaceId={id}
workspaceTransforms={workspaceTransforms}
onChange={handleTransformChange}
onOpenTransform={(transformId) =>
setTab(String(transformId))
}
/>
)}
</Tabs.Panel>
</Box>
</Tabs>
</Box>
<Box
data-testid="workspace-sidebar"
style={{ flex: "1 0 auto", width: "30%" }}
>
<Tabs defaultValue="code">
<Flex
px="md"
align="center"
style={{ borderBottom: "1px solid var(--mb-color-border)" }}
>
<Tabs.List className={styles.tabsPanel}>
<Tabs.Tab value="code">{t`Code`}</Tabs.Tab>
<Tabs.Tab value="data">{t`Data`}</Tabs.Tab>
</Tabs.List>
{sourceDb && (
<AddTransformMenu
databaseId={sourceDb.id}
workspaceId={id}
onCreate={(transform) => {
addOpenedTransform(transform);
}}
/>
)}
</Flex>
<Tabs.Panel value="code" p="md">
<CodeTab
activeTransformId={activeTransform?.id}
transforms={dbTransforms}
workspaceId={workspace.id}
workspaceTransforms={workspaceTransforms}
onTransformClick={(transform) => {
addOpenedTransform(transform);
}}
/>
</Tabs.Panel>
<Tabs.Panel value="data" p="md">
<DataTabSidebar
tables={workspaceTables}
workspaceTransforms={workspaceTransforms}
dbTransforms={dbTransforms}
selectedTableId={activeTable?.tableId}
onTransformClick={(transform) => {
addOpenedTransform(transform);
}}
onTableSelect={handleTableSelect}
/>
</Tabs.Panel>
</Tabs>
</Box>
</Group>
</Stack>
);
}
export const WorkspacePage = ({ params }: WorkspacePageProps) => {
const workspaceId = Number(params.workspaceId);
return (
<WorkspaceProvider workspaceId={workspaceId}>
<WorkspacePageContent key={workspaceId} params={params} />
</WorkspaceProvider>
);
};

View File

@@ -0,0 +1,572 @@
import type { ReactNode } from "react";
import {
createContext,
useCallback,
useContext,
useMemo,
useState,
} from "react";
import { checkNotNull } from "metabase/lib/types";
import { isSameSource } from "metabase-enterprise/transforms/utils";
import type {
DraftTransformSource,
TableId,
Transform,
TransformTargetType,
} from "metabase-types/api";
export interface OpenTable {
tableId: TableId;
name: string;
schema?: string | null;
}
export interface EditedTransform {
name: string;
source: DraftTransformSource;
target: {
name: string;
type: TransformTargetType;
};
}
export interface Tab {
id: string;
name: string;
type: "transform" | "table";
}
export interface TransformTab extends Tab {
type: "transform";
transform: Transform;
}
export interface TableTab extends Tab {
type: "table";
table: OpenTable;
}
export type WorkspaceTab = TransformTab | TableTab;
export interface WorkspaceContextValue {
workspaceId: number;
openedTabs: WorkspaceTab[];
activeTransform?: Transform;
activeEditedTransform?: EditedTransform;
activeTable?: OpenTable;
activeTab?: WorkspaceTab;
setActiveTab: (tab: WorkspaceTab | undefined) => void;
setActiveTransform: (transform: Transform | undefined) => void;
setActiveTable: (table: OpenTable | undefined) => void;
addOpenedTab: (tab: WorkspaceTab) => void;
removeOpenedTab: (tabId: string) => void;
setOpenedTabs: (tabs: WorkspaceTab[]) => void;
addOpenedTransform: (transform: Transform) => void;
removeOpenedTransform: (transformId: number) => void;
editedTransforms: Map<number, EditedTransform>;
patchEditedTransform: (
transformId: number,
patch: Partial<EditedTransform>,
) => void;
removeEditedTransform: (transformId: number) => void;
runTransforms: Set<number>;
updateTransformState: (transform: Transform) => void;
hasUnsavedChanges: () => boolean;
hasTransformEdits: (originalTransform: Transform) => boolean;
}
const WorkspaceContext = createContext<WorkspaceContextValue | undefined>(
undefined,
);
interface WorkspaceState {
openedTabs: WorkspaceTab[];
activeTransform?: Transform;
activeEditedTransform?: Transform;
activeTable?: OpenTable;
activeTab?: WorkspaceTab;
editedTransforms: Map<number, EditedTransform>;
runTransforms: Set<number>;
}
interface WorkspaceProviderProps {
children: ReactNode;
workspaceId: number;
}
const createEmptyWorkspaceState = (): WorkspaceState => ({
openedTabs: [],
activeTransform: undefined,
activeTable: undefined,
activeTab: undefined,
editedTransforms: new Map(),
runTransforms: new Set(),
});
export const WorkspaceProvider = ({
children,
workspaceId,
}: WorkspaceProviderProps) => {
const [workspaceStates, setWorkspaceStates] = useState<
Map<number, WorkspaceState>
>(new Map());
const currentState = useMemo(() => {
const existing = workspaceStates.get(workspaceId);
if (existing) {
return existing;
}
const newState = createEmptyWorkspaceState();
setWorkspaceStates((prev) => new Map(prev).set(workspaceId, newState));
return newState;
}, [workspaceId, workspaceStates]);
const { openedTabs, activeTransform, activeTable, activeTab } = currentState;
const updateWorkspaceState = useCallback(
(updater: (state: WorkspaceState) => WorkspaceState) => {
setWorkspaceStates((prev) => {
const currentState =
prev.get(workspaceId) ?? createEmptyWorkspaceState();
const newState = updater(currentState);
return new Map(prev).set(workspaceId, newState);
});
},
[workspaceId],
);
const setActiveTab = useCallback(
(tab: WorkspaceTab | undefined) => {
updateWorkspaceState((state) => {
const newActiveTab = tab;
let newActiveTransform = state.activeTransform;
let newActiveTable = state.activeTable;
if (tab?.type === "transform") {
newActiveTransform = tab.transform;
newActiveTable = undefined;
} else if (tab?.type === "table") {
newActiveTable = tab.table;
newActiveTransform = undefined;
} else {
newActiveTransform = undefined;
newActiveTable = undefined;
}
return {
...state,
activeTab: newActiveTab,
activeTransform: newActiveTransform,
activeTable: newActiveTable,
};
});
},
[updateWorkspaceState],
);
const addOpenedTab = useCallback(
(tab: WorkspaceTab) => {
updateWorkspaceState((state) => {
const exists = state.openedTabs.some((item) => item.id === tab.id);
if (exists) {
return {
...state,
activeTab: tab,
activeTransform:
tab.type === "transform" ? tab.transform : state.activeTransform,
activeTable: tab.type === "table" ? tab.table : state.activeTable,
};
}
const newOpenedTabs = [...state.openedTabs, tab];
return {
...state,
openedTabs: newOpenedTabs,
activeTab: tab,
activeTransform:
tab.type === "transform" ? tab.transform : state.activeTransform,
activeTable: tab.type === "table" ? tab.table : state.activeTable,
};
});
},
[updateWorkspaceState],
);
const removeOpenedTab = useCallback(
(tabId: string) => {
updateWorkspaceState((state) => {
const filteredTabs = state.openedTabs.filter(
(item) => item.id !== tabId,
);
const removedTab = state.openedTabs.find((item) => item.id === tabId);
const wasActive = state.activeTab?.id === tabId;
let newActiveTab = state.activeTab;
let newActiveTransform = state.activeTransform;
let newActiveTable = state.activeTable;
if (wasActive && removedTab) {
const currentIndex = state.openedTabs.findIndex(
(item) => item.id === tabId,
);
const preferPrevious = currentIndex > 0;
const fallbackTab = preferPrevious
? state.openedTabs[currentIndex - 1]
: filteredTabs[0];
if (fallbackTab) {
newActiveTab = fallbackTab;
newActiveTransform =
fallbackTab.type === "transform"
? fallbackTab.transform
: undefined;
newActiveTable =
fallbackTab.type === "table" ? fallbackTab.table : undefined;
} else {
newActiveTab = undefined;
newActiveTransform = undefined;
newActiveTable = undefined;
}
}
return {
...state,
openedTabs: filteredTabs,
activeTab: newActiveTab,
activeTransform: newActiveTransform,
activeTable: newActiveTable,
};
});
},
[updateWorkspaceState],
);
const setOpenedTabs = useCallback(
(tabs: WorkspaceTab[]) => {
updateWorkspaceState((state) => ({
...state,
openedTabs: tabs,
}));
},
[updateWorkspaceState],
);
const setActiveTransform = useCallback(
(transform: Transform | undefined) => {
updateWorkspaceState((state) => {
if (transform) {
const existingTab = state.openedTabs.find(
(tab) =>
tab.type === "transform" && tab.transform.id === transform.id,
);
if (existingTab) {
return {
...state,
activeTab: existingTab,
activeTransform: transform,
activeTable: undefined,
};
} else {
const newTransformTab: TransformTab = {
id: `transform-${transform.id}`,
name: transform.name,
type: "transform",
transform,
};
return {
...state,
openedTabs: [...state.openedTabs, newTransformTab],
activeTab: newTransformTab,
activeTransform: transform,
activeTable: undefined,
};
}
} else {
return {
...state,
activeTab:
state.activeTab?.type === "transform"
? undefined
: state.activeTab,
activeTransform: undefined,
};
}
});
},
[updateWorkspaceState],
);
const addOpenedTransform = useCallback(
(transform: Transform) => {
const transformTab: TransformTab = {
id: `transform-${transform.id}`,
name: transform.name,
type: "transform",
transform,
};
addOpenedTab(transformTab);
},
[addOpenedTab],
);
const removeOpenedTransform = useCallback(
(transformId: number) => {
removeOpenedTab(`transform-${transformId}`);
},
[removeOpenedTab],
);
const patchEditedTransform = useCallback(
(transformId: number, patch: Partial<EditedTransform>) => {
updateWorkspaceState((state) => {
const activeTransform = checkNotNull(state.activeTransform);
const currentTransform =
state.editedTransforms.get(transformId) ?? activeTransform;
const newEditedTransform = {
name: patch.name ? patch.name : currentTransform.name,
source: patch.source ? patch.source : currentTransform.source,
target: patch.target ? patch.target : currentTransform.target,
};
const hasChanges =
!isSameSource(newEditedTransform.source, activeTransform.source) ||
newEditedTransform.name !== activeTransform.name ||
newEditedTransform.target.name !== activeTransform.target.name;
const newEditedTransforms = new Map(state.editedTransforms);
if (hasChanges) {
newEditedTransforms.set(transformId, newEditedTransform);
} else {
newEditedTransforms.delete(transformId);
}
const newRunTransforms = new Set(state.runTransforms);
newRunTransforms.delete(transformId);
const newOpenedTabs = state.openedTabs.map((tab) => {
if (tab.type === "transform" && tab.transform.id === transformId) {
return {
...tab,
name: newEditedTransform.name,
};
}
return tab;
});
return {
...state,
openedTabs: newOpenedTabs,
editedTransforms: newEditedTransforms,
runTransforms: newRunTransforms,
};
});
},
[updateWorkspaceState],
);
const removeEditedTransform = useCallback(
(transformId: number) => {
updateWorkspaceState((state) => {
const newEditedTransforms = new Map(state.editedTransforms);
newEditedTransforms.delete(transformId);
return {
...state,
editedTransforms: newEditedTransforms,
};
});
},
[updateWorkspaceState],
);
const updateTransformState = useCallback(
(transform: Transform) => {
updateWorkspaceState((state) => {
const newOpenedTabs = state.openedTabs.map((tab) => {
if (tab.type === "transform" && tab.transform.id === transform.id) {
return {
...tab,
transform,
name: transform.name,
};
}
return tab;
});
const newEditedTransforms = new Map(state.editedTransforms);
const currentEdit = state.editedTransforms.get(transform.id);
if (currentEdit) {
const hasNameChanged = currentEdit.name !== transform.name;
const hasSourceChanged = !isSameSource(
currentEdit.source,
transform.source,
);
if (hasNameChanged || hasSourceChanged) {
newEditedTransforms.set(transform.id, {
...currentEdit,
target: transform.target,
});
} else {
newEditedTransforms.delete(transform.id);
}
}
const newRunTransforms = new Set(state.runTransforms);
newRunTransforms.delete(transform.id);
return {
...state,
openedTabs: newOpenedTabs,
activeTransform:
state.activeTransform?.id === transform.id
? transform
: state.activeTransform,
editedTransforms: newEditedTransforms,
runTransforms: newRunTransforms,
};
});
},
[updateWorkspaceState],
);
const hasUnsavedChanges = useCallback(() => {
return currentState.editedTransforms.size > 0;
}, [currentState.editedTransforms]);
const hasTransformEdits = useCallback(
(originalTransform: Transform) => {
const edited = currentState.editedTransforms.get(originalTransform.id);
return (
edited != null &&
(!isSameSource(edited.source, originalTransform.source) ||
edited.name !== originalTransform.name ||
edited.target.name !== originalTransform.target.name)
);
},
[currentState.editedTransforms],
);
const setActiveTable = useCallback(
(table: OpenTable | undefined) => {
updateWorkspaceState((state) => {
if (table) {
const existingTab = state.openedTabs.find(
(tab) =>
tab.type === "table" && tab.table.tableId === table.tableId,
);
if (existingTab) {
return {
...state,
activeTab: existingTab,
activeTable: table,
activeTransform: undefined,
};
} else {
const newTableTab: TableTab = {
id: `table-${table.tableId}`,
name: table.schema ? `${table.schema}.${table.name}` : table.name,
type: "table",
table,
};
return {
...state,
openedTabs: [...state.openedTabs, newTableTab],
activeTab: newTableTab,
activeTable: table,
activeTransform: undefined,
};
}
} else {
return {
...state,
activeTab:
state.activeTab?.type === "table" ? undefined : state.activeTab,
activeTable: undefined,
};
}
});
},
[updateWorkspaceState],
);
const activeEditedTransform = activeTransform
? (currentState.editedTransforms.get(activeTransform.id) ?? activeTransform)
: activeTransform;
const value = useMemo(
() => ({
workspaceId,
openedTabs,
activeTransform,
activeTable,
activeTab,
activeEditedTransform,
setActiveTab,
setActiveTransform,
setActiveTable,
addOpenedTab,
removeOpenedTab,
setOpenedTabs,
addOpenedTransform,
removeOpenedTransform,
editedTransforms: currentState.editedTransforms,
patchEditedTransform,
removeEditedTransform,
runTransforms: currentState.runTransforms,
updateTransformState,
hasUnsavedChanges,
hasTransformEdits,
}),
[
workspaceId,
openedTabs,
activeTransform,
activeTable,
activeTab,
activeEditedTransform,
setActiveTab,
setActiveTransform,
setActiveTable,
addOpenedTab,
removeOpenedTab,
setOpenedTabs,
addOpenedTransform,
removeOpenedTransform,
currentState.editedTransforms,
currentState.runTransforms,
patchEditedTransform,
removeEditedTransform,
updateTransformState,
hasUnsavedChanges,
hasTransformEdits,
],
);
return (
<WorkspaceContext.Provider value={value}>
{children}
</WorkspaceContext.Provider>
);
};
export const useWorkspace = (): WorkspaceContextValue => {
const context = useContext(WorkspaceContext);
if (!context) {
throw new Error("useWorkspace must be used within a WorkspaceProvider");
}
return context;
};
export const useOptionalWorkspace = (): WorkspaceContextValue | undefined => {
return useContext(WorkspaceContext);
};

View File

@@ -0,0 +1,7 @@
.statusDot {
width: 0.5rem;
height: 0.5rem;
border-radius: 50%;
background-color: var(--mb-color-warning);
flex-shrink: 0;
}

View File

@@ -0,0 +1,15 @@
import { Box } from "metabase/ui";
import S from "./StatusDot.module.css";
type StatusDotProps = {
status?: "changed";
};
export const StatusDot = ({ status }: StatusDotProps) => {
if (status !== "changed") {
return null;
}
return <Box className={S.statusDot} />;
};

View File

@@ -0,0 +1,13 @@
import { IndexRoute, Route } from "react-router";
import { WorkspaceListPage } from "./pages/WorkspaceListPage/WorkspaceListPage";
import { WorkspacePage } from "./pages/WorkspacePage/WorkspacePage";
export function getDataStudioWorkspaceRoutes() {
return (
<>
<IndexRoute component={WorkspaceListPage} />
<Route path=":workspaceId" component={WorkspacePage} />
</>
);
}

View File

@@ -1,3 +1,5 @@
.suggestionPaper {
min-height: fit-content;
max-height: 50vh;
overflow-y: auto;
}

View File

@@ -3,6 +3,7 @@ import { tinykeys } from "tinykeys";
import ErrorBoundary from "metabase/ErrorBoundary";
import { useSelector } from "metabase/lib/redux";
import { Sidebar } from "metabase/nav/containers/MainNavbar/MainNavbar.styled";
import { getUser } from "metabase/selectors/user";
import type { SuggestionModel } from "metabase-enterprise/rich_text_editing/tiptap/extensions/shared/types";
@@ -55,7 +56,14 @@ export const MetabotAuthenticated = ({ hide, config }: MetabotProps) => {
return (
<ErrorBoundary errorComponent={() => null}>
<MetabotChat config={config} />
<Sidebar
isOpen={visible}
side="right"
width="30rem"
aria-hidden={!visible}
>
<MetabotChat config={config} />
</Sidebar>
</ErrorBoundary>
);
};

View File

@@ -2,16 +2,19 @@ import { unifiedMergeView } from "@codemirror/merge";
import { useDisclosure } from "@mantine/hooks";
import type { UnknownAction } from "@reduxjs/toolkit";
import cx from "classnames";
import { useMemo } from "react";
import { useMemo, useState } from "react";
import { push } from "react-router-redux";
import { useLocation, useMount } from "react-use";
import { P, match } from "ts-pattern";
import { t } from "ttag";
import _ from "underscore";
import { getErrorMessage } from "metabase/api/utils";
import { CodeMirror } from "metabase/common/components/CodeMirror";
import { slugify } from "metabase/lib/formatting";
import { useDispatch, useSelector } from "metabase/lib/redux";
import * as Urls from "metabase/lib/urls";
import { useMetadataToasts } from "metabase/metadata/hooks";
import EditorS from "metabase/query_builder/components/NativeQueryEditor/CodeMirrorEditor/CodeMirrorEditor.module.css";
import { getMetadata } from "metabase/selectors/metadata";
import {
@@ -24,17 +27,24 @@ import {
Paper,
Text,
} from "metabase/ui";
import { useLazyGetTransformQuery } from "metabase-enterprise/api";
import {
useCreateTransformMutation,
useLazyGetTransformQuery,
} from "metabase-enterprise/api";
import { useOptionalWorkspace } from "metabase-enterprise/data-studio/workspaces/pages/WorkspacePage/WorkspaceProvider";
import {
type MetabotAgentEditSuggestionChatMessage,
activateSuggestedTransform,
getIsSuggestedTransformActive,
} from "metabase-enterprise/metabot/state";
import * as Lib from "metabase-lib";
import Question from "metabase-lib/v1/Question";
import type Metadata from "metabase-lib/v1/metadata/Metadata";
import type {
DraftTransformSource,
MetabotTransformInfo,
SuggestedTransform,
Transform,
} from "metabase-types/api";
import S from "./MetabotAgentSuggestionMessage.module.css";
@@ -104,8 +114,17 @@ export const AgentSuggestionMessage = ({
}) => {
const dispatch = useDispatch();
const metadata = useSelector(getMetadata);
const workspace = useOptionalWorkspace();
const [createTransform] = useCreateTransformMutation();
const { sendErrorToast, sendSuccessToast } = useMetadataToasts();
const [isApplying, setIsApplying] = useState(false);
const [hasAppliedInWorkspace, setHasAppliedInWorkspace] = useState(false);
const { suggestedTransform, editorTransform } = message.payload;
const existingTransformId =
typeof suggestedTransform.id === "number"
? suggestedTransform.id
: undefined;
const isActive = useSelector((state) =>
getIsSuggestedTransformActive(state, suggestedTransform.suggestionId),
);
@@ -117,8 +136,10 @@ export const AgentSuggestionMessage = ({
getTransformUrl(suggestedTransform),
);
const canApply = !isViewing || !isActive;
const isNew = !isViewing && !editorTransform && suggestedTransform.id == null;
const canApply = workspace
? !hasAppliedInWorkspace && !isApplying
: !isViewing || !isActive;
const isNew = !isViewing && !editorTransform && existingTransformId == null;
const {
data: originalTransform,
@@ -131,8 +152,133 @@ export const AgentSuggestionMessage = ({
: "";
const newSource = getSourceCode(suggestedTransform, metadata);
const handleApply = () => {
const handleApply = async () => {
dispatch(activateSuggestedTransform(suggestedTransform));
if (workspace) {
const targetTransform: Transform | undefined =
(editorTransform as Transform | undefined) ??
(originalTransform as Transform | undefined) ??
(existingTransformId ? (suggestedTransform as Transform) : undefined);
if (existingTransformId != null && targetTransform) {
workspace.addOpenedTransform(targetTransform);
workspace.setActiveTransform(targetTransform);
setHasAppliedInWorkspace(true);
return;
}
if (existingTransformId == null && suggestedTransform.target) {
setIsApplying(true);
try {
const normalizeSource = (
source: DraftTransformSource,
): DraftTransformSource => {
if (source.type !== "query") {
return source;
}
const question = Question.create({
dataset_query: source.query,
metadata,
});
const query = question.query();
const { isNative } = Lib.queryDisplayInfo(query);
const normalizedQuery = isNative
? Lib.withNativeQuery(query, Lib.rawNativeQuery(query))
: query;
return {
type: "query",
query: question.setQuery(normalizedQuery).datasetQuery(),
};
};
const normalizedSource = normalizeSource(suggestedTransform.source);
const targetWithDatabase =
suggestedTransform.target &&
suggestedTransform.target.type === "table"
? {
...suggestedTransform.target,
database:
suggestedTransform.target.database ??
(normalizedSource.type === "query"
? normalizedSource.query.database
: normalizedSource.type === "python"
? normalizedSource["source-database"]
: undefined),
}
: suggestedTransform.target;
const sanitizedTarget =
targetWithDatabase?.type === "table"
? (() => {
const fallbackName = slugify(suggestedTransform.name);
const trimmedName =
targetWithDatabase.name?.trim() || fallbackName;
if (!trimmedName) {
sendErrorToast(
t`Suggestion is missing a target table name to create the transform.`,
);
setIsApplying(false);
return null;
}
return {
...targetWithDatabase,
name: trimmedName,
schema:
targetWithDatabase.schema &&
targetWithDatabase.schema.trim() !== ""
? targetWithDatabase.schema.trim()
: null,
};
})()
: targetWithDatabase;
if (sanitizedTarget === null) {
return;
}
if (
targetWithDatabase?.type === "table" &&
targetWithDatabase.database == null
) {
sendErrorToast(
t`Suggestion is missing a target database to create the transform.`,
);
setIsApplying(false);
return;
}
const transform = await createTransform({
name: suggestedTransform.name,
description: suggestedTransform.description ?? null,
source: normalizedSource,
target: sanitizedTarget,
}).unwrap();
workspace.addOpenedTransform(transform);
workspace.setActiveTransform(transform);
setHasAppliedInWorkspace(true);
sendSuccessToast(t`Transform created`);
return;
} catch (error) {
sendErrorToast(
getErrorMessage(error) ??
t`Failed to create transform from suggestion`,
);
} finally {
setIsApplying(false);
}
} else if (existingTransformId == null) {
sendErrorToast(t`Suggestion is missing a target table`);
}
return;
}
dispatch(push(getTransformUrl(suggestedTransform)) as UnknownAction);
};
@@ -215,11 +361,13 @@ export const AgentSuggestionMessage = ({
disabled={!canApply}
onClick={handleApply}
>
{match({ isNew, canApply })
.with({ canApply: false }, () => t`Applied`)
.with({ isNew: true }, () => t`Create`)
.with({ canApply: true }, () => t`Apply`)
.exhaustive()}
{isApplying
? t`Applying...`
: match({ isNew, canApply })
.with({ canApply: false }, () => t`Applied`)
.with({ isNew: true }, () => t`Create`)
.with({ canApply: true }, () => t`Apply`)
.exhaustive()}
</Button>
</Flex>
</Group>

View File

@@ -3,7 +3,6 @@ import { useCallback, useMemo } from "react";
import { t } from "ttag";
import EmptyDashboardBot from "assets/img/dashboard-empty.svg?component";
import { Sidebar } from "metabase/nav/containers/MainNavbar/MainNavbar.styled";
import {
ActionIcon,
Box,
@@ -38,11 +37,11 @@ const defaultConfig: MetabotConfig = {
],
};
export const MetabotChat = ({
config = defaultConfig,
}: {
interface Props {
config?: MetabotConfig;
}) => {
}
export const MetabotChat = ({ config = defaultConfig }: Props) => {
const metabot = useMetabotAgent();
const { handleSubmitInput, handleRetryMessage, handleResetInput } =
useMetabotChatHandlers(config);
@@ -77,137 +76,127 @@ export const MetabotChat = ({
}, [handleSubmitInput, metabot.prompt]);
return (
<Sidebar
isOpen={metabot.visible}
side="right"
width="30rem"
aria-hidden={!metabot.visible}
>
<Box className={Styles.container} data-testid="metabot-chat">
{/* header */}
<Box ref={headerRef} className={Styles.header}>
<Flex align-items="center">
<Text lh={1} fz="sm" c="text-secondary">
{metabot.profileOverride
? t`Using profile: ${metabot.profileOverride}`
: t`Metabot isn't perfect. Double-check results.`}
</Text>
</Flex>
<Box className={Styles.container} data-testid="metabot-chat">
{/* header */}
<Box ref={headerRef} className={Styles.header}>
<Flex align-items="center">
<Text lh={1} fz="sm" c="text-secondary">
{metabot.profileOverride
? t`Using profile: ${metabot.profileOverride}`
: t`Metabot isn't perfect. Double-check results.`}
</Text>
</Flex>
<Flex gap="sm">
<Tooltip label={t`Clear conversation`} position="bottom">
<ActionIcon
onClick={() => metabot.resetConversation()}
data-testid="metabot-reset-chat"
>
<Icon c="text-primary" name="revert" />
</ActionIcon>
</Tooltip>
{!config.preventClose && (
<ActionIcon
onClick={handleClose}
data-testid="metabot-close-chat"
>
<Icon c="text-primary" name="close" />
</ActionIcon>
)}
</Flex>
</Box>
{/* chat messages */}
<Box
ref={scrollContainerRef}
className={Styles.messagesContainer}
data-testid="metabot-chat-messages"
>
{!hasMessages && !metabot.isDoingScience && (
<>
{/* empty state */}
<Flex
h="100%"
gap="md"
direction="column"
align="center"
justify="center"
data-testid="metabot-empty-chat-info"
>
<Box component={EmptyDashboardBot} w="6rem" />
<Text c="text-light" maw="12rem" ta="center">
{config.emptyText ??
t`I can help you explore your metrics and models.`}
</Text>
</Flex>
{!config.hideSuggestedPrompts && (
<Stack
gap="sm"
className={Styles.promptSuggestionsContainer}
data-testid="metabot-prompt-suggestions"
>
<>
{suggestedPrompts.map(({ prompt }, index) => (
<Box key={index}>
<Button
fz="sm"
size="xs"
onClick={() => handleSubmitInput(prompt)}
className={Styles.promptSuggestionButton}
>
{prompt}
</Button>
</Box>
))}
</>
</Stack>
)}
</>
)}
{(hasMessages || metabot.isDoingScience) && (
<Box
className={Styles.messages}
data-testid="metabot-chat-inner-messages"
<Flex gap="sm">
<Tooltip label={t`Clear conversation`} position="bottom">
<ActionIcon
onClick={() => metabot.resetConversation()}
data-testid="metabot-reset-chat"
>
{/* conversation messages */}
<Messages
messages={metabot.messages}
errorMessages={metabot.errorMessages}
onRetryMessage={handleRetryMessage}
isDoingScience={metabot.isDoingScience}
showFeedbackButtons
/>
{/* loading */}
{metabot.isDoingScience && (
<MetabotThinking toolCalls={metabot.activeToolCalls} />
)}
{/* filler - height gets set via ref mutation */}
<div ref={fillerRef} data-testid="metabot-message-filler" />
{/* long convo warning */}
{metabot.isLongConversation && <MetabotResetLongChatButton />}
</Box>
<Icon c="text-primary" name="revert" />
</ActionIcon>
</Tooltip>
{!config.preventClose && (
<ActionIcon onClick={handleClose} data-testid="metabot-close-chat">
<Icon c="text-primary" name="close" />
</ActionIcon>
)}
</Box>
<Box className={Styles.textInputContainer}>
<Paper
className={cx(
Styles.inputContainer,
metabot.isDoingScience && Styles.inputContainerLoading,
)}
>
<MetabotChatEditor
ref={metabot.promptInputRef}
value={metabot.prompt}
autoFocus
isResponding={metabot.isDoingScience}
placeholder={t`Tell me to do something, or ask a question`}
onChange={handleEditorChange}
onSubmit={handleEditorSubmit}
onStop={metabot.cancelRequest}
suggestionModels={config.suggestionModels}
/>
</Paper>
</Box>
</Flex>
</Box>
</Sidebar>
{/* chat messages */}
<Box
ref={scrollContainerRef}
className={Styles.messagesContainer}
data-testid="metabot-chat-messages"
>
{!hasMessages && !metabot.isDoingScience && (
<>
{/* empty state */}
<Flex
h="100%"
gap="md"
direction="column"
align="center"
justify="center"
data-testid="metabot-empty-chat-info"
>
<Box component={EmptyDashboardBot} w="6rem" />
<Text c="text-light" maw="12rem" ta="center">
{config.emptyText ??
t`I can help you explore your metrics and models.`}
</Text>
</Flex>
{!config.hideSuggestedPrompts && (
<Stack
gap="sm"
className={Styles.promptSuggestionsContainer}
data-testid="metabot-prompt-suggestions"
>
<>
{suggestedPrompts.map(({ prompt }, index) => (
<Box key={index}>
<Button
fz="sm"
size="xs"
onClick={() => handleSubmitInput(prompt)}
className={Styles.promptSuggestionButton}
>
{prompt}
</Button>
</Box>
))}
</>
</Stack>
)}
</>
)}
{(hasMessages || metabot.isDoingScience) && (
<Box
className={Styles.messages}
data-testid="metabot-chat-inner-messages"
>
{/* conversation messages */}
<Messages
messages={metabot.messages}
errorMessages={metabot.errorMessages}
onRetryMessage={handleRetryMessage}
isDoingScience={metabot.isDoingScience}
showFeedbackButtons
/>
{/* loading */}
{metabot.isDoingScience && (
<MetabotThinking toolCalls={metabot.activeToolCalls} />
)}
{/* filler - height gets set via ref mutation */}
<div ref={fillerRef} data-testid="metabot-message-filler" />
{/* long convo warning */}
{metabot.isLongConversation && <MetabotResetLongChatButton />}
</Box>
)}
</Box>
<Box className={Styles.textInputContainer}>
<Paper
className={cx(
Styles.inputContainer,
metabot.isDoingScience && Styles.inputContainerLoading,
)}
>
<MetabotChatEditor
ref={metabot.promptInputRef}
value={metabot.prompt}
autoFocus
isResponding={metabot.isDoingScience}
placeholder={t`Tell me to do something, or ask a question`}
onChange={handleEditorChange}
onSubmit={handleEditorSubmit}
onStop={metabot.cancelRequest}
suggestionModels={config.suggestionModels}
/>
</Paper>
</Box>
</Box>
);
};

View File

@@ -14,7 +14,7 @@ export const useMetabotReactions = () => {
) as ReturnType<typeof getMetabotReactionsState>;
const setNavigateToPath = useCallback(
async (navigateToPath: string) => {
async (navigateToPath: string | null) => {
dispatch(setNavigateToPathAction(navigateToPath));
},
[dispatch],

View File

@@ -12,6 +12,7 @@ import { Metabot } from "./components/Metabot";
import { MetabotAdminPage } from "./components/MetabotAdmin/MetabotAdminPage";
import { MetabotTrialPage } from "./components/MetabotAdmin/MetabotTrialPage";
import { MetabotAppBarButton } from "./components/MetabotAppBarButton";
import { MetabotChat } from "./components/MetabotChat";
import { MetabotDataStudioButton } from "./components/MetabotDataStudioButton";
import { getMetabotQuickLinks } from "./components/MetabotQuickLinks";
import { MetabotToggleButton } from "./components/MetabotToggleButton";
@@ -38,6 +39,7 @@ export function initializePlugin() {
if (hasPremiumFeature("metabot_v3")) {
PLUGIN_METABOT.isEnabled = () => true;
PLUGIN_METABOT.Metabot = Metabot;
PLUGIN_METABOT.MetabotChat = MetabotChat;
PLUGIN_METABOT.getMetabotRoutes = getMetabotQuickLinks;
PLUGIN_METABOT.getAdminPaths = () => [

View File

@@ -267,13 +267,45 @@ export const metabot = createSlice({
state.reactions.suggestedTransforms = [];
state.experimental.metabotReqIdOverride = undefined;
},
setConversationSnapshot: (
state,
action: PayloadAction<{
messages: MetabotChatMessage[];
history: MetabotHistory;
state: any;
reactions: MetabotReactionsState;
activeToolCalls: MetabotToolCall[];
errorMessages: MetabotErrorMessage[];
conversationId: string;
}>,
) => {
const {
messages,
history,
state: snapshotState,
reactions,
activeToolCalls,
errorMessages,
conversationId,
} = action.payload;
state.messages = messages ?? [];
state.history = history ?? [];
state.state = snapshotState ?? {};
state.reactions =
reactions ?? ({ navigateToPath: null, suggestedTransforms: [] } as any);
state.activeToolCalls = activeToolCalls ?? [];
state.errorMessages = errorMessages ?? [];
state.conversationId = conversationId ?? uuid();
state.isProcessing = false;
},
resetConversationId: (state) => {
state.conversationId = uuid();
},
setIsProcessing: (state, action: PayloadAction<boolean>) => {
state.isProcessing = action.payload;
},
setNavigateToPath: (state, action: PayloadAction<string>) => {
setNavigateToPath: (state, action: PayloadAction<string | null>) => {
state.reactions.navigateToPath = action.payload;
},
setVisible: (state, action: PayloadAction<boolean>) => {
@@ -379,3 +411,4 @@ export const metabot = createSlice({
});
export const metabotReducer = metabot.reducer;
export const metabotActions = metabot.actions;

View File

@@ -144,7 +144,8 @@ export const getProfile = createSelector(
return profileOverride;
}
return location.pathname.startsWith(Urls.transformList())
return location.pathname.startsWith(Urls.transformList()) ||
location.pathname.startsWith(Urls.dataStudioWorkspaceList())
? "transforms_codegen"
: undefined;
},

View File

@@ -162,6 +162,14 @@ export function useEntitySuggestions({
const isInModelSelectionMode =
!selectedSearchModel && hasSearchModels && hasMatchingFilteredModels;
// If searchModels are provided (e.g., only transforms), skip recents and always search.
const shouldFetchRecents =
enabled &&
query.length === 0 &&
!isInModelSelectionMode &&
!selectedSearchModel &&
!searchModels;
const {
menuItems: entityMenuItems,
isLoading,
@@ -172,11 +180,7 @@ export function useEntitySuggestions({
onSelectSearchResult: handleSearchResultSelect,
onSelectUser: handleUserSelect,
enabled: enabled && !isInModelSelectionMode,
shouldFetchRecents:
enabled &&
query.length === 0 &&
!isInModelSelectionMode &&
!selectedSearchModel,
shouldFetchRecents,
searchModels: effectiveSearchModels,
});

View File

@@ -30,6 +30,8 @@ import {
type PythonDataPickerProps = {
database?: DatabaseId;
disabled?: boolean;
canChangeDatabase?: boolean;
tables: PythonTransformTableAliases;
onChange: (
database: number,
@@ -40,8 +42,10 @@ type PythonDataPickerProps = {
export function PythonDataPicker({
database,
disabled,
tables,
onChange,
canChangeDatabase = true,
}: PythonDataPickerProps) {
const [tableSelections, setTableSelections] = useState<TableSelection[]>(
getInitialTableSelections(tables),
@@ -150,22 +154,25 @@ export function PythonDataPicker({
className={S.dataPicker}
data-testid="python-data-picker"
>
<Box>
<Text fw="bold">{t`Source database`}</Text>
<Text size="sm" c="text-light" mb="sm">
{t`Select the database that contains your source data.`}
</Text>
{canChangeDatabase && (
<Box>
<Text fw="bold">{t`Source database`}</Text>
<Text size="sm" c="text-light" mb="sm">
{t`Select the database that contains your source data.`}
</Text>
<DatabaseDataSelector
className={S.databaseSelector}
selectedDatabaseId={database}
setDatabaseFn={handleDatabaseChange}
databases={databases?.data ?? []}
databaseIsDisabled={(database: Database) =>
!hasFeature(database, "transforms/python")
}
/>
</Box>
<DatabaseDataSelector
className={S.databaseSelector}
readOnly={disabled}
selectedDatabaseId={database}
setDatabaseFn={handleDatabaseChange}
databases={databases?.data ?? []}
databaseIsDisabled={(database: Database) =>
!hasFeature(database, "transforms/python")
}
/>
</Box>
)}
{database && (
<Box>
<Text fw="bold">{t`Pick tables and alias them`}</Text>
@@ -185,12 +192,12 @@ export function PythonDataPicker({
handleSelectionChange(index, selection)
}
onRemove={() => handleRemoveTable(index)}
disabled={isLoadingTables}
disabled={disabled || isLoadingTables}
/>
))}
<AddTableButton
onClick={handleAddTable}
disabled={availableTables.length === 0}
disabled={disabled || availableTables.length === 0}
/>
</Stack>
</Box>

View File

@@ -98,6 +98,7 @@ export function TableSelector({
<Tooltip label={t`Remove this table`}>
<ActionIcon
disabled={disabled}
onClick={onRemove}
pr="sm"
aria-label={t`Remove this table`}

View File

@@ -24,6 +24,7 @@ import { ResizableBoxHandle } from "./ResizableBoxHandle";
import { hasImport, insertImport, removeImport } from "./utils";
type PythonEditorBodyProps = {
disabled?: boolean;
source: string;
proposedSource?: string;
isRunnable: boolean;
@@ -41,6 +42,7 @@ type PythonEditorBodyProps = {
const EDITOR_HEIGHT = 400;
export function PythonEditorBody({
disabled,
source,
proposedSource,
onChange,
@@ -57,6 +59,7 @@ export function PythonEditorBody({
<MaybeResizableBox resizable={withDebugger}>
<Flex h="100%" align="end" bg="bg-light" pos="relative">
<PythonEditor
readOnly={disabled}
value={source}
proposedValue={proposedSource}
onChange={onChange}
@@ -106,7 +109,11 @@ export function PythonEditorBody({
</Box>
)}
</Stack>
<SharedLibraryActions source={source} onChange={onChange} />
<SharedLibraryActions
disabled={disabled}
source={source}
onChange={onChange}
/>
</Flex>
</MaybeResizableBox>
);
@@ -140,24 +147,32 @@ function MaybeResizableBox({
}
function SharedLibraryActions({
disabled,
source,
onChange,
}: {
disabled?: boolean;
source: string;
onChange: (source: string) => void;
}) {
return (
<Group className={S.libraryActions} p="md" gap="sm">
<SharedLibraryImportButton source={source} onChange={onChange} />
<SharedLibraryImportButton
disabled={disabled}
source={source}
onChange={onChange}
/>
<SharedLibraryEditLink />
</Group>
);
}
function SharedLibraryImportButton({
disabled,
source,
onChange,
}: {
disabled?: boolean;
source: string;
onChange: (source: string) => void;
}) {
@@ -173,7 +188,11 @@ function SharedLibraryImportButton({
return (
<Tooltip label={label}>
<ActionIcon aria-label={label} onClick={handleToggleSharedLib}>
<ActionIcon
aria-label={label}
disabled={disabled}
onClick={handleToggleSharedLib}
>
<Icon name="reference" c="text-dark" />
</ActionIcon>
</Tooltip>

View File

@@ -2,7 +2,11 @@ import { useHotkeys } from "@mantine/hooks";
import type { PythonTransformEditorProps } from "metabase/plugins";
import { Flex, Stack } from "metabase/ui";
import type { PythonTransformTableAliases, Table } from "metabase-types/api";
import type {
DatabaseId,
PythonTransformTableAliases,
Table,
} from "metabase-types/api";
import { PythonDataPicker } from "./PythonDataPicker";
import { PythonEditorBody } from "./PythonEditorBody";
@@ -17,6 +21,7 @@ import {
export function PythonTransformEditor({
source,
proposedSource,
uiOptions,
isDirty,
onChangeSource,
onAcceptProposed,
@@ -34,7 +39,7 @@ export function PythonTransformEditor({
};
const handleDataChange = (
database: number,
database: DatabaseId,
sourceTables: PythonTransformTableAliases,
tableInfo: Table[],
) => {
@@ -71,12 +76,15 @@ export function PythonTransformEditor({
return (
<Flex h="100%" w="100%">
<PythonDataPicker
disabled={uiOptions?.readOnly}
database={source["source-database"]}
canChangeDatabase={uiOptions?.canChangeDatabase}
tables={source["source-tables"]}
onChange={handleDataChange}
/>
<Stack w="100%" h="100%" gap={0}>
<PythonEditorBody
disabled={uiOptions?.readOnly}
isRunnable={isPythonTransformSource(source)}
isRunning={isRunning}
isDirty={isDirty}

View File

@@ -3,6 +3,7 @@ import { useMemo } from "react";
import { useSelector } from "metabase/lib/redux";
import {
QueryEditor,
type QueryEditorUiOptions,
type QueryEditorUiState,
} from "metabase/querying/editor/components/QueryEditor";
import { getMetadata } from "metabase/selectors/metadata";
@@ -11,9 +12,10 @@ import type { Database, QueryTransformSource } from "metabase-types/api";
import { getEditorOptions } from "./utils";
type TransformEditorProps = {
export type TransformEditorProps = {
source: QueryTransformSource;
uiState: QueryEditorUiState;
uiOptions?: QueryEditorUiOptions;
proposedSource: QueryTransformSource | undefined;
databases: Database[];
onChangeSource: (source: QueryTransformSource) => void;
@@ -27,6 +29,7 @@ export function TransformEditor({
proposedSource,
databases,
uiState,
uiOptions,
onChangeSource,
onChangeUiState,
onAcceptProposed,
@@ -44,7 +47,10 @@ export function TransformEditor({
: undefined,
[proposedSource, metadata],
);
const uiOptions = useMemo(() => getEditorOptions(databases), [databases]);
const mergedUiOptions = useMemo(
() => ({ ...getEditorOptions(databases), ...uiOptions }),
[databases, uiOptions],
);
const handleQueryChange = (query: Lib.Query) => {
const newSource: QueryTransformSource = {
@@ -60,7 +66,7 @@ export function TransformEditor({
<QueryEditor
query={query}
uiState={uiState}
uiOptions={uiOptions}
uiOptions={mergedUiOptions}
proposedQuery={proposedQuery}
onChangeQuery={handleQueryChange}
onChangeUiState={onChangeUiState}

View File

@@ -0,0 +1,233 @@
import dayjs from "dayjs";
import { useMemo, useState } from "react";
import { push } from "react-router-redux";
import { t } from "ttag";
import { useListDatabasesQuery } from "metabase/api";
import { useDispatch } from "metabase/lib/redux";
import { useMetadataToasts } from "metabase/metadata/hooks";
import {
Box,
Button,
Flex,
Icon,
Loader,
Menu,
ScrollArea,
Stack,
Text,
} from "metabase/ui";
import {
useCreateWorkspaceMutation,
useGetTransformDownstreamMappingQuery,
useGetWorkspacesQuery,
useUpdateWorkspaceContentsMutation,
} from "metabase-enterprise/api";
import { CreateWorkspaceModal } from "metabase-enterprise/data-studio/workspaces/components/CreateWorkspaceModal/CreateWorkspaceModal";
import type { Transform, Workspace } from "metabase-types/api";
type EditTransformMenuProps = {
transform: Transform;
};
export function EditTransformMenu({ transform }: EditTransformMenuProps) {
const dispatch = useDispatch();
const { sendErrorToast } = useMetadataToasts();
const sourceDatabaseId = getTransformDatabaseId(transform);
const { data: workspacesData, isLoading: isLoadingWorkspaces } =
useGetWorkspacesQuery();
const [updateWorkspaceContents, { isLoading: isUpdatingWorkspace }] =
useUpdateWorkspaceContentsMutation();
const [createWorkspace, { isLoading: isCreatingWorkspace }] =
useCreateWorkspaceMutation();
const { data: downstreamMapping } = useGetTransformDownstreamMappingQuery(
transform.id,
);
const [isCreateModalOpen, setIsCreateModalOpen] = useState(false);
const [addedWorkspaceIds, setAddedWorkspaceIds] = useState<Set<number>>(
() => new Set(),
);
const { data: databasesData } = useListDatabasesQuery({
include_analytics: true,
});
const workspaces = useMemo(
() => workspacesData?.items ?? [],
[workspacesData],
);
const existingWorkspaceIds = useMemo(() => {
const ids = new Set<number>();
downstreamMapping?.transforms?.forEach((item) => {
if (item.workspace?.id != null) {
ids.add(item.workspace.id);
}
});
addedWorkspaceIds.forEach((id) => ids.add(id));
return ids;
}, [downstreamMapping, addedWorkspaceIds]);
const matchingWorkspaces = useMemo(
() =>
workspaces
.filter((workspace) => !existingWorkspaceIds.has(workspace.id))
.filter((workspace) =>
sourceDatabaseId == null
? true
: workspace.database_id === sourceDatabaseId,
),
[workspaces, sourceDatabaseId, existingWorkspaceIds],
);
const isBusy = isUpdatingWorkspace || isCreatingWorkspace;
const emptyMessage =
workspaces.length === 0 || sourceDatabaseId == null
? t`No workspaces yet`
: t`No workspaces for this database yet`;
const databaseOptions = useMemo(
() =>
(databasesData?.data ?? []).map((db) => ({
value: String(db.id),
label: db.name,
})),
[databasesData],
);
const handleOpenCreateModal = () => {
setIsCreateModalOpen(true);
};
const handleCloseCreateModal = () => {
setIsCreateModalOpen(false);
};
const handleWorkspaceSelect = async (workspace: Workspace) => {
try {
await updateWorkspaceContents({
id: workspace.id,
add: { transforms: [transform.id] },
}).unwrap();
setAddedWorkspaceIds((prev) => new Set(prev).add(workspace.id));
} catch (error) {
sendErrorToast(t`Failed to add transform to the workspace`);
}
};
const handleCreateWorkspace = async ({
name,
databaseId,
}: {
name: string;
databaseId: string;
}) => {
try {
const workspace = await createWorkspace({
name: name.trim() || t`New workspace`,
database_id: Number(databaseId),
upstream: { transforms: [transform.id] },
}).unwrap();
setAddedWorkspaceIds((prev) => new Set(prev).add(workspace.id));
handleCloseCreateModal();
dispatch(push(`/data-studio/workspaces/${workspace.id}`));
} catch (error) {
sendErrorToast(t`Failed to create workspace`);
}
};
return (
<Menu position="bottom-end" width={280}>
<Menu.Target>
<Button
size="sm"
variant="filled"
leftSection={<Icon name="pencil" />}
rightSection={<Icon name="chevrondown" />}
loading={isBusy}
>
{t`Edit transform`}
</Button>
</Menu.Target>
<Menu.Dropdown>
<Menu.Label>{t`Add to workspace`}</Menu.Label>
<Menu.Item
disabled={isBusy}
leftSection={<Icon name="add" />}
onClick={handleOpenCreateModal}
>
<Stack gap={0} align="flex-start">
<Text fw={600}>{t`New workspace`}</Text>
<Text size="sm" c="text-light">
{t`Create a new workspace`}
</Text>
</Stack>
</Menu.Item>
<Menu.Divider />
{isLoadingWorkspaces ? (
<Flex justify="center" align="center" py="md">
<Loader size="sm" />
</Flex>
) : matchingWorkspaces.length === 0 ? (
<Box px="md" py="sm">
<Text size="sm" c="text-light">
{emptyMessage}
</Text>
</Box>
) : (
<ScrollArea.Autosize mah={320} type="scroll">
<Stack gap={0}>
{matchingWorkspaces.map((workspace) => (
<Menu.Item
key={workspace.id}
leftSection={<Icon name="sparkles" />}
onClick={() => handleWorkspaceSelect(workspace)}
disabled={isBusy}
>
<Stack gap={2} align="flex-start">
<Text fw={600}>{workspace.name}</Text>
<Text size="sm" c="text-light">
{formatWorkspaceDate(workspace.created_at)}
</Text>
</Stack>
</Menu.Item>
))}
</Stack>
</ScrollArea.Autosize>
)}
</Menu.Dropdown>
<CreateWorkspaceModal
opened={isCreateModalOpen}
onClose={handleCloseCreateModal}
onSubmit={handleCreateWorkspace}
databaseOptions={databaseOptions}
isSubmitting={isBusy}
defaultDatabaseId={
sourceDatabaseId != null ? String(sourceDatabaseId) : null
}
/>
</Menu>
);
}
function getTransformDatabaseId(transform: Transform) {
if (
transform.source_type === "python" &&
"source-database" in transform.source
) {
return transform.source["source-database"];
}
if ("query" in transform.source) {
return transform.source.query.database;
}
return undefined;
}
function formatWorkspaceDate(createdAt: string) {
return dayjs(createdAt).format("DD/MM/YYYY");
}

View File

@@ -0,0 +1 @@
export * from "./EditTransformMenu";

View File

@@ -1,16 +1,19 @@
import { useRegisterMetabotContextProvider } from "metabase/metabot";
import type {
PythonTransformSourceDraft,
DraftTransformSource,
SuggestedTransform,
Transform,
TransformSource,
} from "metabase-types/api";
export const useRegisterMetabotTransformContext = (
transform: Transform | SuggestedTransform | undefined,
source: TransformSource | PythonTransformSourceDraft,
source?: DraftTransformSource,
) => {
useRegisterMetabotContextProvider(async () => {
if (!source) {
return {};
}
return {
user_is_viewing: [{ type: "transform", ...(transform || {}), source }],
};

View File

@@ -32,30 +32,40 @@ import { trackTransformCreated } from "../../../analytics";
import { SchemaFormSelect } from "./../../../components/SchemaFormSelect";
function getValidationSchema() {
return Yup.object({
name: Yup.string().required(Errors.required),
targetName: Yup.string().required(Errors.required),
targetSchema: Yup.string().nullable().defined(),
incremental: Yup.boolean().required(),
// For native queries, use checkpointFilter (plain string)
checkpointFilter: Yup.string().nullable(),
// For MBQL/Python queries, use checkpointFilterUniqueKey (prefixed format)
checkpointFilterUniqueKey: Yup.string().nullable(),
sourceStrategy: Yup.mixed<"checkpoint">().oneOf(["checkpoint"]).required(),
targetStrategy: Yup.mixed<"append">().oneOf(["append"]).required(),
});
}
const DEFAULT_VALIDATION_SCHEMA = Yup.object({
name: Yup.string().required(Errors.required),
targetName: Yup.string().required(Errors.required),
targetSchema: Yup.string().nullable().defined(),
incremental: Yup.boolean().required(),
// For native queries, use checkpointFilter (plain string)
checkpointFilter: Yup.string().nullable(),
// For MBQL/Python queries, use checkpointFilterUniqueKey (prefixed format)
checkpointFilterUniqueKey: Yup.string().nullable(),
sourceStrategy: Yup.mixed<"checkpoint">().oneOf(["checkpoint"]).required(),
targetStrategy: Yup.mixed<"append">().oneOf(["append"]).required(),
});
export type NewTransformValues = Yup.InferType<
ReturnType<typeof getValidationSchema>
>;
export type NewTransformValues = Yup.InferType<typeof DEFAULT_VALIDATION_SCHEMA>;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
export type ValidationSchemaExtension = Record<string, Yup.AnySchema>;
function getValidationSchema(extension?: ValidationSchemaExtension) {
if (!extension) {
return DEFAULT_VALIDATION_SCHEMA;
}
return DEFAULT_VALIDATION_SCHEMA.shape(extension);
}
type CreateTransformModalProps = {
source: TransformSource;
defaultValues: Partial<NewTransformValues>;
onCreate: (transform: Transform) => void;
onCreate?: (transform: Transform) => void;
onClose: () => void;
schemas?: string[] | null;
showIncrementalSettings?: boolean;
validationSchemaExtension?: ValidationSchemaExtension;
handleSubmit?: (values: NewTransformValues) => Promise<Transform>;
};
export function CreateTransformModal({
@@ -63,6 +73,10 @@ export function CreateTransformModal({
defaultValues,
onCreate,
onClose,
schemas,
showIncrementalSettings = true,
validationSchemaExtension,
handleSubmit,
}: CreateTransformModalProps) {
return (
<Modal title={t`Save your transform`} opened padding="xl" onClose={onClose}>
@@ -71,6 +85,10 @@ export function CreateTransformModal({
defaultValues={defaultValues}
onCreate={onCreate}
onClose={onClose}
schemas={schemas}
showIncrementalSettings={showIncrementalSettings}
validationSchemaExtension={validationSchemaExtension}
handleSubmit={handleSubmit}
/>
</Modal>
);
@@ -79,8 +97,12 @@ export function CreateTransformModal({
type CreateTransformFormProps = {
source: TransformSource;
defaultValues: Partial<NewTransformValues>;
onCreate: (transform: Transform) => void;
onCreate?: (transform: Transform) => void;
onClose: () => void;
schemas?: string[] | null;
showIncrementalSettings?: boolean;
validationSchemaExtension?: ValidationSchemaExtension;
handleSubmit?: (values: NewTransformValues) => Promise<Transform>;
};
function CreateTransformForm({
@@ -88,11 +110,18 @@ function CreateTransformForm({
defaultValues,
onCreate,
onClose,
schemas: schemasProp,
showIncrementalSettings = true,
validationSchemaExtension,
handleSubmit,
}: CreateTransformFormProps) {
const [sendToast] = useToast();
const databaseId =
source.type === "query" ? source.query.database : source["source-database"];
const shouldFetchSchemas = schemasProp === undefined;
const showSchemaField = schemasProp !== null;
const {
data: database,
isLoading: isDatabaseLoading,
@@ -100,15 +129,22 @@ function CreateTransformForm({
} = useGetDatabaseQuery(databaseId ? { id: databaseId } : skipToken);
const {
data: schemas = [],
data: fetchedSchemas = [],
isLoading: isSchemasLoading,
error: schemasError,
} = useListDatabaseSchemasQuery(
databaseId ? { id: databaseId, include_hidden: true } : skipToken,
shouldFetchSchemas && databaseId
? { id: databaseId, include_hidden: true }
: skipToken,
);
const isLoading = isDatabaseLoading || isSchemasLoading;
const error = databaseError ?? schemasError;
const schemas = useMemo(
() => schemasProp ?? fetchedSchemas ?? [],
[schemasProp, fetchedSchemas],
);
const isLoading =
isDatabaseLoading || (shouldFetchSchemas && isSchemasLoading);
const error = databaseError ?? (shouldFetchSchemas ? schemasError : null);
const [createTransform] = useCreateTransformMutation();
const supportsSchemas = database && hasFeature(database, "schemas");
@@ -118,13 +154,16 @@ function CreateTransformForm({
[schemas, defaultValues],
);
const validationSchema = useMemo(() => getValidationSchema(), []);
const validationSchema = useMemo(
() => getValidationSchema(validationSchemaExtension),
[validationSchemaExtension],
);
if (isLoading || error != null) {
return <LoadingAndErrorWrapper loading={isLoading} error={error} />;
}
const handleSubmit = async (values: NewTransformValues) => {
const defaultHandleSubmit = async (values: NewTransformValues) => {
if (!databaseId) {
throw new Error("Database ID is required");
}
@@ -132,7 +171,7 @@ function CreateTransformForm({
try {
const transform = await createTransform(request).unwrap();
trackTransformCreated({ transformId: transform.id });
onCreate(transform);
onCreate?.(transform);
} catch (error) {
sendToast({
message: getErrorMessage(error, t`Failed to create transform`),
@@ -145,7 +184,7 @@ function CreateTransformForm({
<FormProvider
initialValues={initialValues}
validationSchema={validationSchema}
onSubmit={handleSubmit}
onSubmit={handleSubmit || defaultHandleSubmit}
>
<Form>
<Stack gap="lg" mt="sm">
@@ -155,7 +194,7 @@ function CreateTransformForm({
placeholder={t`My Great Transform`}
data-autofocus
/>
{supportsSchemas && (
{showSchemaField && supportsSchemas && (
<SchemaFormSelect
name="targetSchema"
label={t`Schema`}
@@ -167,7 +206,9 @@ function CreateTransformForm({
label={t`Table name`}
placeholder={t`descriptive_name`}
/>
<IncrementalTransformSettings source={source} />
{showIncrementalSettings && (
<IncrementalTransformSettings source={source} />
)}
<Group>
<Box flex={1}>
<FormErrorMessage />
@@ -182,7 +223,7 @@ function CreateTransformForm({
}
function getInitialValues(
schemas: string[],
schemas: string[] | null,
defaultValues: Partial<NewTransformValues>,
): NewTransformValues {
return {

Some files were not shown because too many files have changed in this diff Show More