Compare commits

...

432 Commits

Author SHA1 Message Date
Ngoc Khuat
593fff9d13 use provisional ready where possible 2025-12-26 18:47:48 +07:00
Ngoc Khuat
82e9d416b2 add a flag for create-ws-ready 2025-12-26 18:27:24 +07:00
Ngoc Khuat
dcd0317d9f add a way to skip workspace isolation init in tests 2025-12-26 18:16:57 +07:00
Ngoc Khuat
1ffecf3d9e move driver implementation of isolation to its driver (#67530) 2025-12-26 17:10:33 +07:00
Uladzimir Havenchyk
c4a847322f open tab automatically 2025-12-25 22:02:08 +03:00
Uladzimir Havenchyk
a7a38fd4ca smaller loader 2025-12-25 21:55:15 +03:00
Uladzimir Havenchyk
486bc64112 show new transforms during merge 2025-12-25 21:29:53 +03:00
Stas Gavrylov
3101e32edd Fix e2e test 2025-12-25 17:36:00 +01:00
Uladzimir Havenchyk
b51c20c9b0 run tranform before opening a preview 2025-12-25 13:27:56 +03:00
Uladzimir Havenchyk
8cdf482f1f add problems 2025-12-24 15:29:46 +03:00
Oleksandr Solovyov
5d173e849d use transform.target_db_id to filter transforms directly in db 2025-12-24 12:22:55 +02:00
Oleksandr Solovyov
aabd4344fb merge master 2025-12-24 10:53:44 +02:00
Uladzimir Havenchyk
f94822749b fix last run 2025-12-23 23:50:15 +03:00
Uladzimir Havenchyk
d3a30cc4ab add ability to preview query results in a new tab 2025-12-23 23:35:26 +03:00
Ngoc Khuat
c8958bea54 mark drivers support for workspaces and use it in tests (#67447) 2025-12-24 00:04:34 +07:00
lbrdnk
dd579af20c Ws 22 drivers CI (#67377)
* Adjust sql jdbc tests

* sql jdbc support should be false

* Fix filter-transforms-combined-test

* Fix different-swap-details-get-separate-pools-test for databricks

* Run ws tests only  for drivers that support it

* Adjust tables-endpoint-test to work with ch

* Remove redundant supports call

* Fix filter-transforms-by-type-test

* Fix filter-transforms-combined-test
2025-12-23 17:17:38 +01:00
Stas Gavrylov
0d57823e39 Update 'Create Workspace' flow according to latest UX requirements 2025-12-23 05:23:15 -08:00
Stas Gavrylov
d0b96d16ed Remove hardcoded name for new workspace creation 2025-12-23 05:16:01 -08:00
Stas Gavrylov
3bea2940a6 Update redirect URL 2025-12-23 05:16:01 -08:00
Stas Gavrylov
7b0f4cfb9e wip 2025-12-23 05:16:01 -08:00
Stas Gavrylov
925d8d76ee Remove 'Create Workspace' modal 2025-12-23 05:16:01 -08:00
Oleksandr Solovyov
f1a8e6a623 sync db selection logic 2025-12-23 15:06:03 +02:00
Oleksandr Solovyov
8a9b4fbd05 do not choose sample database 2025-12-23 14:57:47 +02:00
Oleksandr Solovyov
1deee13b0c less chances 13371337 will be the db of workspace 2025-12-23 14:28:10 +02:00
Oleksandr Solovyov
b89c4b6431 make tests work again 2025-12-23 14:25:29 +02:00
Oleksandr Solovyov
a0f9335ec0 ability to filter external transforms by different db id 2025-12-23 13:49:41 +02:00
Oleksandr Solovyov
8d717aded4 store error message for workspace runs (#67366) 2025-12-23 13:25:03 +02:00
lbrdnk
2c7848edb7 require 2025-12-23 12:24:45 +01:00
Oleksandr Solovyov
edeac448d2 cutesy name generation for workspaces 2025-12-23 12:31:13 +02:00
Oleksandr Solovyov
290b486e1a Correctly set transform.creator_id on workspace merge (#67363)
Closes BOT-691
2025-12-23 11:48:28 +02:00
Ngoc Khuat
67345120b6 grant ownership to fix drop owned 2025-12-23 15:58:11 +07:00
Uladzimir Havenchyk
24ee602364 useless message 2025-12-22 20:33:10 +03:00
Stas Gavrylov
7bdffa72a6 Fix Edit Transform in checked out workspaces 2025-12-22 08:42:16 -08:00
Ngoc Khuat
4b228cd956 init is idempotent 2025-12-22 22:02:17 +07:00
Ngoc Khuat
7da7ca3839 api test fixes 2025-12-22 21:30:21 +07:00
Uladzimir Havenchyk
9e3af41bda remove run all transforms button 2025-12-22 17:14:19 +03:00
Uladzimir Havenchyk
b2ca4d492a move counter back and adjust font size 2025-12-22 16:32:54 +03:00
Ngoc Khuat
c0636ba772 Workspaces archive shouldn't fail if failed to destroy isolation (#67361) 2025-12-22 20:01:51 +07:00
Uladzimir Havenchyk
506fcd0970 show a message that target is isolated and update types 2025-12-22 15:39:45 +03:00
lbrdnk
4a8d9e2085 Ws 22 CI fixes (#67269)
* Throw on missing ref_id

* Add ref_id to workspace transform insertion code

* comment

* Update with-temp-defaults-fns

* Fix mysql appdb tests

* Hack around cljfmt problems

* all-models-accounted-for

* Linter

* Comment

* resolution

* entity-id tests

* Address every-model-is-supported-test failure

* Address all-models-accounted-for-test failure

* Fallback to source for transforms.i/target-db-id :query
2025-12-22 13:13:01 +01:00
Stas Gavrylov
1188beea7d Update 'Edit transform' dropdown to list all matching workspaces 2025-12-22 03:15:23 -08:00
Stas Gavrylov
538318a38d Update Workspaces list to allow deleting only for archived workspaces 2025-12-22 02:28:04 -08:00
Oleksandr Solovyov
bd0387cd60 always store last time WorkspaceTransform has been triggered to run 2025-12-22 11:52:33 +02:00
Chris Truter
0471190c74 Use enclosed transform for list tables and re-mapping execution (#67284) 2025-12-19 18:26:33 +02:00
Ngoc Khuat
3407b668a3 Workspace Merge history (#67088) 2025-12-19 23:06:45 +07:00
lbrdnk
ec826befeb Attempt to fix deployment 2025-12-19 17:02:17 +01:00
lbrdnk
092bf6d679 Fix migrations 2025-12-19 15:58:46 +01:00
Uladzimir Havenchyk
16a258250c fix workspaces but break types 2025-12-19 17:00:22 +03:00
Oleksandr Solovyov
8c9e5886ab Allow workspace to be created without a database_id (#67260) 2025-12-19 15:59:21 +02:00
Uladzimir Havenchyk
d161b663ae add diff view for a merge modal 2025-12-19 16:52:38 +03:00
Oleksandr Solovyov
95a86c4acc Merge branch 'workspaces-2025-11' of https://github.com/metabase/metabase into workspaces-2025-11 2025-12-19 15:42:38 +02:00
Oleksandr Solovyov
51b02e259b merge master 2025-12-19 15:36:59 +02:00
lbrdnk
8ef14ebcf1 [WIP] WS 22 broken ci 2 (#67084)
* Fix type-filter-xf-test

* Fix database-id-filter-xf-test

* Fix blank-schema-identifier-test

* Fix isolation-e2e-test

* Fix drop-entity-ids-test

* Fix python transform target-db-id

* Fix get-job-transforms-test

* Fix table-with-db-and-fields-hydration-test

* Fix openapi-all-routes (add description to endpoint)

* Fix all-models-accounted-for-test

* Fix module-uses-should-be-sorted-test

* Address failure of modules-config-up-to-date-test

* Dependency ordering in deps.edn

* Address failure of driver-multimethods-in-changelog-test

* Order modules config

* Adjust model->namespace, fix all-models-are-accounted-for-test

* Adjustments for comprehensive-entity-id-test to pass

* Adjustments to make every-model-is-supported-test pass

* Adjust expectations in sync-table-test -- workspace_id

* Adjustment to make search-model-cascade-test green

* Fix baseline-comparison-test

* Adjust search-model-cascade-test

* Adjustments sync-table-test

* Licenses

* Make the fks-are-indexed-test pass
2025-12-19 14:09:08 +01:00
Uladzimir Havenchyk
d7e0b28be5 preselect last used db or first db 2025-12-19 15:47:58 +03:00
Oleksandr Solovyov
75231939b5 bin/mage! 2025-12-19 14:44:51 +02:00
Chris Truter
5f91a003bb Add description to problem response 2025-12-19 14:43:37 +02:00
Chris Truter
278fc0c46c Return block_merge (underscored) via problem api 2025-12-19 14:23:30 +02:00
Chris Truter
c7086253b6 Fix test expected status code 2025-12-19 13:17:31 +02:00
Chris Truter
c83ea462d2 DRY up getting graph, so we can cache it 2025-12-19 13:17:16 +02:00
Ngoc Khuat
306d80de92 Fix destroy schema for redshift (#67156) 2025-12-19 16:27:09 +07:00
Ngoc Khuat
a345fb257d Move description infer to BE (#66876) 2025-12-19 15:26:12 +07:00
Stas Gavrylov
2f65a02744 Make 'Results' link open results table tab 2025-12-18 07:54:40 -08:00
Uladzimir Havenchyk
b4a9ece8d8 use correct db id 2025-12-18 18:32:50 +03:00
Uladzimir Havenchyk
75b3a5b509 cleanup 2025-12-18 18:20:43 +03:00
Uladzimir Havenchyk
18db799cd0 automatically pick up db id from a transform 2025-12-18 18:12:32 +03:00
Stas Gavrylov
ea1b32a7c1 Disable unsupported DWHs in Create Workspace modal 2025-12-18 05:32:37 -08:00
Chris Truter
fe8f2dbe37 Fix for a hallucination. Add a test later... 2025-12-18 14:32:06 +02:00
Chris Truter
1c9f3d4fe1 Some workspace validation (#67120)
Closes BOT-724
References BOT-584

Outlined the full set of issues I could think of validating, and implemented the subset of cases we care about for our weekly demo.
2025-12-18 14:17:15 +02:00
lbrdnk
667563b11b fix migration 2025-12-18 11:46:30 +01:00
Chris Truter
d697be5665 Remap python sources as well (#67113) 2025-12-17 20:30:31 +02:00
Chris Truter
c1030a16f7 Remove workspace_id from table, field, and card (#67111) 2025-12-17 20:08:36 +02:00
Chris Truter
b5e5bd328a Remove transform.workspace_id (#67105) 2025-12-17 20:01:01 +02:00
Chris Truter
777f3a47da Misc fix-ups
- De-weirdify function that was morphed to accept #p
- Comment and warning
- Update execution test to properly track scope
- Remove unused imports
2025-12-17 19:45:22 +02:00
Stas Gavrylov
9042477db1 Fix unsaved transforms display 2025-12-17 08:17:37 -08:00
Stas Gavrylov
52f602f0c2 Add MergeWorkspaceModal to get commit message from user 2025-12-17 08:05:52 -08:00
lbrdnk
b1f42b9497 Temporarily run all driver tests 2025-12-17 16:48:30 +01:00
Chris Truter
704bbd280e Graph aware execution and transitive renaming (#67090) 2025-12-17 17:27:50 +02:00
Stas Gavrylov
5e370f937c Add listTag for externam transforms 2025-12-17 06:34:36 -08:00
lbrdnk
662fc910b6 fix broken require 2025-12-17 14:44:03 +01:00
Chris Truter
4448f190fa Finish sentence 2025-12-17 15:41:16 +02:00
Ngoc Khuat
00f58316a7 Workspaces: Fix failed to archive workspace with contents (#67076) 2025-12-17 20:33:34 +07:00
lbrdnk
d956fa61d5 ws dag eastwood 2025-12-17 14:17:06 +01:00
lbrdnk
ca6372be38 ws bigquery eastwood 2025-12-17 14:17:06 +01:00
Uladzimir Havenchyk
5e226fee6e handle unsupported tranforms 2025-12-17 15:39:09 +03:00
Chris Truter
89e92ee71f Remove exports from workspaces.core 2025-12-17 14:25:43 +02:00
Chris Truter
a111938611 Whoops, I made a hashp of it 2025-12-17 14:19:32 +02:00
Chris Truter
fc95d79d04 Add endpoint for listing supported databases 2025-12-17 13:59:20 +02:00
Chris Truter
d8d32ecb32 Add endpoint for checking workspace support 2025-12-17 13:26:19 +02:00
Stas Gavrylov
e4b1d46e78 Fix workspaces list redirect 2025-12-17 03:21:40 -08:00
Stas Gavrylov
f8a7f0f2d8 Redesign table results link 2025-12-17 03:18:22 -08:00
lbrdnk
ca9b70efbe h2 cros db migration fix attempt 2025-12-17 12:16:30 +01:00
lbrdnk
c1841b41cb kondo ignore 2025-12-17 11:56:56 +01:00
lbrdnk
070009d6aa Fix kondo 2025-12-17 11:52:53 +01:00
lbrdnk
d7078d30a1 Remove stale require 2025-12-17 11:39:14 +01:00
lbrdnk
a22f42d04e WS 22 Fix migration tests and other minor CI failures (#67005)
* Fix mysql rollback

* failOnError

* Drop broken rollback

* whitespace

* Fix bigquery typehints

* h2: only do admin check in prod

---------

Co-authored-by: Ngoc Khuat <qn.khuat@gmail.com>
2025-12-17 11:26:23 +01:00
Chris Truter
c50d16407f Delete some of the obsolete models from v2.1 (#67073) 2025-12-17 12:17:46 +02:00
Chris Truter
c46e93e25f Resurrect another bigger dag test, and fix test helper bugs 2025-12-17 12:03:30 +02:00
Chris Truter
8de115c0d6 Fix some fiendish edge cases around nils and SQL normalization (#67066) 2025-12-17 11:11:10 +02:00
Uladzimir Havenchyk
e18f364ab2 do not allow editing of archived workspaces 2025-12-17 11:28:20 +03:00
Uladzimir Havenchyk
73a024c794 redirect to an existing workspace from transforms page keeping related tranform opened 2025-12-17 09:42:16 +03:00
Chris Truter
58da27dc1a Leave comment about desired test 2025-12-17 06:44:25 +02:00
Chris Truter
dee89e8f31 Naive brute force workspace graph calculation (#66940) 2025-12-17 06:41:21 +02:00
Uladzimir Havenchyk
934d5dfd3f open correct transform after creating a new workspace 2025-12-16 22:54:39 +03:00
Stas Gavrylov
47b00082ff Add output and input table labels 2025-12-16 08:38:59 -08:00
lbrdnk
328b9a8988 Unblock FE: transforms return schema 2025-12-16 16:07:35 +01:00
Stas Gavrylov
d16d5c77eb Fix Save transform modal infinite loop 2025-12-16 06:54:48 -08:00
Uladzimir Havenchyk
5d62ce224d no checkout. wip 2025-12-16 17:41:09 +03:00
Uladzimir Havenchyk
4d8df063ae fix react warnings 2025-12-16 15:30:54 +03:00
Stas Gavrylov
e70210ede1 Update empty workspaces list wording 2025-12-16 04:29:35 -08:00
Stas Gavrylov
30baf69e2c Refactor Workspaces List page 2025-12-16 04:26:26 -08:00
Stas Gavrylov
9070d8592f Add output table link next to the Run button 2025-12-16 04:13:12 -08:00
Stas Gavrylov
57709669e1 Add sorting for workspaces list putting archived at the bottom 2025-12-16 03:25:51 -08:00
Chris Truter
17df352531 Fix graph mock API 2025-12-16 12:48:55 +02:00
Stas Gavrylov
ff524bc410 Add transform_id query param to initialize transform tab 2025-12-16 01:31:50 -08:00
Chris Truter
544cacc4a5 Whoops, missed the requires were different 2025-12-16 08:20:27 +02:00
Chris Truter
4603d73aa5 Delete obsolete rich comment 2025-12-16 08:19:30 +02:00
Chris Truter
16952f5d4c And yet more lints 2025-12-16 08:18:23 +02:00
Chris Truter
0cb4318906 More lint fixes 2025-12-16 08:15:42 +02:00
Chris Truter
836dbe993f More lint fixes 2025-12-16 08:07:03 +02:00
Chris Truter
3b5d5a5e6b Fix more lints, delete obsolete code and tests 2025-12-16 08:03:40 +02:00
Chris Truter
ace6b58330 Fixing lints and tests 2025-12-16 07:56:39 +02:00
Uladzimir Havenchyk
c2525e9fdb merge workspaces-master into workspaces-2025-11 2025-12-15 23:45:14 +03:00
Uladzimir Havenchyk
da7385a275 add workspaces back to data studio 2025-12-15 23:42:01 +03:00
Uladzimir Havenchyk
e09bba85ef make app working 2025-12-15 22:26:03 +03:00
lbrdnk
31549a9745 WS22 Adjustments to merge functionality (#66783)
* Extract create-transform!

* Add create-transform-body-schema

* Comment

* Use create-transform! while merging ws only

* require

* Stub for endpoint for merging single transform

* Handle merging of archived transform

* Update endpoint schema

* Merge many return schema

* Use transaction for error handling in merge many

* Test stub showing transactions alone are not sufficient

* merge-transform! ex-infos

* Update test stub

* Nested transaction mess things up

* Lazy seq was causing problems

* Avoid bubbling

* merge-transform! result schema

* Dummy exception to perform rollback visible in test

* glue for single merge

* Merge rollback wtf

* merge-workspace! savepoint

* Cleanup

* Savepoint cleanup

* Cleanup

* API result shape

* Comment out short circuit info

* Schema temp off

* Merge many endpoint update

* relax schema

* fix source serialization

* fix source serialization

* tmp shutdown schema for create-transform

* Tests

* pass object to delete

* fix archive stub

* Poll limit back to original

* comment

* Single transform failure test

* Update single endpoint

* Schema merge many

* fn is in tu now

* Adjust schemas and tests

* Use transforms schema only for API

* Update enterprise/backend/src/metabase_enterprise/workspaces/merge.clj

Co-authored-by: Chris Truter <crisptrutski@users.noreply.github.com>

* TODO

* remarks on endpoints

* test comments

---------

Co-authored-by: Chris Truter <crisptrutski@users.noreply.github.com>
2025-12-15 19:25:29 +01:00
lbrdnk
3e26069789 Resolve: migrations 2025-12-15 17:47:11 +01:00
lbrdnk
0124c006a7 ee src 2025-12-15 15:33:16 +01:00
lbrdnk
3143e4bddb resolve: mage test runner 2025-12-15 15:15:04 +01:00
lbrdnk
1aefab5285 resolve: oss src 2025-12-15 15:14:03 +01:00
lbrdnk
31bb4085dd resolve: deps.edn 2025-12-15 15:08:07 +01:00
lbrdnk
7fb2dc1b2a Resolve: test_resources 2025-12-15 15:03:26 +01:00
lbrdnk
ba5860f72d Resolve: routes.clj 2025-12-15 14:20:05 +01:00
Ngoc Khuat
a839e0948b add ws.tu/with-workspaces for proper workspace initiliation and cleanup in tests (#66938) 2025-12-15 20:08:10 +07:00
Uladzimir Havenchyk
750fce2617 resolve frontend merge conflicts 2025-12-15 15:31:15 +03:00
Uladzimir Havenchyk
455bf8f6ed Revert "update files with conflicts"
This reverts commit 290e5dc5d4.
2025-12-15 15:10:04 +03:00
Uladzimir Havenchyk
290e5dc5d4 update files with conflicts 2025-12-15 15:08:54 +03:00
Ngoc Khuat
e255be5b45 Destroy workspace on archive or delete (#66872)
* destroy workspace on archive or delete

* fix tranforms openning and running

* Fix refetching output dataset

* Fix refetching external transforms on remove transform

* Close table tabs if they no longer exist in the workspace

* Highlight selected input table

* fix last run status and remove polling

* connect mapping to api and execution

* fix checkout api schema

* fix destroy test

* fix clj-kondo

* Make /table default value stable

* Restrict fetching workspace transforms

* Add feedback for workspace merging

* Add proper rtk tag for workspace tables

* fix add transform to a workspace from transforms page

* use dynamic redefs

---------

Co-authored-by: Uladzimir Havenchyk <uladzimir@metabase.com>
Co-authored-by: Stas Gavrylov <stasgavrylov@gmail.com>
Co-authored-by: Oleksandr Solovyov <o@solovyov.net>
2025-12-15 18:09:04 +07:00
Chris Truter
a1bb788f46 Fix ws.execute boundary (#66936) 2025-12-15 13:02:34 +02:00
Chris Truter
bb05aceda2 Fix for missing table id scenarios (#66933) 2025-12-15 12:58:51 +02:00
Uladzimir Havenchyk
1d56cce877 use - in param 2025-12-15 11:34:48 +03:00
Ngoc Khuat
591db0554f Isolation: bigquery (#66644) 2025-12-15 15:23:18 +07:00
Oleksandr Solovyov
13b26368cd fix rollback failing 2025-12-15 09:45:00 +02:00
Uladzimir Havenchyk
70486e25d5 fix add transform to a workspace from transforms page 2025-12-12 19:43:27 +03:00
Stas Gavrylov
5fd6d2477d Add proper rtk tag for workspace tables 2025-12-12 08:05:55 -08:00
Stas Gavrylov
c6b0a63cd0 Add feedback for workspace merging 2025-12-12 07:54:53 -08:00
Stas Gavrylov
f644c57b7b Restrict fetching workspace transforms 2025-12-12 07:33:06 -08:00
Stas Gavrylov
a27baf8ea1 Make /table default value stable 2025-12-12 07:27:04 -08:00
Oleksandr Solovyov
cd2a7f6519 fix clj-kondo 2025-12-12 17:02:33 +02:00
Ngoc Khuat
e0930a4019 fix destroy test 2025-12-12 21:58:31 +07:00
Oleksandr Solovyov
203c115810 fix checkout api schema 2025-12-12 16:56:41 +02:00
Oleksandr Solovyov
2885b08b35 connect mapping to api and execution 2025-12-12 16:46:09 +02:00
Uladzimir Havenchyk
a30a4c9652 fix last run status and remove polling 2025-12-12 17:40:02 +03:00
Stas Gavrylov
e9682a4907 Highlight selected input table 2025-12-12 05:31:10 -08:00
Stas Gavrylov
85eb1b5076 Close table tabs if they no longer exist in the workspace 2025-12-12 05:28:39 -08:00
Stas Gavrylov
1b310378c4 Fix refetching external transforms on remove transform 2025-12-12 04:44:58 -08:00
Stas Gavrylov
01d481c618 Fix refetching output dataset 2025-12-12 04:33:48 -08:00
Uladzimir Havenchyk
cc21ddb5d0 fix tranforms openning and running 2025-12-12 15:33:08 +03:00
Uladzimir Havenchyk
9d740a86aa add external transforms call 2025-12-12 14:29:30 +03:00
Oleksandr Solovyov
86c5570385 last run at 2025-12-12 13:27:06 +02:00
Ngoc Khuat
c63accec35 isolation cleanup (#66823) 2025-12-12 18:01:19 +07:00
Stas Gavrylov
74a5d9cdd8 Prefer cache value for table tranform 2025-12-12 02:59:58 -08:00
Stas Gavrylov
019c7fdd8b Update displayed name for output tables 2025-12-12 02:55:18 -08:00
Stas Gavrylov
46f3302c7d Fix output table results display 2025-12-12 02:52:35 -08:00
Stas Gavrylov
8dd28bdc68 Fix sending content of new transform 2025-12-12 02:35:16 -08:00
Stas Gavrylov
ed3df65de7 Fix input dataset opening 2025-12-12 01:27:39 -08:00
Chris Truter
4646272bc0 Disable must-archive-to-delete validation 2025-12-12 11:00:29 +02:00
Ngoc Khuat
aa334a09ec do not throw an error when swapping in tests 2025-12-12 15:43:41 +07:00
Chris Truter
ed70fe2e6c Wire up ws.execute to ws/:id/run (#66862) 2025-12-12 02:46:11 +02:00
Chris Truter
8d90453a16 Strip some detritus 2025-12-12 01:04:05 +02:00
Chris Truter
5a3c32cbf0 Fix a metric ton of Workspaces stuff (#66853) 2025-12-12 00:53:50 +02:00
Chris Truter
633d8b80b2 Fix analysis and permission grants for postgres workspaces (#66840) 2025-12-12 00:06:47 +02:00
Chris Truter
3c7116b01c External Transforms API for Workspaces (#66828) 2025-12-12 00:06:24 +02:00
Chris Truter
8d3ffad856 Return global ids from transform listing
This is a useful distinction to indicate in the transform browser (new versus updated)
2025-12-11 21:21:58 +02:00
Uladzimir Havenchyk
57159b6017 migrate add to workspace from a transforms page to a new endpoint 2025-12-11 20:35:19 +03:00
Oleksandr Solovyov
afecaddbd3 remove #p 2025-12-11 19:15:03 +02:00
Stas Gavrylov
496b420f34 Fix Change target modal 2025-12-11 08:49:24 -08:00
Oleksandr Solovyov
6b1da0b2b5 Add last_run_at tracking and fix N+1 queries in workspace tables 2025-12-11 18:21:11 +02:00
Stas Gavrylov
d763760ddc Move TransformTab to its folder 2025-12-11 08:05:55 -08:00
Ngoc Khuat
2c9d550ba2 Grant access for new input tables (#66814) 2025-12-11 22:48:26 +07:00
Stas Gavrylov
58fbea310e Update Workspaces list page 2025-12-11 07:43:25 -08:00
Stas Gavrylov
264dde2418 Update Save transform flow 2025-12-11 07:26:34 -08:00
Chris Truter
8df148bac2 Add stale field for transform tracking, but don't update it yet. 2025-12-11 16:54:04 +02:00
Chris Truter
c994dc7fc8 Leave a placeholder for target_stale 2025-12-11 16:54:04 +02:00
Uladzimir Havenchyk
daf6357fc6 fix checkout 2025-12-11 17:36:04 +03:00
Chris Truter
0c58d96ace Fix MySQL migrations, hopefully 2025-12-11 16:28:30 +02:00
Chris Truter
af2bcec859 Implement delete. Consistent POST and CREATE responses. 2025-12-11 16:28:30 +02:00
Chris Truter
8e488b7527 Fix OpenAPI generation 2025-12-11 16:28:30 +02:00
Kamil Mielnik
964e9c12cf Make deletion red 2025-12-11 21:00:07 +07:00
Kamil Mielnik
fa2033d129 Remove comment 2025-12-11 20:51:50 +07:00
Kamil Mielnik
495c5c63de Add test for deleting workspace 2025-12-11 20:25:38 +07:00
Kamil Mielnik
a0b3e19657 Add confirmation modal for deleting a workspace 2025-12-11 20:25:38 +07:00
Kamil Mielnik
76ccbe1622 Show archived workspaces in workspaces page 2025-12-11 20:25:38 +07:00
Chris Truter
7907a6ccb4 Rollback fix only applies (and works) for MySQL (I think) 2025-12-11 15:24:10 +02:00
Chris Truter
f2c159652c Fix whitespace 2025-12-11 15:24:10 +02:00
Stas Gavrylov
909331f485 Fix removing transform from a workspace 2025-12-11 05:15:04 -08:00
Stas Gavrylov
7855a5b795 Fix update transform 2025-12-11 04:46:47 -08:00
Stas Gavrylov
6ba64985da Few more type fixes 2025-12-11 04:46:47 -08:00
Kamil Mielnik
69d5973321 Add test for unarchiving a workspace 2025-12-11 19:24:17 +07:00
Kamil Mielnik
48597d14b4 Update test 2025-12-11 19:04:59 +07:00
Stas Gavrylov
5fd46596cb Fix clicking on transform link in Data tab 2025-12-11 04:00:33 -08:00
Uladzimir Havenchyk
6d5fec7122 show workspace statuses in a sidebar 2025-12-11 13:40:25 +03:00
Stas Gavrylov
783e2254d6 Fix isSaved condition for workspace transforms 2025-12-11 02:29:44 -08:00
Stas Gavrylov
0ab3735bf8 Fix get transform call for workspace transforms 2025-12-11 02:21:06 -08:00
Stas Gavrylov
631800884f Fix editor for new transforms 2025-12-11 02:15:04 -08:00
Ngoc Khuat
4e992e8909 Dependencies v2 (#66761) 2025-12-11 16:42:08 +07:00
Chris Truter
a3e14d9641 Fix for mysql rollback 2025-12-11 10:57:45 +02:00
Chris Truter
6af8f6e242 Fix migration for h2 and incompatible workspaces 2025-12-11 10:50:32 +02:00
Oleksandr Solovyov
05c94b0ce3 fix some clj-kondo issues 2025-12-11 10:45:44 +02:00
Oleksandr Solovyov
7bb0bbaa8c bare-bones execute (#66758) 2025-12-11 10:36:46 +02:00
Uladzimir Havenchyk
a6fb7c76c3 adopt changes 2025-12-10 23:40:33 +03:00
Uladzimir Havenchyk
e1a8b7e90b merge transform method 2025-12-10 23:35:41 +03:00
Uladzimir Havenchyk
705dd79908 all transforms execute -> run 2025-12-10 23:27:30 +03:00
Uladzimir Havenchyk
ba739ee291 add data validation and update types 2025-12-10 23:24:19 +03:00
Uladzimir Havenchyk
40ee74bb0b add crud rtk wrappers for workspace transforms 2025-12-10 23:21:23 +03:00
Uladzimir Havenchyk
4d8a03d04a adopt be changes for crud operations 2025-12-10 23:13:09 +03:00
Uladzimir Havenchyk
2fff970042 show preview in a better way 2025-12-10 22:44:51 +03:00
Stas Gavrylov
d8f39ea54f Fix opening workspace transforms 2025-12-10 09:08:17 -08:00
Stas Gavrylov
2d69e365bb Add fetching specific workspace transforms 2025-12-10 08:56:46 -08:00
Stas Gavrylov
24141d20d0 Continue API migration. Update /tables endpoint 2025-12-10 08:16:11 -08:00
Stas Gavrylov
42c3fbe7e3 Upate types for workspace transforms 2025-12-10 07:57:04 -08:00
Stas Gavrylov
e1a77204d0 Update workspace contents fetching logic 2025-12-10 07:47:13 -08:00
Stas Gavrylov
584f10c2c5 First iteration of moving to new Add transform flow 2025-12-10 07:23:51 -08:00
Stas Gavrylov
c0484509b8 Fix log polling 2025-12-10 06:12:13 -08:00
Chris Truter
8eb100fa52 Remove require for deleted namespace 2025-12-10 15:40:30 +02:00
Chris Truter
5e4b1619f8 Remove duplicate-output-table! again (came back from the dead) 2025-12-10 15:38:32 +02:00
Chris Truter
d8cf6d4ebb More lints 2025-12-10 15:24:44 +02:00
Chris Truter
2b92e87ac6 Lints 2025-12-10 15:20:54 +02:00
Chris Truter
a05468ffcf Workspace Backend version 2.2 skeleton (#66749) 2025-12-10 13:49:15 +02:00
Uladzimir Havenchyk
38ad323d39 transform: add run button feedback and status 2025-12-10 14:48:37 +03:00
Stas Gavrylov
359c63bfc7 Add 'available transforms' header 2025-12-10 03:05:30 -08:00
Stas Gavrylov
70a7f318c8 Merge branch 'workspaces-dataset-refetch' into workspaces-2025-11 2025-12-10 02:49:20 -08:00
Stas Gavrylov
a5f12bd5cc Refactor to use mocked metadata for DataTab 2025-12-10 02:48:04 -08:00
Oleksandr Solovyov
610eb93c6c fix test for workspace creation 2025-12-10 12:17:21 +02:00
Kamil Mielnik
c7b6f06b57 Fix run workspace button not disabled, add assertions for it 2025-12-10 15:59:08 +07:00
Kamil Mielnik
6c8268adb2 Typo 2025-12-10 15:50:34 +07:00
Kamil Mielnik
8e776d2b66 Update helpers 2025-12-10 15:44:44 +07:00
Ngoc Khuat
37da925746 Workspaces: extend driver support (#66389) 2025-12-10 14:37:26 +07:00
Chris Truter
49a00dea09 Strip isolated schemas out of target picker 2025-12-09 22:35:50 +02:00
Chris Truter
ce44815e8e Implement basic workspace execution, for simple graphs 2025-12-09 17:53:59 +02:00
Chris Truter
a4c37283fc Fix types of stub /execute data 2025-12-09 17:41:43 +02:00
Kamil Mielnik
d191b6c0a1 Assert merging workspace 2025-12-09 22:28:37 +07:00
Uladzimir Havenchyk
0cfd5936c5 add bulk transfrom run 2025-12-09 18:27:40 +03:00
Uladzimir Havenchyk
573ba81c1e move workspace modal 2025-12-09 17:34:31 +03:00
Uladzimir Havenchyk
de0616bd90 simplify change target modal logic 2025-12-09 16:50:50 +03:00
Kamil Mielnik
8f7fa76f87 Add transform checkout test 2025-12-09 20:04:33 +07:00
Kamil Mielnik
7f9886044a Fix closing tabs 2025-12-09 19:31:10 +07:00
Kamil Mielnik
becf8db5df Test renaming workspaces 2025-12-09 19:07:51 +07:00
Kamil Mielnik
c78784afef Test archiving 2025-12-09 18:59:39 +07:00
Kamil Mielnik
ac2c41b83f Add 1st e2e test for workspaces + helpers boilerplate + data-testid 2025-12-09 18:44:07 +07:00
Oleksandr Solovyov
c38c69059e fix results of POST /ee/workspace/:id/transform not appearing in contents 2025-12-09 12:37:18 +02:00
Oleksandr Solovyov
f83890077a copy transforms apis inside workspaces 2025-12-08 18:29:18 +02:00
Kamil Mielnik
a5922d9323 Add tooltip with error message 2025-12-08 21:49:59 +07:00
Uladzimir Havenchyk
866dbe4be9 metabot: add db into context and use conversation snapshot per workspace 2025-12-08 16:33:56 +03:00
Kamil Mielnik
bd3c618303 Avoid showing status of other workspaces 2025-12-08 20:21:04 +07:00
Kamil Mielnik
ffbe648b7b Add setup log 2025-12-08 20:15:49 +07:00
Ngoc Khuat
e080d9e1ff fix not savign the database details 2025-12-08 16:36:35 +07:00
Oleksandr Solovyov
af88617937 async api for workspace creation (#66579) 2025-12-08 11:02:03 +02:00
Ngoc Khuat
7bff4573e3 # is unwanted 2025-12-08 15:11:54 +07:00
Uladzimir Havenchyk
098370174a metabot in workspaces - alpha version 2025-12-05 23:13:46 +03:00
Kamil Mielnik
14facfdbb7 Move change target button 2025-12-05 20:37:25 +07:00
Kamil Mielnik
ddc9c7c6c3 Fix clicking Metabot tab (first time) opens Setup tab
- Fixes GDGT-1355
2025-12-05 20:24:16 +07:00
Kamil Mielnik
63f4dcbf7c Fix metabot overflow
- Fixes GDGT-1354
2025-12-05 20:14:36 +07:00
Kamil Mielnik
7f35c17f3e Remove workspace_id attribute from Transform
- in CodeTab the usage was redundant
  - GET /api/ee/transform will not return transforms that belong to a workspace
- in EditTransformMenu the usage was redundant
  - you can't open a transform that belongs to a workspace in this page
- in TransformTab the usage can be replaced by looking up workspace contents
- Closes GDGT-1349
2025-12-05 19:27:56 +07:00
lbrdnk
4f98f13971 Hack to make transforms exec work until there is ws specific api 2025-12-05 12:49:22 +01:00
Ngoc Khuat
651918f3e8 remove duplicated schema database_id 2025-12-05 17:41:05 +07:00
Stas Gavrylov
59c652d537 Add sorting for tabs 2025-12-05 11:34:02 +01:00
Ngoc Khuat
bc3687c4a5 use admin connection for h2 test-data 2025-12-05 16:03:37 +07:00
Ngoc Khuat
05bb1daa42 fix a few transforms api and workspace api tests 2025-12-05 15:52:38 +07:00
Ngoc Khuat
6b6d55cd22 fix serdes tests 2025-12-05 15:31:51 +07:00
Stas Gavrylov
1361b9bda0 Merge branch 'workspaces-merge-tabs' into workspaces-2025-11 2025-12-05 09:26:38 +01:00
Ngoc Khuat
6644f846e8 there are too many migrations on fk_collection_workspace_id, anyway this attempts to fix the cross branch migration CI 2025-12-05 15:20:47 +07:00
Stas Gavrylov
98b5096eda Refactor tabs logic to keep a single array, allowing better UX 2025-12-05 09:19:04 +01:00
Uladzimir Havenchyk
0cc951e1a4 replace target input with change target modal 2025-12-04 23:58:02 +03:00
Ngoc Khuat
24910610c4 fix dag cte for for h2, h2 is stricter and requires column declaration
for h2
2025-12-04 23:32:30 +07:00
Stas Gavrylov
12fd210a41 Squashed commit of the following:
commit 303e05ec59
Author: Stas Gavrylov <stasgavrylov@gmail.com>
Date:   Thu Dec 4 17:25:58 2025 +0100

    merge main branch

commit a67169c5e5
Author: Stas Gavrylov <stasgavrylov@gmail.com>
Date:   Thu Dec 4 16:15:35 2025 +0100

    Cleanup. Add tables refetching on transform run

commit 14e7b758f4
Author: lbrdnk <lbrdnk@users.noreply.github.com>
Date:   Thu Dec 4 14:51:28 2025 +0100

    input tables adj

commit a0feade8ae
Author: lbrdnk <lbrdnk@users.noreply.github.com>
Date:   Thu Dec 4 14:46:11 2025 +0100

    /tables no graph, alt impl

commit 3509629d84
Author: Stas Gavrylov <stasgavrylov@gmail.com>
Date:   Thu Dec 4 11:45:56 2025 +0100

    remove questionWithMetadata

commit bccfd459c5
Author: Stas Gavrylov <stasgavrylov@gmail.com>
Date:   Thu Dec 4 11:39:21 2025 +0100

    wip4

commit f733db75b1
Author: Stas Gavrylov <stasgavrylov@gmail.com>
Date:   Wed Dec 3 16:59:42 2025 +0100

    wip3

commit 4bc044c577
Author: Stas Gavrylov <stasgavrylov@gmail.com>
Date:   Wed Dec 3 16:37:26 2025 +0100

    wip2

commit 2ca514359c
Author: Stas Gavrylov <stasgavrylov@gmail.com>
Date:   Wed Dec 3 16:22:29 2025 +0100

    wip

commit 6a29d09d17
Author: Stas Gavrylov <stasgavrylov@gmail.com>
Date:   Wed Dec 3 13:20:52 2025 +0100

    wip

commit 15c9599bfa
Author: Stas Gavrylov <stasgavrylov@gmail.com>
Date:   Wed Dec 3 11:45:01 2025 +0100

    Refactor edits detection logic

commit d1ac164c32
Author: Stas Gavrylov <stasgavrylov@gmail.com>
Date:   Tue Dec 2 16:11:02 2025 +0100

    Add Data tab with a list of input and output tables
2025-12-04 17:26:55 +01:00
Kamil Mielnik
7484249328 Fix - Changing transform output table always shows an error
- Fixes GDGT-1344
- Simplifies useTransformValidation
2025-12-04 22:00:23 +07:00
Oleksandr Solovyov
d034244735 expose upstream transform details to fe 2025-12-04 15:58:51 +02:00
Ngoc Khuat
3b837f7593 Isolation v2 (#66374) 2025-12-04 20:49:19 +07:00
Ngoc Khuat
36419de6e7 swapped pool supports mutli pools of the same db (#66498) 2025-12-04 15:06:43 +02:00
Kamil Mielnik
b488c1b082 Fix state management issue (changing output table resets source tables and code inputs)
- Fixes GDGT-1328
2025-12-04 19:40:19 +07:00
Kamil Mielnik
869cfcc024 Improve types 2025-12-04 19:39:55 +07:00
Kamil Mielnik
81a7572e0a Fix endpoint usage 2025-12-04 19:11:52 +07:00
Kamil Mielnik
2ff73cd864 Update validate-target endpoint mapping 2025-12-04 04:07:37 -08:00
Uladzimir Havenchyk
3ebdecc2d2 add change target modal to workspace transform 2025-12-04 08:19:44 +03:00
Oleksandr Solovyov
f0bfd6898f transform filtering (#66411) 2025-12-03 17:30:19 +02:00
Oleksandr Solovyov
1b6528c9f1 don't import unexported stuff 2025-12-03 17:28:35 +02:00
Oleksandr Solovyov
f8f2e235f6 very sorry 2025-12-03 17:15:32 +02:00
Oleksandr Solovyov
9086aa10e9 more useful validate-target 2025-12-03 17:14:19 +02:00
Uladzimir Havenchyk
cfa8c389b2 add transform edit menu 2025-12-02 23:45:47 +03:00
Uladzimir Havenchyk
c670809704 remove unused files 2025-12-02 23:45:47 +03:00
lbrdnk
1f78793830 Add remove add remove test 2025-12-02 19:50:41 +01:00
lbrdnk
4e342e8481 Tables docstring 2025-12-02 19:50:41 +01:00
lbrdnk
b61455bb53 /tables, empty handling 2025-12-02 19:50:41 +01:00
lbrdnk
18c743403e /tables endpoint tests 2025-12-02 19:50:40 +01:00
Stas Gavrylov
b1fd062915 Add ability to rename workspace 2025-12-02 16:55:14 +01:00
Chris Truter
15b446ab04 Remove inline config that's breaking cljfmt 2025-12-02 17:39:23 +02:00
Oleksandr Solovyov
0f61a79a6b make merge work if there were no remapped tables (#66467) 2025-12-02 17:30:51 +02:00
lbrdnk
9ebbf353f6 unblock fe ignoring brokne graph 2025-12-02 16:12:26 +01:00
Stas Gavrylov
be25ff92a1 Fix docs 2025-12-02 15:38:18 +01:00
Kamil Mielnik
11ef81fdee Autofocus 2025-12-02 21:33:04 +07:00
Kamil Mielnik
ce08daf26f Use form in new workspace modal 2025-12-02 21:32:29 +07:00
Kamil Mielnik
e2b5e9dd9d Empty state 2025-12-02 21:29:52 +07:00
lbrdnk
49bbcf7467 Avoid calling delete with nil ids 2025-12-02 15:20:25 +01:00
Kamil Mielnik
757cca20db Remove transform from workspace
- Closes GDGT-1319
2025-12-02 21:19:02 +07:00
Oleksandr Solovyov
b196d10ad3 fix error reporting so that tests are passing 2025-12-02 15:52:24 +02:00
Kamil Mielnik
3e92c2d2ad Disable "Merge" button when there are no transforms in the workspace
- Closes GDGT-1326
2025-12-02 20:30:45 +07:00
lbrdnk
f1d495dfde whitespace 2025-12-02 14:30:31 +01:00
lbrdnk
a14b9c1b3b Add support for transform deletion 2025-12-02 14:28:33 +01:00
Kamil Mielnik
6cffef70f5 Fix alignment 2025-12-02 20:15:51 +07:00
Kamil Mielnik
a5a8fa1f1e Fix crash 2025-12-02 20:14:47 +07:00
Kamil Mielnik
6c7102aa37 Fix app-wide transforms filtering 2025-12-02 20:14:47 +07:00
Oleksandr Solovyov
809e5b6724 api for workspace name change and test fix 2025-12-02 15:11:57 +02:00
Stas Gavrylov
5f75951717 Remove tabs centered positioning 2025-12-02 14:10:23 +01:00
Stas Gavrylov
8ff78faef0 Add scrollIntoView for tabs 2025-12-02 14:09:24 +01:00
Stas Gavrylov
2bd68fc928 Add output table name input validation 2025-12-02 14:04:27 +01:00
Kamil Mielnik
288f07c709 Disable transform editor when transform is not a part of the workspace 2025-12-02 20:00:53 +07:00
Ngoc Khuat
1f199c1dfb Workspaces swapping db details (#66324) 2025-12-02 19:50:40 +07:00
Stas Gavrylov
51efac3d49 Fix workspace transforms origin 2025-12-02 13:35:48 +01:00
Kamil Mielnik
a9ccb8def6 Extract CheckOutTransformButton & SaveTransformButton 2025-12-02 19:31:31 +07:00
Stas Gavrylov
b2dacc7a92 Add transform creation within current workspace 2025-12-02 13:22:33 +01:00
Stas Gavrylov
d769903e13 Refactor table name validation to use Yup schema extension 2025-12-02 13:22:32 +01:00
Stas Gavrylov
167af6a806 Extend CreateTransformModal with external schemas and async table name validation. Integrate to WorkspacePage 2025-12-02 13:22:32 +01:00
Oleksandr Solovyov
e2ad85b627 upstream_id is optional 2025-12-02 14:21:44 +02:00
Chris Truter
c1c1926751 Fix some tests 2025-12-02 10:05:05 +02:00
Chris Truter
40dbb2dd5e WIP 2025-12-02 09:57:13 +02:00
Chris Truter
f206ac8d9b Disallow transforms with card dependencies in workspaces 2025-12-02 09:57:11 +02:00
lbrdnk
c93f52c810 Workspaces GET /tables endpoint (#66421)
* /tables endpoint

* Update keys

* Adjust test

* Update test

* remarks

* Avoid using :outputs or :transforms from the graph structure
2025-12-01 22:28:41 +01:00
lbrdnk
fac1eda9fa Workspaces dependent python transforms id adjustments (#66399)
* Python transitive transforms

* wording
2025-12-01 22:27:12 +01:00
Uladzimir Havenchyk
5baa9d4709 allow merging workspaces when all transforms saved 2025-12-01 22:30:45 +03:00
Uladzimir Havenchyk
78272586f8 redirect to the transform list after merge of a workspace 2025-12-01 21:47:19 +03:00
Ngoc Khuat
86926f69d3 return global transforms only by default and add exclude_workspace_id filter 2025-12-01 22:58:47 +07:00
Kamil Mielnik
0ee7e2fa48 Target table name input 2025-12-01 22:12:15 +07:00
Ngoc Khuat
c3f7c0e3f8 hydrate full transform contents 2025-12-01 21:00:11 +07:00
Chris Truter
1774564beb API to check whether transform target is OK 2025-12-01 15:46:44 +02:00
Chris Truter
557ce7c3c6 Fix linter warnings 2025-11-30 14:54:11 +02:00
Chris Truter
a60fa0c2cb Fix whitespace 2025-11-30 14:50:06 +02:00
Chris Truter
39e4f68e31 Add linter (overkill?) and fix for bad claude migration yaml 2025-11-30 14:43:29 +02:00
Uladzimir Havenchyk
072c06e2bd add a skeleton for merge functionality 2025-11-29 01:15:32 +03:00
Chris Truter
e781690132 Fix assertion to actually check something 2025-11-28 16:49:44 +02:00
Kamil Mielnik
68b19966fc Run transform button 2025-11-28 21:24:41 +07:00
Kamil Mielnik
8b5db401e8 Save button 2025-11-28 21:21:29 +07:00
Oleksandr Solovyov
946fc08c43 complain for wrong args to api 2025-11-28 15:20:10 +02:00
Stas Gavrylov
f5cbb8cc48 Update edited transforms display logic 2025-11-28 13:17:00 +01:00
Oleksandr Solovyov
44d6075e8c do not ask for workspace transform to provide db/schema for target 2025-11-28 14:12:23 +02:00
Chris Truter
fa2ae8e884 Fix broken SQL error 2025-11-28 14:09:51 +02:00
Stas Gavrylov
36e3b6ec3a Update WS transforms styling 2025-11-28 12:43:56 +01:00
Oleksandr Solovyov
ba9a841466 fix graph queries 2025-11-28 13:32:36 +02:00
Oleksandr Solovyov
5ebc738f29 less obtuse names 2025-11-28 13:18:22 +02:00
Stas Gavrylov
045e9a224f Refactor Setup tab 2025-11-28 12:00:34 +01:00
Stas Gavrylov
a6e22daac1 Add onChange for TransformEditor. Refactoring 2025-11-28 11:48:08 +01:00
Kamil Mielnik
6bdd48ffe3 Check if transform is in the workspace 2025-11-28 17:37:16 +07:00
Kamil Mielnik
5bb0f1f560 Style 2025-11-28 17:36:54 +07:00
Kamil Mielnik
2d263d19e3 Save button adds transform into the workspace 2025-11-28 17:30:12 +07:00
Kamil Mielnik
5e98f44b80 Add updateWorkspaceContents API mapping 2025-11-28 17:24:46 +07:00
Kamil Mielnik
da6555f5cf Stub handlers 2025-11-28 17:24:46 +07:00
Kamil Mielnik
937f3f7b03 Add TransformTab 2025-11-28 17:24:46 +07:00
Stas Gavrylov
71d065f0a9 Fix Python sidebar 2025-11-28 11:23:39 +01:00
Oleksandr Solovyov
b30071a69a adding entities and fresh transforms to workspace (#66364) 2025-11-28 12:14:40 +02:00
Stas Gavrylov
407fc2a5ab Update Code tab styling 2025-11-28 11:13:16 +01:00
Stas Gavrylov
de113f72ce Merge branch 'workspaces-master' into workspaces-2025-11 2025-11-28 11:09:08 +01:00
Stas Gavrylov
6199e0056f Merge master 2025-11-28 10:51:57 +01:00
lbrdnk
fd6b176446 Dependecy support for python transforms (#66239)
* Python transform dependencies support

* Upstream deps test

* Pass the transform
2025-11-28 10:16:24 +01:00
lbrdnk
42d1202c18 Support for multiple interdependent transforms in workspaces (#66348)
* Interdependent native Xs not executed test

WIP

Fix X mirroring as per huddle

Test: use atom for cleanup

transform-node->data

Use new mirroring and remap code

typo

typo

Make isolation naming functions public

Fix adding workspace namespace, not tested yet

Fix source-for-dup-transform

Dirty interedependent native transforms test

Instead of writing over graph structure, wrap it in context

Cleanup

Cleanup test, not final

Remove redundant test

Consolidate schema set into single update

Remove juxt in favor of select-fn->fn

nits

* Fix stuff after merging feat branch

graph :dependencies now hold transforms not tables

this change adjusts the rest of mirroring to work with that

* Better test cleanup

* hack to avoid failing when provisioning ws with no entities to checkout
2025-11-28 08:31:36 +01:00
Uladzimir Havenchyk
d1a4656cb8 move archive action to workspace item 2025-11-27 23:13:17 +03:00
Uladzimir Havenchyk
cc934ea766 fix layout 2025-11-27 20:32:00 +03:00
Stas Gavrylov
8bf117b047 Fix tabs header wrap 2025-11-27 17:31:55 +01:00
Uladzimir Havenchyk
caea0b31cf move routes out of transforms 2025-11-27 19:25:15 +03:00
Stas Gavrylov
1d6dd0d16d Add tabs icons 2025-11-27 16:51:41 +01:00
Stas Gavrylov
dca47362f8 Disable DB selection in Python editor 2025-11-27 16:23:30 +01:00
Stas Gavrylov
2f8b666797 Add transforms filtering by DB type and display of source DB name 2025-11-27 16:09:33 +01:00
Stas Gavrylov
f3aef4f4ad Some types refactoring. Make transforms names Ellipsified 2025-11-27 15:52:20 +01:00
Stas Gavrylov
5ac6249eb3 Filter transforms. Add tabs layout improvements 2025-11-27 15:35:15 +01:00
Stas Gavrylov
8acbee8000 tabs improvements 2025-11-27 15:21:41 +01:00
Kamil Mielnik
bfd2ad8f0f Better Metabot isolation 2025-11-27 21:13:49 +07:00
Stas Gavrylov
a982db6bf5 Fix for resetting active transform 2025-11-27 15:13:39 +01:00
Stas Gavrylov
3e6d266a33 Add tabs for transforms 2025-11-27 15:11:37 +01:00
Stas Gavrylov
e197cf97af Minor tabs styling updates 2025-11-27 14:47:41 +01:00
Kamil Mielnik
97d89cc855 Make tabs controlled, open transform tab when opening a transform 2025-11-27 20:44:05 +07:00
Kamil Mielnik
955cc0b4be Render Metabot tab only when plugin is enabled 2025-11-27 20:39:28 +07:00
Kamil Mielnik
1b4424393d Change data studio sidebar width 2025-11-27 20:37:29 +07:00
Kamil Mielnik
3337769411 Add Metabot tab 2025-11-27 20:27:26 +07:00
Stas Gavrylov
0b07779984 Initial version of transforms editor integrated to workspaces 2025-11-27 13:06:04 +01:00
Chris Truter
2e54d68fb2 Put mirrored transforms in the WS collection (#66358)
In anticipation of transforms moving into collection in master, let's get our code ready for that.
2025-11-27 14:01:47 +02:00
Chris Truter
e67c3e625d Defend against table name collissions 2025-11-27 13:32:19 +02:00
Chris Truter
325fe07ab1 Actually database is on the target also 2025-11-27 13:20:24 +02:00
Chris Truter
dc53fec17e Fixes to dag + workaround for deps gotcha 2025-11-27 13:04:17 +02:00
Chris Truter
17c3ed6635 Preconditions and checksum overrides for updated migrations 2025-11-27 13:04:17 +02:00
Chris Truter
f661f0c21e Fix module test 2025-11-27 13:04:17 +02:00
Chris Truter
89d1394fb7 Fix unused linter for stub API 2025-11-27 13:04:17 +02:00
Stas Gavrylov
0cac006357 Fix transforms list rendering 2025-11-27 11:07:46 +01:00
Chris Truter
c9a5ea4dd1 Better multi tx test 2025-11-27 10:30:20 +02:00
Chris Truter
79f7d83c4d Add a less trivial path-induced-subgraph test 2025-11-27 09:39:52 +02:00
Uladzimir Havenchyk
56a1ac2862 add archive method 2025-11-26 23:55:39 +03:00
Uladzimir Havenchyk
8d0acd1436 basic ui 2025-11-26 18:48:18 +03:00
Uladzimir Havenchyk
b6628af72f fix merge conflict 2025-11-26 12:37:34 +03:00
Uladzimir Havenchyk
0169970a8b Merge remote-tracking branch 'origin/workspaces-master' into workspaces-2025-11 2025-11-26 12:32:40 +03:00
Chris Truter
9cc27dfa59 Mode cosmetics 2025-11-25 17:15:55 +02:00
Oleksandr Solovyov
9221d59b7e wrong default 2025-11-25 16:56:40 +02:00
Oleksandr Solovyov
22047bb890 claude fixes tests (#66244) 2025-11-25 16:55:45 +02:00
Stas Gavrylov
772216505c Merge branch 'workspaces-list-transforms' into workspaces-2025-11 2025-11-25 12:57:50 +01:00
Stas Gavrylov
3d7541f3c7 Display list of all transforms and workspace transforms 2025-11-25 12:57:31 +01:00
Oleksandr Solovyov
5a59eef925 oops, do not hydrate for the list 2025-11-25 13:52:13 +02:00
Chris Truter
dc05f59514 Stub Workspaces API (#66124)
Co-authored-by: Oleksandr Solovyov <o@solovyov.net>
2025-11-25 13:47:10 +02:00
Stas Gavrylov
f64ab4ebab Adjust workspace page layout. Extract route 2025-11-25 12:45:17 +01:00
Kamil Mielnik
19497a7470 Fix redirect after creating a workspace from a transform (use imperative code instead of a React effect) 2025-11-25 16:54:08 +07:00
Kamil Mielnik
823690bc76 Fix skip tokens 2025-11-25 16:49:24 +07:00
Kamil Mielnik
82930d7f02 Fix workspaces API types and RTK query tags 2025-11-25 16:49:15 +07:00
Chris Truter
fa9b90beb1 Fix cross rollback test, hopefully 2025-11-21 18:38:19 +02:00
Chris Truter
d4d0c64400 Basic Merge functionality (#66137) 2025-11-21 18:22:44 +02:00
Chris Truter
cb70117c05 Fixes for Workspace merge (#66136) 2025-11-21 18:20:59 +02:00
Oleksandr Solovyov
3d9c45e126 Basic workspace merging (#66118) 2025-11-21 18:18:23 +02:00
Chris Truter
516df0c70e Bug fix for output table remapping 2025-11-21 11:05:51 +02:00
Chris Truter
4cd2aa7c3e Add list and show page for workspaces (#66112) 2025-11-21 10:44:18 +02:00
Chris Truter
e6943a7389 Max UX more fancy (links between mirrors) (#66101) 2025-11-21 10:43:33 +02:00
Chris Truter
a43c90f357 More idempotent migration fixes 2025-11-21 00:09:30 +02:00
Chris Truter
91a42b4fac Add constraint that each downstream entity has a unique mapping 2025-11-21 00:09:20 +02:00
Chris Truter
34689d9a05 Ignore checksums, for luck 2025-11-20 18:49:03 +02:00
Chris Truter
528b264f87 Fix migration lints, hopefully without breaking pr env 2025-11-20 18:47:01 +02:00
Chris Truter
0580e2dc34 Fix migration ordering 2025-11-20 18:41:06 +02:00
Chris Truter
91c985eca9 Adding missing indexes and cascade behavior 2025-11-20 18:06:43 +02:00
Chris Truter
3bb34fa1c9 Fix small lints and sanity tests
Fix deadcode lint
Fix serialization tests
Fix revision description test (workspace_id column doesn't change)
2025-11-20 18:06:15 +02:00
Chris Truter
f539b334dd Support multiple transforms (mostly) (#66079) 2025-11-20 17:56:43 +02:00
Chris Truter
9cfe59b1a2 Whoops, need plurals 2025-11-20 15:26:51 +02:00
Chris Truter
fb59bf9325 Graph Analysis for Workspace dependencies (#66069) 2025-11-20 15:15:48 +02:00
Chris Truter
2b2ff2dae5 Use join tables to track workspace entity mappings (#66062)
First class joins are kinda nice:

1.  Detection of relationships and navigation back-and-forth in the FE becomes easy. 
2. Database constraints might catch bugs.
3. Clean-up of obsolete mappings is trivial.
4. Debugging and testing is potentially more convenient too.

Maintenance on them will be low, we just need to handle creating them, and 

I haven't covered `metabase_field` _yet_, as that would be more complex. Waiting to confirm they're really necessary, and if so, whether having their mapping in JSON is sufficient.
2025-11-20 12:53:15 +02:00
Chris Truter
0dd7d78c07 Crude toposorted dag calculation 2025-11-20 10:25:51 +02:00
lbrdnk
33daf1d822 Mirror single transform (#65982)
* Mirror single transform

Creates dwh namespace
Creates mirror target table
Creates mirror transform artifact
with adjusted
target
name

Abusing toucan models
sending those in entity-info

Most of the code incrementally
updates entity-info
with checkout progress
and we should continue with that
with more reasonable way
(i.e. not toucan models but some
subset of relevant keys)

Testing code at the end of
copying ns

Namespaces will change
into different subsystems probably
for now this is poc

* Assert toplevel, docstring

* Adjust namespaces and cleanup

* drop old file

* Tweak for rich comment, incomplete

* Vibe UX

* Tighten up workspace demo-0 code (#66044)

* Update FE to track API changes. Fix some small, subtle FE and BE issues.

* Critical error

---------

Co-authored-by: Chris Truter <chris@metabase.com>
Co-authored-by: Chris Truter <crisptrutski@users.noreply.github.com>
2025-11-20 08:57:21 +01:00
Oleksandr Solovyov
9faad636cb add namespace to workspace collections 2025-11-19 18:47:16 +02:00
dan sutton
b201dca6cf empty for CI 2025-11-19 09:20:17 -06:00
Chris Truter
f1982e56e3 Fix contents API shape 2025-11-19 12:13:13 +02:00
Chris Truter
439c3f3be5 Add API endpoint to return the contents of a workspace 2025-11-19 12:11:01 +02:00
Oleksandr Solovyov
37449925fa migration and basic model file (#65956)
* migration and basic model file

* basic api
2025-11-19 12:04:35 +02:00
190 changed files with 19424 additions and 530 deletions

View File

@@ -1,7 +1,7 @@
;; -*- comment-column: 100; -*-
{:config-paths ["macros" "src" "config"]
:linters
{ ;;
{;;
;; enabled optional linters (these are disabled by default)
;;
@@ -178,6 +178,11 @@
metabase.notification.models/create-notification!
metabase.pulse.send/send-pulse!
metabase-enterprise.transforms-python.python-runner-test/execute!
;; Workspace isolation functions are safe in parallel tests because they operate on
;; isolated, test-specific workspaces and databases that don't affect other tests
metabase-enterprise.workspaces.isolation/destroy-workspace-isolation!
metabase-enterprise.workspaces.isolation/ensure-database-isolation!
metabase-enterprise.workspaces.test-util/create-ready-ws!
;; Safe in tests, where we typically use a private, temporary index per thread.
metabase.search.core/reindex!
metabase.util/run-count!
@@ -191,7 +196,7 @@
:metabase/disallow-hardcoded-driver-names-in-tests
;; don't include `:h2` since that will probably lead to a mountain of tests where we are specifically testing
;; middleware or compilation behavior with just H2 since it's the default.
{:drivers #{ ;; core drivers
{:drivers #{;; core drivers
#_:h2
:postgres
:mysql
@@ -422,6 +427,7 @@
metabase-enterprise.transforms.api-test/with-worker-runs clojure.core/let
metabase-enterprise.transforms.test-util/with-transform-cleanup! clojure.core/let
metabase-enterprise.transforms.execute/with-transform-lifecycle clojure.core/let
metabase-enterprise.workspaces.test-util/with-workspaces! clojure.core/let
metabase.actions.test-util/with-actions clojure.core/let
metabase.api.common/let-404 clojure.core/let
metabase.app-db.custom-migrations.util/with-temp-scheduler! clojure.core/fn

View File

@@ -534,6 +534,11 @@
metabase.driver.sql-jdbc.execute
metabase.driver.sql-jdbc.execute.diagnostic
metabase.driver.sql-jdbc.sync
;; TODO (lbrdnk 2025/12/17): We should revisit addition of the following line. I've added it to make
;; tests pass as we are using it directly with ee/workspaces module. I believe later, when workspaces
;; are in better state quality wise, it should become part of the driver api and that should be used
;; instead of calling directly into the normalize module.
metabase.driver.sql.normalize
metabase.driver.sql.query-processor
metabase.driver.sql.util
metabase.driver.sync
@@ -2279,6 +2284,7 @@
{:team "Querying"
:api #{metabase-enterprise.transforms.api
metabase-enterprise.transforms.core
metabase-enterprise.transforms.execute
metabase-enterprise.transforms.init
metabase-enterprise.transforms.instrumentation
metabase-enterprise.transforms.interface
@@ -2312,6 +2318,7 @@
app-db
config
driver
events
lib
lib-be
query-processor
@@ -2326,7 +2333,24 @@
models
premium-features
upload
util}}}
util}}
enterprise/workspaces
{:team "Metabot"
:api #{metabase-enterprise.workspaces.api
metabase-enterprise.workspaces.init}
:uses #{api
api-keys
app-db
driver
lib
lib-be
models
queries
request
system
util
enterprise/transforms}}}
;; namespaces matching these patterns (with `re-find`) are excluded from the module linter. Since regex literals
;; aren't allowed in EDN just used the `(str <regex>)` version i.e. two slashes instead of one.

View File

@@ -90,6 +90,7 @@
metabase-enterprise.test/with-gtaps! [[:inner 0]]
metabase-enterprise.test/with-gtaps-for-user! [[:inner 0]]
metabase-enterprise.test/with-user-attributes [[:block 2]]
metabase-enterprise.workspaces.models.workspace-log/track! [[:block 2]]
metabase.actions.actions-test/with-actions-test-data-and-actions-permissively-enabled [[:block 0]]
metabase.actions.test-util/with-actions [[:block 1]]
metabase.actions.test-util/with-actions-disabled [[:block 0]]

View File

@@ -61,6 +61,7 @@
(eval . (put-clojure-indent 'u/prog1 1))
(eval . (put-clojure-indent 'u/select-keys-when '(:form)))
(eval . (put-clojure-indent 'with-meta '(:form)))
(eval . (put-clojure-indent 'ws.log/track! 2))
;; these ones have to be done with `define-clojure-indent' for now because of upstream bug
;; https://github.com/clojure-emacs/clojure-mode/issues/600 once that's resolved we should use `put-clojure-indent'
;; instead. Please don't add new entries unless they don't work with `put-clojure-indent'

View File

@@ -1,7 +1,5 @@
# execute this in your shell to enable custom merge driver:
# echo '\n[include]\npath=../.gitconfig' >> .git/config
#
# TODO (Bryan 12/12/25) -- reenable this when we can completely verify correctness
# [merge "yaml-migrations"]
# name = YAML migration changeset-aware merge
# driver = mage -merge-yaml-migrations %O %A %B %L %P
[merge "yaml-migrations"]
name = YAML migration changeset-aware merge
driver = bin/mage -merge-yaml-migrations %O %A %B %L %P

View File

@@ -24,6 +24,9 @@
"dev.failsafe" {"failsafe" {:resource "LICENSE"}}
"colorize" {"colorize" {:resource "EPL.txt"}}
"com.github.jnr" {"jffi$native" {:resource "apache2_0.txt"}}
"com.google.api" {"gax" {:resource "BSD.txt"}
"gax-grpc" {:resource "BSD.txt"}
"gax-httpjson" {:resource "BSD.txt"}}
"com.google.api-client" {"google-api-client" {:resource "apache2_0.txt"}}
"com.google.auto.value" {"auto-value-annotations" {:resource "apache2_0.txt"}}
"com.google.http-client" {"google-http-client" {:resource "apache2_0.txt"}
@@ -60,6 +63,7 @@
"net.thisptr" {"jackson-jq" {:resource "apache2_0.txt"}}
"org.antlr" {"antlr-runtime" {:resource "BSD.txt"}
"antlr4-runtime" {:resource "BSD.txt"}}
"org.codehaus.mojo" {"animal-sniffer-annotations" {:resource "MIT.txt"}}
"org.eclipse.jgit" {"org.eclipse.jgit" {:resource "BSD.txt"}}
"org.eclipse.jetty" {"jetty-webapp" {:resource "EPL.txt"}}
"org.eclipse.jetty.toolchain" {"jetty-jakarta-servlet-api" {:resource "EPL.txt"}}

View File

@@ -144,6 +144,18 @@
change-log
#(pos? (compare % "v49.00-000"))))
(defn require-primary-key-exists-has-table-name
"Ensures that all primaryKeyExists preconditions specify a tableName."
[change-log]
(doseq [{:keys [changeSet]} change-log
:when changeSet
:let [s (pr-str (:preConditions changeSet))]
:when (and (str/includes? s ":primaryKeyExists")
(not (str/includes? s ":tableName")))]
(throw (validation-error
(format "Migration '%s' has primaryKeyExists precondition without tableName" (:id changeSet))
{:id (:id changeSet)}))))
(s/def ::changeSet
(s/spec :change-set.strict/change-set))
@@ -163,6 +175,7 @@
(require-no-bare-blob-or-text-types change-log)
(require-no-bare-boolean-types change-log)
(require-no-datetime-type change-log)
(require-primary-key-exists-has-table-name change-log)
(let [{:keys [changeSet]
:as all} (group-by only-key change-log)]
(when-not (set/subset? (set (keys all)) #{:property :objectQuotingStrategy :changeSet})

View File

@@ -36,7 +36,7 @@
com.clearspring.analytics/stream {:mvn/version "2.9.8" ; Various sketching algorithms
:exclusions [it.unimi.dsi/fastutil
org.slf4j/slf4j-api]}
com.clojure-goes-fast/jvm-alloc-rate-meter{:mvn/version "0.1.4"} ; Monitor heap allocation rate.
com.clojure-goes-fast/jvm-alloc-rate-meter {:mvn/version "0.1.4"} ; Monitor heap allocation rate.
com.clojure-goes-fast/jvm-hiccup-meter {:mvn/version "0.1.1"} ; Monitor GC pauses and other system-induced pauses.
com.draines/postal {:mvn/version "2.0.5" ; SMTP library
:exclusions [com.sun.mail/jakarta.mail]}
@@ -45,7 +45,7 @@
com.github.albfernandez/juniversalchardet {:mvn/version "2.5.0"}
com.github.blagerweij/liquibase-sessionlock {:mvn/version "1.6.9"} ; session lock for liquibase migrations
com.github.jknack/handlebars ^:antq/exclude ; 4.4.0 requires java 17+
{:mvn/version "4.3.1"} ; templating engine
{:mvn/version "4.3.1"} ; templating engine
com.github.oliyh/martian-clj-http {:mvn/version "0.2.1"} ; openapi client
com.github.seancorfield/honeysql {:mvn/version "2.7.1350"} ; Honey SQL 2. SQL generation from Clojure data maps
com.github.seancorfield/next.jdbc {:mvn/version "1.3.1070"} ; Talk to JDBC DBs
@@ -53,12 +53,12 @@
com.github.vertical-blank/sql-formatter {:mvn/version "2.0.5"} ; Java SQL formatting library https://github.com/vertical-blank/sql-formatter
com.google.guava/guava {:mvn/version "33.5.0-jre"} ; dep for BigQuery, Spark, and GA. Require here rather than letting different dep versions stomp on each other — see comments on #9697
com.h2database/h2 ^:antq/exclude ; https://metaboat.slack.com/archives/CKZEMT1MJ/p1727191010259979
{:mvn/version "2.1.214"} ; embedded SQL database
{:mvn/version "2.1.214"} ; embedded SQL database
com.ibm.icu/icu4j {:mvn/version "78.1"} ; Used to transliterate Unicode characters into Latin characters
com.knuddels/jtokkit {:mvn/version "1.1.0"} ; Java tokenizer library for OpenAI models, used for token counting in semantic search
com.mchange/c3p0 {:mvn/version "0.11.2"} ; DB connection pool
com.snowplowanalytics/snowplow-java-tracker ^:antq/exclude ; 2+ depend on apache httpclient 5.x, but saml and clj-http depend on 4.x
{:mvn/version "1.0.1"} ; Snowplow analytics
{:mvn/version "1.0.1"} ; Snowplow analytics
com.sun.mail/jakarta.mail ^:antq/exclude ; 2+ changes imports from javax.mail to jakarta.mail
{:mvn/version "1.6.8"} ; SMTP lib, required by postal
com.taoensso/encore {:mvn/version "3.158.0"} ; Utilities library (used by Nippy)
@@ -891,8 +891,7 @@
{:exec-args {:only ["enterprise/backend/test/metabase_enterprise/transforms"
"enterprise/backend/test/metabase_enterprise/transforms_python"]}}
;; test all MBQL related stuff: MLv2, the legacy shared `metabase.legacy-mbql` code, and the QP
;; test all MBQL related stuff: MLv2, the legacy shared `metabase.legacy-mbql` code, and the QP
;;
;; clj -X:dev:ee:ee-dev:test:test/mbql
:test/mbql

View File

@@ -8,6 +8,11 @@ title: Driver interface changelog
- Added `sql-jdbc.execute/db-type-name` multimethod. Override this if something more than the default is needed in your sql-jdbc-based driver. See the `:mysql` implementation as an example.
- Added workspace isolation multimethods for enterprise workspaces feature:
- `init-workspace-isolation!` - Initialize database isolation for a workspace (create isolated schema/database and user credentials)
- `destroy-workspace-isolation!` - Destroy all database resources created for workspace isolation
- `grant-workspace-read-access!` - Grant read access on specified tables to a workspace's isolated user
## Metabase 0.58.0
- Added a `:collate` feature for drivers that support collation settings on text fields

View File

@@ -78,7 +78,11 @@ export function entityPickerModalItem(level, name) {
}
export function entityPickerModalTab(name) {
return cy.findAllByRole("tab").filter(`:contains(${name})`);
if (typeof name === "string") {
return cy.findAllByRole("tab").filter(`:contains(${name})`);
} else {
return cy.findAllByRole("tab").first().parent().findByText(name);
}
}
// displays at least these tabs:

View File

@@ -0,0 +1,99 @@
import type { WorkspaceId } from "metabase-types/api";
export const Workspaces = {
visit: (id: WorkspaceId) => {
return cy.visit(`/data-studio/workspace/${id}`);
},
visitDataStudio() {
return cy.visit("/data-studio");
},
visitTransformListPage() {
return cy.visit("/data-studio/transforms");
},
getWorkspacesPage() {
return cy.findByTestId("workspaces-page");
},
getWorkspacePage() {
return cy.findByTestId("workspace-page");
},
getWorkspaceContent() {
return cy.findByTestId("workspace-content");
},
getWorkspaceSidebar() {
return cy.findByTestId("workspace-sidebar");
},
getWorkspacesSection() {
return cy.findByTestId("workspaces-section");
},
getWorkspaceItem(name: string | RegExp) {
return Workspaces.getWorkspacesSection().find(`button[name="${name}"]`);
},
getWorkspaceItemStatus(name: string | RegExp) {
return Workspaces.getWorkspaceItem(name).findByTestId("workspace-status");
},
getWorkspaceItemActions(name: string | RegExp) {
return Workspaces.getWorkspaceItem(name).findByLabelText("More actions");
},
getWorkspaceNameInput() {
return cy.findByPlaceholderText("Workspace name");
},
getMergeWorkspaceButton() {
return cy.findByRole("button", { name: /Merge/ });
},
getWorkspaceDatabaseSelect() {
return cy.findByPlaceholderText("Select a database");
},
getRunWorkspaceButton() {
return Workspaces.getWorkspacePage().findByRole("button", {
name: /Run transforms/,
});
},
getTransformTargetButton() {
return cy.findByRole("button", { name: /Change target/ });
},
getRunTransformButton() {
return Workspaces.getWorkspaceContent().findByRole("button", {
name: /Run/,
});
},
getSaveTransformButton() {
return cy.findByRole("button", { name: /Save/ });
},
getWorkspaceTransforms() {
return cy.findByTestId("workspace-transforms");
},
getMainlandTransforms() {
return cy.findByTestId("mainland-transforms");
},
getNewWorkspaceButton() {
return cy.findByRole("button", { name: /New workspace/ });
},
getNewWorkspaceNameInput() {
return cy.findByPlaceholderText("Enter workspace name");
},
getNewWorkspaceDatabaseInput() {
return cy.findByPlaceholderText("Select a database");
},
};

View File

@@ -13,9 +13,9 @@ export * from "./e2e-comment-helpers";
export * from "./e2e-custom-column-helpers";
export * from "./e2e-dashboard-helpers";
export * from "./e2e-dashboard-visualizer-helpers";
export * from "./e2e-data-studio-helpers";
export * from "./e2e-database-metadata-helpers";
export * from "./e2e-datamodel-helpers";
export * from "./e2e-data-studio-helpers";
export * from "./e2e-dependency-graph-helpers";
export * from "./e2e-detail-view-helpers";
export * from "./e2e-dimension-list-helpers";
@@ -60,4 +60,5 @@ export * from "./e2e-upload-helpers";
export * from "./e2e-users-helpers";
export * from "./e2e-visual-tests-helpers";
export * from "./e2e-viz-settings-helpers";
export * from "./e2e-workspaces-helpers";
export * from "./embedding-sdk-helpers/constants";

View File

@@ -0,0 +1,391 @@
import dedent from "ts-dedent";
import { SAMPLE_DB_ID, WRITABLE_DB_ID } from "e2e/support/cypress_data";
import { SAMPLE_DATABASE } from "e2e/support/cypress_sample_database";
import type {
CardType,
PythonTransformTableAliases,
TransformTagId,
} from "metabase-types/api";
const { H } = cy;
const { DataStudio, Workspaces } = H;
const { Transforms } = DataStudio;
const { ORDERS_ID } = SAMPLE_DATABASE;
const DB_NAME = "Writable Postgres12";
const SOURCE_TABLE = "Animals";
const TARGET_TABLE = "transform_table";
const TARGET_TABLE_2 = "transform_table_2";
const TARGET_SCHEMA = "Schema A";
const TARGET_SCHEMA_2 = "Schema B";
const CUSTOM_SCHEMA = "custom_schema";
describe("scenarios > data studio > workspaces", () => {
beforeEach(() => {
H.restore("postgres-writable");
H.resetTestTable({ type: "postgres", table: "many_schemas" });
H.resetSnowplow();
cy.signInAsAdmin();
H.activateToken("bleeding-edge");
H.resyncDatabase({ dbId: WRITABLE_DB_ID, tableName: SOURCE_TABLE });
cy.intercept("POST", "/api/ee/workspace").as("createWorkspace");
// cy.intercept("PUT", "/api/field/*").as("updateField");
// cy.intercept("PUT", "/api/ee/transform/*").as("updateTransform");
// cy.intercept("DELETE", "/api/ee/transform/*").as("deleteTransform");
// cy.intercept("DELETE", "/api/ee/transform/*/table").as(
// "deleteTransformTable",
// );
// cy.intercept("POST", "/api/ee/transform-tag").as("createTag");
// cy.intercept("PUT", "/api/ee/transform-tag/*").as("updateTag");
// cy.intercept("DELETE", "/api/ee/transform-tag/*").as("deleteTag");
});
// afterEach(() => {
// H.expectNoBadSnowplowEvents();
// });
it("should be able to create, navigate, archive, unarchive, rename, and delete workspaces", () => {
Workspaces.visitDataStudio();
Workspaces.getWorkspacesSection()
.findByText("No workspaces yet")
.should("be.visible");
createWorkspace();
cy.wait("@createWorkspace").then((interception) => {
const workspaceName = interception.response?.body?.name;
// Backend returns randomized name.
cy.wrap(workspaceName).as("workspaceNameA");
});
cy.location("pathname").should("match", /data-studio\/workspaces\/\d+/);
cy.log("shows workspace name");
cy.get("@workspaceNameA").then((workspaceName) => {
Workspaces.getWorkspaceNameInput().should("have.value", workspaceName);
});
Workspaces.getMergeWorkspaceButton().should("be.disabled");
Workspaces.getWorkspaceContent().within(() => {
cy.log("Starts on setup tab, and has only 2 tabs");
H.tabsShouldBe("Setup", ["Setup", "Agent Chat"]);
cy.log("shows workspace db");
Workspaces.getWorkspaceDatabaseSelect()
.should("have.value", "Writable Postgres12")
.should("be.enabled");
cy.log("Doesn't show workspace setup logs");
cy.findByText("Setting up the workspace").should("not.exist");
});
Workspaces.getWorkspaceSidebar().within(() => {
cy.log("starts on Code tab, and has only 2 tabs");
H.tabsShouldBe("Code", ["Code", "Data"]);
cy.log("shows transforms lists");
cy.findByText("Workspace is empty").should("be.visible");
cy.findByText("No available transforms").should("be.visible");
});
createWorkspace();
// Workspaces.getNewWorkspaceButton().click();
cy.wait("@createWorkspace").then((interception) => {
const workspaceName = interception.response?.body?.name;
// Backend returns randomized name.
cy.wrap(workspaceName).as("workspaceNameB");
});
cy.get("@workspaceNameB").then((workspaceNameB: string) => {
Workspaces.getWorkspaceNameInput().should("have.value", workspaceNameB);
cy.get("@workspaceNameA").then((workspaceNameA) => {
Workspaces.getWorkspacesSection().within(() => {
cy.findByText(workspaceNameB).should("be.visible");
cy.findByText(workspaceNameA).should("be.visible").click();
});
cy.log("can archive a workspace");
Workspaces.getWorkspaceItemStatus(workspaceNameA).should(
"contain.text",
"Ready",
);
Workspaces.getWorkspaceItemActions(workspaceNameA).click();
H.popover().findByText("Archive").click();
verifyAndCloseToast("Workspace archived successfully");
cy.log("should show archived workspaces and their status");
Workspaces.getWorkspaceItem(workspaceNameA).should(
"contain.text",
"Archived",
);
// TODO: Move to another test, because we can't unarchive uninitialized workspace
// cy.log("can unarchive a workspace");
// Workspaces.getWorkspaceItemActions(workspaceNameA).click();
// H.popover().findByText("Restore").click();
// verifyAndCloseToast("Workspace restored successfully");
// Workspaces.getWorkspaceItem(workspaceNameA).should(
// "contain.text",
// "Ready",
// );
cy.log("can delete a workspace");
Workspaces.getWorkspaceItemActions(workspaceNameA).click();
H.popover().findByText("Delete").click();
H.modal().findByText("Delete").click();
verifyAndCloseToast("Workspace deleted successfully");
Workspaces.getWorkspaceItem(workspaceNameA).should("not.exist");
cy.location("pathname").should("match", /data-studio\/workspaces\/\d+/);
Workspaces.getWorkspaceNameInput().should("have.value", workspaceNameB);
Workspaces.getWorkspaceNameInput()
.clear()
.type("Renamed workspace")
.blur();
Workspaces.getWorkspaceItem("Renamed workspace").should("be.visible");
});
});
});
it("should be able to check out exisitng transform into a new workspace from the transform page", () => {
const sourceTable = `${TARGET_SCHEMA}.${SOURCE_TABLE}`;
createTransforms({ visit: true });
cy.findByRole("button", { name: /Edit transform/ }).click();
H.popover().within(() => {
cy.findByText("No workspaces yet").should("be.visible");
cy.findByText("New workspace").should("be.visible").click();
});
Workspaces.getNewWorkspaceNameInput().should("have.value", "New workspace");
Workspaces.getNewWorkspaceDatabaseInput().should(
"have.value",
"Writable Postgres12",
);
H.modal().findByText("Create").click();
cy.location("pathname").should("match", /data-studio\/workspaces\/\d+/);
Workspaces.getWorkspaceNameInput().should("have.value", "New workspace");
Workspaces.getWorkspacePage()
.findByText("Workspace is empty")
.should("not.exist");
Workspaces.getMainlandTransforms()
.findByText("Python transform")
.should("be.visible");
Workspaces.getMainlandTransforms()
.findByText("MBQL transform")
.should("not.exist");
Workspaces.getWorkspaceTransforms()
.findByText("SQL transform")
.should("be.visible")
.click();
Workspaces.getWorkspaceContent().within(() => {
H.tabsShouldBe("SQL transform", ["Setup", "Agent Chat", "SQL transform"]);
});
Workspaces.getMergeWorkspaceButton().should("be.enabled");
Workspaces.getRunWorkspaceButton().should("be.enabled");
Workspaces.getTransformTargetButton().should("be.enabled");
Workspaces.getRunTransformButton().should("be.enabled");
Workspaces.getSaveTransformButton().should("be.disabled");
H.NativeEditor.type(" LIMIT 2");
Workspaces.getMergeWorkspaceButton().should("be.disabled");
Workspaces.getRunWorkspaceButton().should("be.disabled");
Workspaces.getTransformTargetButton().should("be.disabled");
Workspaces.getRunTransformButton().should("be.disabled");
Workspaces.getSaveTransformButton().should("be.enabled").click();
Workspaces.getMergeWorkspaceButton().should("be.enabled");
Workspaces.getRunWorkspaceButton().should("be.enabled");
Workspaces.getTransformTargetButton().should("be.enabled");
Workspaces.getSaveTransformButton().should("be.disabled");
Workspaces.getRunTransformButton().should("be.enabled").click();
Workspaces.getWorkspaceSidebar().within(() => {
cy.findByRole("tab", { name: "Data" }).click();
cy.findByLabelText(sourceTable).should("be.visible").click();
});
Workspaces.getWorkspaceContent().within(() => {
H.tabsShouldBe(sourceTable, [
"Setup",
"Agent Chat",
"SQL transform",
sourceTable,
]);
H.assertTableData({
columns: ["Name", "Score"],
firstRows: [
["Duck", "10"],
["Horse", "20"],
["Cow", "30"],
],
});
});
Workspaces.getWorkspaceSidebar().within(() => {
cy.findByRole("tab", { name: "Data" }).click();
cy.findByLabelText(/mb__isolation_/)
.should("be.visible")
.click();
});
Workspaces.getWorkspaceContent().within(() => {
H.tabsShouldBe(/mb__isolation_/, [
"Setup",
"Agent Chat",
"SQL transform",
sourceTable,
/mb__isolation_/,
]);
H.assertTableData({
columns: ["Name", "Score"],
firstRows: [
["Duck", "10"],
["Horse", "20"],
],
});
cy.findByText("Cow").should("not.exist");
cy.findByText("30").should("not.exist");
});
Workspaces.getMergeWorkspaceButton().click();
cy.findByRole("link", { name: /SQL transform/ }).click();
H.NativeEditor.value().should(
"eq",
'SELECT * FROM "Schema A"."Animals" LIMIT 2',
);
Transforms.runTab().click();
runTransformAndWaitForSuccess();
Transforms.targetTab().click();
getTableLink().should("contain.text", "transform_table").click();
H.assertTableData({
columns: ["Name", "Score"],
firstRows: [
["Duck", "10"],
["Horse", "20"],
],
});
H.tableInteractiveBody().findByText("Cow").should("not.exist");
H.tableInteractiveBody().findByText("30").should("not.exist");
});
// it("should be able to check out existing transform into a new workspace from the workspace page", () => {
// createTransforms({ visit: true });
// });
});
function createWorkspace() {
Workspaces.getNewWorkspaceButton().click();
// H.modal().findByText("Create new workspace").should("be.visible");
// Workspaces.getNewWorkspaceNameInput().clear().type(name);
// Workspaces.getNewWorkspaceDatabaseInput().click();
// H.popover().within(() => {
// // cy.findByText("Internal Metabase Database").should("not.exist"); // TODO: uncomment once it works
// // cy.findByText("Sample Database").should("not.exist"); // TODO: uncomment once it works
// cy.findByText("Writable Postgres12").should("be.visible").click();
// });
// H.modal().findByText("Create").click();
}
function createTransforms({ visit }: { visit?: boolean } = {}) {
createMbqlTransform({
targetTable: TARGET_TABLE,
visitTransform: true,
});
H.getTableId({ name: "Animals", databaseId: WRITABLE_DB_ID }).then((id) => {
createPythonTransform({
body: dedent`
import pandas as pd
def transform(foo):
return pd.DataFrame([{"foo": 42 }])
`,
sourceTables: { foo: id },
});
});
createSqlTransform({
sourceQuery: `SELECT * FROM "${TARGET_SCHEMA}"."${SOURCE_TABLE}"`,
visitTransform: visit,
});
}
function createMbqlTransform(
opts: {
sourceTable?: string;
targetTable?: string;
targetSchema?: string | null;
tagIds?: TransformTagId[];
name?: string;
databaseId?: number;
visitTransform?: boolean;
} = {},
) {
return H.createMbqlTransform({
sourceTable: SOURCE_TABLE,
targetTable: TARGET_TABLE,
targetSchema: TARGET_SCHEMA,
name: "MBQL transform",
...opts,
});
}
function createSqlTransform(opts: {
sourceQuery: string;
targetTable?: string;
targetSchema?: string;
tagIds?: TransformTagId[];
visitTransform?: boolean;
}) {
return H.createSqlTransform({
targetTable: TARGET_TABLE,
targetSchema: TARGET_SCHEMA,
...opts,
});
}
function createPythonTransform(opts: {
body: string;
sourceTables: PythonTransformTableAliases;
targetTable?: string;
targetSchema?: string;
tagIds?: TransformTagId[];
visitTransform?: boolean;
}) {
return H.createPythonTransform({
targetTable: TARGET_TABLE,
targetSchema: TARGET_SCHEMA,
...opts,
});
}
function getTableLink({ isActive = true }: { isActive?: boolean } = {}) {
return cy
.findByTestId("table-link")
.should("have.attr", "aria-disabled", String(!isActive));
}
function runTransformAndWaitForSuccess() {
getRunButton().click();
getRunButton().should("have.text", "Ran successfully");
}
function getRunButton(options: { timeout?: number } = {}) {
return cy.findAllByTestId("run-button").eq(0, options);
}
function verifyAndCloseToast(message: string) {
H.undoToast().should("contain.text", message);
H.undoToast().icon("close").click({ force: true });
}

View File

@@ -40,6 +40,7 @@
[metabase-enterprise.transforms-python.api]
[metabase-enterprise.transforms.api]
[metabase-enterprise.upload-management.api]
[metabase-enterprise.workspaces.api]
[metabase.api.macros :as api.macros]
[metabase.api.util.handlers :as handlers]
[metabase.util.i18n :refer [deferred-tru]]))
@@ -134,7 +135,8 @@
"/transform-job" (premium-handler metabase-enterprise.transforms.api/transform-job-routes :transforms)
"/transform-tag" (premium-handler metabase-enterprise.transforms.api/transform-tag-routes :transforms)
"/tenant" (premium-handler metabase-enterprise.tenants.api/routes :tenants)
"/upload-management" (premium-handler metabase-enterprise.upload-management.api/routes :upload-management)})
"/upload-management" (premium-handler metabase-enterprise.upload-management.api/routes :upload-management)
"/workspace" metabase-enterprise.workspaces.api/routes})
;;; ↑↑↑ KEEP THIS SORTED OR ELSE ↑↑↑
(def ^:private routes-map

View File

@@ -22,4 +22,5 @@
[metabase-enterprise.stale.init]
[metabase-enterprise.support-access-grants.init]
[metabase-enterprise.transforms-python.init]
[metabase-enterprise.transforms.init]))
[metabase-enterprise.transforms.init]
[metabase-enterprise.workspaces.init]))

View File

@@ -42,6 +42,12 @@
query-deps
param-card-ids)))
(mu/defn upstream-deps:python-transform :- ::deps.schema/upstream-deps
"Given a Toucan `:model/Transform`, return its upstream dependencies as a map from the kind to a set of IDs."
[{{tables :source-tables} :source :as _py-transform}
:- [:map [:source-tables {:optional true} [:map-of :string :int]]]]
{:table (set (vals tables))})
(mu/defn upstream-deps:transform :- ::deps.schema/upstream-deps
"Given a Transform (in Toucan form), return its upstream dependencies."
[{{:keys [query], source-type :type} :source :as transform} :-
@@ -50,9 +56,15 @@
[:query
[:map [:query ::lib.schema/query]]]
[:python
[:map]]]]]]
(if (= (keyword source-type) :query)
[:map [:source-tables {:optional true} [:map-of :string :int]]]]]]]]
(cond
(= (keyword source-type) :query)
(upstream-deps:query query)
(= (keyword source-type) :python)
(upstream-deps:python-transform transform)
:else
(do (log/warnf "Don't know how to analyze the deps of Transform %d with source type '%s'" (:id transform) source-type)
{})))

View File

@@ -116,4 +116,16 @@
"User"
"UserKeyValue"
"UserParameterValue"
"ViewLog"])
"ViewLog"
"Workspace"
;; TODO (lbrdnk 2025/12/17): I've added rest of the workspace models. I believe some of those will be available for
;; serialization, hence we should revisit this later in the project.
"WorkspaceDependency"
"WorkspaceInput"
"WorkspaceInputExternal"
"WorkspaceLog"
"WorkspaceMerge"
"WorkspaceMergeTransform"
"WorkspaceOutput"
"WorkspaceOutputExternal"
"WorkspaceTransform"])

View File

@@ -81,9 +81,13 @@
(defn get-transforms
"Get a list of transforms."
[& {:keys [last_run_start_time last_run_statuses tag_ids]}]
[& {:keys [last_run_start_time last_run_statuses tag_ids database_id type]}]
(api/check-superuser)
(let [transforms (t2/select :model/Transform {:order-by [[:id :asc]]})]
(let [type (when type (map #(if (= % "query") "mbql" %) type))
where (cond-> [:and]
type (conj [:in :source_type type])
database_id (conj [:= :target_db_id database_id]))
transforms (t2/select :model/Transform {:where where :order-by [[:id :asc]]})]
(into []
(comp (transforms.util/->date-field-filter-xf [:last_run :start_time] last_run_start_time)
(transforms.util/->status-filter-xf [:last_run :status] last_run_statuses)
@@ -106,14 +110,35 @@
[:map
[:last_run_start_time {:optional true} [:maybe ms/NonBlankString]]
[:last_run_statuses {:optional true} [:maybe (ms/QueryVectorOf [:enum "started" "succeeded" "failed" "timeout"])]]
[:tag_ids {:optional true} [:maybe (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]]]]
[:tag_ids {:optional true} [:maybe (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]]
[:database_id {:optional true} [:maybe ms/PositiveInt]]
[:type {:optional true} [:maybe (ms/QueryVectorOf [:enum "query" "native" "python"])]]]]
(get-transforms query-params))
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
;;
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
(api.macros/defendpoint :post "/"
(defn create-transform!
"Create new transform in the appdb.
Optionally accepts a creator-id to use instead of the current user (for workspace merges)."
([body]
(create-transform! body nil))
([body creator-id]
(let [creator-id (or creator-id api/*current-user-id*)
transform (t2/with-transaction [_]
(let [tag-ids (:tag_ids body)
transform (t2/insert-returning-instance!
:model/Transform
(assoc (select-keys body [:name :description :source :target :run_trigger])
:creator_id creator-id))]
;; Add tag associations if provided
(when (seq tag-ids)
(transform.model/update-transform-tags! (:id transform) tag-ids))
;; Return with hydrated tag_ids
(t2/hydrate transform :transform_tag_ids :creator)))]
(events/publish-event! :event/transform-create {:object transform :user-id api/*current-user-id*})
transform)))
;; TODO (chris 2025/12/16) fully populate the result schema
;; TODO (lbrdnk 2025/12/16) relaxed result schema to unblock FE. This should be properly handled later.
(api.macros/defendpoint :post "/" :- [:map [:name :string] [:description {:optional true} [:maybe :string]]]
"Create a new transform."
[_route-params
_query-params
@@ -183,6 +208,34 @@
dependencies (map id->transform dep-ids)]
(t2/hydrate dependencies :creator)))
(def ^:private MergeHistoryEntry
[:map
[:id ms/PositiveInt]
[:workspace_merge_id ms/PositiveInt]
[:commit_message :string]
[:workspace_id [:maybe ms/PositiveInt]]
[:workspace_name :string]
[:merging_user_id ms/PositiveInt]
[:created_at :any]])
(api.macros/defendpoint :get "/:id/merge-history"
:- [:sequential MergeHistoryEntry]
"Get merge history for a transform. Returns all merge events that affected this transform,
ordered by created_at descending (newest first)."
[{:keys [id]} :- [:map [:id ms/PositiveInt]]]
(api/check-superuser)
(api/check-404 (t2/select-one :model/Transform id))
(t2/select [:model/WorkspaceMergeTransform
:id
:workspace_merge_id
:commit_message
:workspace_id
:workspace_name
:merging_user_id
:created_at]
{:where [:= :transform_id id]
:order-by [[:created_at :desc]]}))
;; TODO (Cam 10/28/25) -- fix this endpoint so it uses kebab-case for query parameters for consistency with the rest
;; of the REST API
;;
@@ -210,24 +263,10 @@
:limit (request/limit)))
(update :data #(map transforms.util/localize-run-timestamps %))))
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
;;
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
(api.macros/defendpoint :put "/:id"
"Update a transform."
[{:keys [id]} :- [:map
[:id ms/PositiveInt]]
_query-params
body :- [:map
[:name {:optional true} :string]
[:description {:optional true} [:maybe :string]]
[:source {:optional true} ::transforms.schema/transform-source]
[:target {:optional true} ::transforms.schema/transform-target]
[:run_trigger {:optional true} ::run-trigger]
[:tag_ids {:optional true} [:sequential ms/PositiveInt]]
[:collection_id {:optional true} [:maybe ms/PositiveInt]]]]
(api/check-superuser)
(defn update-transform!
"Update a transform. Validates features, database support, cycles, and target conflicts.
Returns the updated transform with hydrated associations."
[id body]
(let [transform (t2/with-transaction [_]
;; Cycle detection should occur within the transaction to avoid race
(let [old (t2/select-one :model/Transform id)
@@ -252,6 +291,35 @@
(events/publish-event! :event/transform-update {:object transform :user-id api/*current-user-id*})
transform))
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
;;
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
(api.macros/defendpoint :put "/:id"
"Update a transform."
[{:keys [id]} :- [:map
[:id ms/PositiveInt]]
_query-params
body :- [:map
[:name {:optional true} :string]
[:description {:optional true} [:maybe :string]]
[:source {:optional true} ::transforms.schema/transform-source]
[:target {:optional true} ::transforms.schema/transform-target]
[:run_trigger {:optional true} ::run-trigger]
[:tag_ids {:optional true} [:sequential ms/PositiveInt]]
[:collection_id {:optional true} [:maybe ms/PositiveInt]]]]
(api/check-superuser)
(update-transform! id body))
(defn delete-transform!
"Delete a transform and publish the delete event."
[transform]
(t2/delete! :model/Transform (:id transform))
(events/publish-event! :event/transform-delete
{:object transform
:user-id api/*current-user-id*})
nil)
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
;;
@@ -261,12 +329,7 @@
[{:keys [id]} :- [:map
[:id ms/PositiveInt]]]
(api/check-superuser)
(let [transform (api/check-404 (t2/select-one :model/Transform id))]
(t2/delete! :model/Transform id)
(events/publish-event! :event/transform-delete
{:object transform
:user-id api/*current-user-id*}))
nil)
(delete-transform! (api/check-404 (t2/select-one :model/Transform id))))
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
@@ -296,18 +359,12 @@
(transforms.canceling/cancel-run! (:id run))))
nil)
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
;;
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
(api.macros/defendpoint :post "/:id/run"
"Run a transform."
[{:keys [id]} :- [:map
[:id ms/PositiveInt]]]
(api/check-superuser)
(let [transform (api/check-404 (t2/select-one :model/Transform id))
_ (check-feature-enabled! transform)
start-promise (promise)]
(defn run-transform!
"Run a transform. Returns a 202 response with run_id.
The transform must already be fetched and validated."
[transform]
(check-feature-enabled! transform)
(let [start-promise (promise)]
(u.jvm/in-virtual-thread*
(transforms.execute/execute! transform {:start-promise start-promise
:run-method :manual}))
@@ -320,6 +377,17 @@
:run_id run-id})
(assoc :status 202)))))
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
;;
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
(api.macros/defendpoint :post "/:id/run"
"Run a transform."
[{:keys [id]} :- [:map
[:id ms/PositiveInt]]]
(api/check-superuser)
(run-transform! (api/check-404 (t2/select-one :model/Transform id))))
(defn- extract-columns-from-query
"Attempts to extract column names from an MBQL query without executing it.

View File

@@ -1,6 +1,7 @@
(ns metabase-enterprise.transforms.core
"API namespace for the `metabase-enterprise.transform` module."
(:require
[metabase-enterprise.transforms.models.transform]
[metabase-enterprise.transforms.models.transform-run]
[metabase-enterprise.transforms.settings]
[metabase-enterprise.transforms.util]
@@ -12,6 +13,9 @@
[metabase-enterprise.transforms.util
native-query-transform?
python-transform?
query-transform?]
query-transform?
transform-source-type]
[metabase-enterprise.transforms.models.transform-run
timeout-run!])
timeout-run!]
[metabase-enterprise.transforms.models.transform
update-transform-tags!])

View File

@@ -60,7 +60,7 @@
(collection/check-allowed-content :model/Transform collection_id))
(assoc transform
:source_type (transforms.util/transform-source-type source)
:target_db_id (or (:database target) (:target_db_id transform))))
:target_db_id (or (transforms.i/target-db-id transform) (:database target) (:target_db_id transform))))
(t2/define-before-update :model/Transform
[{:keys [source target] :as transform}]

View File

@@ -21,7 +21,9 @@
(defmethod transforms.i/target-db-id :query
[transform]
(or (get-in transform [:target :database])
(:target_db_id transform)))
(:target_db_id transform)
;; Fallback to source. https://github.com/metabase/metabase/pull/67084#discussion_r2630147855
(-> transform :source :query :database)))
(mr/def ::transform-details
[:map

View File

@@ -45,14 +45,14 @@
[:map
[:database {:optional true} :int]
[:type [:= "table"]]
[:schema {:optional true} [:or ms/NonBlankString :nil]]
[:schema {:optional true} [:maybe ms/NonBlankString]]
[:name :string]])
(mr/def ::table-incremental-target
[:map
[:database {:optional true} :int]
[:type [:= "table-incremental"]]
[:schema {:optional true} [:or ms/NonBlankString :nil]]
[:schema {:optional true} [:maybe ms/NonBlankString]]
[:name :string]
[:target-incremental-strategy ::target-incremental-strategy]])

View File

@@ -436,33 +436,28 @@
(defn- matching-timestamp?
[job field-path {:keys [start end]}]
(when-let [field-instant (->instant (get-in job field-path))]
(let [start-instant (some-> start u.date/parse ->instant)
end-instant (some-> end u.date/parse ->instant)]
(and (or (nil? start)
(not (.isBefore field-instant start-instant)))
(or (nil? end)
(.isAfter end-instant field-instant))))))
(let [parse #(-> % u.date/parse ->instant)]
;; logic here is to find when it's not matching and invert this
(not (or (and start (.isBefore field-instant (parse start)))
(and end (.isAfter field-instant (parse end))))))))
(defn ->date-field-filter-xf
"Returns an xform for a date filter."
[field-path filter-value]
(let [range (some-> filter-value (params.dates/date-string->range {:inclusive-end? false}))]
(if range
(filter #(matching-timestamp? % field-path range))
identity)))
(if-let [range (some-> filter-value (params.dates/date-string->range {:inclusive-end? false}))]
(filter #(matching-timestamp? % field-path range))
identity))
(defn ->status-filter-xf
"Returns an xform for a transform run status filter."
[field-path statuses]
(let [statuses (->> statuses (map keyword) set not-empty)]
(if statuses
(filter #(statuses (get-in % field-path)))
identity)))
(if-let [statuses (->> statuses (map keyword) set not-empty)]
(filter #(statuses (get-in % field-path)))
identity))
(defn ->tag-filter-xf
"Returns an xform for a transform tag filter."
[field-path tag-ids]
(let [tag-ids (-> tag-ids set not-empty)]
(if tag-ids
(filter #(some tag-ids (get-in % field-path)))
identity)))
(if-let [tag-ids (-> tag-ids set not-empty)]
(filter #(some tag-ids (get-in % field-path)))
identity))

View File

@@ -12,6 +12,7 @@
[metabase.app-db.core :as app-db]
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
[metabase.events.core :as events]
[metabase.util :as u]
[metabase.util.format :as u.format]
[metabase.util.i18n :as i18n]
@@ -360,6 +361,7 @@
(log/info "Executing Python transform" transform-id "with target" (pr-str target))
(let [start-ms (u/start-timer)
transform-details {:db-id (:id db)
:transform-id transform-id
:transform-type (keyword (:type target))
:conn-spec (driver/connection-spec driver db)
:output-schema (:schema target)
@@ -372,7 +374,8 @@
result (transforms.instrumentation/with-stage-timing [run-id [:computation :python-execution]]
(transforms.util/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn))]
(transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]
(transforms.util/sync-target! target db))
(transforms.util/sync-target! target db)
(events/publish-event! :event/transform-run-complete {:object transform-details}))
{:run_id run-id
:result result}))
(catch Throwable t

View File

@@ -10,7 +10,7 @@
(defmethod transforms.i/source-db-id :python
[transform]
(-> transform :source :database))
(-> transform :source :source-database))
#_{:clj-kondo/ignore [:discouraged-var]}
(defmethod transforms.i/execute! :python

View File

@@ -0,0 +1,872 @@
(ns metabase-enterprise.workspaces.api
"`/api/ee/workspace/` routes"
(:require
[clojure.set :as set]
[clojure.string :as str]
[honey.sql.helpers :as sql.helpers]
[metabase-enterprise.transforms.core :as transforms]
[metabase-enterprise.transforms.util :as transforms.util]
[metabase-enterprise.workspaces.common :as ws.common]
[metabase-enterprise.workspaces.impl :as ws.impl]
[metabase-enterprise.workspaces.merge :as ws.merge]
[metabase-enterprise.workspaces.models.workspace :as ws.model]
[metabase-enterprise.workspaces.models.workspace-input-external]
[metabase-enterprise.workspaces.models.workspace-log]
[metabase-enterprise.workspaces.models.workspace-output-external]
[metabase-enterprise.workspaces.types :as ws.t]
[metabase-enterprise.workspaces.validation :as ws.validation]
[metabase.api.common :as api]
[metabase.api.macros :as api.macros]
[metabase.api.routes.common :refer [+auth]]
[metabase.driver.util :as driver.u]
[metabase.lib.core :as lib]
[metabase.queries.schema :as queries.schema]
[metabase.request.core :as request]
[metabase.util :as u]
[metabase.util.i18n :as i18n :refer [deferred-tru tru]]
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
(def ^:private log-limit "Maximum number of recent workspace log items to show" 20)
(mr/def ::appdb-or-ref-id [:or ::ws.t/appdb-id ::ws.t/ref-id])
(mr/def ::status [:enum :uninitialized :pending :ready] #_[:enum :uninitialized :database-not-read :graph-not-ready :ready])
(def ^:private Workspace
[:map
[:id ::ws.t/appdb-id]
[:name :string]
[:collection_id ::ws.t/appdb-id]
[:database_id ::ws.t/appdb-id]
[:status ::status]
[:created_at ms/TemporalInstant]
[:updated_at ms/TemporalInstant]
[:archived_at [:maybe :any]]])
;; Transform-related schemas (adapted from transforms/api.clj)
;; TODO we should reuse these schemas, by exposing common types from the transforms module. they *can* match exactly.
(mr/def ::transform-source
[:multi {:dispatch (comp keyword :type)}
[:query
[:map
[:type [:= "query"]]
[:query ::queries.schema/query]]]
[:python
[:map {:closed true}
[:source-database {:optional true} :int]
[:source-tables [:map-of [:string {:min 1}] :int]]
[:type [:= "python"]]
[:body :string]]]])
(mr/def ::transform-target
[:map {:closed true}
[:database {:optional true} ::ws.t/appdb-id]
[:type [:enum "table"]]
[:schema {:optional true} [:maybe [:string {:min 1}]]]
[:name [:string {:min 1}]]])
(mr/def ::run-trigger
[:enum "none" "global-schedule"])
(defn- check-transforms-enabled!
[db-id]
(let [database (api/check-400 (t2/select-one :model/Database db-id)
(deferred-tru "The target database cannot be found."))]
(api/check (transforms.util/check-feature-enabled nil)
[402 (deferred-tru "Premium features required for transforms are not enabled.")])
(api/check-400 (not (:is_sample database))
(deferred-tru "Cannot run transforms on the sample database."))
(api/check-400 (not (:is_audit database))
(deferred-tru "Cannot run transforms on audit databases."))
(api/check-400 (driver.u/supports? (:engine database) :transforms/table database)
(deferred-tru "The database does not support the requested transform target type."))
(api/check-400 (not (transforms.util/db-routing-enabled? database))
(deferred-tru "Transforms are not supported on databases with DB routing enabled."))))
(defn- ws->response [ws]
(select-keys ws [:id :name :collection_id :database_id :status :created_at :updated_at :archived_at]))
;;; routes
(def ^:private WorkspaceListing
[:map {:closed true}
[:id ::ws.t/appdb-id]
[:database_id ::ws.t/appdb-id]
[:name :string]
[:archived :boolean]])
(api.macros/defendpoint :get "/" :- [:map {:closed true}
[:items [:sequential WorkspaceListing]]
[:limit [:maybe :int]]
[:offset [:maybe :int]]]
"Get a list of all workspaces"
[_route-params
_query-params]
{:items (t2/select [:model/Workspace :id :name :database_id [[:not= nil :archived_at] :archived]]
(cond-> {:order-by [[:created_at :desc]]}
(request/limit) (sql.helpers/limit (request/limit))
(request/offset) (sql.helpers/offset (request/offset))))
:limit (request/limit)
:offset (request/offset)})
(mr/def ::input-table
[:map
;; Future-proof with cross-db Python transforms
[:db_id ::ws.t/appdb-id]
[:schema [:maybe :string]]
[:table :string]
[:table_id [:maybe ::ws.t/appdb-id]]])
(mr/def ::output-table
[:map
;; Future-proof with multi-db Workspaces, plus necessary to resolve table references when id is null.
[:db_id ::ws.t/appdb-id]
[:global [:map
;; transform_id is nil for workspace outputs, int for external outputs
[:transform_id [:maybe ::ws.t/appdb-id]]
[:schema [:maybe :string]]
[:table :string]
[:table_id [:maybe ::ws.t/appdb-id]]]]
[:isolated [:map
;; transform_id is ref_id (string) for workspace outputs, int for external outputs
[:transform_id [:or ::ws.t/ref-id ::ws.t/appdb-id]]
[:schema :string]
[:table :string]
[:table_id [:maybe ::ws.t/appdb-id]]]]])
(defn- batch-lookup-table-ids
"Given a bounded list of tables, all within the same database, return an association list of [db schema table] => id"
[db-id schema-key table-key table-refs]
(when (seq table-refs)
(t2/select-fn-vec (juxt (juxt (constantly db-id) :schema :name) :id)
[:model/Table :id :schema :name]
:db_id db-id
{:where (into [:or] (for [tr table-refs]
[:and
[:= :schema (get tr schema-key)]
[:= :name (get tr table-key)]]))})))
(defn- table-ids-fallbacks
"Given a list of maps holding [db_id schema table], return a mapping from those tuples => table_id"
[schema-key table-key id-key table-refs]
(when-let [table-refs (seq (remove id-key table-refs))]
;; These are ordered by db, so this will partition fine.
(u/for-map [table-refs (partition-by :db_id table-refs)
:let [db_id (:db_id (first table-refs))]
;; Guesstimating a number that prevents this query being too large.
table-refs (partition-all 20 table-refs)
map-entry (batch-lookup-table-ids db_id schema-key table-key table-refs)]
map-entry)))
(api.macros/defendpoint :get "/:id/table"
:- [:map {:closed true}
[:inputs [:sequential ::input-table]]
[:outputs [:sequential ::output-table]]]
"Get workspace tables"
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params]
(api/check-404 (t2/select-one :model/Workspace :id id))
(let [order-by {:order-by [:db_id :global_schema :global_table]}
outputs (t2/select [:model/WorkspaceOutput
:db_id :global_schema :global_table :global_table_id
:isolated_schema :isolated_table :isolated_table_id :ref_id]
:workspace_id id order-by)
external-outputs (t2/select [:model/WorkspaceOutputExternal
:db_id :global_schema :global_table :global_table_id
:isolated_schema :isolated_table :isolated_table_id :transform_id]
:workspace_id id order-by)
all-outputs (concat outputs external-outputs)
raw-inputs (t2/select [:model/WorkspaceInput :db_id :schema :table :table_id]
:workspace_id id {:order-by [:db_id :schema :table]})
external-inputs (t2/select [:model/WorkspaceInputExternal :db_id :schema :table :table_id]
:workspace_id id {:order-by [:db_id :schema :table]})
all-raw-inputs (concat raw-inputs external-inputs)
;; Some of our inputs may be shadowed by the outputs of other transforms. We only want external inputs.
shadowed? (into #{} (map (juxt :db_id :global_schema :global_table)) all-outputs)
inputs (remove (comp shadowed? (juxt :db_id :schema :table)) all-raw-inputs)
;; Build a map of [d s t] => id for every table that has been synced since the output row was written.
fallback-map (merge
(table-ids-fallbacks :global_schema :global_table :global_table_id all-outputs)
(table-ids-fallbacks :isolated_schema :isolated_table :isolated_table_id all-outputs))]
{:inputs (sort-by (juxt :db_id :schema :table) inputs)
:outputs (sort-by
(juxt :db_id (comp (juxt :schema :table) :global))
(concat
;; Workspace transform outputs
(for [{:keys [ref_id db_id global_schema global_table global_table_id
isolated_schema isolated_table isolated_table_id]} outputs]
{:db_id db_id
:global {:transform_id nil
:schema global_schema
:table global_table
:table_id (or global_table_id (get fallback-map [db_id global_schema global_table]))}
:isolated {:transform_id ref_id
:schema isolated_schema
:table isolated_table
:table_id (or isolated_table_id (get fallback-map [db_id isolated_schema isolated_table]))}})
;; External transform outputs
(for [{:keys [transform_id db_id global_schema global_table global_table_id
isolated_schema isolated_table isolated_table_id]} external-outputs]
{:db_id db_id
:global {:transform_id transform_id
:schema global_schema
:table global_table
:table_id (or global_table_id (get fallback-map [db_id global_schema global_table]))}
:isolated {:transform_id transform_id
:schema isolated_schema
:table isolated_table
:table_id (or isolated_table_id (get fallback-map [db_id isolated_schema isolated_table]))}})))}))
(api.macros/defendpoint :get "/:id" :- Workspace
"Get a single workspace by ID"
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params]
(-> (t2/select-one :model/Workspace :id id) api/check-404 ws->response))
(api.macros/defendpoint :get "/:id/log"
:- [:map
[:workspace_id ms/PositiveInt]
[:status ::status]
[:updated_at :any]
[:last_completed_at [:maybe :any]]
[:logs [:sequential [:map
[:id ms/PositiveInt]
[:task :keyword]
[:description ms/LocalizedString]
[:started_at :any]
[:updated_at :any]
[:completed_at [:maybe :any]]
[:status [:maybe :keyword]]
[:message [:maybe :string]]]]]]
"Get workspace creation status and recent log entries for polling during async setup"
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params]
(let [workspace (api/check-404 (t2/select-one :model/Workspace :id id))
logs (t2/select [:model/WorkspaceLog
:id :task :started_at :completed_at :status :message
:updated_at]
:workspace_id id
{:order-by [[:started_at :desc]]
:limit log-limit})]
{:workspace_id id
:status (:status workspace)
:logs logs
:updated_at (->> (map :updated_at logs) sort reverse first)
:last_completed_at (->> (seq (keep :completed_at logs)) sort reverse first)}))
(def ^:private CreateWorkspace
[:map
[:name {:optional true} [:string {:min 1}]]
[:database_id {:optional true} ::ws.t/appdb-id]])
(defn- first-supported-database-id
"Return the ID of the first database that supports workspaces, or nil if none."
[]
(:id
(u/seek #(driver.u/supports? (:engine %) :workspace %)
(t2/select :model/Database :is_audit false :is_sample false {:order-by [:name]}))))
(api.macros/defendpoint :post "/" :- Workspace
"Create a new workspace
Potential payload:
{:name \"a\" :database_id 2}}"
[_route-params
_query-params
{:keys [database_id] :as body} :- CreateWorkspace]
(when database_id
(check-transforms-enabled! database_id))
;; If no database_id provided, use first supported DB as provisional default (uninitialized workspace)
(let [provisional? (not database_id)
database-id (or database_id
(api/check-400 (first-supported-database-id)
(tru "No supported databases configured. Please add a database that supports workspaces.")))]
(ws->response (ws.common/create-workspace! api/*current-user-id*
(assoc body
:database_id database-id
:provisional? provisional?)))))
(defn- db-unsupported-reason [db]
(when (not (driver.u/supports? (:engine db) :workspace db))
"Database type not supported."))
(api.macros/defendpoint :get "/enabled" :- [:map [:supported :boolean] [:reason {:optional true} :string]]
"Test whether the current user can use Workspaces. Optionally takes a specific database.
Factors include: driver support, database user privileges, metabase permissions."
[_url-params
{:keys [database-id]} :- [:map [:database-id {:optional true} ms/PositiveInt]]]
(if-let [reason (or (when (not api/*is-superuser?*)
(tru "Not allowed."))
(when database-id
(db-unsupported-reason (api/check-404 (t2/select-one :model/Database database-id)))))]
{:supported false, :reason reason}
{:supported true}))
(api.macros/defendpoint :get "/database" :- [:map
[:databases
[:sequential [:map
[:id ms/PositiveInt]
[:name :string]
[:supported :boolean]
[:reason {:optional true} :string]]]]]
"Get a list of databases to show in the workspace picker, along with whether they're supported."
[_url-params
_query-params]
{:databases (->> (t2/select :model/Database :is_audit false :is_sample false {:order-by [:name]})
;; Omit those we don't even support
(filter #(driver.u/supports? (:engine %) :workspace %))
(mapv (fn [db]
(merge (select-keys db [:id :name])
(if-let [reason (db-unsupported-reason db)]
{:supported false, :reason reason}
{:supported true})))))})
(api.macros/defendpoint :put "/:id" :- Workspace
"Update simple workspace properties.
Can set database_id only on uninitialized workspaces."
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params
{:keys [name database_id]} :- [:map {:closed true}
[:name {:optional true} [:string {:min 1}]]
[:database_id {:optional true} ::ws.t/appdb-id]]]
(let [workspace (api/check-404 (t2/select-one :model/Workspace :id id))
_ (api/check-400 (nil? (:archived_at workspace)) "Cannot update an archived workspace")
data (cond-> {}
database_id (-> (u/prog1
(api/check-400 (= :uninitialized (:status workspace))
"Can only set database_id on uninitialized workspace")
(check-transforms-enabled! database_id))
(assoc :database_id database_id))
name (assoc :name name))]
(ws->response
(if (seq data)
(do
(t2/update! :model/Workspace id data)
(t2/select-one :model/Workspace :id id))
workspace))))
(api.macros/defendpoint :post "/:id/archive" :- Workspace
"Archive a workspace. Deletes the isolated schema and tables, but preserves mirrored entities."
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params
_body-params]
(let [ws (api/check-404 (t2/select-one :model/Workspace :id id))]
(api/check-400 (nil? (:archived_at ws)) "You cannot archive an archived workspace")
(ws.model/archive! ws)
(-> (t2/select-one :model/Workspace :id id)
ws->response)))
(api.macros/defendpoint :post "/:id/unarchive" :- Workspace
"Restore an archived workspace. Recreates the isolated schema and tables."
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params
_body-params]
(let [ws (api/check-404 (t2/select-one :model/Workspace :id id))]
(api/check-400 (some? (:archived_at ws)) "You cannot unarchive a workspace that is not archived")
(ws.model/unarchive! ws)
(-> (t2/select-one :model/Workspace :id id)
ws->response)))
(api.macros/defendpoint :delete "/:ws-id" :- [:map [:ok [:= true]]]
"Delete a workspace and all its contents, including mirrored entities."
[{:keys [ws-id]} :- [:map [:ws-id ms/PositiveInt]]
_query-params]
(let [ws (api/check-404 (t2/select-one :model/Workspace :id ws-id))]
(api/check-400 (some? (:archived_at ws)) "You cannot delete a workspace without first archiving it")
(ws.model/delete! ws)
{:ok true}))
(def ^:private ExternalTransform
;; Might be interesting to show whether they're enclosed, once we have the graph.
;; When they're enclosed, it could also be interesting to know whether they're stale.
[:map
[:id ::ws.t/appdb-id]
[:name :string]
[:source_type :keyword]
[:checkout_disabled [:maybe :string]]])
(api.macros/defendpoint :get "/:ws-id/external/transform" :- [:map [:transforms [:sequential ExternalTransform]]]
"Get transforms that are external to the workspace, i.e. no matching workspace_transform row exists."
[{:keys [ws-id]} :- [:map [:ws-id ::ws.t/appdb-id]]
{:keys [database-id]} :- [:map [:database-id {:optional true} ::ws.t/appdb-id]]]
(api/check-superuser)
(let [db-id (or database-id
(:database_id (api/check-404 (t2/select-one [:model/Workspace :database_id] ws-id))))
transforms (t2/select [:model/Transform :id :name :source_type :source]
{:left-join [[:workspace_transform :wt]
[:and
[:= :transform.id :wt.global_id]
[:= :wt.workspace_id ws-id]]]
;; NULL workspace_id means transform is not checked out into this workspace
:where [:and
[:= nil :wt.workspace_id]
[:= :target_db_id db-id]]
:order-by [[:id :asc]]})]
{:transforms
(into []
(map #(-> %
(dissoc :source)
(assoc :checkout_disabled (case (:source_type %)
:mbql "mbql"
:native (when (seq (lib/template-tags-referenced-cards (:query (:source %))))
"card-reference")
:python nil
"unknown-type"))))
;; (Sanya 2025-12-24) - do we need this line?
#_(map #(update % :last_run transforms.util/localize-run-timestamps))
transforms
;; Perhaps we want to expose some of this later?
#_(t2/hydrate transforms :last_run :creator))}))
(api.macros/defendpoint :post "/:ws-id/run"
:- [:map
[:succeeded [:sequential ::ws.t/ref-id]]
[:failed [:sequential ::ws.t/ref-id]]
[:not_run [:sequential ::ws.t/ref-id]]]
"Execute all transforms in the workspace in dependency order.
Returns which transforms succeeded, failed, and were not run."
[{:keys [ws-id]} :- [:map [:ws-id ::ws.t/appdb-id]]
_query-params
;; Hmmm, I wonder why this isn't a boolean? T_T
{:keys [stale_only]} :- [:map [:stale_only {:optional true} [:or [:= 1] :boolean]]]]
(let [workspace (t2/select-one :model/Workspace :id ws-id)]
(api/check-404 workspace)
(api/check-400 (nil? (:archived_at workspace)) "Cannot execute archived workspace")
(ws.impl/execute-workspace! workspace {:stale-only stale_only})))
(mr/def ::graph-node-type [:enum :input-table :external-transform :workspace-transform])
(mr/def ::graph-node
[:map
[:id ::appdb-or-ref-id]
[:type [:enum :input-table :external-transform :workspace-transform]]
[:dependents_count [:map-of ::graph-node-type ms/PositiveInt]]
[:data :map]])
(mr/def ::graph-edge
[:map
[:from_entity_id ::appdb-or-ref-id]
[:from_entity_type :string]
[:to_entity_id ::appdb-or-ref-id]
[:to_entity_type :string]])
(def ^:private GraphResult
[:map
[:nodes [:sequential ::graph-node]]
[:edges [:sequential ::graph-edge]]])
(defn- node-type [node]
(let [nt (:node-type node)]
(case nt
:table :input-table
nt)))
(defn- node-id [{:keys [node-type id]}]
(case node-type
:workspace-transform id
:external-transform id
:table (str (:db id) "-" (:schema id) "-" (:table id))))
;; TODO we'll want to bulk query this of course...
(defn- node-data [{:keys [node-type id]}]
(case node-type
:table id
:external-transform (t2/select-one [:model/Transform :id :name] id)
;; TODO we'll want to select by workspace as well here, to relax uniqueness assumption
:workspace-transform (t2/select-one [:model/WorkspaceTransform :ref_id :name] :ref_id id)))
(api.macros/defendpoint :get "/:ws-id/graph" :- GraphResult
"Display the dependency graph between the Changeset and the (potentially external) entities that they depend on."
[{:keys [ws-id]} :- [:map [:ws-id ms/PositiveInt]]
_query-params]
(let [workspace (api/check-404 (t2/select-one :model/Workspace :id ws-id))
{:keys [inputs entities dependencies]} (ws.impl/get-or-calculate-graph workspace)
;; TODO Graph analysis doesn't return this currently, we need to invert the deps graph
;; It could be cheaper to build it as we go.
inverted (reduce
(fn [inv [c parents]]
(reduce (fn [inv p] (update inv p (fnil conj #{}) c)) inv parents))
{}
dependencies)
dep-count #(frequencies (map node-type (get inverted %)))]
{:nodes (concat (for [i inputs]
{:type :input-table
;; Use an id that's independent of whether the table exists yet.
:id (node-id {:id i, :node-type :table})
:data i
:dependents_count (dep-count {:node-type :table, :id i})})
(for [e entities]
{:type (node-type e)
:id (:id e)
:data (node-data e)
:dependents_count (dep-count e)}))
:edges (for [[child parents] dependencies, parent parents]
;; Yeah, this graph points to dependents, not dependencies
{:from_entity_type (name (node-type parent))
:from_entity_id (node-id parent)
:to_entity_type (name (node-type child))
:to_entity_id (node-id child)})}))
;;; ---------------------------------------- Problems/Validation ----------------------------------------
(api.macros/defendpoint :get "/:id/problem" :- [:sequential ::ws.t/problem]
"Detect problems in the workspace that would affect downstream transforms after merge.
Returns a list of problems, each with:
- category: the problem category (e.g. 'unused', 'internal-downstream', 'external-downstream')
- problem: the specific problem (e.g. 'not-run', 'stale', 'removed-field')
- severity: :error, :warning, or :info
- block-merge: whether this problem prevents merging
- data: extra information, shape depends on the problem type
See `metabase-enterprise.workspaces.types/problem-types` for the full list."
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params]
(let [workspace (api/check-404 (t2/select-one :model/Workspace :id id))
graph (ws.impl/get-or-calculate-graph workspace)]
(ws.validation/find-downstream-problems id graph)))
(def ^:private db+schema+table (juxt :database :schema :name))
(defn- internal-target-conflict?
"Check whether the given table is the target of another transform within the workspace. Ignores global transforms."
[ws-id target & [tx-id]]
(contains?
(t2/select-fn-set (comp db+schema+table :target)
[:model/WorkspaceTransform :target]
:ref_id [:not= tx-id]
:workspace_id ws-id)
(db+schema+table target)))
(api.macros/defendpoint :post "/:ws-id/transform/validate/target"
:- [:map [:status :int] [:body [:or :string i18n/LocalizedString]]]
"Validate the target table for a workspace transform"
[{:keys [ws-id]} :- [:map [:ws-id ::ws.t/appdb-id]]
{:keys [transform-id]} :- [:map [:transform-id {:optional true} ::ws.t/ref-id]]
{:keys [db_id target]} :- [:map
[:db_id {:optional true} ::ws.t/appdb-id]
[:target [:map
[:database {:optional true} ::ws.t/appdb-id]
[:type :string]
[:schema :string]
[:name :string]]]]]
(let [workspace (api/check-404 (t2/select-one [:model/Workspace :database_id :status] ws-id))
target (update target :database #(or % db_id))
tx-id (when transform-id (parse-long transform-id))
;; For uninitialized workspaces we skip over checks for their db id
ws-db-id (when (not= :uninitialized (:status workspace)) (:database_id workspace))]
(cond
(not= "table" (:type target))
{:status 403 :body (deferred-tru "Unsupported target type")}
(and db_id ws-db-id (not= db_id ws-db-id))
{:status 403 :body (deferred-tru "Must target the workspace database")}
(not (or db_id ws-db-id))
{:status 403 :body (deferred-tru "Must target a database")}
(str/starts-with? (:schema target) "mb__isolation_")
{:status 403 :body (deferred-tru "Must not target an isolated workspace schema")}
;; Within a workspace, we defer blocking on conflicts outside the workspace
#_{:status 403 :body (deferred-tru "A table with that name already exists.")}
;; TODO consider deferring this validation until merge also.
(internal-target-conflict? ws-id target tx-id)
{:status 403 :body (deferred-tru "Another transform in this workspace already targets that table")}
:else
{:status 200 :body "OK"})))
(defn- malli-map-keys [schema]
(into [] (map first) (rest schema)))
(defn- select-malli-keys
"Like select-keys, but with the arguments reversed, and taking the malli schema for the output map.
Nothing smart - for example, it doesn't understand optional verus maybe."
([schema m]
(select-keys m (malli-map-keys schema)))
([schema alias-map m]
(merge (select-malli-keys schema m)
(u/for-map [[to from] alias-map] [to (from m)]))))
(defn- select-model-malli-keys
"Build the model-fields vector for a t2/select from a malli schema for the output row(s)"
([model schema]
(into [model] (malli-map-keys schema)))
([model schema alias-map]
(into [model]
(map (fn [to] (if-let [from (to alias-map)] [from to] to)))
(malli-map-keys schema))))
(def ^:private WorkspaceTransform
[:map
[:ref_id ::ws.t/ref-id]
[:global_id [:maybe ::ws.t/appdb-id]]
[:name :string]
[:description [:maybe :string]]
[:source :map]
[:target :map]
;; Not yet calculated, see https://linear.app/metabase/issue/BOT-684/mark-stale-transforms-workspace-only
[:target_stale :boolean]
[:workspace_id ::ws.t/appdb-id]
[:creator_id [:maybe ::ws.t/appdb-id]]
[:archived_at :any]
[:created_at :any]
[:updated_at :any]
[:last_run_at :any]
[:last_run_message [:maybe :string]]])
(def ^:private workspace-transform-alias {:target_stale :stale})
(api.macros/defendpoint :post "/:id/transform"
:- WorkspaceTransform
"Add another transform to the Changeset. This could be a fork of an existing global transform, or something new."
[{:keys [id]} :- [:map [:id ::ws.t/appdb-id]]
_query-params
body :- [:map #_{:closed true}
[:name :string]
[:description {:optional true} [:maybe :string]]
[:source ::transform-source]
;; Not sure why this schema is giving trouble
#_[:target ::transform-target]]]
(api/check (transforms.util/check-feature-enabled body)
[402 (deferred-tru "Premium features required for this transform type are not enabled.")])
(t2/with-transaction [_tx]
(let [workspace (u/prog1 (api/check-404 (t2/select-one :model/Workspace :id id))
(api/check-400 (nil? (:archived_at <>)) "Cannot create transforms in an archived workspace"))
;; TODO why 400 here and 403 in the validation route? T_T
_ (api/check-400 (not (internal-target-conflict? id (:target body)))
(deferred-tru "Another transform in this workspace already targets that table"))
global-id (:global_id body (:id body))
;; For uninitialized workspaces, preserve the target database from the request body
;; (add-to-changeset! will reinitialize the workspace with it if different from provisional)
;; For initialized workspaces, ensure the target database matches the workspace's database
body (-> body (dissoc :global_id)
(cond-> (not= :uninitialized (:status workspace))
(update :target assoc :database (:database_id workspace))))
transform (ws.common/add-to-changeset! api/*current-user-id* workspace :transform global-id body)]
(select-malli-keys WorkspaceTransform workspace-transform-alias transform))))
;; TODO Confirm precisely which fields are needed by the FE
(def ^:private WorkspaceTransformListing
"Schema for a transform in a workspace"
[:map {:closed true}
[:ref_id ::ws.t/ref-id]
[:global_id [:maybe ::ws.t/appdb-id]]
[:name :string]
[:source_type [:maybe :keyword]]
[:creator_id [:maybe ::ws.t/appdb-id]]
;[:last_run :map]
; See https://metaboat.slack.com/archives/C099RKNLP6U/p1765205882655869?thread_ts=1765205222.888209&cid=C099RKNLP6U
#_[:target_stale :boolean]])
;; Global transforms precalculate these in a toucan hook - maybe we should do that too? Let's review later.
(defn- map-source-type [ws-tx]
(-> ws-tx
(assoc :source_type (transforms/transform-source-type (:source ws-tx)))
(dissoc :source)))
(api.macros/defendpoint :get "/:id/transform" :- [:map [:transforms [:sequential WorkspaceTransformListing]]]
"Get all transforms in a workspace."
[{:keys [id]} :- [:map [:id ::ws.t/appdb-id]]]
(api/check-404 (t2/select-one :model/Workspace :id id))
{:transforms (->> (t2/select [:model/WorkspaceTransform :ref_id :global_id :name :source :creator_id]
:workspace_id id {:order-by [:created_at]})
(map map-source-type))})
(defn- fetch-ws-transform [ws-id tx-id]
;; TODO We still need to do some hydration, e.g. of the target table (both internal and external)
(-> (select-model-malli-keys :model/WorkspaceTransform WorkspaceTransform workspace-transform-alias)
(t2/select-one :ref_id tx-id :workspace_id ws-id)
api/check-404))
(api.macros/defendpoint :get "/:id/transform/:tx-id" :- WorkspaceTransform
"Get a specific transform in a workspace."
[{:keys [id tx-id]} :- [:map [:id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]
(fetch-ws-transform id tx-id))
(api.macros/defendpoint :put "/:ws-id/transform/:tx-id" :- WorkspaceTransform
"Update a transform in a workspace."
[{:keys [ws-id tx-id]} :- [:map [:ws-id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]
_query-params
body :- [:map
[:name {:optional true} :string]
[:description {:optional true} [:maybe :string]]
[:source {:optional true} ::transform-source]
[:target {:optional true} ::transform-target]]]
(t2/with-transaction [_tx]
(api/check-404 (t2/select-one :model/WorkspaceTransform :ref_id tx-id :workspace_id ws-id))
(t2/update! :model/WorkspaceTransform tx-id body)
;; Being cheeky and using the API response value for the pre-validation, to save a query.
(u/prog1 (fetch-ws-transform ws-id tx-id)
;; NOTE: FE may send these fields even when unchanged, causing unnecessary re-syncs.
;; This is acceptable for now, but using t2/changes in hooks might catch false positives?
;; The most reliable thing would be to have a clear, tested contract with the FE to NOT send them if unchanged.
(when (or (:source body) (:target body))
;; Note that we do NOT want to couple ourselves to the response shape of this API.
;; We want to be extremely mindful of the fields we depend on, in case we remove them from the response.
(let [transform (select-keys <> [:ref_id :source :source_type :target])
workspace (t2/select-one :model/Workspace :id ws-id)]
;; Re-sync dependencies if source or target changed.
(ws.impl/sync-transform-dependencies! workspace transform))))))
(api.macros/defendpoint :post "/:id/transform/:tx-id/archive" :- :nil
"Mark the given transform to be archived when the workspace is merged.
For provisional transforms we will skip even creating it in the first place."
[{:keys [id tx-id]} :- [:map [:id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]
(api/check-404 (pos? (t2/update! :model/WorkspaceTransform {:ref_id tx-id :workspace_id id} {:archived_at [:now]})))
nil)
(api.macros/defendpoint :post "/:id/transform/:tx-id/unarchive" :- :nil
"Unmark the given transform for archival. This will recall the last definition it had within the workspace."
[{:keys [id tx-id]} :- [:map [:id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]
(api/check-404 (pos? (t2/update! :model/WorkspaceTransform {:ref_id tx-id :workspace_id id} {:archived_at nil})))
nil)
(api.macros/defendpoint :delete "/:id/transform/:tx-id" :- :nil
"Discard a transform from the changeset.
Equivalent to resetting a checked-out transform to its global definition, or deleting a provisional transform."
[{:keys [id tx-id]} :- [:map [:id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]
(api/check-404 (pos? (t2/delete! :model/WorkspaceTransform :ref_id tx-id :workspace_id id)))
nil)
(api.macros/defendpoint :post "/:id/transform/:tx-id/run"
:- ::ws.t/execution-result
"Run a transform in a workspace.
App DB changes are rolled back. Warehouse DB changes persist."
[{:keys [id tx-id]} :- [:map [:id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]
(let [workspace (api/check-404 (t2/select-one :model/Workspace id))
transform (api/check-404 (t2/select-one :model/WorkspaceTransform :ref_id tx-id :workspace_id id))]
(api/check-400 (nil? (:archived_at workspace)) "Cannot execute archived workspace")
(check-transforms-enabled! (:database_id workspace))
(ws.impl/run-transform! workspace transform)))
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-route-uses-kebab-case]}
(api.macros/defendpoint :get "/checkout"
:- [:map
[:transforms [:sequential
[:map
[:id ::ws.t/ref-id]
[:name :string]
[:workspace [:map
[:id ms/PositiveInt]
[:name :string]]]]]]]
"Get all downstream transforms for a transform that is not in a workspace.
Returns the transforms that were mirrored from this upstream transform, with workspace info."
[_route-params
{:keys [transform-id]} :- [:map {:closed true} [:transform-id ms/PositiveInt]]]
(let [transforms (t2/select [:model/WorkspaceTransform :ref_id :name :workspace_id] :global_id transform-id)
workspace-ids (map :workspace_id transforms)
workspaces-by-id (when (seq transforms)
(t2/select-fn->fn :id identity [:model/Workspace :id :name] :id [:in workspace-ids]))]
{:transforms (for [{:keys [ref_id name workspace_id]} transforms]
{:id ref_id
:name name
:workspace (get workspaces-by-id workspace_id)})}))
(api.macros/defendpoint :post "/:ws-id/merge"
:- [:or
[:map
[:merged
[:map
[:transforms [:sequential
[:map
[:op [:enum :create :delete :update :noop]]
[:global_id {:optional true} [:maybe ::ws.t/appdb-id]]
[:ref_id ::ws.t/ref-id]]]]]]
[:errors
[:sequential
[:map
[:op [:enum :create :delete :update :noop]]
[:global_id {:optional true} [:maybe ::ws.t/appdb-id]]
[:ref_id ::ws.t/ref-id]]]]
[:workspace [:map [:id ::ws.t/appdb-id] [:name :string]]]
[:archived_at [:maybe :any]]]
;; error message from check-404 or check-400
:string]
"This will:
1. Update original transforms with workspace versions
2. Delete the workspace and clean up isolated resources
Returns a report of merged entities, or error in errors key.
Request body may include:
- commit-message: A description of what changes are being merged (optional, will be required in future)"
[{:keys [ws-id] :as _query-params} :- [:map [:ws-id ::ws.t/appdb-id]]
_query-params
{:keys [commit-message]
:or {commit-message "Placeholder for merge commit message. Should be required on FE"}} :- [:map
[:commit-message {:optional true} [:string {:min 1}]]]]
(let [ws (u/prog1 (t2/select-one :model/Workspace :id ws-id)
(api/check-404 <>)
(api/check-400 (nil? (:archived_at <>)) "Cannot merge an archived workspace"))
{:keys [merged
errors]} (-> (ws.merge/merge-workspace! ws api/*current-user-id* commit-message)
(update :errors
(partial mapv #(-> %
(update :error (fn [e] (.getMessage ^Throwable e)))
(set/rename-keys {:error :message})))))]
(u/prog1
{:merged merged
:errors errors
:workspace {:id ws-id, :name (:name ws)}
:archived_at (when-not (seq errors)
;; TODO call a ws.common method, which can handle the clean-up too
(t2/update! :model/Workspace :id ws-id {:archived_at [:now]})
(t2/select-one-fn :archived_at [:model/Workspace :archived_at] :id ws-id))}
(when-not (seq errors)
;; Most of the APIs and the FE are not respecting when a Workspace is archived yet.
(ws.model/delete! ws)))))
(api.macros/defendpoint :post "/:ws-id/transform/:tx-id/merge"
:- [:map
[:op [:enum :create :delete :update :noop]]
[:global_id [:maybe ::ws.t/appdb-id]]
[:ref_id ::ws.t/ref-id]
[:message {:optional true} :string]]
"Merge single transform from workspace back to the core. If workspace transform is archived
the corresponding core transform is deleted.
Request body may include:
- commit-message: A description of what changes are being merged (optional, will be required in future)"
[{:keys [ws-id tx-id] :as _query-params} :- [:map
[:ws-id ::ws.t/appdb-id]
[:tx-id ::ws.t/ref-id]]
_query-params
{:keys [commit-message]
:or {commit-message "Placeholder for merge commit message. Should be required on FE"}} :- [:map
[:commit-message {:optional true} [:string {:min 1}]]]]
(let [ws (api/check-404 (t2/select-one [:model/Workspace :id :name] :id ws-id))
ws-transform (api/check-404 (t2/select-one :model/WorkspaceTransform :workspace_id ws-id :ref_id tx-id))
ws-merge-id (t2/insert-returning-pk!
:model/WorkspaceMerge
{:workspace_id (:id ws)
:workspace_name (:name ws)
:commit_message commit-message
:creator_id api/*current-user-id*})
{:keys [error] :as result} (ws.merge/merge-transform! {:ws-transform ws-transform
:workspace ws
:workspace-merge-id ws-merge-id
:merging-user-id api/*current-user-id*
:commit-message commit-message})]
(if error
(throw (ex-info "Failed to merge transform."
(-> result
(dissoc :error)
(assoc :status-code 500))
error))
result)))
(def ^{:arglists '([request respond raise])} routes
"`/api/ee/workspace/` routes."
(api.macros/ns-handler *ns* api/+check-superuser +auth))

View File

@@ -0,0 +1,183 @@
(ns metabase-enterprise.workspaces.common
(:require
[clojure.string :as str]
[metabase-enterprise.workspaces.dag :as ws.dag]
[metabase-enterprise.workspaces.impl :as ws.impl]
[metabase-enterprise.workspaces.isolation :as ws.isolation]
[metabase-enterprise.workspaces.models.workspace-log :as ws.log]
[metabase-enterprise.workspaces.util :as ws.u]
[metabase.api-keys.core :as api-key]
[metabase.api.common :as api]
[metabase.util :as u]
[metabase.util.log :as log]
[metabase.util.quick-task :as quick-task]
[toucan2.core :as t2]))
;; should be encapsulated in our dag namespace, or dependency module
(defn check-no-card-dependencies!
"Check that transforms don't depend on cards. Throws 400 if they do."
[transform-ids]
(when-let [card-ids (seq (ws.dag/unsupported-dependency? transform-ids))]
(api/check-400 false
(format "Cannot add transforms that depend on saved questions (cards). Found dependencies on card IDs: %s"
(pr-str (vec card-ids))))))
(defn- extract-suffix-number
"Extract the numeric suffix from a workspace name like 'Foo (3)', or nil if no valid suffix."
[ws-name base-name]
(when (= (str/replace ws-name #"\s*\(\d+\)$" "") base-name)
(when-let [[_ n] (re-find #"\((\d+)\)$" ws-name)]
(parse-long n))))
(defn- generate-unique-workspace-name
"Generate a unique workspace name by appending (N) if the name already exists.
Strategy:
1. If the base name doesn't exist, use it as-is
2. If it exists, strip any existing (N) suffix and find the highest N among similar names
3. Return the base name with (N+1) appended
Note: This function has a potential race condition between checking existence and insertion.
Callers should handle unique constraint violations by retrying."
[base-name]
(when (str/blank? base-name)
(throw (ex-info "Workspace name cannot be empty" {:status-code 400})))
(if-not (t2/exists? :model/Workspace :name base-name)
base-name
(let [stripped-name (str/replace base-name #"\s*\(\d+\)$" "")
existing (t2/select-fn-set :name :model/Workspace
:name [:like (str stripped-name " (%)")])
numbers (keep #(extract-suffix-number % stripped-name) existing)
next-num (inc (apply max 0 numbers))]
(str stripped-name " (" next-num ")"))))
;; TODO: Generate new metabase user for the workspace
;; TODO: Should we move this to model as per the diagram?
(defn- create-workspace-container!
"Create the workspace and its related collection, user, and api key."
[creator-id db-id workspace-name status]
;; TODO (Chris 2025-11-19) Unsure API key name is unique, and remove this (insecure) workaround.
(let [api-key (let [key-name (format "API key for Workspace %s" workspace-name)]
(or (t2/select-one :model/ApiKey :name key-name)
(api-key/create-api-key-with-new-user! {:key-name key-name})))
;; TODO (Chris 2025-11-19) Associate the api-key user with the workspace as well.
ws (t2/insert-returning-instance! :model/Workspace
{:name workspace-name
:creator_id creator-id
:database_id db-id
:api_key_id (:id api-key)
:execution_user (:user_id api-key)
:status status})
coll (t2/insert-returning-instance! :model/Collection
{:name (format "Collection for Workspace %s" workspace-name)
:namespace "workspace"
:workspace_id (:id ws)})
;; Only set schema for initialized workspaces (not uninitialized)
schema (when (not= status :uninitialized) (ws.u/isolation-namespace-name ws))
ws (assoc ws
:collection_id (:id coll)
:schema schema)]
;; Set the backlink from the workspace to the collection inside it and set the schema.
(t2/update! :model/Workspace (:id ws) (cond-> {:collection_id (:id coll)}
schema (assoc :schema schema)))
;; TODO (Sanya 2025-11-18) - not sure how to transfer this api key to agent
#_(log/infof "Generated API key for workspace: %s" (u.secret/expose (:unmasked_key api-key)))
ws))
(defn- unique-constraint-violation?
"Check if an exception is due to a unique constraint violation."
[e]
(let [msg (str (ex-message e) (some-> (ex-cause e) ex-message))]
(or (str/includes? msg "unique")
(str/includes? msg "duplicate")
(str/includes? msg "UNIQUE"))))
(defn- run-workspace-setup!
"Background job: runs isolation, grants. Updates status to :ready when done."
[{ws-id :id :as workspace} database]
(try
(ws.log/track! ws-id :workspace-setup
(let [isolation-details (ws.log/track! ws-id :database-isolation
(ws.isolation/ensure-database-isolation! workspace database))]
(t2/update! :model/Workspace ws-id (merge (select-keys isolation-details [:schema :database_details])
{:status :ready}))))
(catch Exception e
(log/error e "Failed to setup workspace")
(throw e))))
(defn initialize-workspace!
"Initialize an uninitialized workspace with the given database_id.
Updates database_id (if different from provisional), sets schema, creates isolation resources async,
and transitions to :pending status. Returns the updated workspace with schema set."
[workspace database-id]
(let [database (t2/select-one :model/Database :id database-id)
schema (ws.u/isolation-namespace-name workspace)
res (t2/update! :model/Workspace {:id (:id workspace)
:status :uninitialized}
{:database_id database-id
:schema schema
:status :pending})]
(when (zero? res)
(let [new-db-id (t2/select-one-fn :database_id :model/Workspace (:id workspace))]
(when (not= database-id new-db-id)
(throw (ex-info "Workspace has been initialized already with a different database"
{:requested-db-id database-id
:actual-db-id new-db-id})))))
(u/prog1 (t2/select-one :model/Workspace :id (:id workspace))
(quick-task/submit-task! #(run-workspace-setup! <> database)))))
(defn- create-uninitialized-workspace!
"Create a workspace with a provisional database_id but no isolation resources.
Retries on unique constraint violations for workspace name."
[creator-id db-id ws-name max-retries]
(loop [attempt 1]
(let [unique-name (generate-unique-workspace-name ws-name)
{:keys [retry
workspace]} (try
{:workspace (create-workspace-container! creator-id db-id unique-name :uninitialized)}
(catch Exception e
(if (and (< attempt max-retries) (unique-constraint-violation? e))
{:retry true}
(throw e))))]
(if retry
(recur (inc attempt))
workspace))))
;; TODO internal: test!
(defn create-workspace!
"Create workspace. If :provisional is true, creates an uninitialized workspace with a
provisional database_id (no isolation resources yet). Otherwise creates a pending
workspace and kicks off async setup."
[creator-id {ws-name-maybe :name
db-id :database_id
provisional? :provisional?}]
(let [ws-name (or ws-name-maybe (ws.u/generate-name))
ws (create-uninitialized-workspace! creator-id db-id ws-name 5)]
(if provisional?
ws
(initialize-workspace! ws db-id))))
(defn add-to-changeset!
"Add the given transform to the workspace changeset.
If workspace is uninitialized, initializes it with the transform's target database."
[creator-id workspace entity-type global-id body]
(ws.u/assert-transform! entity-type)
;; Initialize workspace if uninitialized (outside transaction so async task can see committed data)
(let [workspace (if (= :uninitialized (:status workspace))
(let [target-db-id (get-in body [:target :database])]
(api/check-400 target-db-id "Transform must have a target database")
(initialize-workspace! workspace target-db-id))
workspace)]
(t2/with-transaction [_]
(let [workspace-id (:id workspace)
workspace-db-id (:database_id workspace)
body (assoc-in body [:target :database] workspace-db-id)
transform (t2/insert-returning-instance!
:model/WorkspaceTransform
(assoc (select-keys body [:name :description :source :target])
:ref_id (ws.u/generate-ref-id)
:creator_id creator-id
:global_id global-id
:workspace_id workspace-id))]
(ws.impl/sync-transform-dependencies! workspace (select-keys transform [:ref_id :source_type :source :target]))
transform))))

View File

@@ -0,0 +1,2 @@
;; For now, we do not anticipate any non-API usage of this module.
(ns metabase-enterprise.workspaces.core)

View File

@@ -0,0 +1,212 @@
;; TODO consider renaming this to dataflow
(ns metabase-enterprise.workspaces.dag
"Used to do graph computations with respect to dataflow within a workspace.
This namespace should encapsulate all coupling to the [[metabase-enterprise.dependencies]] module, including through
database artifacts.
It is purely concerned with calculation, any side effects wanted as a result of the analysis should happen outside."
(:require
[metabase-enterprise.workspaces.util :as ws.u]
[metabase.util :as u]
[toucan2.core :as t2]
[toucan2.realize :as t2.realize]))
;; Is this working with appdb-ids or ref-ids?
(defn unsupported-dependency?
"Return a map of the entities which have unsupported dependencies, or nil if there are none."
[{transform-ids :transforms :as _entity-map}]
;; TODO filter out those that don't have a card dependency
(when (seq transform-ids)
{:transforms transform-ids}))
;; TODO (chris 2025/12/15) Should we leverage data-authority table metadata to speed things up?
(defn- table-id->coord [id]
(t2/select-one [:model/Table :id :schema [:name :table] [:db_id :db]] id))
(defn- global-parents
"Dependencies analyzed by global hooks - may be incomplete. For example, excluding transforms that have not been run."
[ws-id node-type id]
;; Note: from_entity_type is transformed to a keyword by the Dependency model
(t2/select-fn-vec (fn [{id :to_entity_id entity-type :to_entity_type}]
(case entity-type
:card {:node-type :external-card, :id id}
:table {:node-type :table, :id (table-id->coord id)}
:transform {:node-type :external-transform, :id id}))
[:model/Dependency :to_entity_type :to_entity_id]
:from_entity_type node-type
:from_entity_id id
;; Exclude transforms which are being overridden in the workspace.
;; Note: use string "transform" in SQL WHERE clause (db stores strings)
{:where [:not [:and
[:= "transform" :to_entity_type]
[:exists {:select [1]
:from [[:workspace_transform :wt]]
:where [:and
[:= :wt.workspace_id ws-id]
[:= :wt.global_id :to_entity_id]]}]]]}))
(defn- ws-transform-parents [ws-id ref-id]
;; We assume there are no card dependencies yet
(t2/select-fn-vec (fn [table-coord]
{:node-type :table, :id (t2.realize/realize table-coord)})
[:model/WorkspaceInput [:db_id :db] :schema :table [:table_id :id]]
:workspace_id ws-id
:id [:in {:select [:to_entity_id]
:from [:workspace_dependency]
:where [:and
[:= "transform" :from_entity_type]
[:= ref-id :from_entity_id]
[:= "input" :to_entity_type]]}]))
(defn- table-producers [ws-id id-or-coord]
;; Work with either logical co-ords or an id
(let [{:keys [db schema table id]} (if (map? id-or-coord) id-or-coord (table-id->coord id-or-coord))
tx-ref-id (t2/select-one-fn :ref_id [:model/WorkspaceOutput :ref_id]
{:where [:and
[:= :workspace_id ws-id]
[:or
(when id
[:= :global_table_id id])
[:and
[:= :db_id db]
[:= :global_schema schema]
[:= :global_table table]]]]})]
(if tx-ref-id
;; If there is a workspace transform that targets this table, ignore any global transform that also targets it.
[{:node-type :workspace-transform :id tx-ref-id}]
(global-parents ws-id "table" id))))
(defn- table? [{:keys [node-type]}] (= :table node-type))
(defn- node-parents [ws-id {:keys [node-type id]}]
(case node-type
:workspace-transform (ws-transform-parents ws-id id)
:external-transform (global-parents ws-id "transform" id)
:external-card (global-parents ws-id "card" id)
:table (table-producers ws-id id)))
(defn- node->allowed-parents
[pred deps node]
(if-not (pred node)
[node]
;; Collapse to the things that *produce* node. If it is a root node, leave it in.
(or (seq (mapcat (partial node->allowed-parents pred deps) (deps node)))
[node])))
(defn- collapse
"Remove nodes that satisfy pred, in-lining their transitive dependencies instead."
[pred deps]
(u/for-map [[child parents] deps
:when (not (pred child))]
[child (mapcat (partial node->allowed-parents pred deps) parents)]))
(defn- render-graph [entities parents deps & {:keys [table? table-sort unwrap-table]
:or {table? table?
table-sort (juxt :db :schema :table)
unwrap-table :id}}]
(let [table-nodes (filter table? entities)
;; Any table that has a parent in the subgraph is an output
outputs (filter deps table-nodes)
;; Anything other parent table is an input
inputs (->> entities (mapcat parents) (filter table?) (remove (set outputs)) distinct)
entities (->> (ws.u/toposort-dfs deps) (remove table?))
deps+input (reduce
(fn [deps [c parents]]
(if-not (deps c)
deps
(update deps c into parents)))
deps
parents)]
{:inputs (sort-by table-sort (map unwrap-table inputs))
:outputs (sort-by table-sort (map unwrap-table outputs))
:entities entities
;; collapse tables out, directly connecting transforms? smaller graphs are easier for humans to read
:dependencies (collapse table? deps+input)}))
(defn- path-induced-subgraph*
"Implementation for [[path-induced-subgraph]] that takes the lookup functions as arguments. Useful for testing."
[init-nodes fns]
(loop [members (set init-nodes)
cache {}
;; Association list sets from nodes to their direct dependencies
deps (u/for-map [node init-nodes] [node #{}])
;; Paths are vectors sorted from child to parent
[path & paths] (for [node members] [node])]
(if-not path
;; Finished walking all paths, render the result.
(render-graph members cache deps fns)
;; Look-up from cache first
(let [_ (when (some #(not= 1 (val %)) (frequencies path))
(throw (ex-info "Cycle detected" {:path path})))
head (peek path)
node-parents* (:node-parents fns)
parents (cache head (node-parents* head))
cache (assoc cache head parents)
continue (when parents (remove members parents))]
(if (not= parents continue)
;; At least one parent is in the enclosed subgraph, so this entire path is as well.
(let [add-dep (fn [deps [child parent]]
(update deps child (fnil conj #{}) parent))
;; Break up the path into adjacent child-parent pairs.
pairs (concat (partition 2 1 path)
(map (fn [end] [(peek path) end]) (filter members parents)))
;; Since everything along this path will now be added to deps already, we can truncate the paths.
next-paths (into paths (for [c continue] [(peek path) c]))]
(recur (into members path) cache (reduce add-dep deps pairs) next-paths))
;; We have not reached another member of the enclosed subgraph yet, so keep extending.
(recur members cache deps (into paths (for [c continue] (conj path c)))))))))
;;;; Public API
(defn path-induced-subgraph
"Given a map of entity types to IDs, compute the path-induced subgraph.
`entities-by-type` is a map like {:transform [1 2 3]}.
Returns a map with:
- :inputs - tables that the subgraph directly depends on, that are not themselves produced by the subgraph.
ordered lexically by db+schema+table, to be stable across metabase instances, and visibly sorted.
- :outputs - outputs of both the changeset transforms, and the external transforms enclosed by the changeset.
both the external names, and the internal (isolated) names. sorted by the external names.
point back to their relevant transform.
- :entities - the full list of entities, both in the changeset, and those that are enclosed. topo-sorted.
- :dependencies - association list for the subgraph, with keys and values both in topological order"
[ws-id changeset]
(ws.u/assert-transforms! changeset)
(if (empty? changeset)
{:inputs []
:outputs []
:entities []
:dependencies {}}
(let [tx-nodes (for [{:keys [entity-type id]} changeset]
(case entity-type
:transform {:node-type :workspace-transform, :id id}))
outputs (when (seq tx-nodes)
(t2/select-fn-vec (fn [row]
{:node-type :table, :id (t2.realize/realize row)})
[:model/WorkspaceOutput
[:db_id :db]
[:global_schema :schema]
[:global_table :table]
[:global_table_id :id]]
:ref_id [:in (map :id tx-nodes)]))
init-nodes (concat tx-nodes outputs)
fns {:node-parents (partial node-parents ws-id)
:table? table?}]
(path-induced-subgraph* init-nodes fns))))
;; source-table ---------> checked-out-transform
;; \____external-transform__/
;;
;; should external-transform be enclosed? answer: no, because it won't become stale from changes in the workspace
;;
;; source-table ----> model ---> checked-out-transform
;;
;; this time we *need* the model to be serialized, because we need to read it in a fresh instance.
;; it doesn't need to be remapped or "run" independently though, so does it make sense to call it enclosed?
;;
;; enclosed: between changeset only? or including the model in the example above?
;; ancestors: as far back as you can go
;;

View File

@@ -0,0 +1,400 @@
(ns metabase-enterprise.workspaces.dependencies
"Workspace-local dependency tracking.
Unlike the global `dependency` table which links to `table_id` (requiring the table to exist),
workspace dependencies use three tables that support logical references:
- `workspace_input` - external tables transforms consume (may not exist yet)
- `workspace_output` - tables transforms produce
- `workspace_dependency` - edges: transform → input (external) or transform → output (internal)
## Example: Internal Dependency (Transform B depends on Transform A's output)
Transform A produces `analytics.orders_summary`:
```
{:ref_id \"happy-dolphin-a1b2\"
:target {:database 1 :schema \"analytics\" :name \"orders_summary\"}}
```
Transform B reads from `analytics.orders_summary`:
```
{:ref_id \"brave-lion-g7h8\"
:source {:type \"query\" :query \"SELECT * FROM analytics.orders_summary\"}
:target {:database 1 :schema \"analytics\" :name \"orders_report\"}}
```
After processing both transforms:
workspace_output:
| id | ref_id | schema | table |
|-----|--------------------|-----------|----------------|
| 101 | happy-dolphin-a1b2 | analytics | orders_summary |
| 104 | brave-lion-g7h8 | analytics | orders_report |
workspace_dependency:
| from_entity_type | from_entity_id | to_entity_type | to_entity_id | meaning |
|------------------|--------------------| ---------------|--------------|-------------------------|
| transform | brave-lion-g7h8 | output | 101 | B depends on A's output |
Transform B's dependency points to `output:101` (Transform A's output), NOT to a new `input`.
This is because `analytics.orders_summary` matches an existing `workspace_output` record.
If Transform A also depended on an external table ORDERS, we'd also have:
workspace_input:
| id | schema | table | table_id |
|-----|--------|--------|----------|
| 201 | PUBLIC | ORDERS | 42 |
workspace_dependency:
| from_entity_type | from_entity_id | to_entity_type | to_entity_id | meaning |
|------------------|--------------------| ---------------|--------------|------------------------------|
| transform | happy-dolphin-a1b2 | input | 201 | A depends on external ORDERS |
| transform | brave-lion-g7h8 | output | 101 | B depends on A's output |"
(:require
[clojure.set :as set]
[metabase-enterprise.workspaces.models.workspace-dependency]
[metabase-enterprise.workspaces.models.workspace-input]
[metabase-enterprise.workspaces.models.workspace-output]
[metabase-enterprise.workspaces.util :as ws.u]
[metabase.app-db.core :as app-db]
[metabase.driver :as driver]
[metabase.driver.sql :as driver.sql]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.util :as u]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
[metabase.util.performance :as perf]
[toucan2.core :as t2]))
;;; ---------------------------------------- Schemas ----------------------------------------
(mr/def ::table-ref
"A logical reference to a table by schema and name. Includes table_id if the table exists."
[:map
[:db_id :int]
[:schema [:maybe :string]]
[:table :string]
[:table_id {:optional true} [:maybe :int]]])
(mr/def ::output-ref
"A reference to an output table (transform target)."
[:map
[:db_id :int]
[:schema [:maybe :string]]
[:table :string]])
(mr/def ::analysis
"The result of analyzing an entity's dependencies."
[:map
[:output ::output-ref]
[:inputs [:sequential ::table-ref]]])
;;; ---------------------------------------- Batch table lookups ----------------------------------------
(defn- batch-table-refs-from-ids
"Look up table metadata for multiple table IDs in a single query.
Returns a map of table-id -> {:db_id :schema :table :table_id}."
[table-ids]
(when (seq table-ids)
(t2/select-fn->fn :id
(fn [{:keys [id db_id schema name]}]
{:db_id db_id
:schema schema
:table name
:table_id id})
[:model/Table :id :db_id :schema :name]
:id [:in table-ids])))
(defn- batch-lookup-table-ids
"Look up table_ids for multiple logical table references in a single query.
Uses the same matching logic as find-table-or-transform: if parsed schema is nil,
it matches against the driver's default schema.
Returns a map of [db_id search-schema table] -> table_id where search-schema
is the schema used for lookup (defaulted if nil)."
[driver db-id table-refs]
(when (seq table-refs)
(let [default-schema (driver.sql/default-schema driver)
;; Include default-schema in filter so nil refs can match that too
nil-schema? (some (comp nil? :schema) table-refs)
schemas-to-query (into #{} (map #(or (:schema %) default-schema)) table-refs)
db-table-lookup (t2/select-fn->fn (juxt :schema :name) :id
[:model/Table :id :schema :name]
:db_id db-id
:name [:in (map :table table-refs)]
{:where (if nil-schema?
[:or [:= :schema nil] [:in :schema schemas-to-query]]
[:in :schema schemas-to-query])})]
(into {}
(keep (fn [{:keys [schema table]}]
(let [table-id (or (get db-table-lookup [schema table])
(when (nil? schema)
(get db-table-lookup [default-schema table])))]
(when table-id
[[db-id schema table] table-id]))))
table-refs))))
(defn- output-from-target
"Extract output table info from a transform's :target field."
[{:keys [database schema name] :as _target}]
{:db_id database
:schema schema
:table name})
;;; ---------------------------------------- Input extraction ----------------------------------------
(defn- inputs-from-mbql-query
"Extract table refs from an MBQL query. Tables are looked up by ID in batch."
[query]
(let [table-ids (-> #{}
(into (lib/all-source-table-ids query))
(into (lib/all-implicitly-joined-table-ids query)))
table-refs (batch-table-refs-from-ids table-ids)]
(u/keepv table-refs table-ids)))
(defn- inputs-from-native-query
"Extract table refs from a native query. Uses driver/native-query-table-refs to get
schema+table directly, then batch looks up table_ids for tables that exist.
Uses the same default schema logic as find-table-or-transform."
[query db-id]
(let [driver (:engine (lib.metadata/database query))
parsed-refs (driver/native-query-table-refs driver query)
;; Batch lookup existing table IDs (handles default schema logic)
table-id-map (batch-lookup-table-ids driver db-id parsed-refs)]
(perf/for [{:keys [schema table]} parsed-refs
:let [table-id (get table-id-map [db-id schema table])]]
(cond-> {:db_id db-id
:schema schema
:table table}
table-id (assoc :table_id table-id)))))
(defn- inputs-from-query
"Extract table refs from a query (MBQL or native)."
[query db-id]
(if (lib/native-only-query? query)
(inputs-from-native-query query db-id)
(inputs-from-mbql-query query)))
(defn- inputs-from-python-transform
"Extract table refs from a python transform's source-tables.
Python transforms require tables to exist (they map name -> table_id). Batch lookup."
[source-tables]
(let [table-ids (set (vals source-tables))
table-refs (batch-table-refs-from-ids table-ids)]
(u/keepv table-refs (vals source-tables))))
(mu/defn analyze-entity :- ::analysis
"Analyze a workspace entity to find its dependencies.
Arguments:
- entity-type: keyword, must be :transform (asserted)
- entity: the workspace transform map with :source and :target
Returns:
{:output {:db_id int :schema string :table string}
:inputs [{:db_id int :schema string :table string} ...]}"
[entity-type :- :keyword
entity :- :map]
(ws.u/assert-transform! entity-type)
(let [{{:keys [query source-tables], source-type :type} :source
target :target} entity
db-id (:database target)
inputs (cond
(= (keyword source-type) :query)
(inputs-from-query query db-id)
(= (keyword source-type) :python)
(inputs-from-python-transform source-tables)
:else
(do (log/warnf "Don't know how to analyze deps of transform with source type '%s'" source-type)
[]))]
{:output (output-from-target target)
:inputs inputs}))
;;; ---------------------------------------- Write helpers ----------------------------------------
(defn- upsert-workspace-output!
"Upsert a workspace_output record for the transform's target table.
Stores both global (original) and isolated (workspace-specific) table identifiers.
Returns the workspace_output id."
[workspace-id ref-id isolated-schema {:keys [db_id schema table]} normalize-sql]
(app-db/update-or-insert! :model/WorkspaceOutput
{:workspace_id workspace-id
:ref_id ref-id}
(fn [existing]
(let [isolated-table (ws.u/isolated-table-name schema table)
qry-table-id (fn [schema table]
(t2/select-one-fn :id [:model/Table :id]
:db_id db_id
:schema (normalize-sql schema)
:name (normalize-sql table)))
id-if-match (fn [schema-key schema table-key table id-key]
(when (and (= schema (get existing schema-key))
(= table (get existing table-key)))
(get existing id-key)))
id-or-fallback (fn [schema-key schema table-key table id-key]
(or (id-if-match schema-key schema table-key table id-key)
(qry-table-id schema table)))]
{:db_id db_id
:global_schema schema
:global_table table
:global_table_id (id-or-fallback
:global_schema schema
:global_table table
:global_table_id)
:isolated_schema isolated-schema
:isolated_table isolated-table
:isolated_table_id (id-or-fallback
:isolated_schema isolated-schema
:isolated_table isolated-table
:isolated_table_id)}))))
(defn- build-output-lookup
"Build a lookup map for workspace outputs: [db_id global_schema global_table] -> output_id.
Single query to fetch all outputs for the workspace."
[workspace-id]
(t2/select-fn->fn (juxt :db_id :global_schema :global_table) :id
:model/WorkspaceOutput
:workspace_id workspace-id))
(defn- build-input-lookup
"Build a lookup map for workspace inputs: [db_id schema table] -> {:id :table_id}.
Single query to fetch all inputs for the workspace."
[workspace-id]
(t2/select-fn->fn (juxt :db_id :schema :table)
#(select-keys % [:id :table_id])
[:model/WorkspaceInput :id :db_id :schema :table :table_id]
:workspace_id workspace-id))
(defn- normalize-input-schema
"Normalize an input's schema: replace nil with the driver's default schema.
This ensures consistent comparison with output schemas which are always explicit."
[default-schema input]
(update input :schema #(or % default-schema)))
(defn- ensure-workspace-inputs!
"Ensure all external inputs exist in workspace_input table.
Uses batch lookup to avoid N+1. Returns map of [db_id schema table] -> input_id.
Inputs should already have normalized schemas (nil replaced with driver default)."
[workspace-id inputs existing-input-lookup]
(let [{existing true new-inputs false}
(group-by (fn [{:keys [db_id schema table]}]
(contains? existing-input-lookup [db_id schema table]))
inputs)
;; TODO: This N+1 section could be optimized into one batch update using HoneySQL
_ (doseq [{:keys [db_id schema table table_id]} existing]
(let [existing-record (get existing-input-lookup [db_id schema table])]
(when (and existing-record
(not= (:table_id existing-record) table_id))
(t2/update! :model/WorkspaceInput (:id existing-record) {:table_id table_id}))))
new-input-records (when (seq new-inputs)
(t2/insert-returning-instances!
:model/WorkspaceInput
(for [{:keys [db_id schema table table_id]} new-inputs]
{:workspace_id workspace-id
:db_id db_id
:schema schema
:table table
:table_id table_id})))
new-input-lookup (into {} (map (fn [{:keys [id db_id schema table]}]
[[db_id schema table] id])
new-input-records))]
(merge (into {} (map (fn [[k v]] [k (:id v)]) existing-input-lookup))
new-input-lookup)))
(defn- create-dependency-edges!
"Batch create workspace_dependency edges."
[workspace-id ref-id dep-specs]
(when (seq dep-specs)
(t2/insert! :model/WorkspaceDependency
(for [{:keys [to_entity_type to_entity_id]} dep-specs]
{:workspace_id workspace-id
:from_entity_type :transform
:from_entity_id ref-id
:to_entity_type to_entity_type
:to_entity_id to_entity_id}))))
(defn- delete-stale-edges!
"Delete dependency edges that are no longer present.
Uses single delete with OR conditions for batch deletion."
[workspace-id ref-id stale-edges]
(when (seq stale-edges)
(doseq [{:keys [to_entity_type to_entity_id]} stale-edges]
;; TODO: Could optimize further with raw SQL DELETE ... WHERE (type, id) IN (...)
(t2/delete! :model/WorkspaceDependency
:workspace_id workspace-id
:from_entity_type :transform
:from_entity_id ref-id
:to_entity_type to_entity_type
:to_entity_id to_entity_id))))
(defn- current-edge-specs
"Get the current dependency edge specs for a transform."
[workspace-id ref-id]
(into #{}
(map #(select-keys % [:to_entity_type :to_entity_id]))
(t2/select [:model/WorkspaceDependency :to_entity_type :to_entity_id]
:workspace_id workspace-id
:from_entity_type :transform
:from_entity_id ref-id)))
(mu/defn write-dependencies! :- :nil
"Persist dependency analysis for a workspace entity.
Arguments:
- workspace-id: int
- isolated-schema: string, the workspace's isolated schema name
- entity-type: keyword, must be :transform
- ref-id: string, the workspace_transform.ref_id
- analysis: result from analyze-entity
Side effects:
- Upserts workspace_output for this transform (with both global and isolated identifiers)
- Upserts workspace_input for each external table dependency
- Creates workspace_dependency edges
- Deletes stale edges"
[workspace-id :- :int
isolated-schema :- :string
entity-type :- :keyword
ref-id :- :string
{:keys [output inputs]} :- ::analysis]
(ws.u/assert-transform! entity-type)
(t2/with-transaction [_conn]
(let [output-lookup (build-output-lookup workspace-id)
existing-input-lookup (build-input-lookup workspace-id)
current-edges (current-edge-specs workspace-id ref-id)
driver (t2/select-one-fn :engine [:model/Database :engine]
:id [:in {:select [:database_id]
:from [:workspace]
:where [:= :id workspace-id]}])
;; TODO (chris 2025/12/17) I solemnly declare that we will clean up this coupling nightmare for table normalization
;; -> lbrdnk: requring resolve instead of normal :require here to avoid reformat attempt of the cljfmt
;; performed in CI, which if successful moves the ignore where it should not be, hence the linter
;; is failing, or the reformat alone fails the CI.
normalize (partial @(requiring-resolve 'metabase.driver.sql.normalize/normalize-name) driver)]
(upsert-workspace-output! workspace-id ref-id isolated-schema output normalize)
(let [{internal-inputs true external-inputs false} (group-by (fn [{:keys [db_id schema table]}]
(contains? output-lookup [db_id schema table]))
inputs)
default-schema (driver.sql/default-schema driver)
;; Normalize external inputs so lookups use consistent schema keys
normalized-external-inputs (map (partial normalize-input-schema default-schema) external-inputs)
input-id-lookup (ensure-workspace-inputs! workspace-id normalized-external-inputs existing-input-lookup)
new-dep-specs (concat
(for [{:keys [db_id schema table]} internal-inputs]
{:to_entity_type :output
:to_entity_id (get output-lookup [db_id schema table])})
(for [{:keys [db_id schema table]} normalized-external-inputs]
{:to_entity_type :input
:to_entity_id (get input-id-lookup [db_id schema table])}))
new-dep-specs-set (set new-dep-specs)
edges-to-create (set/difference new-dep-specs-set current-edges)
edges-to-delete (set/difference current-edges new-dep-specs-set)]
(create-dependency-edges! workspace-id ref-id edges-to-create)
(delete-stale-edges! workspace-id ref-id edges-to-delete)))))

View File

@@ -0,0 +1,120 @@
(ns metabase-enterprise.workspaces.execute
"This namespace is concerned with executing non-AppDB transforms, with optional reference re-mappings.
It currently lives inside the workspace module, but eventually will become part of the transform module, so it should
not make any assumptions about being run within a workspace, having the input / output semantics of a workspace, and
especially it should not touch any of the Workspace entities inside AppDb, or any of the other workspace namespaces.
For now, it uses AppDB as a side-channel with the transforms module, but in future this module should be COMPLETELY
decoupled from AppDb."
(:require
[macaw.core :as macaw]
[metabase-enterprise.transforms.core :as transforms]
[metabase-enterprise.transforms.execute :as transforms.execute]
[metabase.api.common :as api]
[metabase.util :as u]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
(defn- execution-results
"Extract execution metadata from transform_run and target table info.
Must be called within the transaction before rollback."
[{xf-id :id :keys [target]}]
(let [run (t2/select-one :model/TransformRun :transform_id xf-id)]
(merge
(select-keys run [:status :start_time :end_time :message])
{:table {:name (:name target)
:schema (:schema target)}})))
(defn- remap-python-source
"Remap source-tables in a Python transform source to point to isolated tables.
Handles both legacy integer format and new map format (from PR #66934)."
[table-mapping source]
(letfn [(remap [{:keys [database_id schema table table_id] :as table-ref}]
(if-let [{:keys [id db-id schema table]} (or (table-mapping table-ref)
(some-> table_id table-mapping)
(when table
(table-mapping [database_id schema table])))]
;; Prefer table ID when available; map format won't work until PR #66934 is merged:
;; https://github.com/metabase/metabase/pull/66934
(or id {:database_id db-id, :schema schema, :table table})
;; Leave it un-mapped if we don't have an override.
table-ref))]
(update source :source-tables update-vals remap)))
(defn- remap-sql-source [table-mapping source]
(let [remapping (reduce
(fn [remapping [[_ source-schema source-table] {target-schema :schema, target-table :table}]]
(-> remapping
(assoc-in [:schemas source-schema] target-schema)
(assoc-in [:tables {:schema source-schema
:table source-table}] target-table)))
{:schemas {}
:tables {}}
;; Strip out the numeric keys (table ids)
(filter (comp vector? key) table-mapping))]
;; We may need to set other options, like the case insensitivity (driver dependent)
(update-in source [:query :stages 0 :native] #(macaw/replace-names % remapping {:allow-unused? true}))))
(defn- remap-mbql-source [_table-mapping _field-map source]
(throw (ex-info "Remapping MBQL queries is not supported yet" {:source source})))
(defn- remap-source [table-map field-map source-type source]
(case source-type
:mbql (remap-mbql-source table-map field-map source)
;; TODO make sure it's actually a SQL dialect though..
:native (remap-sql-source table-map source)
:python (remap-python-source table-map source)))
;; You might prefer a multi-method? I certainly would.
(defn- remap-target [table-map target-fallback {d :database, s :schema, t :name :as target}]
(if-let [replacement (or (table-map [d s t]) (target-fallback [d s t]))]
;; Always fallback to tables for re-mapped outputs, regardless of the type used in the original target.
{:type (:type replacement "table")
:database (:db-id replacement)
:schema (:schema replacement)
:name (:table replacement)}
target))
(def ^:private no-mapping {})
(defn run-transform-with-remapping
"Execute a given collection with the given table and field re-mappings.
This is used by Workspaces to re-route the output of each transform to a non-production table, and
to re-write their queries where these outputs transitively become inputs to other transforms.
Returns an ::ws.t/execution-result map with status, timing, and table metadata.
-------
NOTE: currently execution is done using transaction-rollback pattern, as a short-term hack.
1. Creating temporary Transform/TransformRun records in a transaction
2. Executing using existing transform infrastructure
3. Scraping metadata (execution stats + table schema)
4. Rolling back the transaction (no app DB records persist)
The warehouse DB changes (actual table data) DO persist in the isolated schema."
[{:keys [source target] :as transform} remapping]
(t2/with-transaction [_conn]
(let [s-type (transforms/transform-source-type source)
table-mapping (:tables remapping no-mapping)
target-fallback (:target-fallback remapping no-mapping)
field-mapping (:fields remapping no-mapping)
new-xf (-> (select-keys transform [:name :description])
(assoc :creator_id api/*current-user-id*
:source (remap-source table-mapping field-mapping s-type source)
:target (remap-target table-mapping target-fallback target)))
_ (assert (:target new-xf) "Target mapping must not be nil")
temp-xf (t2/insert-returning-instance! :model/Transform new-xf)]
(try
(transforms.execute/execute! temp-xf {:run-method :manual})
(catch Exception _e
;; Execution failed - the TransformRun record has been updated with failure status and message
nil))
;; Return execution results whether succeeded or failed
(u/prog1 (execution-results temp-xf)
(t2/delete! :model/Transform (:id temp-xf))))))

View File

@@ -0,0 +1,364 @@
(ns metabase-enterprise.workspaces.impl
"Glue code connecting workspace subsystems (dependencies, isolation)."
(:require
[clojure.set :as set]
[metabase-enterprise.workspaces.dag :as ws.dag]
[metabase-enterprise.workspaces.dependencies :as ws.deps]
[metabase-enterprise.workspaces.execute :as ws.execute]
[metabase-enterprise.workspaces.isolation :as ws.isolation]
[metabase-enterprise.workspaces.util :as ws.u]
[metabase.driver.sql :as driver.sql]
[metabase.util :as u]
[metabase.util.log :as log]
[toucan2.core :as t2]))
(defn- query-ungranted-external-inputs
"Query for external inputs in a workspace that haven't been granted access yet.
External inputs are tables that are not shadowed by any output.
Returns seq of WorkspaceInput records where access_granted is false."
[workspace-id]
;; NOTE: Could optimize with table_id join if workspace_output gets that column,
;; but not worth it given the small number of rows per workspace.
(t2/select :model/WorkspaceInput
:workspace_id workspace-id
{:where [:and
[:= :workspace_input.access_granted false]
[:not [:exists {:select [1]
:from [[:workspace_output :wo]]
:where [:and
[:= :wo.workspace_id :workspace_input.workspace_id]
[:= :wo.db_id :workspace_input.db_id]
[:or
[:and [:= :wo.global_schema nil] [:= :workspace_input.schema nil]]
[:= :wo.global_schema :workspace_input.schema]]
[:= :wo.global_table :workspace_input.table]]}]]]}))
(defn- external-input->table
[{:keys [schema table]}]
{:schema schema
:name table})
(defn sync-grant-accesses!
"Grant read access to external input tables for a workspace that haven't been granted yet.
External inputs are tables that are read by transforms but not produced by any transform in the workspace.
This should be called after adding transforms to a workspace or when re-initializing workspace isolation
(e.g., after unarchiving)."
[{workspace-id :id :as workspace}]
(let [ungranted-inputs (query-ungranted-external-inputs workspace-id)]
(if-not (:database_details workspace)
;; TODO (chris 2025/12/15) we will want to make this strict before merging to master
#_(throw (ex-info "No database details, unable to grant read only access to the service account." {}))
(log/warn "No database details, unable to grant read only access to the service account.")
(when (seq ungranted-inputs)
(let [database (t2/select-one :model/Database :id (:database_id workspace))
tables (mapv external-input->table ungranted-inputs)]
(try
(ws.isolation/grant-read-access-to-tables! database workspace tables)
;; Mark inputs as granted after successful grant
(t2/update! :model/WorkspaceInput {:id [:in (map :id ungranted-inputs)]}
{:access_granted true})
(catch Exception e
(log/warn e "Error granting RO table permissions"))))))))
(defn sync-transform-dependencies!
"Analyze and persist dependencies for a workspace transform, then grant
read access to external input tables."
[{workspace-id :id, isolated-schema :schema :as workspace} transform]
(let [analysis (ws.deps/analyze-entity :transform transform)]
(ws.deps/write-dependencies! workspace-id isolated-schema :transform (:ref_id transform) analysis)
(sync-grant-accesses! workspace)))
(defn- build-remapping [workspace]
;; Build table remapping from stored WorkspaceOutput and WorkspaceOutputExternal data.
;; Maps [db_id global_schema global_table] -> {:db-id :schema :table :id} for isolated tables.
;; Also maps global_table_id -> same. This is more convenient and reliable for MBQL queries and Python transforms.
;; Also maps [db_id nil global_table] for tables in the default schema, so unqualified SQL references work.
;; This is used to remap queries, sources and targets to reflect the "isolated" tables used to seal the Workspace.
(let [outputs (t2/select [:model/WorkspaceOutput
:db_id :global_schema :global_table :global_table_id
:isolated_schema :isolated_table :isolated_table_id]
:workspace_id (:id workspace))
external-outputs (t2/select [:model/WorkspaceOutputExternal
:db_id :global_schema :global_table :global_table_id
:isolated_schema :isolated_table :isolated_table_id]
:workspace_id (:id workspace))
all-outputs (concat outputs external-outputs)
;; Get default schema for each database involved
db-ids (into #{} (map :db_id) all-outputs)
db-id->default (when (seq db-ids)
(t2/select-fn->fn :id #(driver.sql/default-schema (:engine %))
[:model/Database :id :engine]
:id [:in db-ids]))
table-map (reduce
(fn [m {:keys [db_id global_schema global_table global_table_id
isolated_schema isolated_table isolated_table_id]}]
(let [replacement {:db-id db_id
:schema isolated_schema
:table isolated_table
:id isolated_table_id}
default-schema (get db-id->default db_id)]
(cond-> (assoc m [db_id global_schema global_table] replacement)
global_table_id (assoc global_table_id replacement)
;; Add nil-schema entry for tables in the default schema
(= global_schema default-schema) (assoc [db_id nil global_table] replacement))))
{}
all-outputs)]
{:tables table-map
;; We never want to write to any global tables, so remap on-the-fly if we hit an un-mapped target.
:target-fallback (fn [[d s t]]
(log/warn "Missing remapping for" {:db d :schema s, :table s})
{:db-id d, :schema (:schema workspace), :table (ws.u/isolated-table-name s t), :id nil})
;; We won't need the field-map until we support MBQL.
:fields nil}))
(defn- backfill-isolated-table-id!
"Backfill workspace_output.isolated_table_id FK in the case where we just created the table for the first time."
[ref-id]
(when-let [table-id (:id (t2/query-one
{:from [[:workspace_output :wo]]
:join [[:metabase_table :t] [:and
[:= :t.db_id :wo.db_id]
[:= :t.schema :wo.isolated_schema]
[:= :t.name :wo.isolated_table]]]
:select [:t.id]
:where [:and
[:= :wo.ref_id ref-id]
[:or [:= :wo.isolated_table_id nil] [:not= :t.id :wo.isolated_table_id]]]}))]
(t2/update! :model/WorkspaceOutput {:ref_id ref-id} {:isolated_table_id table-id})))
(defn- backfill-external-isolated-table-id!
"Backfill workspace_output_external.isolated_table_id FK after running an enclosed external transform."
[transform-id]
(when-let [table-id (:id (t2/query-one
{:from [[:workspace_output_external :woe]]
:join [[:metabase_table :t] [:and
[:= :t.db_id :woe.db_id]
[:= :t.schema :woe.isolated_schema]
[:= :t.name :woe.isolated_table]]]
:select [:t.id]
:where [:and
[:= :woe.transform_id transform-id]
[:or [:= :woe.isolated_table_id nil] [:not= :t.id :woe.isolated_table_id]]]}))]
(t2/update! :model/WorkspaceOutputExternal {:transform_id transform-id} {:isolated_table_id table-id})))
(defn run-transform!
"Execute the given workspace transform or enclosed external transform."
([workspace transform]
(run-transform! workspace transform (build-remapping workspace)))
([workspace transform remapping]
(let [ref-id (:ref_id transform)
external-id (:id transform)
result (ws.isolation/with-workspace-isolation workspace (ws.execute/run-transform-with-remapping transform remapping))]
(when ref-id
(t2/update! :model/WorkspaceTransform ref-id {:last_run_at (:end_time result)
:last_run_message (:message result)}))
(when (= :succeeded (:status result))
(if ref-id
;; Workspace transform
(backfill-isolated-table-id! ref-id)
;; External transform (enclosed in workspace)
(backfill-external-isolated-table-id! external-id)))
result)))
;;;; ---------------------------------------- External Transform Sync ----------------------------------------
(defn- extract-external-transform-ids
"Extract IDs of external transforms from graph entities."
[entities]
(into [] (comp (filter #(= :external-transform (:node-type %)))
(map :id))
entities))
(defn- sync-external-outputs!
"Sync workspace_output_external table based on enclosed external transforms in the graph.
Deletes obsolete rows, upserts current ones, and backfills table IDs where possible."
[workspace-id isolated-schema entities]
(let [external-tx-ids (extract-external-transform-ids entities)
existing-rows (when (seq external-tx-ids)
(t2/select-fn->fn :transform_id identity
:model/WorkspaceOutputExternal
:workspace_id workspace-id))
existing-tx-ids (set (keys existing-rows))]
;; Delete rows for transforms no longer in the graph
(let [obsolete-ids (set/difference existing-tx-ids (set external-tx-ids))]
(when (seq obsolete-ids)
(t2/delete! :model/WorkspaceOutputExternal
:workspace_id workspace-id
:transform_id [:in obsolete-ids])))
;; Upsert rows for current external transforms
(when (seq external-tx-ids)
(let [transforms (t2/select [:model/Transform :id :target] :id [:in external-tx-ids])]
(doseq [{tx-id :id, {:keys [database schema name]} :target} transforms]
(let [isolated-table (ws.u/isolated-table-name schema name)
existing (get existing-rows tx-id)
;; Try to find table IDs - reuse existing if schema/table match
global-table-id (or (when (and existing
(= schema (:global_schema existing))
(= name (:global_table existing)))
(:global_table_id existing))
(t2/select-one-fn :id [:model/Table :id]
:db_id database :schema schema :name name))
isolated-table-id (or (when (and existing
(= isolated-schema (:isolated_schema existing))
(= isolated-table (:isolated_table existing)))
(:isolated_table_id existing))
(t2/select-one-fn :id [:model/Table :id]
:db_id database :schema isolated-schema :name isolated-table))]
(if existing
(t2/update! :model/WorkspaceOutputExternal (:id existing)
{:db_id database
:global_schema schema
:global_table name
:global_table_id global-table-id
:isolated_schema isolated-schema
:isolated_table isolated-table
:isolated_table_id isolated-table-id})
(t2/insert! :model/WorkspaceOutputExternal
{:workspace_id workspace-id
:transform_id tx-id
:db_id database
:global_schema schema
:global_table name
:global_table_id global-table-id
:isolated_schema isolated-schema
:isolated_table isolated-table
:isolated_table_id isolated-table-id}))))))))
(defn- normalize-table-schema
"Normalize a table's schema: replace nil with the driver's default schema.
Takes a map of db_id -> default-schema for lookup."
[db-id->default-schema {:keys [db_id schema] :as table}]
(if (some? schema)
table
(assoc table :schema (get db-id->default-schema db_id))))
(defn- sync-external-inputs!
"Sync workspace_input_external table based on enclosed external transforms in the graph.
Tracks external tables consumed by enclosed transforms that aren't outputs of other transforms.
Inputs are deduplicated per workspace. Normalizes nil schemas to driver defaults."
[workspace-id entities]
(let [external-tx-ids (extract-external-transform-ids entities)]
(when (seq external-tx-ids)
;; Query global Dependency table for all inputs of external transforms
(let [input-tables (t2/select [:model/Dependency :to_entity_id]
:from_entity_type :transform
:from_entity_id [:in external-tx-ids]
:to_entity_type :table)
input-table-ids (into #{} (map :to_entity_id) input-tables)
;; Get workspace output table IDs (both workspace and external outputs)
workspace-output-ids (t2/select-fn-set :global_table_id
[:model/WorkspaceOutput :global_table_id]
:workspace_id workspace-id
{:where [:not= :global_table_id nil]})
external-output-ids (t2/select-fn-set :global_table_id
[:model/WorkspaceOutputExternal :global_table_id]
:workspace_id workspace-id
{:where [:not= :global_table_id nil]})
all-output-ids (set/union workspace-output-ids external-output-ids)
;; Filter to only true external inputs (not outputs of transforms in graph)
external-input-ids (set/difference input-table-ids all-output-ids)]
(when (seq external-input-ids)
;; Get table details for external inputs
(let [tables (t2/select [:model/Table :id :db_id :schema :name] :id [:in external-input-ids])
;; Build default schema lookup for each database involved
db-ids (into #{} (map :db_id) tables)
db-id->default-schema (when (seq db-ids)
(into {}
(map (fn [{:keys [id engine]}]
[id (driver.sql/default-schema engine)]))
(t2/select [:model/Database :id :engine] :id [:in db-ids])))
;; Normalize schemas: nil -> default schema for the table's database
normalized-tables (map (partial normalize-table-schema db-id->default-schema) tables)
new-inputs (into {}
(map (fn [{:keys [id db_id schema name]}]
[[db_id schema name] {:table_id id}]))
normalized-tables)
;; Get existing external inputs
existing-inputs (t2/select-fn->fn (juxt :db_id :schema :table)
#(select-keys % [:id :table_id])
[:model/WorkspaceInputExternal :id :db_id :schema :table :table_id]
:workspace_id workspace-id)
existing-keys (set (keys existing-inputs))
new-keys (set (keys new-inputs))
obsolete-keys (set/difference existing-keys new-keys)
missing-keys (set/difference new-keys existing-keys)]
;; Delete obsolete rows
(doseq [[db_id schema table] obsolete-keys]
(t2/delete! :model/WorkspaceInputExternal
:workspace_id workspace-id
:db_id db_id
:schema schema
:table table))
;; Insert missing rows
(when (seq missing-keys)
(t2/insert! :model/WorkspaceInputExternal
(for [[db_id schema table] missing-keys]
{:workspace_id workspace-id
:db_id db_id
:schema schema
:table table
:table_id (:table_id (get new-inputs [db_id schema table]))
:access_granted false})))))))))
;; TODO save graph with invalidation hooks
(defn get-or-calculate-graph
"Return the graph. Also syncs workspace_output_external and workspace_input_external tables."
[{ws-id :id, isolated-schema :schema :as _workspace}]
(let [graph (ws.dag/path-induced-subgraph ws-id (t2/select-fn-vec (fn [{:keys [ref_id]}] {:entity-type :transform :id ref_id})
[:model/WorkspaceTransform :ref_id]
:workspace_id ws-id))]
(sync-external-outputs! ws-id isolated-schema (:entities graph))
(sync-external-inputs! ws-id (:entities graph))
graph))
(defn- transforms-to-execute
"Given a workspace and an optional filter, return the global and workspace definitions to run, in the correct order."
[{ws-id :id :as workspace} & {:keys [stale-only?]}]
;; 1. Depending on what we end up storing in this field, we might not be considering stale ancestors.
;; 2. For now, we never set this field to false, so we'll always run everything, even with the flag.
;; Why is there all this weird code then? To avoid unused references.
(let [stale-clause (if stale-only? {:where [:= :stale true]} {})
entities (:entities (get-or-calculate-graph workspace))
type->ids (u/group-by :node-type :id entities)
id->tx (merge
{}
(when-let [ids (seq (type->ids :external-transform))]
(t2/select-fn->fn :id identity
[:model/Transform :id :name :source :target]
:id [:in ids]
stale-clause))
(when-let [ref-ids (seq (type->ids :workspace-transform))]
(t2/select-fn->fn :ref_id identity
[:model/WorkspaceTransform :ref_id :name :source :target]
:workspace_id ws-id
:ref_id [:in ref-ids]
stale-clause)))]
(keep (comp id->tx :id) entities)))
(defn- id->str [ref-id-or-id]
(if (string? ref-id-or-id)
ref-id-or-id
(str "global-id:" ref-id-or-id)))
(defn execute-workspace!
"Execute all the transforms within a given workspace."
[workspace & {:keys [stale-only?] :or {stale-only? false}}]
(let [ws-id (:id workspace)
remapping (build-remapping workspace)]
(reduce
(fn [acc {external-id :id ref-id :ref_id :as transform}]
(let [node-type (if external-id :external-transform :workspace-transform)
id-str (id->str (or external-id ref-id))]
(try
;; Perhaps we want to return some of the metadata from this as well?
(if (= :succeeded (:status (run-transform! workspace transform remapping)))
(update acc :succeeded conj id-str)
;; Perhaps the status might indicate it never ran?
(update acc :failed conj id-str))
(catch Exception e
(log/error e "Failed to execute transform" {:workspace-id ws-id :node-type node-type :id id-str})
(update acc :failed conj id-str)))))
{:succeeded []
:failed []
:not_run []}
(transforms-to-execute workspace {:stale-only stale-only?}))))

View File

@@ -0,0 +1,6 @@
(ns metabase-enterprise.workspaces.init
"Initialize workspace models. Driver workspace isolation methods are now
defined directly in the driver files."
(:require
[metabase-enterprise.workspaces.models.workspace]
[metabase-enterprise.workspaces.models.workspace-transform]))

View File

@@ -0,0 +1,63 @@
(ns metabase-enterprise.workspaces.isolation
"Workspace database isolation functions.
The actual driver-specific implementations are now in the driver files:
- metabase.driver.h2
- metabase.driver.postgres
- metabase.driver.redshift
- metabase.driver.snowflake
- metabase.driver.sqlserver
- metabase.driver.clickhouse
- metabase.driver.bigquery-cloud-sdk
This namespace provides the public API for workspace isolation."
(:require
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]))
;;;; Delegation to driver methods
;; The actual multimethod implementations are now in the individual driver files.
;; These functions dispatch through the driver multimethods.
(defn init-workspace-database-isolation!
"Create database isolation for a workspace. Return the database details.
Delegates to driver/init-workspace-isolation!"
[database workspace]
(driver/init-workspace-isolation! (driver.u/database->driver database) database workspace))
(defn grant-read-access-to-tables!
"Grant read access to these tables.
Delegates to driver/grant-workspace-read-access!"
[database workspace tables]
(driver/grant-workspace-read-access! (driver.u/database->driver database) database workspace tables))
(defn destroy-workspace-isolation!
"Destroy all database resources created for workspace isolation.
This includes dropping tables, schemas/databases, users, roles, and logins.
Fails fast on first error. Should be called when deleting a workspace.
Delegates to driver/destroy-workspace-isolation!"
[database workspace]
(driver/destroy-workspace-isolation! (driver.u/database->driver database) database workspace))
;;;; Public API
(defn ensure-database-isolation!
"Wrapper around the driver method, to make migrations easier in future."
[workspace database]
(init-workspace-database-isolation! database workspace))
(defn do-with-workspace-isolation
"Impl of [[with-workspace-isolation]]."
[workspace thunk]
(when-not (:database_details workspace)
(throw (ex-info (str "Workspace isolation was not initialized. "
"If this is a test, ensure the workspace was created with isolation enabled.")
{:workspace-id (:id workspace)})))
(driver/with-swapped-connection-details (:database_id workspace)
(:database_details workspace)
(thunk)))
(defmacro with-workspace-isolation
"Execute body with necessary isolation."
[workspace & body]
`(do-with-workspace-isolation ~workspace (fn [] ~@body)))

View File

@@ -0,0 +1,129 @@
(ns metabase-enterprise.workspaces.merge
"Update global entities based on the definitions within a workspace.
All functions assumes that validation of the dependencies AFTER merge is done BEFORE calling any of these methods."
(:require
[metabase-enterprise.transforms.api :as transforms.api]
[metabase-enterprise.workspaces.models.workspace-merge]
[metabase-enterprise.workspaces.models.workspace-merge-transform]
[metabase-enterprise.workspaces.types :as ws.types]
[metabase.util.malli :as mu]
[toucan2.core :as t2])
(:import (java.sql Connection)))
(set! *warn-on-reflection* true)
(defn- create-merge-transform-history!
"Create a workspace_merge_transform record to track this merge operation.
Does nothing for :noop operations (archived transforms that don't exist globally)
or :delete operations (history would be CASCADE deleted with the transform anyway)."
[op merge-map]
(when (#{:create :update} op)
(t2/insert! :model/WorkspaceMergeTransform merge-map)))
;; TODO (crisptrutski 2025/12/10): When there are more entity types support, this should update those too.
(mu/defn merge-transform! :- [:map
[:op [:enum :create :delete :noop :update]]
[:ref_id ::ws.types/ref-id]
[:global_id {:optional true} [:maybe ::ws.types/appdb-id]]
[:error {:optional true} :any]]
"Make the given transform in the Changeset public, i.e. create or update the relevant model/Transform entities.
This should also clear it out from the Changset, as it no longer has any changes.
Creates a WorkspaceMergeTransform history record.
- :ws-transform - The workspace transform to merge
- :workspace - The workspace map (must have :id and :name keys)
- :workspace-merge-id - ID of parent WorkspaceMerge (caller must create this)
- :merging-user-id - user performing the merge
- :commit-message - description of the merge"
[{:keys [ws-transform workspace workspace-merge-id merging-user-id commit-message]}]
;; Problems this may run into:
;; It will recalculate and validate the dag greedily as it inserts each items, and this might fail of temporary conflicts.
;; Some of these conflicts could be solved by ordering things smartly, but in the general case (e.g. reversing a
;; chain) this will not be possible - we would need to defer all validation and calculation, which is a bit scary.
;; Let's just focus on the happy path for now though, as this is a fundamental problem and unrelated to the design.
(let [{:keys [global_id ref_id archived_at]} ws-transform
{:keys [error] :as result}
(cond (and global_id archived_at)
(merge
{:op :delete :global_id global_id :ref_id ref_id}
(try
(transforms.api/delete-transform! (t2/select-one :model/Transform :id global_id))
nil
(catch Throwable e
{:error e})))
global_id
(merge
{:op :update :global_id global_id :ref_id ref_id}
(try
(transforms.api/update-transform!
global_id (select-keys ws-transform [:name :description :source :target]))
nil
(catch Throwable e
{:error e})))
archived_at
{:op :noop :global_id nil :ref_id ref_id}
:else
(merge
{:op :create :global_id nil :ref_id ref_id}
(try
{:global_id (:id (transforms.api/create-transform!
(select-keys ws-transform [:name :description :source :target])
(:creator_id ws-transform)))}
(catch Throwable e
{:error e}))))]
(when-not error
(t2/delete! :model/WorkspaceTransform :ref_id ref_id)
(create-merge-transform-history!
(:op result)
{:workspace_merge_id workspace-merge-id
:workspace_id (:id workspace)
:workspace_name (:name workspace)
:transform_id (:global_id result)
:commit_message commit-message
:merging_user_id merging-user-id}))
result))
;; TODO (crisptrutski 2025-12-XX): When there are more entity types support, this should update those too.
(defn merge-workspace!
"Make all the transforms in the Changeset public, i.e. create or update the relevant model/Transform entities.
This should also clear the entire Changeset, as it no longer has any changes.
- workspace: The workspace map (must have :id and :name keys)
- merging-user-id: user performing the merge
- commit-message: description of the merge"
[workspace merging-user-id commit-message]
;; Perhaps we want to solve the N+1?
;; This will require reworking the lifecycle for validating and recalculating dependencies for the transforms.
(t2/with-transaction [tx]
(let [savepoint (.setSavepoint ^Connection tx)
ws-merge-id (t2/insert-returning-pk!
:model/WorkspaceMerge
{:workspace_id (:id workspace)
:workspace_name (:name workspace)
:commit_message commit-message
:creator_id merging-user-id})
result (reduce
(fn [acc ws-transform]
(let [{:keys [error] :as result} (merge-transform! {:ws-transform ws-transform
:workspace workspace
:workspace-merge-id ws-merge-id
:merging-user-id merging-user-id
:commit-message commit-message})]
(if error
(reduced (-> acc
(update :errors conj result)
(assoc-in [:merged :transforms] [])
#_(assoc :short_circuit true)))
(update-in acc [:merged :transforms] conj result))))
{:merged {:transforms []}
:errors []
#_#_:short_circuit false}
(t2/select :model/WorkspaceTransform :workspace_id (:id workspace)))]
(when (seq (:errors result))
(.rollback ^Connection tx savepoint))
result)))

View File

@@ -0,0 +1,66 @@
(ns metabase-enterprise.workspaces.models.workspace
(:require
[metabase-enterprise.workspaces.impl :as ws.impl]
[metabase-enterprise.workspaces.isolation :as ws.isolation]
[metabase.models.interface :as mi]
[metabase.util.log :as log]
[methodical.core :as methodical]
[toucan2.core :as t2]))
(methodical/defmethod t2/table-name :model/Workspace [_model] :workspace)
(t2/deftransforms :model/Workspace
{:graph mi/transform-json
:database_details mi/transform-encrypted-json
:status mi/transform-keyword})
(doto :model/Workspace
(derive :metabase/model)
(derive :hook/timestamped?))
(defn archive!
"Archive a workspace. Destroys database isolation resources (best-effort), revokes access grants, sets archived_at.
Cleanup failures are logged but don't block archiving - the workspace can be unarchived to retry cleanup,
or deleted once the underlying permission issues are resolved."
[{workspace-id :id :as workspace}]
;; Only destroy isolation if workspace was initialized (not uninitialized status)
(when (not= :uninitialized (:status workspace))
(let [database (t2/select-one :model/Database :id (:database_id workspace))]
;; Best-effort cleanup - don't block archiving on cleanup failures
(try
(ws.isolation/destroy-workspace-isolation! database workspace)
(catch Exception e
(log/warnf e "Failed to cleanup isolation resources for workspace %s, proceeding with archive" workspace-id)))
;; Mark all inputs as un-granted since the user may have been dropped
(t2/update! :model/WorkspaceInput {:workspace_id workspace-id}
{:access_granted false})))
(t2/update! :model/Workspace workspace-id {:archived_at [:now]}))
(defn unarchive!
"Unarchive a workspace. Re-initializes database isolation resources, re-grants read access
to source tables, and clears archived_at.
Cannot unarchive an uninitialized workspace (no isolation resources to restore)."
[workspace]
(when (= :uninitialized (:status workspace))
(throw (ex-info "Cannot unarchive an uninitialized workspace" {:status-code 400})))
(let [database (t2/select-one :model/Database (:database_id workspace))
isolation-result (ws.isolation/ensure-database-isolation! workspace database)
workspace-id (:id workspace)
_ (t2/update! :model/Workspace workspace-id
(merge {:archived_at nil}
(select-keys isolation-result [:schema :database_details])))
;; Re-fetch workspace to get updated database_details and schema for granting permissions
updated-ws (t2/select-one :model/Workspace workspace-id)]
;; Re-grant read access to source tables (inputs were marked un-granted during archive)
(ws.impl/sync-grant-accesses! updated-ws)))
(defn delete!
"Delete a workspace. Destroys database isolation resources and deletes the workspace."
[workspace]
;; Only destroy isolation if workspace was initialized (not uninitialized status)
(when (not= :uninitialized (:status workspace))
(when-let [database (t2/select-one :model/Database :id (:database_id workspace))]
(ws.isolation/destroy-workspace-isolation! database workspace)
(t2/update! :model/WorkspaceInput {:workspace_id (:id workspace)}
{:access_granted false})))
(t2/delete! :model/Workspace (:id workspace)))

View File

@@ -0,0 +1,17 @@
(ns metabase-enterprise.workspaces.models.workspace-dependency
"Model for WorkspaceDependency - edges between workspace transforms and their dependencies."
(:require
[metabase.models.interface :as mi]
[methodical.core :as methodical]
[toucan2.core :as t2]))
(methodical/defmethod t2/table-name :model/WorkspaceDependency [_model] :workspace_dependency)
(doto :model/WorkspaceDependency
(derive :metabase/model)
(derive :hook/created-at-timestamped?))
(t2/deftransforms :model/WorkspaceDependency
{:from_entity_type mi/transform-keyword
:to_entity_type mi/transform-keyword
:from_entity_id mi/transform-trim})

View File

@@ -0,0 +1,11 @@
(ns metabase-enterprise.workspaces.models.workspace-input
"Model for WorkspaceInput - external tables that workspace transforms consume."
(:require
[methodical.core :as methodical]
[toucan2.core :as t2]))
(methodical/defmethod t2/table-name :model/WorkspaceInput [_model] :workspace_input)
(doto :model/WorkspaceInput
(derive :metabase/model)
(derive :hook/timestamped?))

View File

@@ -0,0 +1,11 @@
(ns metabase-enterprise.workspaces.models.workspace-input-external
"Model for WorkspaceInputExternal - external tables that enclosed external transforms consume."
(:require
[methodical.core :as methodical]
[toucan2.core :as t2]))
(methodical/defmethod t2/table-name :model/WorkspaceInputExternal [_model] :workspace_input_external)
(doto :model/WorkspaceInputExternal
(derive :metabase/model)
(derive :hook/timestamped?))

View File

@@ -0,0 +1,70 @@
(ns metabase-enterprise.workspaces.models.workspace-log
(:require
[java-time.api :as t]
[metabase.models.interface :as mi]
[metabase.util.i18n :refer [deferred-tru]]
[methodical.core :as methodical]
[toucan2.core :as t2]))
(methodical/defmethod t2/table-name :model/WorkspaceLog [_model] :workspace_log)
(doto :model/WorkspaceLog
(derive :metabase/model)
(derive :hook/timestamped?))
(t2/deftransforms :model/WorkspaceLog
{:task mi/transform-keyword
:status mi/transform-keyword})
(defn- task->description
"Return a human-readable description for a workspace task."
[task]
(case task
:workspace-setup (deferred-tru "Setting up the workspace")
:database-isolation (deferred-tru "Provisioning database isolation")
:mirror-entities (deferred-tru "Mirroring entities")
:grant-read-access (deferred-tru "Granting permissions")
(name task)))
(t2/define-after-select :model/WorkspaceLog
[{:keys [task] :as log}]
(assoc log :description (task->description task)))
(defn start!
"Create a log entry for a workspace setup task. Returns the created log."
[workspace-id task]
(t2/insert-returning-instance! :model/WorkspaceLog
{:workspace_id workspace-id
:task task
:started_at (t/offset-date-time)
:status :started}))
(defn- complete!
[status log-id & [message]]
(assert (#{:success :failure} status) "It can only be either success or failure")
(let [data (cond-> {:completed_at (t/offset-date-time)
:status status}
message (assoc :message message))]
(t2/update! :model/WorkspaceLog log-id data)))
(defn success!
"Task was completed successfully"
[log-id & [message]]
(complete! :success log-id message))
(defn failure!
"Task has failed"
[log-id & [message]]
(complete! :failure log-id message))
(defmacro track!
"Wrap body with automatic logging: starts a log entry, marks success on completion, failure on exception."
[workspace-id task & body]
`(let [log# (start! ~workspace-id ~task)]
(try
(let [result# (do ~@body)]
(success! (:id log#))
result#)
(catch Exception e#
(failure! (:id log#) (ex-message e#))
(throw e#)))))

View File

@@ -0,0 +1,14 @@
(ns metabase-enterprise.workspaces.models.workspace-merge
"Model for WorkspaceMerge - tracks batch merge events with commit messages.
Survives workspace deletion through inlined workspace_name."
(:require
[methodical.core :as methodical]
[toucan2.core :as t2]))
;;; -------------------------------------------------- Model Setup --------------------------------------------------
(methodical/defmethod t2/table-name :model/WorkspaceMerge [_model] :workspace_merge)
(doto :model/WorkspaceMerge
(derive :metabase/model)
(derive :hook/created-at-timestamped?))

View File

@@ -0,0 +1,15 @@
(ns metabase-enterprise.workspaces.models.workspace-merge-transform
"Model for WorkspaceMergeTransform - tracks individual transform merges.
Linked to WorkspaceMerge for batch merges, standalone for single-transform merges.
Survives workspace deletion through inlined workspace_name."
(:require
[methodical.core :as methodical]
[toucan2.core :as t2]))
;;; -------------------------------------------------- Model Setup --------------------------------------------------
(methodical/defmethod t2/table-name :model/WorkspaceMergeTransform [_model] :workspace_merge_transform)
(doto :model/WorkspaceMergeTransform
(derive :metabase/model)
(derive :hook/created-at-timestamped?))

View File

@@ -0,0 +1,15 @@
(ns metabase-enterprise.workspaces.models.workspace-output
"Model for WorkspaceOutput - tables that workspace transforms produce."
(:require
[metabase.models.interface :as mi]
[methodical.core :as methodical]
[toucan2.core :as t2]))
(methodical/defmethod t2/table-name :model/WorkspaceOutput [_model] :workspace_output)
(doto :model/WorkspaceOutput
(derive :metabase/model)
(derive :hook/timestamped?))
(t2/deftransforms :model/WorkspaceOutput
{:ref_id mi/transform-trim})

View File

@@ -0,0 +1,11 @@
(ns metabase-enterprise.workspaces.models.workspace-output-external
"Model for WorkspaceOutputExternal - tables that enclosed external transforms produce."
(:require
[methodical.core :as methodical]
[toucan2.core :as t2]))
(methodical/defmethod t2/table-name :model/WorkspaceOutputExternal [_model] :workspace_output_external)
(doto :model/WorkspaceOutputExternal
(derive :metabase/model)
(derive :hook/timestamped?))

View File

@@ -0,0 +1,49 @@
(ns metabase-enterprise.workspaces.models.workspace-transform
"Model for WorkspaceTransform - holds the changeset of transforms being created
and edited within a workspace."
(:require
[clojure.string :as str]
[medley.core :as m]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.models.interface :as mi]
[methodical.core :as methodical]
[toucan2.core :as t2]))
(methodical/defmethod t2/table-name :model/WorkspaceTransform [_model] :workspace_transform)
(methodical/defmethod t2/primary-keys :model/WorkspaceTransform [_model] [:ref_id])
(doto :model/WorkspaceTransform
(derive :metabase/model)
(derive :hook/timestamped?)
(derive :hook/entity-id))
;; TODO (chris 2025/12/11) we need to share a bunch of stuff with transforms, i think we'll need to reorganize modules
;; suggestion: add a transforms-interfaces module which both transforms and workspaces depend on.
(defn- transform-source-out-DUPLICATED [m]
(-> m
mi/json-out-without-keywordization
(update-keys keyword)
(m/update-existing :query lib-be/normalize-query)
(m/update-existing :type keyword)
(m/update-existing :source-incremental-strategy #(update-keys % keyword))))
(defn- transform-source-in-DUPLICATED [m]
(-> m
(m/update-existing :query (comp lib/prepare-for-serialization lib-be/normalize-query))
mi/json-in))
(t2/deftransforms :model/WorkspaceTransform
{:ref_id {:in identity :out str/trim}
:source {:out transform-source-out-DUPLICATED, :in transform-source-in-DUPLICATED}
:target mi/transform-json})
(t2/define-before-insert :model/WorkspaceTransform
[instance]
;; Ref id can not be added here due to mysql t2 bug
(when-not (string? (not-empty (:ref_id instance)))
(throw (ex-info "ref_id required for WorkspaceTransform insertion."
{:instance instance})))
instance)

View File

@@ -0,0 +1,161 @@
(ns metabase-enterprise.workspaces.types
(:require
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]))
(mr/def ::appdb-id ms/PositiveInt)
(mr/def ::ref-id [:string {:min 1 :api/regex #".+"}])
(def entity-types
"The kinds of entities we can store within a Workspace."
[{:name "Transform"
:key :transform
:group :transforms
:model :model/Transform}])
(mr/def ::entity-type (into [:enum] (map :key) entity-types))
(mr/def ::entity-grouping (into [:enum] (map :group) entity-types))
;; Map like {:transforms [1 2 3]}
(mr/def ::entity-map
[:map-of ::entity-grouping [:sequential ms/PositiveInt]])
(mr/def ::execution-result
[:map
[:status [:enum :succeeded :failed]]
[:start_time {:optional true} [:maybe some?]]
[:end_time {:optional true} [:maybe some?]]
[:message {:optional true} [:maybe :string]]
[:table [:map
[:name :string]
[:schema {:optional true} [:maybe :string]]]]])
;;; ---------------------------------------- Graph/Problem Types ----------------------------------------
;; Node types used in workspace graphs and problem detection
(mr/def ::node-type [:enum :input-table :external-transform :workspace-transform])
(mr/def ::table-coord
"Logical reference to a table by database, schema, and name."
[:map
[:db_id ::appdb-id]
[:schema [:maybe :string]]
[:table :string]])
(mr/def ::node-ref
"Reference to a node in the graph or a related entity."
[:map
[:type ::node-type]
[:id [:or ::appdb-id ::ref-id]]
[:name {:optional true} :string]])
;;; ---------------------------------------- Problem Types ----------------------------------------
;; Problem type definitions with metadata for FE display and merge-blocking behavior
;; See validation.clj docstring for full documentation table
(def problem-types
"All problem types with their metadata.
Each entry has:
- :description - human-readable explanation
- :severity - :error, :warning, or :info
- :block-merge - whether this problem should prevent merging
- :static? - true if determinable from metadata alone, false if requires runtime info
## Categories
Dependency-based (about output state) - prefix indicates who depends on the output:
- unused-* : nothing depends on this output
- internal-downstream-* : transforms within the workspace graph depend on this
- external-downstream-* : transforms outside the workspace depend on this
Structural (about graph shape) - prefix indicates scope of the issue:
- internal-* : between workspace transforms only
- external-* : involves global transforms (upstream or downstream)"
{;; No dependents (informational)
:unused/not-run {:description "Output hasn't been created yet, but nothing depends on it"
:severity :info
:block-merge false
:static? true}
:unused/stale {:description "Output is stale (needs re-run), but nothing depends on it"
:severity :info
:block-merge false
:static? true}
:unused/failed {:description "Transform failed on last run, but nothing depends on it"
:severity :error
:block-merge false
:static? false}
;; Internal dependents (blocks workspace progress)
:internal-downstream/not-run {:description "Output hasn't been created yet, other workspace transforms need it"
:severity :warning
:block-merge true
:static? true}
:internal-downstream/stale {:description "Output is stale, other workspace transforms need fresh data"
:severity :warning
:block-merge true
:static? true}
:internal-downstream/failed {:description "Transform failed on last run, other workspace transforms need it"
:severity :error
:block-merge true
:static? false}
;; External dependents (would affect things after merge)
:external-downstream/not-run {:description "Output hasn't been created yet, external transforms depend on it"
:severity :warning
:block-merge true
:static? true}
:external-downstream/stale {:description "Output is stale, external transforms are using outdated data"
:severity :warning
:block-merge true
:static? true}
:external-downstream/removed-field {:description "Field was removed that external transforms reference"
:severity :error
:block-merge true
:static? true}
:external-downstream/removed-table {:description "Table was removed, external transforms will stop receiving updates"
:severity :warning
:block-merge false
:static? true}
;; Structural issues (conflicts/cycles)
:internal/target-conflict {:description "Multiple workspace transforms target the same table"
:severity :error
:block-merge true
:static? true}
:internal/cycle {:description "Circular dependency between workspace transforms"
:severity :error
:block-merge true
:static? true}
:external/target-conflict {:description "Workspace transform conflicts with an existing global transform's target"
:severity :error
:block-merge true
:static? true}
:external/cycle {:description "Workspace changes would create a circular dependency with global transforms"
:severity :error
:block-merge true
:static? true}})
(mr/def ::problem-type (into [:enum] (keys problem-types)))
(mr/def ::severity [:enum :error :warning :info])
;; Categories derived from problem-type namespaces
(mr/def ::problem-category
(into [:enum] (distinct (map (comp keyword namespace) (keys problem-types)))))
;; Problem names derived from problem-type names
(mr/def ::problem-name
(into [:enum] (distinct (map (comp keyword name) (keys problem-types)))))
(mr/def ::problem
"A problem detected during workspace validation."
[:map
[:category ::problem-category]
[:problem ::problem-name]
[:severity ::severity]
[:block_merge :boolean]
[:data :map]])

View File

@@ -0,0 +1,167 @@
(ns metabase-enterprise.workspaces.util
(:require
[clojure.string :as str]
[metabase.system.core :as system]))
(defn assert-transform!
"Test whether we support the given entity type within workspaces yet.
Named for the only case we support currently, to make call sites assumptions more obvious."
[entity-type]
(when (not= "transform" (name entity-type))
(throw (ex-info "Only transform entity type is supported"
{:status-code 400
:entity-type entity-type}))))
(defn assert-transforms!
"Test that only supported types are given in the given list.
Named for the only case we support currently, to make call sites assumptions more obvious."
[entities]
(when-let [other-types (seq (remove #{:transform} (map :entity-type entities)))]
(throw (ex-info "Only transform entities are currently supported"
{:status-code 400
:unsupported-types other-types}))))
(defn- toposort-visit [node child->parents visited result]
(cond
(visited node) [visited result]
:else (let [parents (child->parents node [])
[visited' result'] (reduce (fn [[v r] p]
(toposort-visit p child->parents v r))
[(conj visited node) result]
parents)]
[visited' (conj result' node)])))
(defn toposort-dfs
"Perform a topological sort using depth-first search.
Takes a map from child nodes to their parent nodes (dependencies).
Returns nodes in topological order (dependencies before dependents)."
[child->parents]
;; TODO (Chris 2025-11-20): Detect cycles and throw an error. (In practice inputs will never be cyclic, but still.)
(let [all-nodes (set (keys child->parents))]
(loop [visited #{}
result []
remaining all-nodes]
(if (empty? remaining)
result
(let [node (first remaining)
[visited' result'] (toposort-visit node child->parents visited result)]
(recur visited' result' (disj remaining node)))))))
;;; Naming
;; re-using https://github.com/metabase/metabase/pull/61887/commits/c92e4a9cc451c61a13fef19ed9d6107873b17f07
;; (original ws isolation code)
(defn- instance-uuid-slug
"Create a slug from the site UUID, taking the first character of each section."
[site-uuid-string]
(->> (str/split site-uuid-string #"-")
(map first)
(apply str)))
;; WARNING: Changing this prefix requires backwards compatibility handling for existing workspaces.
;; The prefix is used to identify isolation namespaces in the database, and existing workspaces
;; will have namespaces created with the current prefix.
(def ^:private isolated-prefix "mb__isolation")
(defn isolation-namespace-name
"Generate namespace/database name for workspace isolation following mb__isolation_<slug>_<workspace-id> pattern.
Uses 'namespace' as the generic term that maps to 'schema' in Postgres, 'database' in ClickHouse, etc."
[workspace]
(assert (some? (:id workspace)) "Workspace must have an :id")
(let [instance-slug (instance-uuid-slug (str (system/site-uuid)))
clean-workspace-id (str/replace (str (:id workspace)) #"[^a-zA-Z0-9]" "_")]
(format "%s_%s_%s" isolated-prefix instance-slug clean-workspace-id)))
(defn isolated-table-name
"Generate name for a table mirroring transform target table in the isolated database namespace.
Returns schema__name when schema is present, or __name when schema is nil (to distinguish from global tables)."
[schema name]
;; TODO: This naming scheme is not guaranteed to give a unique name, even if it's likely
;; to always be unique in practice. Consider adding a hash suffix or conflict detection
;; before merging to master.
(if schema
(format "%s__%s" schema name)
(format "__%s" name)))
(defn isolation-user-name
"Generate username for workspace isolation."
[workspace]
(let [instance-slug (instance-uuid-slug (str (system/site-uuid)))]
(format "%s_%s_%s" isolated-prefix instance-slug (:id workspace))))
(def ^:private password-char-sets
"Character sets for password generation. Cycles through these to ensure representation from each."
["ABCDEFGHJKLMNPQRSTUVWXYZ"
"abcdefghjkmnpqrstuvwxyz"
"123456789"
"!#$%&*+-="])
(defn random-isolated-password
"Generate a random password suitable for most database engines.
Ensures the password contains characters from all sets (uppercase, lowercase, digits, special)
by cycling through the character sets. Result is shuffled for randomness."
[]
(->> (cycle password-char-sets)
(take (+ 32 (rand-int 32)))
(map rand-nth)
shuffle
(apply str)))
;;; -------------------------------------------------- Cute Names --------------------------------------------------
(def ^:private adjectives
"Adjectives for generating cute ref IDs, inspired by Docker's naming scheme."
["admiring" "adoring" "affectionate" "agitated" "amazing" "angry" "awesome"
"beautiful" "blissful" "bold" "boring" "brave" "busy" "charming" "clever"
"cool" "compassionate" "competent" "condescending" "confident" "cranky"
"crazy" "dazzling" "determined" "distracted" "dreamy" "eager" "ecstatic"
"elastic" "elated" "elegant" "eloquent" "epic" "exciting" "fervent"
"festive" "flamboyant" "focused" "friendly" "frosty" "funny" "gallant"
"gifted" "goofy" "gracious" "great" "happy" "hardcore" "heuristic" "hopeful"
"hungry" "infallible" "inspiring" "intelligent" "interesting" "jolly"
"jovial" "keen" "kind" "laughing" "loving" "lucid" "magical" "modest"
"musing" "mystifying" "naughty" "nervous" "nice" "nifty" "nostalgic"
"objective" "optimistic" "peaceful" "pedantic" "pensive" "practical"
"priceless" "quirky" "quizzical" "recursing" "relaxed" "reverent" "romantic"
"sad" "serene" "sharp" "silly" "sleepy" "stoic" "strange" "stupefied"
"suspicious" "sweet" "tender" "thirsty" "trusting" "unruffled" "upbeat"
"vibrant" "vigilant" "vigorous" "wizardly" "wonderful" "xenodochial" "youthful"
"zealous" "zen"])
(def ^:private animals
"Animals for generating cute ref IDs."
["albatross" "alligator" "alpaca" "ant" "anteater" "antelope" "armadillo"
"baboon" "badger" "bat" "bear" "beaver" "bee" "bison" "boar" "buffalo"
"butterfly" "camel" "capybara" "caribou" "cat" "caterpillar" "cheetah"
"chicken" "chimpanzee" "chinchilla" "cobra" "condor" "cormorant" "coyote"
"crab" "crane" "crocodile" "crow" "deer" "dingo" "dog" "dolphin" "donkey"
"dove" "dragonfly" "duck" "eagle" "echidna" "eel" "elephant" "elk" "emu"
"falcon" "ferret" "finch" "firefly" "flamingo" "fox" "frog" "gazelle"
"gerbil" "giraffe" "goat" "goose" "gopher" "gorilla" "grasshopper" "grouse"
"hamster" "hare" "hawk" "hedgehog" "heron" "hippopotamus" "hornet" "horse"
"hummingbird" "hyena" "ibex" "iguana" "impala" "jackal" "jaguar" "jellyfish"
"kangaroo" "kingfisher" "kiwi" "koala" "komodo" "kookaburra" "lemur"
"leopard" "lion" "lizard" "llama" "lobster" "lynx" "magpie" "mallard"
"manatee" "mandrill" "marmot" "meerkat" "mink" "mole" "mongoose" "monkey"
"moose" "moth" "mouse" "mule" "narwhal" "newt" "nightingale" "ocelot"
"octopus" "okapi" "opossum" "orangutan" "orca" "oryx" "osprey" "ostrich"
"otter" "owl" "ox" "panda" "panther" "parrot" "partridge" "peacock"
"pelican" "penguin" "pheasant" "pig" "pigeon" "platypus" "pony" "porcupine"
"porpoise" "puma" "quail" "rabbit" "raccoon" "ram" "raven" "reindeer"
"rhinoceros" "salamander" "salmon" "sardine" "scorpion" "seahorse" "seal"
"shark" "sheep" "shrew" "skunk" "sloth" "snail" "snake" "sparrow" "spider"
"squid" "squirrel" "starfish" "stingray" "stork" "swallow" "swan" "tapir"
"tiger" "toad" "toucan" "turkey" "turtle" "viper" "vulture" "wallaby"
"walrus" "warthog" "wasp" "weasel" "whale" "wolf" "wolverine" "wombat"
"woodpecker" "yak" "zebra"])
(defn generate-name
"Generate a cute adjective-animal style name. Making sure it's unique is your job."
[]
(str (rand-nth adjectives) "-" (rand-nth animals)))
(defn generate-ref-id
"Generate a cute adjective-animal style ref ID with a random suffix.
Format: adjective-animal-xxxx (where xxxx is a random 4-char hex suffix)"
[]
(str (generate-name) "-" (format "%04x" (rand-int 65536))))

View File

@@ -0,0 +1,279 @@
(ns metabase-enterprise.workspaces.validation
"Validation of workspace changes against downstream dependencies.
ASSUMPTION: All transforms in the workspace have been run, and their outputs are not stale.
This validation checks the *current* isolated table schema against external transform requirements.
## Problem Types
See `metabase-enterprise.workspaces.types/problem-types` for the full list of problem types
with their descriptions, severity levels, and merge-blocking behavior.
## Currently Implemented
| Category | Problem | Description |
|---------------------|---------------|----------------------------------------------------------------|
| unused | not-run | Output hasn't been created, nothing depends on it |
| internal-downstream | not-run | Output hasn't been created, other workspace transforms need it |
| external-downstream | not-run | Output hasn't been created, external transforms need it |
| external-downstream | removed-field | Field was removed that external transforms reference |"
(:require
[metabase-enterprise.workspaces.types :as ws.t]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata.protocols :as lib.metadata.protocols]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
;;;; ---------------------------------------- Metadata Provider Override ----------------------------------------
(defn- make-field-override-provider
"Create a metadata provider that overrides a table's fields with a different set of fields.
Used to simulate what external transforms would see after a merge.
- `base-provider`: the underlying metadata provider
- `target-table-id`: the global table ID whose fields we're overriding
- `replacement-fields`: the fields from the isolated table (what fields will exist after merge)"
[base-provider target-table-id replacement-fields]
(let [replacement-field-names (into #{} (map :name) replacement-fields)
;; TODO we probably want to do one query for *all* the tables we care about, earlier in our validation.
table-field-id? (t2/select-fn-set :id [:model/Field :id] :table_id target-table-id)]
(reify
lib.metadata.protocols/MetadataProvider
(database [_this]
(lib.metadata.protocols/database base-provider))
(metadatas [_this metadata-spec]
;; Yes, for some reason :id is actually a set of ids
(let [{spec-type :lib/type spec-field-ids :id} metadata-spec]
(if (and (= :metadata/column spec-type) (some table-field-id? spec-field-ids))
;; Return replacement fields with :active based on whether they exist
(let [original-fields (lib.metadata.protocols/metadatas base-provider metadata-spec)]
(map (fn [field]
(if (contains? replacement-field-names (:name field))
field
(assoc field :active false)))
original-fields))
;; Delegate everything else
(lib.metadata.protocols/metadatas base-provider metadata-spec))))
(setting [_this setting-key]
(lib.metadata.protocols/setting base-provider setting-key))
lib.metadata.protocols/CachedMetadataProvider
(cached-metadatas [_this metadata-type metadata-ids]
(lib.metadata.protocols/cached-metadatas base-provider metadata-type metadata-ids))
(store-metadata! [_this a-metadata]
(lib.metadata.protocols/store-metadata! base-provider a-metadata))
(cached-value [_this k not-found]
(lib.metadata.protocols/cached-value base-provider k not-found))
(cache-value! [_this k v]
(lib.metadata.protocols/cache-value! base-provider k v))
(has-cache? [_this]
(lib.metadata.protocols/has-cache? base-provider))
(clear-cache! [_this]
(lib.metadata.protocols/clear-cache! base-provider)))))
;;;; ---------------------------------------- External Transform Discovery ----------------------------------------
;; TODO we could join these extra fields onto the graph before calling into this namespace, and save coupling
(defn- workspace-outputs
"Get all workspace outputs with their global and isolated table info.
Includes both workspace transform outputs and enclosed external transform outputs."
[workspace-id]
(let [ws-outputs (t2/select [:model/WorkspaceOutput
:id :ref_id :db_id
:global_schema :global_table :global_table_id
:isolated_schema :isolated_table :isolated_table_id]
:workspace_id workspace-id)
;; External outputs use transform_id (int) instead of ref_id (string)
;; Map transform_id to ref_id for compatibility with the rest of the validation code
ext-outputs (map #(assoc % :ref_id (:transform_id %) :external? true)
(t2/select [:model/WorkspaceOutputExternal
:id :transform_id :db_id
:global_schema :global_table :global_table_id
:isolated_schema :isolated_table :isolated_table_id]
:workspace_id workspace-id))]
(concat ws-outputs ext-outputs)))
(defn- ids-for-type [entities desired-node-type]
(for [{:keys [node-type id]} entities, :when (= node-type desired-node-type)] id))
(defn- checked-out-transform-ids
"Get the global IDs of transforms that are checked out into this workspace.
Extracts workspace-transform ref_ids from the graph and looks up their global_ids."
[{:keys [entities] :as _graph}]
(if-let [ref-ids (seq (ids-for-type entities :workspace-transform))]
;; TODO if we keep track of the global_id properties when building the graph, we can save this query.
(t2/select-fn-set :global_id
[:model/WorkspaceTransform :global_id]
:ref_id [:in ref-ids]
{:where [:not= :global_id nil]})
#{}))
(defn- enclosed-transform-ids
"Get the IDs of external transforms that are enclosed by the workspace graph.
These transforms run within the workspace, so they don't need further static validation."
[{:keys [entities] :as _graph}]
(ids-for-type entities :external-transform))
(def ^:private table? #{:table})
(defn- internal-dependents
"Build a map of non-table ids to the non-table ids that directly depend on it (internal to workspace).
This is essentially an inversion of the dependencies map, with some filtering."
[{:keys [dependencies] :as _graph}]
;; :dependencies is {child #{parents...}}, we need to invert to {parent #{children...}}
(reduce (fn [acc [{child-type :node-type, child-id :id} parents]]
(if (table? child-type)
acc
(reduce (fn [acc {parent-type :node-type, parent-id :id}]
(if (table? parent-type)
acc
(update acc parent-id (fnil conj #{}) child-id)))
acc
parents)))
{}
dependencies))
(defn- external-transform-dependents
"Get external transforms that depend on a given table (by global_table_id).
Excludes transforms that are checked out or enclosed in the workspace."
[global-table-id checked-out-ids enclosed-ids]
(when global-table-id
(let [dependent-transform-ids (t2/select-fn-set :from_entity_id
[:model/Dependency :from_entity_id]
:to_entity_type :table
:to_entity_id global-table-id
:from_entity_type :transform)
;; Remove checked out and enclosed transforms
external-ids (-> dependent-transform-ids
(disj nil)
(as-> ids (apply disj ids checked-out-ids))
(as-> ids (apply disj ids enclosed-ids)))]
(when (seq external-ids)
(t2/select [:model/Transform :id :name :source] :id [:in external-ids])))))
;;;; ---------------------------------------- Validation Logic ----------------------------------------
(defn- validate-transform-against-fields
"Check if a transform would break given a modified set of fields.
Returns bad refs if the transform's query references fields that don't exist in replacement-fields."
[transform global-table-id replacement-fields]
(let [db-id (get-in transform [:source :query :database])]
(when db-id
(let [base-provider (lib-be/application-database-metadata-provider db-id)
override-provider (make-field-override-provider base-provider global-table-id replacement-fields)
query (lib/query override-provider (get-in transform [:source :query]))]
(lib/find-bad-refs query)))))
(defn- get-table-fields
"Get all active fields for a table by ID."
[table-id]
(when table-id
(t2/select [:model/Field :id :name :base_type :semantic_type]
:table_id table-id
:active true)))
(defn- make-problem
"Construct a problem map with type metadata from ws.t/problem-types.
Splits the namespaced problem-type into :category and :problem keys."
[problem-type data]
(let [{:keys [severity block-merge description]} (get ws.t/problem-types problem-type)]
{:category (keyword (namespace problem-type))
:problem (keyword (name problem-type))
:severity severity
:block_merge block-merge
:description description
:data data}))
(defn- problems-for-output
"Find problems related to a single workspace output.
Returns a sequence of problem maps."
[output checked-out-ids enclosed-ids internal-dependents-map]
(let [{:keys [ref_id db_id global_schema global_table global_table_id
isolated_table_id external?]} output
;; For external outputs, ref_id is actually the transform_id (an int)
transform-type (if external? :external-transform :workspace-transform)
external-transforms (external-transform-dependents global_table_id checked-out-ids enclosed-ids)
has-external-dependents? (seq external-transforms)
internal-dependents (get internal-dependents-map ref_id)
has-internal-dependents? (seq internal-dependents)
table-coord {:db_id db_id :schema global_schema :table global_table}]
(cond
;; Case 1: Isolated table doesn't exist yet
(nil? isolated_table_id)
(cond
;; External transforms depend on this output that doesn't exist
has-external-dependents?
[(make-problem :external-downstream/not-run
{:output table-coord
:transform {:type transform-type
:id ref_id}
:dependents (mapv (fn [{:keys [id name]}]
{:type :external-transform
:id id
:name name})
external-transforms)})]
;; Internal workspace transforms depend on this output that doesn't exist
has-internal-dependents?
[(make-problem :internal-downstream/not-run
{:output table-coord
:transform {:type transform-type
:id ref_id}
:dependents (mapv (fn [dep-ref-id]
{:type :workspace-transform
:id dep-ref-id})
internal-dependents)})]
;; No dependents - just informational
:else
[(make-problem :unused/not-run
{:output table-coord
:transform {:type transform-type
:id ref_id}})])
;; Case 2: Isolated table exists - check for field breakages
has-external-dependents?
(let [isolated-fields (get-table-fields isolated_table_id)]
(into []
(mapcat (fn [transform]
(when-let [bad-refs (validate-transform-against-fields
transform global_table_id isolated-fields)]
[(make-problem :external-downstream/removed-field
{:output table-coord
:transform {:type :external-transform
:id (:id transform)
:name (:name transform)}
:bad-refs bad-refs})])))
external-transforms))
;; Case 3: No problems
:else [])))
;;;; ---------------------------------------- Public API ----------------------------------------
(defn find-downstream-problems
"Find problems that would affect transforms outside the workspace after merge.
Takes the workspace graph (from `ws.dag/path-induced-subgraph`) as an argument.
Returns a sequence of problem maps, each with:
- :category - the problem category (e.g. :unused, :internal-downstream, :external-downstream)
- :problem - the specific problem (e.g. :not-run, :removed-field)
- :severity - :error, :warning, or :info
- :block-merge - whether this problem should prevent merging
- :data - further data, depending on type
Currently implemented problem types (category + problem):
- :unused/not-run - output table hasn't been created (informational, no dependents)
- :internal-downstream/not-run - output table hasn't been created, other workspace transforms need it
- :external-downstream/not-run - output table hasn't been created, but external transforms depend on it
- :external-downstream/removed-field - a field was removed that external transforms need"
[workspace-id graph]
(let [outputs (workspace-outputs workspace-id)
checked-out-ids (checked-out-transform-ids graph)
enclosed-ids (enclosed-transform-ids graph)
dependents (internal-dependents graph)]
(into []
(mapcat #(problems-for-output % checked-out-ids enclosed-ids dependents))
outputs)))

View File

@@ -181,6 +181,20 @@
:table #{checkins-id venues-id}}
(calculation/upstream-deps:transform transform)))))))
(deftest ^:parallel upstream-deps-python-transform-test
(mt/with-premium-features #{:transforms}
(let [products-id (mt/id :products)
orders-id (mt/id :orders)]
(mt/with-temp [:model/Transform transform {:name "Test Transform"
:source {:type :python
:source-tables {"PRODUCTS" products-id
"ORDERS" orders-id}
:body "..."}
:target {:schema "PUBLIC"
:name "test_output"}}]
(is (= {:table #{products-id orders-id}}
(calculation/upstream-deps:transform transform)))))))
(deftest ^:parallel upstream-deps-card-native-with-parameter-source-test
(let [mp (mt/metadata-provider)
products-id (mt/id :products)

View File

@@ -107,7 +107,19 @@
:model/ConnectionImpersonation
:model/CloudMigration
:model/Comment
:model/CommentReaction})
:model/CommentReaction
;; TODO (lbrdnk 2025/12/17): I've added rest of the workspace models here as Workspace was present. I believe
;; going forward all of that will be available for export. We should revisit this later in the project.
:model/Workspace
:model/WorkspaceDependency
:model/WorkspaceInput
:model/WorkspaceInputExternal
:model/WorkspaceLog
:model/WorkspaceMerge
:model/WorkspaceMergeTransform
:model/WorkspaceOutput
:model/WorkspaceOutputExternal
:model/WorkspaceTransform})
(deftest ^:parallel comprehensive-entity-id-test
(let [entity-id-models (->> (v2.entity-ids/toucan-models)

View File

@@ -65,9 +65,11 @@
(testing "GET /api/ee/transform-job/:id/transforms"
(mt/with-premium-features #{:transforms}
(let [crowberto-id (mt/user->id :crowberto)]
(mt/with-temp [:model/Transform {transform1-id :id} {:name "tr1" :creator_id crowberto-id}
:model/Transform {transform2-id :id} {:name "tr2" :creator_id crowberto-id}
:model/Transform _transform3 {}
(mt/with-temp [:model/Transform {transform1-id :id} {:name "tr1" :creator_id crowberto-id
:target {:database (mt/id)}}
:model/Transform {transform2-id :id} {:name "tr2" :creator_id crowberto-id
:target {:database (mt/id)}}
:model/Transform _transform3 {:target {:database (mt/id)}}
:model/TransformTag {tag-id :id} {}
:model/TransformTransformTag _ {:transform_id transform1-id :tag_id tag-id :position 0}
:model/TransformTransformTag _ {:transform_id transform2-id :tag_id tag-id :position 0}

View File

@@ -11,6 +11,7 @@
with-transform-cleanup!]]
[metabase-enterprise.transforms.util :as transforms.util]
[metabase.driver :as driver]
[metabase.lib-be.metadata.jvm :as lib.metadata.jvm]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.query-processor :as qp]
@@ -278,6 +279,139 @@
(is (=? [{:id t2-id}]
(mt/user-http-request :crowberto :get 200 "ee/transform" :tag_ids [tag2-id])))))))))
(deftest filter-transforms-by-type-test
(testing "should be able to filter transforms by source type"
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/with-premium-features #{:transforms :transforms-python}
(let [mp (mt/metadata-provider)
nq (lib/native-query mp "select 1")]
(mt/with-temp [:model/Transform {native-transform-id :id}
{:name "Native Transform"
:source {:type "query"
:query nq}
:target {:type "table"
:name (str "test_native_" (u/generate-nano-id))}}
:model/Transform {python-transform-id :id}
{:name "Python Transform"
:source {:type "python"
:body "print('hello')"
:source-tables {}}
:target {:type "table"
:name (str "test_python_" (u/generate-nano-id))
:database (mt/id)}}]
(testing "filter by native type"
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :type ["native"])]
(is (some #(= native-transform-id (:id %)) results))
(is (not (some #(= python-transform-id (:id %)) results)))))
(testing "filter by python type"
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :type ["python"])]
(is (some #(= python-transform-id (:id %)) results))
(is (not (some #(= native-transform-id (:id %)) results)))))
(testing "filter by both types returns all"
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :type ["native" "python"])]
(is (some #(= native-transform-id (:id %)) results))
(is (some #(= python-transform-id (:id %)) results))))))))))
(deftest filter-transforms-by-database-id-test
(testing "should be able to filter transforms by database_id"
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/with-premium-features #{:transforms}
(mt/with-temp [:model/Database {other-db-id :id} {:engine :h2 :details {}}
:model/Transform {transform1-id :id} {:name "Transform for main DB"
:source {:type "query"
:query {:database (mt/id)
:type "native"
:native {:query "SELECT 1"
:template-tags {}}}}
:target_db_id (mt/id)
:target {:type "table"
:name (str "test_main_" (u/generate-nano-id))}}
:model/Transform {transform2-id :id} {:name "Transform for other DB"
:source {:type "query"
:query {:database other-db-id
:type "native"
:native {:query "SELECT 2"
:template-tags {}}}}
:target_db_id other-db-id
:target {:type "table"
:name (str "test_other_" (u/generate-nano-id))}}]
(testing "filter by main database id"
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :database_id (mt/id))]
(is (some #(= transform1-id (:id %)) results))
(is (not (some #(= transform2-id (:id %)) results)))))
(testing "filter by other database id"
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :database_id other-db-id)]
(is (some #(= transform2-id (:id %)) results))
(is (not (some #(= transform1-id (:id %)) results)))))
(testing "filter by non-existent database returns empty for our transforms"
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :database_id 999999)]
(is (not (some #(= transform1-id (:id %)) results)))
(is (not (some #(= transform2-id (:id %)) results))))))))))
(deftest filter-transforms-by-database-id-target-test
(testing "should match transforms where target database matches"
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/with-premium-features #{:transforms :transforms-python}
(mt/with-temp [:model/Database {target-db-id :id} {:engine :h2 :details {}}
:model/Transform {transform-id :id} {:name "Python Transform with target DB"
:source {:type "python"
:body "print('hello')"
:source-tables {}}
:target_db_id target-db-id
:target {:type "table"
:name (str "test_target_" (u/generate-nano-id))
:database target-db-id}}]
(testing "filter by target database id"
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :database_id target-db-id)]
(is (some #(= transform-id (:id %)) results)))))))))
(deftest filter-transforms-combined-test
(testing "should be able to combine database_id and type filters"
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/with-premium-features #{:transforms :transforms-python}
(let [nq (lib/native-query (mt/metadata-provider) "select 1")]
(mt/with-temp [:model/Database {other-db-id :id} {:engine :h2 :details {}}
:model/Transform {query-main-id :id}
{:name "Query on main DB"
:source_type :native
:source {:type "query"
:query nq}
:target {:type "table"
:name (str "test_qm_" (u/generate-nano-id))}}
:model/Transform {python-main-id :id}
{:name "Python on main DB"
:source_type :python
:source {:type "python"
:body "print('hello')"
:source-tables {}}
:target {:type "table"
:name (str "test_pm_" (u/generate-nano-id))
:database (mt/id)}}
:model/Transform {query-other-id :id}
{:name "Query on other DB"
:source_type :native
:source {:type "query"
:query (lib/native-query
(lib.metadata.jvm/application-database-metadata-provider other-db-id)
"select 2")}
:target {:type "table"
:name (str "test_qo_" (u/generate-nano-id))
:database other-db-id}}]
(testing "filter by main database and query type"
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform"
:database_id (mt/id)
:type ["native"])]
(is (some #(= query-main-id (:id %)) results))
(is (not (some #(= python-main-id (:id %)) results)))
(is (not (some #(= query-other-id (:id %)) results)))))
(testing "filter by main database and python type"
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform"
:database_id (mt/id)
:type ["python"])]
(is (some #(= python-main-id (:id %)) results))
(is (not (some #(= query-main-id (:id %)) results)))
(is (not (some #(= query-other-id (:id %)) results)))))))))))
(deftest get-transforms-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/with-premium-features #{:transforms}

View File

@@ -108,3 +108,44 @@
(catch Exception _e
;; Ignore cleanup errors
nil)))))))))
;;; ------------------------------------------------------------
;;; Filter xf tests
;;; ------------------------------------------------------------
(deftest ^:parallel matching-timestamp?-test
(testing "matching-timestamp? checks if a timestamp falls within a date range [start, end)"
(let [matching-timestamp? #'transforms.util/matching-timestamp?
field-path [:start_time]
range-jan-feb {:start "2024-01-01T00:00:00Z" :end "2024-02-01T00:00:00Z"}
range-start-only {:start "2024-01-01T00:00:00Z" :end nil}
range-end-only {:start nil :end "2024-02-01T00:00:00Z"}]
(testing "with both start and end bounds"
(are [expected timestamp]
(= expected (matching-timestamp? {:start_time timestamp} field-path range-jan-feb))
nil nil ; missing field returns nil
true "2024-01-15T12:00:00Z" ; timestamp in middle of range
false "2023-12-15T12:00:00Z" ; timestamp before range
false "2024-02-15T12:00:00Z" ; timestamp after range
true "2024-01-01T00:00:00Z" ; start boundary is inclusive
true "2024-02-01T00:00:00Z")) ; end boundary is inclusive too 🤷
(testing "with only start bound"
(are [expected timestamp]
(= expected (matching-timestamp? {:start_time timestamp} field-path range-start-only))
true "2024-01-15T12:00:00Z" ; timestamp after start
true "2024-02-15T12:00:00Z" ; any timestamp after start
false "2023-12-15T12:00:00Z")) ; timestamp before start
(testing "with only end bound"
(are [expected timestamp]
(= expected (matching-timestamp? {:start_time timestamp} field-path range-end-only))
true "2024-01-15T12:00:00Z" ; timestamp before end
true "2023-12-15T12:00:00Z" ; any timestamp before end
false "2024-02-15T12:00:00Z")) ; timestamp after end
(testing "returns nil when field value is missing"
(are [job] (nil? (matching-timestamp? job field-path range-jan-feb))
{}
{:other "value"})))))

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,113 @@
(ns metabase-enterprise.workspaces.dag-abstract
"Abstract graph solver using shorthand notation for testing.
Shorthand uses keywords like :x1, :t2, :m3 where:
- :x<n> is a transform
- :t<n> is a table (source/generated)
- :m<n> is a model (card)"
(:require
[clojure.set :as set]
[flatland.ordered.map :as ordered-map]
[metabase-enterprise.workspaces.util :as ws.u]
[metabase.util :as u]))
;;;; Shorthand notation helpers
(defn- related [t id]
(keyword (str (name t) (subs (name id) 1))))
(def ^:private x->t (partial related :t))
(defn is?
"Check if id is of type t (e.g., (is? :x :x1) => true)."
[t id]
(= (name t) (str (first (name id)))))
(defn- lexico [id]
(let [n (name id)]
[(parse-long (subs n 1))
({\x 1 \t 2 \m 3} (first n))]))
(defn expand-shorthand
"As shorthand, we can use transforms in the place of their induced tables.
This function expands the graph to insert the induced tables, and replace the corresponding references."
[dependencies]
(let [xs (set (filter #(is? :x %) (concat (keys dependencies) (mapcat val dependencies))))]
(into (sorted-map-by #(compare (lexico %1) (lexico %2)))
(merge (zipmap (map x->t xs) (map vector xs))
(into {} (filter (comp #(is? :t %) key)) dependencies)
(update-vals
(into {} (remove (comp #(is? :t %) key)) dependencies)
(fn [deps] (->> deps
(map (fn [id]
(if (is? :x id)
(x->t id)
id)))
(sort-by lexico)
(vec))))))))
;;;; Graph traversal helpers
(defn- children [dependencies]
(reduce
(fn [acc [child parents]]
(reduce (fn [acc p] (update acc p u/conjv child)) acc parents))
(sorted-map-by #(compare (lexico %1) (lexico %2)))
dependencies))
(defn- ->ts [ids]
(into #{}
(keep (fn [id]
(cond (is? :t id) id
(is? :x id) (x->t id))))
ids))
;;;; Toposort
(defn- toposort-map [child->parents]
(let [child->parents (if (map? child->parents) child->parents (into {} child->parents))]
(into (ordered-map/ordered-map)
(keep (fn [n] (when-let [deps (get child->parents n)] [n deps])))
(ws.u/toposort-dfs child->parents))))
;;;; Abstract path-induced subgraph solver
(def ^:private max-iterations 100)
(defn path-induced-subgraph
"Abstract solver for path-induced subgraph using shorthand notation."
[{:keys [check-outs dependencies] :as _graph}]
(let [->parents dependencies
->children (children dependencies)]
(loop [iteration 1
below check-outs
above check-outs
seen-down? check-outs
seen-up? check-outs
to-descend check-outs
to-ascend check-outs]
(if (and (empty? to-descend) (empty? to-ascend))
(let [nodes (set/intersection above below)
entities (remove #(is? :t %) nodes)
tables (sort-by lexico (filter #(is? :t %) nodes))]
{:check-outs (sort-by lexico check-outs)
:inputs (sort-by lexico (remove nodes (->ts (mapcat ->parents check-outs))))
:tables tables
:transforms (sort-by lexico (filter #(is? :x %) nodes))
:entities (sort-by lexico entities)
:dependencies (apply dissoc
(toposort-map
(for [n nodes]
[n (vec (filter nodes (get dependencies n)))]))
tables)})
(if (> iteration max-iterations)
{:error "max iterations exceeded"}
(let [next-down (into #{} (comp (mapcat ->children) (remove seen-down?)) to-descend)
next-up (into #{} (comp (mapcat ->parents) (remove seen-up?)) to-ascend)]
(recur (inc iteration)
(into below next-down)
(into above next-up)
(into seen-down? next-down)
(into seen-up? next-up)
next-down
next-up)))))))

View File

@@ -0,0 +1,313 @@
(ns metabase-enterprise.workspaces.dag-test
(:require
[clojure.string :as str]
[clojure.test :refer :all]
[flatland.ordered.map :as ordered-map]
[metabase-enterprise.workspaces.dag :as ws.dag]
[metabase-enterprise.workspaces.dag-abstract :as dag-abstract]
[metabase-enterprise.workspaces.impl :as ws.impl]
[metabase.app-db.core :as app-db]
[metabase.driver.sql.normalize :as sql.normalize]
[metabase.test :as mt]
[metabase.util :as u]
[toucan2.core :as t2]))
(use-fixtures
:once
(fn [f]
(mt/with-premium-features [:dependencies :transforms :workspaces]
(mt/with-model-cleanup [:model/Dependency :model/Transform :model/Table]
(f)))))
;;;; Example graphs for testing
(def ^:private example-graph
{:x3 [:x1 :t2]
:x4 [:x3]
:m6 [:x4 :t5]
:m10 [:t9]
:x11 [:m10]
:m12 [:x11]
:m13 [:x11 :m12]})
;;;; Test data helpers
(defn- transform? [kw] (= \x (first (name kw))))
(defn- table? [kw] (= \t (first (name kw))))
(defn- kw->id [kw] (parse-long (subs (name kw) 1)))
(defn- create-test-graph!
"Create test transforms and tables from shorthand notation, returning id mappings.
Shorthand uses keywords like :x1, :t2, :m3 where:
- :x<n> is a transform
- :t<n> is a table (source/generated)
- :m<n> is a model (card)
Returns a map from shorthand id to real database id, e.g. {:x1 123, :t2 456}
NOTE: Caller should wrap in mt/with-model-cleanup for proper cleanup."
[dependencies]
(let [schema (str/replace (str (random-uuid)) "-" "_")
all-ids (set (concat (keys dependencies)
(mapcat val dependencies)))
transforms (filter transform? all-ids)
driver (t2/select-one-fn :engine [:model/Database :engine] (mt/id))
normalize #(sql.normalize/normalize-name driver %)
tables (filter table? all-ids)
table-ids (u/for-map [t tables]
[t (t2/insert-returning-pk! :model/Table
{:db_id (mt/id)
:schema (some-> schema normalize)
:name (normalize (str "test_table_" (kw->id t)))
:active true})])
;; Create transforms that reference their input tables
tx-ids (u/for-map [tx transforms]
(let [parent-tables (->> (get dependencies tx []) (filter table?) (map table-ids))
id (t2/insert-returning-pk! :model/Transform
{:name (str "Test " (name tx))
:source {:type :query
:query (if (seq parent-tables)
#_{:database (mt/id)
:type :query
:query {:source-table (first parent-tables)
:joins (for [pt (rest parent-tables)]
{:source-table pt
:condition [:= 1 1]})}}
{:database (mt/id)
:type :native
:native {:query (->> (t2/select-fn-vec #(str (:schema %) "." (:name %))
[:model/Table :schema :name]
:id [:in parent-tables])
(str/join ", ")
(str "SELECT * FROM "))}}
{:database (mt/id)
:type :native
:native {:query "SELECT 1"}})}
:target {:type "table"
:database (mt/id)
:schema schema
:name (str "test_table_" (kw->id tx))}})]
[tx id]))]
;; This is a workaround for the dependency between transforms and their output only being inserted on run.
(doseq [[tx-kw tx-id] tx-ids]
(app-db/update-or-insert! :model/Dependency
{:to_entity_type "transform"
:to_entity_id tx-id
:from_entity_type "table"
:from_entity_id (table-ids (keyword (str "t" (kw->id tx-kw))))}))
(merge tx-ids table-ids)))
(defn- translate-result
"Translate result from real IDs back to shorthand notation for easier comparison."
[{:keys [inputs outputs entities dependencies] :as _result} id-map]
(let [reverse-map (u/for-map [[k v] id-map] [v k])
table->kw (comp reverse-map :id)
node->kw (fn [{:keys [node-type id]}]
(reverse-map (case node-type
:table (:id id)
:external-transform id
:workspace-transform (t2/select-one-fn :global_id [:model/WorkspaceTransform :global_id] :ref_id id))))]
{:inputs (into #{} (map table->kw) inputs)
:outputs (into #{} (map table->kw) outputs)
:entities (into #{} (map node->kw) entities)
:dependencies (u/for-map [[child parents] dependencies]
[(node->kw child) (into #{} (map node->kw parents))])}))
;;;; Tests
(deftest path-induced-subgraph-shorthand-test
(testing "graph built from shorthand matches abstract solver"
(let [shorthand {:x2 [:t1]}
id-map (create-test-graph! (dag-abstract/expand-shorthand shorthand))
gtx (t2/select-one :model/Transform (id-map :x2))]
;; TODO make this nice and declarative too
(mt/with-temp [:model/Workspace ws {:name "Test Workspace", :database_id (mt/id)}
:model/WorkspaceTransform wtx (merge
(select-keys gtx [:name :source :target])
{:workspace_id (:id ws)
:global_id (:id gtx)})]
(ws.impl/sync-transform-dependencies! ws wtx)
(let [entity {:entity-type :transform, :id (:ref_id wtx)}
result (ws.dag/path-induced-subgraph (:id ws) [entity])
translated (translate-result result id-map)]
(is (=? {:inputs #{:t1}
:outputs #{:t2}
:entities #{:x2}
:dependencies {:x2 #{:t1}}}
translated)))))))
(deftest path-induced-subgraph-larger-test
(testing "graph built from shorthand matches abstract solver"
;; check-outs x2, x4
(let [shorthand {:x1 [:t0]
:x2 [:x1, :t10]
:x3 [:x2, :t8]
:x4 [:x3]
:x5 [:x2, :x4, :t9]}
id-map (create-test-graph! (dag-abstract/expand-shorthand shorthand))
gtx-2 (t2/select-one :model/Transform (id-map :x2))
gtx-4 (t2/select-one :model/Transform (id-map :x4))
fork (fn [ws gtx]
(merge
(select-keys gtx [:name :source :target])
{:workspace_id (:id ws)
:global_id (:id gtx)}))]
;; TODO make a convenience method / method for doing these checkouts too
(mt/with-temp [:model/Workspace ws {:name "Test Workspace", :database_id (mt/id)}
:model/WorkspaceTransform wtx-2 (fork ws gtx-2)
:model/WorkspaceTransform wtx-4 (fork ws gtx-4)]
(ws.impl/sync-transform-dependencies! ws wtx-2)
(ws.impl/sync-transform-dependencies! ws wtx-4)
(let [entities (for [wtx [wtx-2 wtx-4]]
{:entity-type :transform, :id (:ref_id wtx)})
result (ws.dag/path-induced-subgraph (:id ws) entities)
translated (translate-result result id-map)]
(is (=? {:inputs #{:t1 :t8 :t10}
:outputs #{:t2 :t3 :t4}
:entities #{:x2 :x3 :x4}
:dependencies {:x2 #{:t1 :t10}
:x3 #{:x2 :t8}
:x4 #{:x3}}}
translated)))))))
(deftest expand-solver-test
(testing "expand-shorthand inserts interstitial nodes for transform output tables"
(is (= {:t1 [:x1]
:x3 [:t1 :t2]
:t3 [:x3]
:x4 [:t3]
:t4 [:x4]
:m6 [:t4 :t5]
:m10 [:t9]
:x11 [:m10]
:t11 [:x11]
:m12 [:t11]
:m13 [:t11 :m12]}
(dag-abstract/expand-shorthand example-graph)))))
(deftest abstract-path-induced-subgraph-test
(testing "path-induced-subgraph computes correct result for example graph"
(is (= {:check-outs [:x3 :m6 :m10 :m13]
:inputs [:t1 :t2 :t5 :t9]
:tables [:t3 :t4 :t11]
:transforms [:x3 :x4 :x11]
:entities [:x3 :x4 :m6 :m10 :x11 :m12 :m13]
:dependencies (ordered-map/ordered-map
:m10 []
:x11 [:m10]
:m12 [:t11]
:m13 [:t11 :m12]
:x3 []
:x4 [:t3]
:m6 [:t4])}
(dag-abstract/path-induced-subgraph
{:check-outs #{:x3 :m6 :m10 :m13}
:dependencies (dag-abstract/expand-shorthand example-graph)})))))
;;;; Card dependency detection tests
(deftest unsupported-dependency-no-transforms-test
(testing "empty input returns nil"
(is (nil? (ws.dag/unsupported-dependency? {})))
(is (nil? (ws.dag/unsupported-dependency? {:transforms []})))))
(deftest unsupported-dependency-mbql-card-test
#_(testing "transform with card dependencies are unsupported"
(let [tx-with-no-dependencies 1
;; These tests transforms must seem pretty redundant - but it's intentional: we may relax them one case at
;; at time in the future.
;; ----------------------
;; We don't allow MBQL card dependencies, as we do not support re-mapping their references on execution.
;; This is only a problem if the card depends on a table that is shadowed in the isolated schema, but we
;; want to keep the semantics simple.
tx-with-direct-mbql-card-dependency 2
;; Even a transitive dependency would need to be remapped.
tx-with-indirect-mbql-card-dependency 3
;; In fact, we don't allow *any* card dependency for two reasons:
;; 1. We don't yet support reference re-mapping across entity references.
;; 2. It's an anti-pattern to build transforms on models, the relationship is intended to be the other way
;; around.
tx-with-sql-card-dependency 4]
(is (= nil (ws.dag/unsupported-dependency? {:transforms [1]})))
(is (= {:transforms [2 3 4]} (ws.dag/unsupported-dependency? {:transforms [1 2 3 4]}))))))
(deftest collapse-test
(is (= {:x1 [:x2 :x3]
:x2 [:t3]
:x3 [:x5]
:x4 []
:x5 []}
(#'ws.dag/collapse
table?
{:x1 [:t1 :t2]
:t1 [:x2]
:t2 [:x3]
:x2 [:t3]
:x3 [:t4]
:t4 [:x5]
:t5 [:x4]
:x4 []
:x5 []}))))
(defn tx->table [kw]
(when (transform? kw)
(keyword (str "t" (kw->id kw)))))
(defn- solve-in-memory [init-nodes graph]
(let [tx-nodes (filter transform? init-nodes)
tables (map tx->table tx-nodes)]
(#'ws.dag/path-induced-subgraph*
;; Include all changeset targets in the init-nodes
(distinct (into init-nodes tables))
{:node-parents (dag-abstract/expand-shorthand graph)
:table? table?
:table-sort kw->id
:unwrap-table identity})))
(defn- chain->deps [chain]
(reduce
(fn [deps [from to]]
(assoc deps from [to]))
{}
(partition 2 1 (reverse chain))))
(deftest in-memory-path-induced-subgraph-test
(testing "singleton"
(is (= {:inputs [:t1]
:outputs [:t2]
:entities [:x2]
:dependencies {:x2 [:t1]}}
(solve-in-memory [:x2] {:x2 [:t1]}))))
(testing "encloses middle of a chain"
(is (= {:inputs [:t1]
:outputs [:t2 :t3 :t4]
:entities [:x2 :x3 :x4]
:dependencies {:x2 [:t1]
:x3 [:x2]
:x4 [:x3]}}
(solve-in-memory [:x2 :x4] (chain->deps [:x1 :x2 :x3 :x4 :x5])))))
(testing "larger graph"
(is (= {:inputs [:t1 :t2 :t5 :t9]
:outputs [:t3 :t4 :t11]
:entities [:m10 :x11 :m12 :m13 :x3 :x4 :m6]
:dependencies {:m10 [:t9]
:m12 [:x11]
:m13 [:m12 :x11]
:m6 [:x4 :t5]
:x11 [:m10]
:x3 [:t2 :t1]
:x4 [:x3]}}
(solve-in-memory
[:x3 :m6 :m10 :m13]
{:x3 [:x1 :t2]
:x4 [:x3]
:m6 [:x4 :t5]
:m10 [:t9]
:x11 [:m10]
:m12 [:x11]
:m13 [:x11 :m12]})))))

View File

@@ -0,0 +1,318 @@
(ns metabase-enterprise.workspaces.dependencies-test
(:require
[clojure.test :refer :all]
[metabase-enterprise.workspaces.dependencies :as ws.deps]
[metabase-enterprise.workspaces.test-util :as ws.tu]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.test :as mt]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
(ws.tu/ws-fixtures!)
;;; ---------------------------------------- analyze-entity tests ----------------------------------------
(deftest ^:parallel analyze-entity-query-transform-test
(testing "analyze-entity extracts dependencies from a query transform"
(let [mp (mt/metadata-provider)
query (lib/query mp (lib.metadata/table mp (mt/id :orders)))
transform {:source {:type "query"
:query query}
:target {:database (mt/id)
:schema "public"
:name "orders_with_products"}}
result (ws.deps/analyze-entity :transform transform)]
(testing "output is extracted from target"
(is (= {:db_id (mt/id)
:schema "public"
:table "orders_with_products"}
(:output result))))
(testing "inputs contain the source table with table_id (since table exists)"
(let [inputs (:inputs result)
table-names (set (map :table inputs))]
(is (= 1 (count inputs)))
(is (contains? table-names (mt/format-name :orders)))
;; MBQL queries have table_id since the tables must exist
(is (every? #(and (:db_id %)
(:table %)
(:table_id %))
inputs)))))))
(deftest ^:parallel analyze-entity-native-query-existing-table-test
(testing "analyze-entity extracts dependencies from native query referencing existing table"
(let [mp (mt/metadata-provider)
query (lib/native-query mp "SELECT * FROM ORDERS WHERE total > 100")
transform {:source {:type "query"
:query query}
:target {:database (mt/id)
:schema "analytics"
:name "high_value_orders"}}
result (ws.deps/analyze-entity :transform transform)]
(testing "output is extracted from target"
(is (= {:db_id (mt/id)
:schema "analytics"
:table "high_value_orders"}
(:output result))))
(testing "inputs include table_id since ORDERS table exists"
(let [inputs (:inputs result)]
;; Native query parsing should detect ORDERS table reference and look up table_id
(is (pos? (count inputs)))
(is (every? #(and (:db_id %)
(:table %)
(:table_id %))
inputs)))))))
(deftest ^:parallel analyze-entity-native-query-nonexistent-table-test
(testing "analyze-entity handles native query referencing non-existent table"
(let [mp (mt/metadata-provider)
;; Reference a table that doesn't exist in the database
query (lib/native-query mp "SELECT * FROM future_sales_data WHERE region = 'US'")
transform {:source {:type "query"
:query query}
:target {:database (mt/id)
:schema "analytics"
:name "regional_sales"}}
result (ws.deps/analyze-entity :transform transform)]
(testing "output is extracted from target"
(is (= {:db_id (mt/id)
:schema "analytics"
:table "regional_sales"}
(:output result))))
(testing "inputs have logical reference but no table_id (table doesn't exist)"
(let [inputs (:inputs result)]
;; Should still parse the table reference even though table doesn't exist
(is (pos? (count inputs)))
;; Has db_id and table name but NOT table_id
(is (every? #(and (:db_id %)
(:table %))
inputs))
;; Verify table_id is nil or missing for non-existent tables
(is (every? #(nil? (:table_id %)) inputs)))))))
(deftest ^:parallel analyze-entity-python-transform-test
(testing "analyze-entity extracts dependencies from a python transform"
(let [transform {:source {:type "python"
:source-tables {"orders" (mt/id :orders)
"products" (mt/id :products)}
:body "def transform(orders, products): return orders"}
:target {:database (mt/id)
:schema "public"
:name "python_result"}}
result (ws.deps/analyze-entity :transform transform)]
(testing "output is extracted from target"
(is (= {:db_id (mt/id)
:schema "public"
:table "python_result"}
(:output result))))
(testing "inputs contain the source tables with table_ids"
(let [inputs (:inputs result)
table-names (set (map :table inputs))]
(is (= 2 (count inputs)))
(is (contains? table-names (mt/format-name :orders)))
(is (contains? table-names (mt/format-name :products)))
;; Python transforms have table_ids since source-tables maps to existing table IDs
(is (every? :table_id inputs)))))))
(deftest analyze-entity-asserts-transform-type-test
(testing "analyze-entity asserts entity-type is :transform"
(is (thrown-with-msg? Exception #"Only transform entity type is supported"
(ws.deps/analyze-entity :card {})))))
;;; ---------------------------------------- write-dependencies! tests ----------------------------------------
(deftest write-dependencies-creates-output-test
(testing "write-dependencies! creates workspace_output record"
(mt/with-temp [:model/Workspace workspace {:name "Test Workspace"
:database_id (mt/id)
:schema "test_isolated_schema"}
:model/WorkspaceTransform wt {:workspace_id (:id workspace)
:name "Test Transform"
:source {:type "query" :query {}}
:target {:database (mt/id)
:schema "public"
:name "test_output"}}]
(let [analysis {:output {:db_id (mt/id)
:schema "public"
:table "test_output"}
:inputs []}]
(ws.deps/write-dependencies! (:id workspace) "test_isolated_schema" :transform (:ref_id wt) analysis)
(let [output (t2/select-one :model/WorkspaceOutput
:workspace_id (:id workspace)
:ref_id (:ref_id wt))]
(is (some? output))
(is (= (mt/id) (:db_id output)))
(is (= "public" (:global_schema output)))
(is (= "test_output" (:global_table output)))
(is (= "test_isolated_schema" (:isolated_schema output)))
(is (= "public__test_output" (:isolated_table output))))))))
(deftest write-dependencies-creates-inputs-test
(testing "write-dependencies! creates workspace_input records for external dependencies"
(let [iso-schema "test_isolated_schema"]
(mt/with-temp [:model/Workspace workspace {:name "Test Workspace"
:database_id (mt/id)
:schema iso-schema}
:model/WorkspaceTransform wt {:workspace_id (:id workspace)
:name "Test Transform"
:source {:type "query" :query {}}
:target {:database (mt/id)
:schema "public"
:name "test_output"}}]
(let [orders-table (t2/select-one [:model/Table :db_id :schema :name] :id (mt/id :orders))
analysis {:output {:db_id (mt/id)
:schema "public"
:table "test_output"}
:inputs [{:db_id (:db_id orders-table)
:schema (:schema orders-table)
:table (:name orders-table)}]}]
(ws.deps/write-dependencies! (:id workspace) iso-schema :transform (:ref_id wt) analysis)
(let [input (t2/select-one :model/WorkspaceInput
:workspace_id (:id workspace)
:table (:name orders-table))]
(is (some? input))
(is (= (:db_id orders-table) (:db_id input)))
(is (= (:name orders-table) (:table input)))))))))
(deftest write-dependencies-creates-edges-test
(testing "write-dependencies! creates workspace_dependency edges"
(let [iso-schema "test_isolated_schema"]
(mt/with-temp [:model/Workspace workspace {:name "Test Workspace"
:database_id (mt/id)
:schema iso-schema}
:model/WorkspaceTransform wt {:workspace_id (:id workspace)
:name "Test Transform"
:source {:type "query" :query {}}
:target {:database (mt/id)
:schema "public"
:name "test_output"}}]
(let [orders-table (t2/select-one [:model/Table :db_id :schema :name] :id (mt/id :orders))
analysis {:output {:db_id (mt/id)
:schema "public"
:table "test_output"}
:inputs [{:db_id (:db_id orders-table)
:schema (:schema orders-table)
:table (:name orders-table)}]}]
(ws.deps/write-dependencies! (:id workspace) iso-schema :transform (:ref_id wt) analysis)
(let [edges (t2/select :model/WorkspaceDependency
:workspace_id (:id workspace)
:from_entity_type :transform
:from_entity_id (:ref_id wt))]
(is (= 1 (count edges)))
(is (= :input (:to_entity_type (first edges))))))))))
(deftest write-dependencies-internal-dependency-test
(testing "write-dependencies! links to workspace_output for internal dependencies"
(mt/with-temp [:model/Workspace workspace {:name "Test Workspace"
:database_id (mt/id)
:schema "test_isolated_schema"}
:model/WorkspaceTransform wt1 {:workspace_id (:id workspace)
:name "Upstream Transform"
:source {:type "query" :query {}}
:target {:database (mt/id)
:schema "public"
:name "upstream_output"}}
:model/WorkspaceTransform wt2 {:workspace_id (:id workspace)
:name "Downstream Transform"
:source {:type "query" :query {}}
:target {:database (mt/id)
:schema "public"
:name "downstream_output"}}]
;; First, write dependencies for the upstream transform (creates the output)
(ws.deps/write-dependencies! (:id workspace) "test_isolated_schema" :transform (:ref_id wt1)
{:output {:db_id (mt/id)
:schema "public"
:table "upstream_output"}
:inputs []})
;; Now write dependencies for downstream that depends on upstream's output
(ws.deps/write-dependencies! (:id workspace) "test_isolated_schema" :transform (:ref_id wt2)
{:output {:db_id (mt/id)
:schema "public"
:table "downstream_output"}
:inputs [{:db_id (mt/id)
:schema "public"
:table "upstream_output"}]})
(let [edges (t2/select :model/WorkspaceDependency
:workspace_id (:id workspace)
:from_entity_type :transform
:from_entity_id (:ref_id wt2))]
(testing "creates edge to output (internal dependency)"
(is (= 1 (count edges)))
(is (= :output (:to_entity_type (first edges)))))
(testing "no workspace_input created for internal dependency"
(is (nil? (t2/select-one :model/WorkspaceInput
:workspace_id (:id workspace)
:table "upstream_output"))))))))
(deftest write-dependencies-updates-on-change-test
(testing "write-dependencies! updates records and removes stale edges"
(mt/with-temp [:model/Workspace workspace {:name "Test Workspace"
:database_id (mt/id)
:schema "test_isolated_schema"}
:model/WorkspaceTransform wt {:workspace_id (:id workspace)
:name "Test Transform"
:source {:type "query" :query {}}
:target {:database (mt/id)
:schema "public"
:name "test_output"}}]
(let [orders-table (t2/select-one [:model/Table :db_id :schema :name] :id (mt/id :orders))
products-table (t2/select-one [:model/Table :db_id :schema :name] :id (mt/id :products))]
;; First write with orders dependency
(ws.deps/write-dependencies! (:id workspace) "test_isolated_schema" :transform (:ref_id wt)
{:output {:db_id (mt/id)
:schema "public"
:table "test_output"}
:inputs [{:db_id (:db_id orders-table)
:schema (:schema orders-table)
:table (:name orders-table)}]})
(is (= 1 (t2/count :model/WorkspaceDependency :workspace_id (:id workspace))))
;; Update to depend on products instead
(ws.deps/write-dependencies! (:id workspace) "test_isolated_schema" :transform (:ref_id wt)
{:output {:db_id (mt/id)
:schema "public"
:table "test_output"}
:inputs [{:db_id (:db_id products-table)
:schema (:schema products-table)
:table (:name products-table)}]})
(testing "old edge is removed"
(is (= 1 (t2/count :model/WorkspaceDependency :workspace_id (:id workspace)))))
(testing "new input is created"
(is (some? (t2/select-one :model/WorkspaceInput
:workspace_id (:id workspace)
:table (:name products-table)))))))))
;;; ---------------------------------------- Integration test ----------------------------------------
(deftest analyze-and-write-integration-test
(testing "Full flow: analyze a real transform and write its dependencies"
(let [mp (mt/metadata-provider)
query (lib/query mp (lib.metadata/table mp (mt/id :orders)))]
(mt/with-temp [:model/Workspace workspace {:name "Integration Test Workspace"
:database_id (mt/id)
:schema "test_isolated_schema"}
:model/WorkspaceTransform wt {:workspace_id (:id workspace)
:name "Orders Analysis"
:source {:type "query" :query {}}
:target {:database (mt/id)
:schema "analytics"
:name "orders_analysis"}}]
;; Create the transform entity with the proper query (not from DB)
(let [transform-entity {:source {:type "query"
:query query}
:target {:database (mt/id)
:schema "analytics"
:name "orders_analysis"}}
analysis (ws.deps/analyze-entity :transform transform-entity)]
(ws.deps/write-dependencies! (:id workspace) "test_isolated_schema" :transform (:ref_id wt) analysis)
(testing "output record created"
(is (= 1 (t2/count :model/WorkspaceOutput :workspace_id (:id workspace)))))
(testing "input record created for ORDERS table"
(is (= 1 (t2/count :model/WorkspaceInput :workspace_id (:id workspace)))))
(testing "dependency edge created"
(is (= 1 (t2/count :model/WorkspaceDependency :workspace_id (:id workspace))))))))))

View File

@@ -0,0 +1,20 @@
(ns ^:mb/driver-tests metabase-enterprise.workspaces.drivers-test
(:require
[clojure.test :refer :all]
[metabase.driver :as driver]
[metabase.test :as mt]))
(deftest drivers-support-workspaces-has-to-support-db-swapping-test
(mt/test-drivers (mt/normal-drivers-with-feature :workspace)
(let [db-swapped? (fn []
(try
(mt/run-mbql-query venues {:limit 1})
false
(catch Exception _e
true)))]
(testing "db is swapped if being executed inside a macros"
(driver/with-swapped-connection-details (mt/id) {:user "unicorn"}
(is (db-swapped?))))
(testing "sanity check that it's not swapped outside of the macros"
(is (not (db-swapped?)))))))

View File

@@ -0,0 +1,79 @@
(ns ^:mb/driver-tests metabase-enterprise.workspaces.e2e-test
(:require
[clojure.test :refer :all]
[metabase-enterprise.transforms.execute :as transforms.execute]
[metabase-enterprise.transforms.test-util :as transforms.tu]
[metabase-enterprise.workspaces.common :as ws.common]
[metabase-enterprise.workspaces.isolation :as ws.isolation]
[metabase-enterprise.workspaces.test-util :as ws.tu]
[metabase.driver :as driver]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.query-processor.preprocess :as qp.preprocess]
^{:clj-kondo/ignore [:deprecated-namespace]}
[metabase.query-processor.store :as qp.store]
[metabase.test :as mt]
[metabase.util :as u]
[toucan2.core :as t2]))
(defn- execute-workspace-transform!
"Execute a transform within workspace isolation context. For testing purposes."
[workspace ws-transform opts]
(ws.isolation/with-workspace-isolation workspace
(let [result (atom nil)]
(t2/with-transaction [_tx]
(let [transform (t2/insert-returning-instance!
:model/Transform
(select-keys ws-transform [:name :description :source :target]))]
(reset! result (transforms.execute/execute! transform opts))
;; rather abort transaction
(t2/delete! :model/Transform (:id transform))))
@result)))
(defn- mbql->native [query]
(qp.store/with-metadata-provider (mt/id)
(sql.qp/mbql->native driver/*driver* (qp.preprocess/preprocess query))))
(deftest isolation-e2e-test
(mt/test-drivers (mt/normal-drivers-with-feature :workspace)
(transforms.tu/with-transform-cleanup! [output-table-name (str "test_table_1_" (mt/random-name))]
(mt/with-model-cleanup [:model/Transform :model/Workspace]
(mt/with-temp [:model/Transform {transform-id :id} {:name "Transform 1"
:source {:type "query"
:query (mt/native-query (mbql->native (mt/mbql-query orders {:limit 3})))}
:target {:type "table"
:name output-table-name
:database (mt/id)
:schema (t2/select-one-fn :schema :model/Table (mt/id :orders))}}]
(ws.tu/with-workspaces! [workspace {:name (mt/random-name)
:database_id (mt/id)
:upstream {:transforms [transform-id]}}]
(ws.common/add-to-changeset! (mt/user->id :crowberto) workspace
:transform transform-id
(t2/select-one :model/Transform transform-id))
(let [_workspace (u/poll {:thunk #(t2/select-one :model/Workspace (:id workspace))
:done? #(= :ready (:status %))
:timeout-ms 5000})
isolated-transform (t2/select-one :model/WorkspaceTransform :workspace_id (:id workspace))
#_#_executed-transform (mt/with-current-user (mt/user->id :crowberto)
(ws.impl/run-transform! workspace isolated-transform))
#_#_output-table (-> executed-transform :table)]
;; TODO: the table is synced but there are no fields even though the table and fields exist in the db
#_(testing "execute the transform with the original query is fine and the output is in an isolated schema"
(u/poll {:thunk #(t2/select-one :model/Table :name (:name output-table))
:done? some?
:timeout-ms 1000})
(transforms.tu/wait-for-table (:name output-table) 1000)
(is (str/starts-with? (:schema output-table) "mb__isolation"))
(is (= 1 (count (transforms.tu/table-rows (name output-table))))))
(testing "changing the query without granting access will fail"
(t2/update! :model/WorkspaceTransform
{:ref_id (:ref_id isolated-transform)}
{:source {:type "query"
:query (mt/native-query (mbql->native (mt/mbql-query venues {:limit 1})))}})
(is (thrown-with-msg?
Exception
#"ERROR: permission denied for table.*"
(execute-workspace-transform! workspace
(t2/select-one :model/WorkspaceTransform :ref_id (:ref_id isolated-transform))
{:run-method :manual})))))))))))

View File

@@ -0,0 +1,196 @@
(ns ^:mb/driver-tests metabase-enterprise.workspaces.execute-test
(:require
[clojure.string :as str]
[clojure.test :refer :all]
[metabase-enterprise.transforms.test-util :as transforms.tu]
[metabase-enterprise.workspaces.common :as ws.common]
[metabase-enterprise.workspaces.execute :as ws.execute]
[metabase-enterprise.workspaces.impl :as ws.impl]
[metabase-enterprise.workspaces.test-util :as ws.tu]
[metabase.test :as mt]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
(ws.tu/ws-fixtures!)
(deftest run-workspace-transform-no-input-test
(testing "Executing a workspace transform returns results and rolls back app DB records"
(transforms.tu/with-transform-cleanup! [output-table "ws_execute_test"]
(let [workspace (ws.tu/create-ready-ws! "Execute Test Workspace")
db-id (:database_id workspace)
ws-schema (:schema workspace)
body {:name "Test Transform"
:source {:type "query"
:query (mt/native-query {:query "SELECT 1 as id, 'hello' as name"})}
:target {:type "table"
:database db-id
:schema nil
:name output-table}}
ws-transform (ws.common/add-to-changeset! (mt/user->id :crowberto) workspace :transform nil body)
before {:xf (t2/count :model/Transform)
:xfrun (t2/count :model/TransformRun)}]
(testing "execution returns expected result structure"
(is (=? {:status :succeeded
:start_time some?
:end_time some?
:table {:name #(str/includes? % output-table)
:schema ws-schema}}
(mt/with-current-user (mt/user->id :crowberto)
(ws.impl/run-transform! workspace ws-transform))))
(is (=? {:last_run_at some?}
(t2/select-one :model/WorkspaceTransform :ref_id (:ref_id ws-transform))))
(is (=? [{:workspace_id (:id workspace)
:global_table output-table
:global_schema nil
:global_table_id nil
:isolated_schema string?
:isolated_table string?
;; TODO I think this is broken because of normalization breaking case equality
#_#_:isolated_table_id number?}]
(t2/select :model/WorkspaceOutput :ref_id (:ref_id ws-transform)))))
(testing "app DB records are rolled back"
(is (= before
{:xf (t2/count :model/Transform)
:xfrun (t2/count :model/TransformRun)})))))))
(deftest remap-python-source-test
(let [remap-python-source #'ws.execute/remap-python-source]
(testing "remaps to table ID when mapping has :id"
(let [table-mapping {[1 "public" "orders"] {:db-id 1
:schema "ws_isolated_123"
:table "orders_isolated"
:id 456}}
source {:type "python"
:body "import pandas as pd"
:source-tables {"orders" {:database_id 1
:schema "public"
:table "orders"
:table_id 123}}}]
(is (= {:type "python"
:body "import pandas as pd"
:source-tables {"orders" 456}}
(remap-python-source table-mapping source)))))
(testing "remaps to map format when mapping has nil :id"
(let [table-mapping {[1 "public" "orders"] {:db-id 1
:schema "ws_isolated_123"
:table "orders_isolated"
:id nil}}
source {:type "python"
:body "import pandas as pd"
:source-tables {"orders" {:database_id 1
:schema "public"
:table "orders"
:table_id 123}}}]
(is (= {:type "python"
:body "import pandas as pd"
:source-tables {"orders" {:database_id 1
:schema "ws_isolated_123"
:table "orders_isolated"}}}
(remap-python-source table-mapping source)))))
(testing "leaves source-tables unchanged when no mapping exists"
(let [table-mapping {}
source {:type "python"
:body "import pandas as pd"
:source-tables {"orders" {:database_id 1
:schema "public"
:table "orders"
:table_id 123}}}]
(is (= source (remap-python-source table-mapping source)))))
(testing "remaps legacy integer to table ID when mapping has :id"
(let [table-mapping {123 {:db-id 1
:schema "ws_isolated_123"
:table "orders_isolated"
:id 456}}
source {:type "python"
:body "import pandas as pd"
:source-tables {"orders" 123}}]
(is (= {:type "python"
:body "import pandas as pd"
:source-tables {"orders" 456}}
(remap-python-source table-mapping source)))))
(testing "remaps legacy integer to map format when mapping has nil :id"
(let [table-mapping {123 {:db-id 1
:schema "ws_isolated_123"
:table "orders_isolated"
:id nil}}
source {:type "python"
:body "import pandas as pd"
:source-tables {"orders" 123}}]
(is (= {:type "python"
:body "import pandas as pd"
:source-tables {"orders" {:database_id 1
:schema "ws_isolated_123"
:table "orders_isolated"}}}
(remap-python-source table-mapping source)))))
(testing "leaves legacy integer table-id unchanged when no mapping exists"
(let [table-mapping {}
source {:type "python"
:body "import pandas as pd"
:source-tables {"orders" 123}}]
(is (= source (remap-python-source table-mapping source)))))
(testing "handles mixed formats with mappings for both"
(let [table-mapping {[1 "public" "orders"] {:db-id 1
:schema "ws_isolated_123"
:table "orders_isolated"
:id 456}
789 {:db-id 1
:schema "ws_isolated_123"
:table "customers_isolated"
:id 999}}
source {:type "python"
:body "import pandas as pd"
:source-tables {"orders" {:database_id 1
:schema "public"
:table "orders"
:table_id 123}
"customers" 789}}]
(is (= {:type "python"
:body "import pandas as pd"
:source-tables {"orders" 456
"customers" 999}}
(remap-python-source table-mapping source)))))
(testing "prefers table_id lookup over triple lookup for map format"
(let [table-mapping {123 {:db-id 1
:schema "ws_isolated_123"
:table "orders_by_id"
:id 456}
[1 "public" "orders"] {:db-id 1
:schema "ws_isolated_123"
:table "orders_by_triple"
:id 789}}
source {:type "python"
:body "import pandas as pd"
:source-tables {"orders" {:database_id 1
:schema "public"
:table "orders"
:table_id 123}}}]
(is (= {:type "python"
:body "import pandas as pd"
:source-tables {"orders" 456}}
(remap-python-source table-mapping source)))))
(testing "falls back to triple lookup when table_id not in mapping"
(let [table-mapping {[1 "public" "orders"] {:db-id 1
:schema "ws_isolated_123"
:table "orders_isolated"
:id 456}}
source {:type "python"
:body "import pandas as pd"
:source-tables {"orders" {:database_id 1
:schema "public"
:table "orders"
:table_id 999}}}]
(is (= {:type "python"
:body "import pandas as pd"
:source-tables {"orders" 456}}
(remap-python-source table-mapping source)))))))

View File

@@ -0,0 +1,84 @@
(ns metabase-enterprise.workspaces.impl-test
(:require
[clojure.test :refer :all]
[metabase-enterprise.workspaces.common :as ws.common]
[metabase-enterprise.workspaces.impl :as ws.impl]
[metabase-enterprise.workspaces.isolation :as ws.isolation]
[metabase-enterprise.workspaces.test-util :as ws.tu]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.test :as mt]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
(ws.tu/ws-fixtures!)
(deftest sync-transform-dependencies-test
(testing "sync-transform-dependencies! writes dependencies and grants access to external inputs"
(let [mp (mt/metadata-provider)
query-a (lib/query mp (lib.metadata/table mp (mt/id :orders)))
query-b (lib/query mp (lib.metadata/table mp (mt/id :products)))
workspace (ws.tu/create-provisional-ws! "Test Workspace")
grant-calls (atom [])]
;; Set up mock BEFORE calling add-to-changeset! since it calls sync-transform-dependencies! internally
(mt/with-dynamic-fn-redefs [ws.isolation/grant-read-access-to-tables!
(fn [_database _workspace tables]
(swap! grant-calls conj (set (map :name tables))))]
(testing "Transform A depends on ORDERS"
(ws.common/add-to-changeset! (mt/user->id :crowberto) workspace
:transform nil
{:name "Transform A"
:source {:type "query" :query query-a}
:target {:database (mt/id)
:schema "analytics"
:name "table_a"}})
(testing "creates output and input records"
(is (= 1 (t2/count :model/WorkspaceOutput :workspace_id (:id workspace))))
(is (= 1 (t2/count :model/WorkspaceInput :workspace_id (:id workspace)))))
(testing "input record has correct table name"
(is (= (mt/format-name :orders)
(t2/select-one-fn :table :model/WorkspaceInput :workspace_id (:id workspace)))))
(testing "grants access to ORDERS"
(is (= 1 (count @grant-calls)))
(is (contains? (first @grant-calls) (mt/format-name :orders)))))
(testing "Transform B depends on PRODUCTS, table_a is internal output"
(reset! grant-calls [])
(ws.common/add-to-changeset! (mt/user->id :crowberto) workspace
:transform nil
{:name "Transform B"
:source {:type "query" :query query-b}
:target {:database (mt/id)
:schema "analytics"
:name "table_b"}})
(testing "creates records for both transforms"
(is (= 2 (t2/count :model/WorkspaceOutput :workspace_id (:id workspace))))
(is (= 2 (t2/count :model/WorkspaceInput :workspace_id (:id workspace)))))
(testing "only grants access to newly discovered inputs (PRODUCTS), not already-granted ones"
(is (= 1 (count @grant-calls)))
;; ORDERS was already granted in Transform A, so only PRODUCTS is in this grant call
(is (not (contains? (first @grant-calls) (mt/format-name :orders))))
(is (contains? (first @grant-calls) (mt/format-name :products)))
(is (not (contains? (first @grant-calls) "table_a")))))))))
(deftest sync-transform-dependencies-idempotent-test
(testing "sync-transform-dependencies! is idempotent"
(let [mp (mt/metadata-provider)
query (lib/query mp (lib.metadata/table mp (mt/id :orders)))
workspace (ws.tu/create-provisional-ws! "Test Workspace")]
(mt/with-dynamic-fn-redefs [ws.isolation/grant-read-access-to-tables! (constantly nil)]
(let [wt (ws.common/add-to-changeset! (mt/user->id :crowberto) workspace
:transform nil
{:name "Transform"
:source {:type "query" :query query}
:target {:database (mt/id)
:schema "analytics"
:name "output_table"}})]
;; Call sync again to test idempotency
(ws.impl/sync-transform-dependencies! workspace wt)
(ws.impl/sync-transform-dependencies! workspace wt)
(testing "still has exactly one of each after multiple syncs"
(is (= 1 (t2/count :model/WorkspaceOutput :workspace_id (:id workspace))))
(is (= 1 (t2/count :model/WorkspaceInput :workspace_id (:id workspace))))
(is (= 1 (t2/count :model/WorkspaceDependency :workspace_id (:id workspace))))))))))

View File

@@ -0,0 +1,121 @@
(ns ^:mb/driver-tests metabase-enterprise.workspaces.isolation-test
(:require
[clojure.java.jdbc :as jdbc]
[clojure.test :refer :all]
[metabase-enterprise.workspaces.common :as ws.common]
[metabase-enterprise.workspaces.isolation :as isolation]
[metabase-enterprise.workspaces.test-util :as ws.tu]
[metabase-enterprise.workspaces.util :as ws.u]
[metabase.driver :as driver]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.test :as mt]))
(ws.tu/ws-fixtures!)
(set! *warn-on-reflection* true)
(def isolated-prefix @#'ws.u/isolated-prefix)
;;; Test helper multimethod to verify cleanup
(defmulti workspace-isolation-resources-exist?
"Check if isolation resources still exist for the workspace.
Returns a map with keys indicating which resources exist."
{:arglists '([database workspace])}
(fn [database _workspace]
(driver/the-driver (:engine database)))
:hierarchy #'driver/hierarchy)
(defmethod workspace-isolation-resources-exist? :postgres
[database workspace]
(let [schema-name (ws.u/isolation-namespace-name workspace)
username (ws.u/isolation-user-name workspace)
conn-spec (sql-jdbc.conn/db->pooled-connection-spec (:id database))]
{:schema (-> (jdbc/query conn-spec
["SELECT 1 FROM information_schema.schemata WHERE schema_name = ?" schema-name])
seq boolean)
:user (-> (jdbc/query conn-spec
["SELECT 1 FROM pg_user WHERE usename = ?" username])
seq boolean)}))
(defmethod workspace-isolation-resources-exist? :h2
[database workspace]
(let [schema-name (ws.u/isolation-namespace-name workspace)
username (ws.u/isolation-user-name workspace)
conn-spec (sql-jdbc.conn/db->pooled-connection-spec (:id database))]
{:schema (-> (jdbc/query conn-spec
["SELECT 1 FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = ?" schema-name])
seq boolean)
:user (-> (jdbc/query conn-spec
["SELECT 1 FROM INFORMATION_SCHEMA.USERS WHERE NAME = ?" username])
seq boolean)}))
(defmethod workspace-isolation-resources-exist? :snowflake
[database workspace]
(let [schema-name (ws.u/isolation-namespace-name workspace)
db-name (-> database :details :db)
role-name (format "MB_ISOLATION_ROLE_%s" (:id workspace))
username (ws.u/isolation-user-name workspace)
conn-spec (sql-jdbc.conn/db->pooled-connection-spec (:id database))]
{:schema (-> (jdbc/query conn-spec
[(format "SHOW SCHEMAS LIKE '%s' IN DATABASE \"%s\"" schema-name db-name)])
seq boolean)
:user (-> (jdbc/query conn-spec
[(format "SHOW USERS LIKE '%s'" username)])
seq boolean)
:role (-> (jdbc/query conn-spec
[(format "SHOW ROLES LIKE '%s'" role-name)])
seq boolean)}))
(defmethod workspace-isolation-resources-exist? :sqlserver
[database workspace]
(let [schema-name (ws.u/isolation-namespace-name workspace)
login-name (format "%s_%s_login" isolated-prefix (:id workspace))
username (ws.u/isolation-user-name workspace)
conn-spec (sql-jdbc.conn/db->pooled-connection-spec (:id database))]
{:schema (-> (jdbc/query conn-spec
["SELECT 1 FROM sys.schemas WHERE name = ?" schema-name])
seq boolean)
:user (-> (jdbc/query conn-spec
["SELECT 1 FROM sys.database_principals WHERE name = ?" username])
seq boolean)
:login (-> (jdbc/query conn-spec
["SELECT 1 FROM master.sys.server_principals WHERE name = ?" login-name])
seq boolean)}))
(defmethod workspace-isolation-resources-exist? :clickhouse
[database workspace]
(let [db-name (ws.u/isolation-namespace-name workspace)
username (ws.u/isolation-user-name workspace)
conn-spec (sql-jdbc.conn/db->pooled-connection-spec (:id database))]
{:database (-> (jdbc/query conn-spec
["SELECT 1 FROM system.databases WHERE name = ?" db-name])
seq boolean)
:user (-> (jdbc/query conn-spec
["SELECT 1 FROM system.users WHERE name = ?" username])
seq boolean)}))
;;; Tests
(deftest destroy-workspace-isolation-test
(mt/test-drivers (mt/normal-drivers-with-feature :workspace)
(ws.tu/with-workspaces! [workspace {:name "Test destroy isolation"}]
(ws.common/add-to-changeset! (mt/user->id :crowberto) workspace
:transform nil
{:name "Transform A"
:source {:type "query" :query (mt/mbql-query orders {:limit 1})}
:target {:database (mt/id)
:schema "analytics"
:name "table_a"}})
(let [database (mt/db)]
(testing "resources are creeated during workspace initialization"
(let [resources (workspace-isolation-resources-exist? database workspace)]
(is (every? true? (vals resources)))))
(testing "destroy removes all isolation resources"
(isolation/destroy-workspace-isolation! database workspace)
(let [resources (workspace-isolation-resources-exist? database workspace)]
(is (every? false? (vals resources)))))
(testing "destroy is idempotent"
(isolation/destroy-workspace-isolation! database workspace))))))

View File

@@ -0,0 +1,141 @@
(ns ^:mb/driver-tests metabase-enterprise.workspaces.test-util
(:require
[clojure.test :refer :all]
[metabase-enterprise.workspaces.common :as ws.common]
[metabase-enterprise.workspaces.isolation :as ws.isolation]
[metabase-enterprise.workspaces.models.workspace :as ws.model]
[metabase-enterprise.workspaces.util :as ws.u]
[metabase.config.core :as config]
[metabase.search.test-util :as search.tu]
[metabase.test :as mt]
[metabase.util :as u]
[metabase.util.log :as log]
[toucan2.core :as t2]))
(defn ws-fixtures!
"Sets up test fixtures for workspace tests. Must be called at the top level of test namespaces."
[]
(use-fixtures :once (fn [tests]
;; E.g. app-db.yml tests perorm driver tests. Workspaces are not supported on mysql.
;; Following disj suppresses those runs destined for failure.
(mt/test-drivers (mt/normal-drivers-with-feature :workspace)
(mt/with-premium-features [:workspaces :dependencies :transforms]
(search.tu/with-index-disabled
(tests))))))
(use-fixtures :each (fn [tests]
(mt/with-model-cleanup [:model/Collection
:model/Transform
:model/TransformRun
:model/Workspace
:model/WorkspaceTransform
:model/WorkspaceInput
:model/WorkspaceOutput
:model/WorkspaceDependency]
(tests)))))
(derive :model/Workspace :model/WorkspaceCleanUpInTest)
(t2/define-before-delete :model/WorkspaceCleanUpInTest
[workspace]
(try
(log/infof "Cleaningup workspace %d in tests" (:id workspace))
(when (:database_details workspace)
(ws.isolation/destroy-workspace-isolation! (t2/select-one :model/Database (:database_id workspace)) workspace))
(catch Exception e
(log/warn e "Failed to destroy isolation" {:workspace workspace})))
workspace)
(defn ws-ready
"Poll until workspace status becomes :ready or timeout.
Note: uninitialized workspaces will never become ready without adding a transform."
[ws-or-id]
(let [ws-id (cond-> ws-or-id
(map? ws-or-id) :id)]
(or (u/poll {:thunk #(t2/select-one :model/Workspace :id ws-id)
:done? #(not= :pending (:status %))
;; some cloud drivers are really slow
:timeout-ms (if config/is-dev? 5000 60000)})
(throw (ex-info "Timeout waiting for workspace to be ready" {:workspace-id ws-id})))))
(defn create-workspace-for-test!
"Create a workspace for testing using proper initialization.
Returns the workspace after waiting for it to be ready."
[props]
(let [creator-id (or (:creator_id props) (mt/user->id :crowberto))
props (-> props
(dissoc :creator_id)
(update :database_id #(or % (mt/id))))]
(ws-ready (mt/with-current-user creator-id
(ws.common/create-workspace! creator-id props)))))
(defn create-ready-ws!
"Create a workspace with full isolation and wait for it to be ready.
Use [[create-provisional-ws!]] for tests that don't need isolation."
[name]
(create-workspace-for-test! {:name name}))
(defn create-provisional-ws!
"Create a workspace without database isolation for testing.
The workspace will have :ready status but no isolation resources.
Use this for tests that don't need actual database isolation, such as API
listing tests or tests that mock isolation functions.
For tests that need a :ready workspace with real isolation, use [[create-ready-ws!]]."
[name]
(let [creator-id (mt/user->id :crowberto)
db-id (mt/id)]
(mt/with-current-user creator-id
(let [ws (ws.common/create-workspace! creator-id {:name name
:database_id db-id
:provisional? true})]
;; Set schema and status to :ready so add-to-changeset! doesn't try to initialize
(t2/update! :model/Workspace (:id ws) {:schema (ws.u/isolation-namespace-name ws)
:status :ready})
(t2/select-one :model/Workspace :id (:id ws))))))
(defn do-with-workspaces!
"Function that sets up workspaces for testing and cleans up afterwards.
Takes a sequence of props for workspace creation and a thunk that receives
the created workspaces as a vector. Each workspace is cleaned up by its own
stack frame, so cleanup naturally happens in reverse creation order."
[props-list thunk]
(if (empty? props-list)
(thunk [])
(let [ws (create-workspace-for-test! (first props-list))]
(try
(do-with-workspaces! (rest props-list)
(fn [rest-workspaces]
(thunk (into [ws] rest-workspaces))))
(finally
(when ws
(try
(ws.model/delete! (t2/select-one :model/Workspace :id (:id ws)))
(catch Exception _
;; Workspace may already be deleted by the test
nil))))))))
(defmacro with-workspaces!
"Execute body with properly initialized workspaces that are cleaned up afterward.
Creates each workspace using `ws.common/create-workspace!` and waits for it to
be ready. After body execution (or on error), cleans up using `ws.model/delete!`
which properly destroys database isolation resources.
Usage:
(with-workspaces [ws1 {:name \"Test WS 1\"}
ws2 {:name \"Test WS 2\" :database_id (mt/id)}]
(testing \"workspace operations\"
(is (= :ready (:status ws1)))))"
[bindings & body]
(assert (vector? bindings) "bindings must be a vector")
(assert (even? (count bindings)) "bindings must have an even number of forms")
(let [pairs (partition 2 bindings)
syms (mapv first pairs)
props-list (mapv second pairs)]
`(do-with-workspaces!
[~@props-list]
(fn [[~@syms]]
~@body))))

View File

@@ -0,0 +1,178 @@
(ns ^:mb/driver-tests metabase-enterprise.workspaces.validation-test
(:require
[clojure.test :refer :all]
[metabase-enterprise.workspaces.dag :as ws.dag]
[metabase-enterprise.workspaces.test-util :as ws.tu]
[metabase-enterprise.workspaces.validation :as ws.validation]
[metabase.test :as mt]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
(ws.tu/ws-fixtures!)
(defn- build-graph
"Build the workspace graph for validation tests."
[ws-id]
(let [changeset (t2/select-fn-vec (fn [{:keys [ref_id]}] {:entity-type :transform, :id ref_id})
[:model/WorkspaceTransform :ref_id] :workspace_id ws-id)]
(when (seq changeset)
(ws.dag/path-induced-subgraph ws-id changeset))))
;;; ---------------------------------------- Basic API Tests ----------------------------------------
(deftest find-downstream-problems-empty-workspace-test
(testing "find-downstream-problems returns empty for workspace with no transforms"
(let [workspace (ws.tu/create-provisional-ws! "Empty Workspace")]
(is (= [] (ws.validation/find-downstream-problems (:id workspace) nil))))))
(deftest problems-endpoint-returns-empty-for-new-workspace-test
(testing "GET /api/ee/workspace/:id/problems returns empty list for workspace with no transforms"
(let [workspace (ws.tu/create-provisional-ws! "Test Workspace")
response (mt/user-http-request :crowberto :get 200
(str "ee/workspace/" (:id workspace) "/problem"))]
(is (= [] response)))))
(deftest problems-endpoint-requires-superuser-test
(testing "GET /api/ee/workspace/:id/problems requires superuser"
(let [workspace (ws.tu/create-provisional-ws! "Private Workspace")]
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :get 403
(str "ee/workspace/" (:id workspace) "/problem")))))))
(deftest problems-endpoint-404-for-nonexistent-workspace-test
(testing "GET /api/ee/workspace/:id/problems returns 404 for non-existent workspace"
(is (= "Not found."
(mt/user-http-request :crowberto :get 404 "ee/workspace/999999/problem")))))
;;; ---------------------------------------- Problem Detection Tests ----------------------------------------
(deftest unused-output-not-yet-created-test
(testing "detects when a workspace output table hasn't been created yet (no external dependents)"
(ws.tu/with-workspaces! [workspace {:name "Test Workspace"}]
(let [ws-id (:id workspace)
;; Create a workspace transform
{ref-id :ref_id} (mt/user-http-request :crowberto :post 200
(str "ee/workspace/" ws-id "/transform")
{:name "Test Transform"
:source {:type "query"
:query {:database (mt/id)
:type :native
:native {:query "SELECT 1 as col"}}}
:target {:type "table"
:schema "public"
:name "test_output_table"}})
;; The transform hasn't been run, so isolated_table_id is nil
;; No external transforms depend on it
graph (build-graph ws-id)
problems (ws.validation/find-downstream-problems ws-id graph)]
(is (= [{:category :unused
:problem :not-run
:severity :info
:block_merge false
:description "Output hasn't been created yet, but nothing depends on it"
:data {:output {:db_id (mt/id), :schema "public", :table "test_output_table"}
:transform {:type :workspace-transform, :id ref-id}}}]
problems))))))
(deftest dependent-transform-output-not-yet-created-test
(testing "detects when output table hasn't been created but external transforms depend on it"
(ws.tu/with-workspaces! [workspace {:name "Test Workspace"}]
(let [ws-id (:id workspace)]
;; First, create the global output table that we'll have a dependency on
(mt/with-temp [:model/Table output-table {:db_id (mt/id)
:schema "public"
:name "global_output_table"
:active true}]
;; Create an external transform that depends on this table
;; Note: The dependency will be created automatically by the Transform model hooks
(mt/with-temp [:model/Transform external-tx {:name "External Transform"
:source {:type :query
:query {:database (mt/id)
:type :query
:query {:source-table (:id output-table)}}}
:target {:type "table"
:database (mt/id)
:schema "public"
:name "external_output"}}]
;; Create a workspace transform that targets the same table
(let [{ref-id :ref_id} (mt/user-http-request :crowberto :post 200
(str "ee/workspace/" ws-id "/transform")
{:name "Workspace Transform"
:source {:type "query"
:query {:database (mt/id)
:type :native
:native {:query "SELECT 1 as col"}}}
:target {:type "table"
:schema "public"
:name "global_output_table"}})]
;; Manually update the WorkspaceOutput to point to the global table
;; (simulating a checked-out transform scenario)
(t2/update! :model/WorkspaceOutput
{:workspace_id ws-id :ref_id ref-id}
{:global_table_id (:id output-table)})
(is (= [{:category :external-downstream
:problem :not-run
:severity :warning
:block_merge true
:description "Output hasn't been created yet, external transforms depend on it"
:data {:output {:db_id (mt/id), :schema "public", :table "global_output_table"}
:transform {:type :workspace-transform, :id ref-id}
:dependents [{:type :external-transform
:id (:id external-tx)
:name "External Transform"}]}}]
(ws.validation/find-downstream-problems ws-id (build-graph ws-id)))))))))))
(deftest no-problems-when-output-exists-and-no-field-changes-test
(testing "no problems when isolated table exists and has all required fields"
(ws.tu/with-workspaces! [workspace {:name "Test Workspace"}]
(let [ws-id (:id workspace)]
;; Create both global and isolated tables with matching fields
(mt/with-temp [:model/Table global-table {:db_id (mt/id)
:schema "public"
:name "output_table"
:active true}
:model/Field _ {:table_id (:id global-table)
:name "col1"
:database_type "TEXT"
:base_type :type/Text
:active true}
:model/Table isolated-table {:db_id (mt/id)
:schema "mb__isolation_test"
:name "public__output_table"
:active true}
:model/Field _ {:table_id (:id isolated-table)
:name "col1"
:database_type "TEXT"
:base_type :type/Text
:active true}]
;; External transform that uses col1 via MBQL source-table
(mt/with-temp [:model/Transform _external-tx {:name "External Transform"
:source {:type :query
:query {:database (mt/id)
:type :query
:query {:source-table (:id global-table)}}}
:target {:type "table"
:database (mt/id)
:schema "public"
:name "external_output"}}]
;; Create workspace transform and output
(let [{ref-id :ref_id} (mt/user-http-request :crowberto :post 200
(str "ee/workspace/" ws-id "/transform")
{:name "Workspace Transform"
:source {:type "query"
:query {:database (mt/id)
:type :native
:native {:query "SELECT 'test' as col1"}}}
:target {:type "table"
:schema "public"
:name "output_table"}})]
;; Update WorkspaceOutput to have both global and isolated table IDs
(t2/update! :model/WorkspaceOutput
{:workspace_id ws-id :ref_id ref-id}
{:global_table_id (:id global-table)
:isolated_table_id (:id isolated-table)})
;; Should have no problems since isolated table has all required fields
(is (empty? (ws.validation/find-downstream-problems ws-id (build-graph ws-id)))))))))))

View File

@@ -5,12 +5,27 @@ import { useMetabotAgent } from "metabase-enterprise/metabot/hooks";
import { trackQueryFixClicked } from "../../analytics";
export function FixSqlQueryButton() {
type FixSqlQueryButtonProps = {
rawSql?: string | null;
errorMessage?: string | null;
};
export function FixSqlQueryButton({
rawSql,
errorMessage,
}: FixSqlQueryButtonProps) {
const { submitInput } = useMetabotAgent("omnibot");
const handleClick = () => {
trackQueryFixClicked();
submitInput("Fix this SQL query");
const promptParts = ["Fix this SQL query."];
if (errorMessage) {
promptParts.push(`The database returned this error: ${errorMessage}`);
}
if (rawSql) {
promptParts.push(`SQL:\n${rawSql}`);
}
submitInput(promptParts.join("\n\n"));
};
return <Button onClick={handleClick}>{t`Have Metabot fix it`}</Button>;

View File

@@ -20,3 +20,4 @@ export * from "./transform";
export * from "./transform-job";
export * from "./transform-tag";
export * from "./upload-management";
export * from "./workspace";

View File

@@ -28,6 +28,8 @@ import type {
TransformJob,
TransformRun,
TransformTag,
Workspace,
WorkspaceItem,
} from "metabase-types/api";
export const ENTERPRISE_TAG_TYPES = [
@@ -42,6 +44,10 @@ export const ENTERPRISE_TAG_TYPES = [
"transform-job",
"transform-job-via-tag",
"transform-run",
"workspace-transforms",
"workspace-transform",
"workspace-tables",
"external-transforms",
"git-tree",
"git-file-content",
"collection-dirty-entities",
@@ -49,6 +55,7 @@ export const ENTERPRISE_TAG_TYPES = [
"remote-sync-branches",
"remote-sync-current-task",
"python-transform-library",
"workspace",
"support-access-grant",
"support-access-grant-current",
"library-collection",
@@ -82,6 +89,24 @@ export function invalidateTags(
return !error ? tags : [];
}
export function provideWorkspacesTags(
workspaces: Workspace[],
): TagDescription<EnterpriseTagType>[] {
return [listTag("workspace"), ...workspaces.flatMap(provideWorkspaceTags)];
}
export function provideWorkspaceTags(
workspace: Workspace | WorkspaceItem,
): TagDescription<EnterpriseTagType>[] {
return [idTag("workspace", workspace.id)];
}
export function provideWorkspaceContentItemsTags(
items: WorkspaceContentItem[],
): TagDescription<EnterpriseTagType>[] {
return items.flatMap((transform) => idTag("transform", transform.id));
}
export function provideTransformTags(
transform: Transform,
): TagDescription<EnterpriseTagType>[] {

View File

@@ -151,7 +151,10 @@ export const transformApi = EnterpriseApi.injectEndpoints({
body,
}),
invalidatesTags: (_, error) =>
invalidateTags(error, [listTag("transform")]),
invalidateTags(error, [
listTag("transform"),
listTag("external-transforms"),
]),
}),
updateTransform: builder.mutation<Transform, UpdateTransformRequest>({
query: ({ id, ...body }) => ({

View File

@@ -0,0 +1,405 @@
import type {
CreateWorkspaceRequest,
CreateWorkspaceTransformRequest,
CreateWorkspaceTransformResponse,
ExternalTransformsRequest,
ExternalTransformsResponse,
TransformDownstreamMapping,
TransformId,
TransformUpstreamMapping,
UpdateWorkspaceTransformRequest,
ValidateTableNameRequest,
ValidateTableNameResponse,
Workspace,
WorkspaceAllowedDatabasesResponse,
WorkspaceCheckoutResponse,
WorkspaceGraphResponse,
WorkspaceId,
WorkspaceListResponse,
WorkspaceLogResponse,
WorkspaceMergeResponse,
WorkspaceProblem,
WorkspaceRunRequest,
WorkspaceRunResponse,
WorkspaceTablesResponse,
WorkspaceTransform,
WorkspaceTransformItem,
WorkspaceTransformMergeResponse,
WorkspaceTransformRef,
WorkspaceTransformRunResponse,
WorkspaceTransformsResponse,
} from "metabase-types/api";
import { EnterpriseApi } from "./api";
import {
idTag,
invalidateTags,
listTag,
provideWorkspaceContentItemsTags,
provideWorkspaceTags,
provideWorkspacesTags,
tag,
} from "./tags";
export const workspaceApi = EnterpriseApi.injectEndpoints({
endpoints: (builder) => ({
getWorkspaces: builder.query<WorkspaceListResponse, void>({
query: () => ({
method: "GET",
url: "/api/ee/workspace",
}),
providesTags: (response) =>
response ? provideWorkspacesTags(response.items) : [],
}),
getWorkspace: builder.query<Workspace, WorkspaceId>({
query: (id) => ({
method: "GET",
url: `/api/ee/workspace/${id}`,
}),
providesTags: (workspace) =>
workspace ? provideWorkspaceTags(workspace) : [],
}),
createWorkspace: builder.mutation<Workspace, void>({
query: () => ({
method: "POST",
url: "/api/ee/workspace",
}),
invalidatesTags: (_, error) =>
invalidateTags(error, [listTag("workspace"), listTag("transform")]),
}),
updateWorkspace: builder.mutation<
Workspace,
{ id: WorkspaceId; name?: string; database_id?: number }
>({
query: ({ id, ...body }) => ({
method: "PUT",
url: `/api/ee/workspace/${id}`,
body,
}),
invalidatesTags: (_, error, { id }) =>
invalidateTags(error, [idTag("workspace", id)]),
}),
createWorkspaceTransform: builder.mutation<
CreateWorkspaceTransformResponse,
{ id: WorkspaceId } & CreateWorkspaceTransformRequest
>({
query: ({ id, ...body }) => ({
method: "POST",
url: `/api/ee/workspace/${id}/transform`,
body,
}),
invalidatesTags: (_, error, { id, global_id }) =>
invalidateTags(error, [
idTag("workspace", id),
idTag("workspace-transforms", id),
idTag("external-transforms", id),
idTag("workspace-tables", id),
listTag("transform"),
...(global_id != null ? [idTag("transform", global_id)] : []),
]),
transformResponse: (t: WorkspaceTransform) => ({ ...t, id: t.ref_id }),
}),
getTransformUpstreamMapping: builder.query<
TransformUpstreamMapping,
TransformId
>({
query: (id) => ({
method: "GET",
url: `/api/ee/workspace/mapping/transform/${id}/upstream`,
}),
providesTags: (mapping) =>
mapping?.transform
? provideWorkspaceContentItemsTags([mapping.transform])
: [],
}),
getTransformDownstreamMapping: builder.query<
TransformDownstreamMapping,
TransformId
>({
query: (id) => ({
method: "GET",
url: `/api/ee/workspace/mapping/transform/${id}/downstream`,
}),
providesTags: (mapping) =>
mapping ? provideWorkspaceContentItemsTags(mapping.transforms) : [],
}),
getWorkspaceCheckout: builder.query<WorkspaceCheckoutResponse, TransformId>(
{
query: (transformId) => ({
method: "GET",
url: `/api/ee/workspace/checkout`,
params: { "transform-id": transformId },
}),
providesTags: (_, __, transformId) => [idTag("transform", transformId)],
},
),
mergeWorkspace: builder.mutation<
WorkspaceMergeResponse,
{ id: WorkspaceId; commit_message: string }
>({
query: ({ id, commit_message }) => ({
method: "POST",
url: `/api/ee/workspace/${id}/merge`,
body: { commit_message },
}),
invalidatesTags: (_, error) =>
invalidateTags(error, [tag("workspace"), tag("transform")]),
}),
mergeWorkspaceTransform: builder.mutation<
WorkspaceTransformMergeResponse,
WorkspaceTransformRef
>({
query: ({ workspaceId, transformId }) => ({
method: "POST",
url: `/api/ee/workspace/${workspaceId}/transform/${transformId}/merge`,
}),
invalidatesTags: (_, error) =>
invalidateTags(error, [tag("workspace"), tag("transform")]),
}),
archiveWorkspace: builder.mutation<Workspace, WorkspaceId>({
query: (id) => ({
method: "POST",
url: `/api/ee/workspace/${id}/archive`,
}),
invalidatesTags: (_, error) =>
invalidateTags(error, [tag("workspace"), tag("transform")]),
}),
unarchiveWorkspace: builder.mutation<Workspace, WorkspaceId>({
query: (id) => ({
method: "POST",
url: `/api/ee/workspace/${id}/unarchive`,
}),
invalidatesTags: (_, error) =>
invalidateTags(error, [tag("workspace"), tag("transform")]),
}),
deleteWorkspace: builder.mutation<void, WorkspaceId>({
query: (id) => ({
method: "DELETE",
url: `/api/ee/workspace/${id}`,
}),
invalidatesTags: (_, error) =>
invalidateTags(error, [tag("workspace"), tag("transform")]),
}),
validateTableName: builder.mutation<
ValidateTableNameResponse,
ValidateTableNameRequest
>({
query: ({ id, ...body }) => ({
method: "POST",
url: `/api/ee/workspace/${id}/transform/validate/target`,
body,
}),
}),
getWorkspaceTables: builder.query<WorkspaceTablesResponse, WorkspaceId>({
query: (id) => ({
method: "GET",
url: `/api/ee/workspace/${id}/table`,
}),
providesTags: (_, __, id) => [idTag("workspace-tables", id)],
}),
getWorkspaceGraph: builder.query<WorkspaceGraphResponse, WorkspaceId>({
query: (id) => ({
method: "GET",
url: `/api/ee/workspace/${id}/graph`,
}),
providesTags: (_, __, id) => [idTag("workspace", id)],
}),
getWorkspaceProblems: builder.query<WorkspaceProblem[], WorkspaceId>({
query: (id) => ({
method: "GET",
url: `/api/ee/workspace/${id}/problem`,
}),
providesTags: (_, __, id) => [idTag("workspace", id)],
}),
getWorkspaceTransforms: builder.query<
WorkspaceTransformItem[],
WorkspaceId
>({
query: (id) => ({
method: "GET",
url: `/api/ee/workspace/${id}/transform`,
}),
providesTags: (_, __, id) => [idTag("workspace-transforms", id)],
transformResponse: (response: WorkspaceTransformsResponse) =>
response.transforms,
}),
getExternalTransforms: builder.query<
ExternalTransformsResponse["transforms"],
ExternalTransformsRequest
>({
query: ({ workspaceId, databaseId }) => ({
method: "GET",
url: `/api/ee/workspace/${workspaceId}/external/transform`,
params: { database_id: databaseId },
}),
providesTags: (_, __, { workspaceId: id }) => [
listTag("external-transforms"),
idTag("external-transforms", id),
],
transformResponse: (response: ExternalTransformsResponse) =>
response.transforms,
}),
getWorkspaceTransform: builder.query<
WorkspaceTransform,
{ workspaceId: WorkspaceId; transformId: string }
>({
query: ({ workspaceId, transformId }) => ({
method: "GET",
url: `/api/ee/workspace/${workspaceId}/transform/${transformId}`,
}),
providesTags: (_, __, { transformId }) => [
// idTag("workspace-transforms", workspaceId),
idTag("workspace-transform", transformId),
],
transformResponse: (response: WorkspaceTransform) => ({
...response,
id: response.ref_id,
}),
}),
getWorkspaceLog: builder.query<WorkspaceLogResponse, WorkspaceId>({
query: (id) => ({
method: "GET",
url: `/api/ee/workspace/${id}/log`,
}),
providesTags: (_, _error, id) => [idTag("workspace", id)],
}),
runWorkspace: builder.mutation<WorkspaceRunResponse, WorkspaceRunRequest>({
query: ({ id, stale_only }) => ({
method: "POST",
url: `/api/ee/workspace/${id}/run`,
params: stale_only ? { stale_only: 1 } : undefined,
}),
invalidatesTags: (_, error, { id }) =>
invalidateTags(error, [
idTag("workspace", id),
idTag("workspace-transforms", id),
idTag("workspace-tables", id),
tag("workspace-transform"),
tag("transform"),
]),
}),
runWorkspaceTransform: builder.mutation<
WorkspaceTransformRunResponse,
WorkspaceTransformRef
>({
query: ({ workspaceId, transformId }) => ({
method: "POST",
url: `/api/ee/workspace/${workspaceId}/transform/${transformId}/run`,
}),
invalidatesTags: (_, error, { workspaceId, transformId }) =>
invalidateTags(error, [
idTag("workspace", workspaceId),
idTag("workspace-transforms", workspaceId),
idTag("workspace-transform", transformId),
idTag("workspace-tables", workspaceId),
]),
}),
updateWorkspaceTransform: builder.mutation<
WorkspaceTransform,
UpdateWorkspaceTransformRequest
>({
query: ({ workspaceId, transformId, ...body }) => ({
method: "PUT",
url: `/api/ee/workspace/${workspaceId}/transform/${transformId}`,
body,
}),
invalidatesTags: (_, error, { workspaceId, transformId }) =>
invalidateTags(error, [
idTag("workspace", workspaceId),
idTag("workspace-transforms", workspaceId),
idTag("workspace-transform", transformId),
]),
transformResponse: (t: WorkspaceTransform) => ({ ...t, id: t.ref_id }),
}),
archiveWorkspaceTransform: builder.mutation<void, WorkspaceTransformRef>({
query: ({ workspaceId, transformId }) => ({
method: "POST",
url: `/api/ee/workspace/${workspaceId}/transform/${transformId}/archive`,
}),
invalidatesTags: (_, error, { workspaceId, transformId }) =>
invalidateTags(error, [
idTag("workspace", workspaceId),
idTag("workspace-transforms", workspaceId),
idTag("workspace-transform", transformId),
idTag("workspace-tables", workspaceId),
tag("transform"),
]),
}),
unarchiveWorkspaceTransform: builder.mutation<void, WorkspaceTransformRef>({
query: ({ workspaceId, transformId }) => ({
method: "POST",
url: `/api/ee/workspace/${workspaceId}/transform/${transformId}/unarchive`,
}),
invalidatesTags: (_, error, { workspaceId, transformId }) =>
invalidateTags(error, [
idTag("workspace", workspaceId),
idTag("workspace-transforms", workspaceId),
idTag("workspace-transform", transformId),
idTag("workspace-tables", workspaceId),
tag("transform"),
]),
}),
deleteWorkspaceTransform: builder.mutation<void, WorkspaceTransformRef>({
query: ({ workspaceId, transformId }) => ({
method: "DELETE",
url: `/api/ee/workspace/${workspaceId}/transform/${transformId}`,
}),
invalidatesTags: (_, error, { workspaceId, transformId }) =>
invalidateTags(error, [
idTag("workspace", workspaceId),
idTag("workspace-transforms", workspaceId),
idTag("external-transforms", workspaceId),
idTag("workspace-transform", transformId),
idTag("workspace-tables", workspaceId),
tag("transform"),
]),
}),
getWorkspaceAllowedDatabases: builder.query<
WorkspaceAllowedDatabasesResponse,
void
>({
query: () => ({
method: "GET",
url: `/api/ee/workspace/database`,
}),
}),
}),
});
export const {
useGetWorkspacesQuery,
useGetWorkspaceQuery,
useGetWorkspaceTransformsQuery,
useGetExternalTransformsQuery,
useGetWorkspaceTransformQuery,
useLazyGetWorkspaceTransformQuery,
useCreateWorkspaceMutation,
useCreateWorkspaceTransformMutation,
useUpdateWorkspaceTransformMutation,
useArchiveWorkspaceTransformMutation,
useUnarchiveWorkspaceTransformMutation,
useDeleteWorkspaceTransformMutation,
useGetTransformUpstreamMappingQuery,
useGetTransformDownstreamMappingQuery,
useGetWorkspaceCheckoutQuery,
useMergeWorkspaceMutation,
useMergeWorkspaceTransformMutation,
useArchiveWorkspaceMutation,
useUnarchiveWorkspaceMutation,
useDeleteWorkspaceMutation,
useValidateTableNameMutation,
useUpdateWorkspaceMutation,
useGetWorkspaceTablesQuery,
useGetWorkspaceGraphQuery,
useGetWorkspaceProblemsQuery,
useGetWorkspaceLogQuery,
useRunWorkspaceMutation,
useRunWorkspaceTransformMutation,
useGetWorkspaceAllowedDatabasesQuery,
} = workspaceApi;
export const DEFAULT_WORKSPACE_TABLES_QUERY_RESPONSE: WorkspaceTablesResponse =
{
inputs: [],
outputs: [],
};

View File

@@ -33,6 +33,7 @@
.toggle {
position: relative;
color: var(--mb-color-text-primary);
font-weight: 600;
&:hover,
&.selected {
@@ -44,14 +45,103 @@
}
}
.hoverButton {
&:hover {
.showOnHover {
opacity: 1;
}
.workspacesSection {
border-top: 1px solid var(--mb-color-border);
padding-top: 0.75rem;
}
.hideOnHover {
opacity: 0;
.workspacesSectionHeader {
background-color: var(--mb-color-bg-secondary);
color: var(--mb-color-text-primary);
font-weight: 600;
transition: background-color 150ms ease;
&:hover {
background-color: var(--mb-color-brand-light);
color: var(--mb-color-brand);
}
&.selected {
background-color: var(--mb-color-brand-light);
color: var(--mb-color-brand);
}
}
.workspaceItem {
background-color: var(--mb-color-bg-white);
color: var(--mb-color-text-primary);
cursor: pointer;
border: 1px solid var(--mb-color-border);
transition:
background-color 150ms ease,
border-color 150ms ease,
color 150ms ease;
&:hover {
background-color: var(--mb-color-bg-light);
border-color: var(--mb-color-border-dark);
}
&:hover .workspaceMenuButton {
opacity: 1;
}
&.selected {
background-color: var(--mb-color-brand-lighter);
border-color: var(--mb-color-brand);
color: var(--mb-color-brand);
}
&.selected .workspaceMenuButton {
opacity: 1;
}
&:disabled {
opacity: 0.5;
cursor: not-allowed;
}
}
.workspaceMenuButton {
color: var(--mb-color-text-secondary);
opacity: 0;
transition:
opacity 150ms ease,
background-color 150ms ease,
color 150ms ease;
flex-shrink: 0;
&:hover {
background-color: var(--mb-color-bg-medium);
color: var(--mb-color-text-primary);
}
}
.newWorkspaceButton {
background-color: transparent;
color: var(--mb-color-text-primary);
cursor: pointer;
border: none;
transition: color 150ms ease;
&:hover {
color: var(--mb-color-brand);
}
&:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.hoverButton {
&:hover {
.showOnHover {
opacity: 1;
}
.hideOnHover {
opacity: 0;
}
}
}
}

View File

@@ -28,6 +28,7 @@ import {
} from "metabase/ui";
import S from "./DataStudioLayout.module.css";
import { WorkspacesSection } from "./WorkspaceSection";
import { getCurrentTab } from "./utils";
type DataStudioLayoutProps = {
@@ -87,7 +88,7 @@ function DataStudioNav({ isNavbarOpened, onNavbarToggle }: DataStudioNavProps) {
justify="space-between"
data-testid="data-studio-nav"
>
<Stack gap="0.75rem">
<Stack gap="0.75rem" flex={1} mih={0}>
<DataStudioNavbarToggle
isNavbarOpened={isNavbarOpened}
onNavbarToggle={onNavbarToggle}
@@ -144,6 +145,9 @@ function DataStudioNav({ isNavbarOpened, onNavbarToggle }: DataStudioNavProps) {
showLabel={isNavbarOpened}
/>
)}
{canAccessTransforms && (
<WorkspacesSection showLabel={isNavbarOpened} />
)}
</Stack>
<Stack gap="0.75rem">
{canAccessTransforms && (
@@ -164,6 +168,7 @@ function DataStudioNav({ isNavbarOpened, onNavbarToggle }: DataStudioNavProps) {
showLabel={isNavbarOpened}
/>
)}
<DataStudioTab
label={t`Exit`}
icon="exit"

View File

@@ -0,0 +1,382 @@
import { useDisclosure } from "@mantine/hooks";
import { h } from "@tiptap/core";
import cx from "classnames";
import dayjs from "dayjs";
import { useCallback, useMemo, useState } from "react";
import { push } from "react-router-redux";
import { t } from "ttag";
import { ConfirmModal } from "metabase/common/components/ConfirmModal/ConfirmModal";
import { ForwardRefLink } from "metabase/common/components/Link";
import { useDispatch, useSelector } from "metabase/lib/redux";
import * as Urls from "metabase/lib/urls";
import { useMetadataToasts } from "metabase/metadata/hooks";
import { getLocation } from "metabase/selectors/routing";
import {
ActionIcon,
FixedSizeIcon,
Flex,
Group,
Icon,
type IconName,
Loader,
Menu,
Skeleton,
Stack,
Text,
Tooltip,
UnstyledButton,
} from "metabase/ui";
import {
useArchiveWorkspaceMutation,
useCreateWorkspaceMutation,
useDeleteWorkspaceMutation,
useGetWorkspacesQuery,
useUnarchiveWorkspaceMutation,
} from "metabase-enterprise/api/workspace";
import { TOOLTIP_OPEN_DELAY } from "metabase-enterprise/dependencies/components/DependencyGraph/constants";
import type { Workspace, WorkspaceId } from "metabase-types/api/workspace";
import S from "./DataStudioLayout.module.css";
type WorkspacesSectionProps = {
showLabel: boolean;
};
function WorkspacesSection({ showLabel }: WorkspacesSectionProps) {
const dispatch = useDispatch();
const [isWorkspacesExpanded, setIsWorkspacesExpanded] = useState(true);
const { pathname } = useSelector(getLocation);
const { data: workspacesData, isLoading: areWorkspacesLoading } =
useGetWorkspacesQuery();
const [createWorkspace, { isLoading: isCreatingWorkspace }] =
useCreateWorkspaceMutation();
const workspaces = useMemo(
() =>
[...(workspacesData?.items ?? [])].sort((a, b) => {
// Archived workspaces should be placed at the end
if (a.archived !== b.archived) {
return a.archived ? 1 : -1;
}
// Within each group (archived/unarchived), sort by updated_at descending
const aDate = a.updated_at ? new Date(a.updated_at).getTime() : 0;
const bDate = b.updated_at ? new Date(b.updated_at).getTime() : 0;
return bDate - aDate;
}),
[workspacesData],
);
const handleOpenWorkspace = useCallback(
(workspaceId: number) => {
dispatch(push(Urls.dataStudioWorkspace(workspaceId)));
},
[dispatch],
);
const { sendErrorToast, sendSuccessToast } = useMetadataToasts();
const handleCreateWorkspace = useCallback(async () => {
try {
const workspace = await createWorkspace().unwrap();
handleOpenWorkspace(workspace.id);
} catch (error) {
sendErrorToast(t`Failed to create workspace`);
}
}, [createWorkspace, handleOpenWorkspace, sendErrorToast]);
const [archiveWorkspace] = useArchiveWorkspaceMutation();
const [unarchiveWorkspace] = useUnarchiveWorkspaceMutation();
const [deleteWorkspace] = useDeleteWorkspaceMutation();
const handleWorkspaceArchive = async (id: WorkspaceId) => {
try {
await archiveWorkspace(id).unwrap();
sendSuccessToast(t`Workspace archived successfully`);
dispatch(push(Urls.dataStudioWorkspaceList()));
} catch (error) {
sendErrorToast(t`Failed to archive workspace`);
}
};
const handleWorkspaceUnarchive = async (id: WorkspaceId) => {
try {
await unarchiveWorkspace(id).unwrap();
sendSuccessToast(t`Workspace restored successfully`);
} catch (error) {
sendErrorToast(t`Failed to restore workspace`);
}
};
const handleWorkspaceDelete = async (id: WorkspaceId) => {
try {
await deleteWorkspace(id).unwrap();
sendSuccessToast(t`Workspace deleted successfully`);
dispatch(push(Urls.dataStudioWorkspaceList()));
} catch (error) {
sendErrorToast(t`Failed to delete workspace`);
}
};
const isWorkspaceListPage = pathname === Urls.dataStudioWorkspaceList();
if (!showLabel) {
// In collapsed mode, show only an icon button with tooltip
return (
<Tooltip
label={t`Workspaces`}
position="right"
openDelay={TOOLTIP_OPEN_DELAY}
>
<UnstyledButton
className={cx(S.tab, { [S.selected]: isWorkspaceListPage })}
component={ForwardRefLink}
to={Urls.dataStudioWorkspaceList()}
p="0.75rem"
bdrs="md"
>
<Flex align="center" justify="center">
<FixedSizeIcon name="git_branch" display="block" />
</Flex>
</UnstyledButton>
</Tooltip>
);
}
return (
<Stack
data-testid="workspaces-section"
gap="0.5rem"
className={S.workspacesSection}
flex={1}
mih={0}
>
<UnstyledButton
className={cx(S.workspacesSectionHeader, {
[S.selected]: isWorkspaceListPage,
})}
onClick={() => setIsWorkspacesExpanded(!isWorkspacesExpanded)}
p="0.75rem"
bdrs="md"
>
<Flex align="center" justify="space-between" w="100%">
<Text size="sm" fw={600}>
{t`Workspaces`}
</Text>
<Icon
name={isWorkspacesExpanded ? "chevrondown" : "chevronright"}
size={16}
/>
</Flex>
</UnstyledButton>
{isWorkspacesExpanded && (
<>
<UnstyledButton
className={S.newWorkspaceButton}
onClick={handleCreateWorkspace}
disabled={isCreatingWorkspace || areWorkspacesLoading}
p="0.5rem"
bdrs="md"
>
<Flex align="center" gap="xs">
{isCreatingWorkspace ? (
<Loader size="xs" />
) : (
<Icon name="add" size={16} />
)}
<Text size="sm" fw={500}>
{t`New workspace`}
</Text>
</Flex>
</UnstyledButton>
<Stack
gap="0.75rem"
style={{ flex: 1, overflowY: "auto", minHeight: 0 }}
>
{areWorkspacesLoading ? (
<>
<Skeleton height={80} radius="md" />
<Skeleton height={80} radius="md" />
</>
) : workspaces.length === 0 ? (
<Text c="text-secondary" size="xs" px="0.5rem">
{t`No workspaces yet`}
</Text>
) : (
workspaces.map((workspace) => {
const isSelected =
pathname === Urls.dataStudioWorkspace(workspace.id);
return (
<WorkspaceItem
key={workspace.id}
workspace={workspace}
isSelected={isSelected}
onOpen={handleOpenWorkspace}
onArchive={handleWorkspaceArchive}
onUnarchive={handleWorkspaceUnarchive}
onDelete={handleWorkspaceDelete}
/>
);
})
)}
</Stack>
</>
)}
</Stack>
);
}
export { WorkspacesSection };
interface WorkspaceItemProps {
workspace: Workspace;
isSelected: boolean;
onOpen: (workspaceId: WorkspaceId) => void;
onArchive: (workspaceId: WorkspaceId) => Promise<void>;
onUnarchive: (workspaceId: WorkspaceId) => Promise<void>;
onDelete: (workspaceId: WorkspaceId) => Promise<void>;
}
function WorkspaceItem({
workspace,
isSelected,
onOpen,
onArchive,
onUnarchive,
onDelete,
}: WorkspaceItemProps) {
const [
deleteModalOpened,
{ open: openDeleteModal, close: closeDeleteModal },
] = useDisclosure();
const timeAgo = workspace.updated_at
? dayjs(workspace.updated_at).fromNow()
: null;
const handleArchive = () => {
onArchive(workspace.id);
};
const handleUnarchive = () => {
onUnarchive(workspace.id);
};
const handleDelete = () => {
openDeleteModal();
};
const handleConfirmDelete = async () => {
await onDelete(workspace.id);
closeDeleteModal();
};
const status = getWorkspaceListStatus(workspace);
return (
<UnstyledButton
className={cx(S.workspaceItem, { [S.selected]: isSelected })}
onClick={() => onOpen(workspace.id)}
p="0.75rem"
bdrs="md"
name={workspace.name}
>
<Flex align="flex-start" justify="space-between" gap="xs">
<Stack gap="xs" style={{ flex: 1, minWidth: 0 }}>
<Text size="sm" fw={600} truncate>
{workspace.name}
</Text>
{status && (
<Group
gap="xs"
align="center"
wrap="nowrap"
data-testid="workspace-status"
>
<Icon name={status.icon} size={10} c={status.color} />
<Text size="xs" fw={500} c={status.color}>
{status.label}
</Text>
</Group>
)}
{timeAgo && (
<Text size="xs" c="text-secondary" truncate>
{t`Updated ${timeAgo}`}
</Text>
)}
</Stack>
<Menu position="right" withinPortal>
<Menu.Target>
<ActionIcon
className={S.workspaceMenuButton}
component="span"
aria-label={t`More actions`}
onClick={(e: React.MouseEvent) => e.stopPropagation()}
size="sm"
variant="subtle"
>
<Icon name="ellipsis" size={16} />
</ActionIcon>
</Menu.Target>
<Menu.Dropdown onClick={(e: React.MouseEvent) => e.stopPropagation()}>
{workspace.archived ? (
<>
<Menu.Item
leftSection={<Icon name="revert" />}
onClick={handleUnarchive}
>
{t`Restore`}
</Menu.Item>
<Menu.Divider />
<Menu.Item
c="error"
leftSection={<Icon name="trash" />}
onClick={handleDelete}
color="danger"
>
{t`Delete`}
</Menu.Item>
</>
) : (
<Menu.Item
leftSection={<Icon name="archive" />}
onClick={handleArchive}
>
{t`Archive`}
</Menu.Item>
)}
</Menu.Dropdown>
</Menu>
</Flex>
<ConfirmModal
confirmButtonText={t`Delete`}
message={t`This can't be undone.`}
opened={deleteModalOpened}
title={t`Delete ${workspace.name}?`}
onClose={closeDeleteModal}
onConfirm={handleConfirmDelete}
/>
</UnstyledButton>
);
}
type WorkspaceListStatus = {
label: string;
icon: IconName;
color: string;
};
function getWorkspaceListStatus(workspace: Workspace): WorkspaceListStatus {
if (workspace.archived) {
return { label: t`Archived`, icon: "archive", color: "text-light" };
}
if (workspace.status === "pending") {
return { label: t`Pending setup`, icon: "clock", color: "warning" };
}
return { label: t`Ready`, icon: "check", color: "success" };
}

View File

@@ -0,0 +1,22 @@
import { t } from "ttag";
import { usePageTitle } from "metabase/hooks/use-page-title";
import { SectionLayout } from "../../components/SectionLayout";
type WorkspacesSectionLayoutProps = {
location: Location;
children?: React.ReactNode;
};
export function WorkspacesSectionLayout({
children,
}: WorkspacesSectionLayoutProps) {
usePageTitle(t`Workspaces`);
return (
<SectionLayout>
{children}
</SectionLayout>
);
}

View File

@@ -0,0 +1 @@
export * from "./WorkspacesSectionLayout";

View File

@@ -95,6 +95,7 @@ type PaneHeaderInputProps = {
"data-testid"?: string;
onChange?: (value: string) => void;
onContentChange?: (value: string) => void;
disabled?: boolean;
};
export function PaneHeaderInput({
@@ -105,6 +106,7 @@ export function PaneHeaderInput({
isOptional,
onChange,
onContentChange,
disabled,
}: PaneHeaderInputProps) {
return (
<EditableText
@@ -118,6 +120,7 @@ export function PaneHeaderInput({
px={isOptional ? "xs" : undefined}
bd={isOptional ? "1px solid var(--mb-color-border)" : undefined}
isOptional={isOptional}
isDisabled={disabled}
data-testid={dataTestId}
onChange={onChange}
onContentChange={onContentChange}

View File

@@ -15,11 +15,13 @@ import { DataStudioLayout } from "./app/pages/DataStudioLayout";
import { DependenciesSectionLayout } from "./app/pages/DependenciesSectionLayout";
import { LibrarySectionLayout } from "./app/pages/LibrarySectionLayout";
import { TransformsSectionLayout } from "./app/pages/TransformsSectionLayout";
import { WorkspacesSectionLayout } from "./app/pages/WorkspacesSectionLayout";
import { getDataStudioMetadataRoutes } from "./data-model/routes";
import { getDataStudioGlossaryRoutes } from "./glossary/routes";
import { getDataStudioMetricRoutes } from "./metrics/routes";
import { getDataStudioSnippetRoutes } from "./snippets/routes";
import { getDataStudioTableRoutes } from "./tables/routes";
import { getDataStudioWorkspaceRoutes } from "./workspaces/routes";
export function getDataStudioRoutes(
store: Store<State>,
@@ -54,6 +56,9 @@ export function getDataStudioRoutes(
{getDataStudioMetricRoutes()}
{getDataStudioSnippetRoutes()}
</Route>
<Route path="workspaces" component={WorkspacesSectionLayout}>
{getDataStudioWorkspaceRoutes()}
</Route>
{PLUGIN_DEPENDENCIES.isEnabled && (
<Route path="dependencies" component={DependenciesSectionLayout}>
{PLUGIN_DEPENDENCIES.getDataStudioDependencyRoutes()}

View File

@@ -0,0 +1,31 @@
.sidebarItem {
cursor: pointer;
border-left: 2px solid transparent;
&:hover {
background: var(--mb-color-bg-light);
}
}
.sidebarItemActive {
background: var(--mb-color-bg-light);
border-left-color: var(--mb-color-brand);
}
.diffEditor {
height: 100%;
:global {
.cm-content {
padding: 0;
}
.cm-collapsedLines {
padding-left: 0;
}
.cm-scroller {
font-family: monospace;
}
}
}

View File

@@ -0,0 +1,652 @@
import { unifiedMergeView } from "@codemirror/merge";
import cx from "classnames";
import { diffLines } from "diff";
import { useEffect, useMemo, useState } from "react";
import { msgid, ngettext, t } from "ttag";
import _ from "underscore";
import * as Yup from "yup";
import { CodeMirror } from "metabase/common/components/CodeMirror";
import {
Form,
FormProvider,
FormSubmitButton,
FormTextarea,
} from "metabase/forms";
import { useSelector } from "metabase/lib/redux";
import { useMetadataToasts } from "metabase/metadata/hooks";
import EditorS from "metabase/query_builder/components/NativeQueryEditor/CodeMirrorEditor/CodeMirrorEditor.module.css";
import { getMetadata } from "metabase/selectors/metadata";
import {
Box,
Collapse,
Flex,
Group,
Icon,
Loader,
Modal,
Stack,
Text,
Title,
} from "metabase/ui";
import {
useGetWorkspaceProblemsQuery,
useLazyGetTransformQuery,
useLazyGetWorkspaceTransformQuery,
} from "metabase-enterprise/api";
import * as Lib from "metabase-lib";
import type Metadata from "metabase-lib/v1/metadata/Metadata";
import type {
TransformSource,
WorkspaceId,
WorkspaceTransformItem,
} from "metabase-types/api";
import S from "./MergeWorkspaceModal.module.css";
import type { ProblemCheckCategory } from "./problemUtils";
import {
formatProblemDetails,
getCheckName,
getCheckStatus,
getCheckStatusWithUnused,
groupProblemsByCategory,
} from "./problemUtils";
const MAX_COMMIT_MESSAGE_LENGTH = 255;
type ReviewChangesModalProps = {
onClose: VoidFunction;
onSubmit: (commitMessage: string) => Promise<void>;
isLoading?: boolean;
workspaceId: WorkspaceId;
workspaceName: string;
workspaceTransforms: WorkspaceTransformItem[];
};
type MergeWorkspaceFormValues = {
commit_message: string;
};
const initialValues: MergeWorkspaceFormValues = {
commit_message: "",
};
const getMergeWorkspaceSchema = () =>
Yup.object().shape({
commit_message: Yup.string()
.trim()
.max(MAX_COMMIT_MESSAGE_LENGTH, t`Commit message is too long.`)
.required(t`Please provide a commit message.`),
});
function getSourceCode(
transform: { source: TransformSource },
metadata: Metadata,
): string {
if (transform.source.type === "python") {
return transform.source.body;
}
if (transform.source.type === "query") {
const metadataProvider = Lib.metadataProvider(
transform.source.query.database,
metadata,
);
const query = Lib.fromJsQuery(metadataProvider, transform.source.query);
if (Lib.queryDisplayInfo(query).isNative) {
return Lib.rawNativeQuery(query);
}
}
return "";
}
function computeDiffStats(
oldSource: string,
newSource: string,
): { additions: number; deletions: number } {
const changes = diffLines(oldSource, newSource);
let additions = 0;
let deletions = 0;
for (const change of changes) {
const lineCount = change.count ?? 0;
if (change.added) {
additions += lineCount;
} else if (change.removed) {
deletions += lineCount;
}
}
return { additions, deletions };
}
const QualityChecksSection = ({
workspaceId,
}: {
workspaceId: WorkspaceId;
}) => {
const { data: problems = [], isLoading } =
useGetWorkspaceProblemsQuery(workspaceId);
const grouped = useMemo(() => groupProblemsByCategory(problems), [problems]);
const [expandedChecks, setExpandedChecks] = useState<
Set<ProblemCheckCategory>
>(new Set());
const checkCategories: ProblemCheckCategory[] = [
"external-dependencies",
"internal-dependencies",
"structural-issues",
"unused-outputs",
];
const toggleCheck = (category: ProblemCheckCategory) => {
setExpandedChecks((prev) => {
const next = new Set(prev);
if (next.has(category)) {
next.delete(category);
} else {
next.add(category);
}
return next;
});
};
return (
<Stack gap="sm">
<Text fw="bold">{t`Quality Checks`}</Text>
<Stack gap="sm">
{checkCategories.map((category) => {
const categoryProblems = grouped[category];
const checkStatus =
category === "unused-outputs"
? getCheckStatusWithUnused(categoryProblems)
: getCheckStatus(categoryProblems);
const isExpanded = expandedChecks.has(category);
const hasProblems = checkStatus.count > 0;
return (
<Box key={category}>
<Group
justify="space-between"
style={{ cursor: hasProblems ? "pointer" : "default" }}
onClick={() => hasProblems && toggleCheck(category)}
>
<Text>{getCheckName(category)}</Text>
{isLoading ? (
<Loader size="xs" />
) : (
<Group gap="xs">
{checkStatus.status === "passed" && (
<Icon name="check" size={14} c="success" />
)}
<Text
c={
checkStatus.status === "passed"
? "success"
: checkStatus.problems.some(
(p) => p.severity === "error",
)
? "error"
: "warning"
}
>
{checkStatus.status === "passed"
? category === "unused-outputs" && checkStatus.count > 0
? t`${checkStatus.count} info`
: t`Passed`
: checkStatus.count === 1
? t`Failed`
: t`${checkStatus.count} issues`}
</Text>
</Group>
)}
</Group>
{hasProblems && (
<Collapse in={isExpanded}>
<Stack gap="xs" mt="xs" pl="md">
{checkStatus.problems.map((problem, idx) => (
<Box key={idx}>
<Text size="sm" c="text-medium">
{formatProblemDetails(problem)}
</Text>
</Box>
))}
</Stack>
</Collapse>
)}
</Box>
);
})}
</Stack>
</Stack>
);
};
const OverviewPanel = ({
commitMessageLength = 0,
hasCommitMessageError,
workspaceId,
transformCount,
}: {
commitMessageLength?: number;
hasCommitMessageError: boolean;
workspaceId: WorkspaceId;
transformCount: number;
}) => {
return (
<Box p="xl" h="100%" style={{ overflowY: "auto" }}>
<Stack>
<Box>
<Text>{t`This will merge all changes from this workspace back to the source transforms.`}</Text>
<Text mt="xs">
{t`The commit message will be used to display the history of transform changes.`}
</Text>
</Box>
<Stack mt="md" gap="xs">
<FormTextarea
data-autofocus
label={t`Commit message`}
name="commit_message"
placeholder={t`Describe the changes you made in this workspace...`}
minRows={4}
required
/>
{!hasCommitMessageError && (
<Text size="xs" c="text-tertiary">
{commitMessageLength}/{MAX_COMMIT_MESSAGE_LENGTH}
</Text>
)}
</Stack>
<QualityChecksSection workspaceId={workspaceId} />
<Box
p="md"
style={{
backgroundColor: "var(--mb-color-bg-light)",
borderRadius: "var(--mb-radius-sm)",
}}
>
<Text>
{ngettext(
msgid`${transformCount} transform will be merged`,
`${transformCount} transforms will be merged`,
transformCount,
)}
</Text>
</Box>
</Stack>
</Box>
);
};
const DiffView = ({
transform,
workspaceId,
}: {
transform: WorkspaceTransformItem;
workspaceId: WorkspaceId;
}) => {
const metadata = useSelector(getMetadata);
const isNewTransform = transform.global_id == null;
const [fetchWorkspaceTransform, workspaceTransformResult] =
useLazyGetWorkspaceTransformQuery();
const [fetchGlobalTransform, globalTransformResult] =
useLazyGetTransformQuery();
useEffect(() => {
fetchWorkspaceTransform({
workspaceId,
transformId: transform.ref_id,
});
if (transform.global_id) {
fetchGlobalTransform(transform.global_id);
}
}, [
transform.global_id,
transform.ref_id,
workspaceId,
fetchWorkspaceTransform,
fetchGlobalTransform,
]);
const isLoading =
workspaceTransformResult.isLoading ||
(!isNewTransform && globalTransformResult.isLoading);
const hasError =
workspaceTransformResult.error ||
(!isNewTransform && globalTransformResult.error);
const oldSource = globalTransformResult.data
? getSourceCode(globalTransformResult.data, metadata)
: "";
const newSource = workspaceTransformResult.data
? getSourceCode(workspaceTransformResult.data, metadata)
: "";
const extensions = useMemo(
() =>
_.compact([
oldSource &&
unifiedMergeView({
original: oldSource,
mergeControls: false,
}),
]),
[oldSource],
);
if (isLoading) {
return (
<Flex
align="center"
justify="center"
h="100%"
direction="column"
gap="sm"
>
<Loader size="sm" />
<Text c="text-medium">{t`Loading diff...`}</Text>
</Flex>
);
}
if (hasError) {
return (
<Flex align="center" justify="center" h="100%">
<Text c="danger">{t`Failed to load diff`}</Text>
</Flex>
);
}
if (!newSource) {
return (
<Flex align="center" justify="center" h="100%">
<Text c="text-medium">{t`No source code to display`}</Text>
</Flex>
);
}
return (
<CodeMirror
className={cx(EditorS.editor, S.diffEditor)}
extensions={extensions}
value={newSource}
readOnly
autoCorrect="off"
/>
);
};
const TransformListItem = ({
transform,
isSelected,
onClick,
workspaceId,
}: {
transform: WorkspaceTransformItem;
isSelected: boolean;
onClick: () => void;
workspaceId: WorkspaceId;
}) => {
const metadata = useSelector(getMetadata);
const isNewTransform = transform.global_id == null;
const [fetchWorkspaceTransform, workspaceTransformResult] =
useLazyGetWorkspaceTransformQuery();
const [fetchGlobalTransform, globalTransformResult] =
useLazyGetTransformQuery();
useEffect(() => {
fetchWorkspaceTransform({
workspaceId,
transformId: transform.ref_id,
});
if (transform.global_id) {
fetchGlobalTransform(transform.global_id);
}
}, [
transform.global_id,
transform.ref_id,
workspaceId,
fetchWorkspaceTransform,
fetchGlobalTransform,
]);
const oldSource = globalTransformResult.data
? getSourceCode(globalTransformResult.data, metadata)
: "";
const newSource = workspaceTransformResult.data
? getSourceCode(workspaceTransformResult.data, metadata)
: "";
const stats = useMemo(() => {
if (isNewTransform && newSource) {
// For new transforms, count all lines as additions
const lineCount = newSource.split("\n").length;
return { additions: lineCount, deletions: 0 };
}
if (oldSource && newSource) {
return computeDiffStats(oldSource, newSource);
}
return null;
}, [isNewTransform, oldSource, newSource]);
return (
<Flex
align="center"
justify="space-between"
px="md"
py="sm"
className={cx(S.sidebarItem, isSelected && S.sidebarItemActive)}
onClick={onClick}
>
<Flex align="center" gap="sm" style={{ overflow: "hidden" }}>
<Icon name="code_block" size={14} c="text-medium" />
<Text truncate>{transform.name}</Text>
</Flex>
{stats && (stats.additions > 0 || stats.deletions > 0) && (
<Flex gap="xs" fz="xs" style={{ flexShrink: 0 }}>
{stats.additions > 0 && <Text c="success">+{stats.additions}</Text>}
{stats.deletions > 0 && <Text c="danger">-{stats.deletions}</Text>}
</Flex>
)}
</Flex>
);
};
export const ReviewChangesModal = ({
onClose,
onSubmit,
isLoading = false,
workspaceId,
workspaceName,
workspaceTransforms,
}: ReviewChangesModalProps) => {
const { sendErrorToast } = useMetadataToasts();
const validationSchema = useMemo(() => getMergeWorkspaceSchema(), []);
const { data: workspaceProblems = [] } =
useGetWorkspaceProblemsQuery(workspaceId);
const hasBlockingProblems = useMemo(
() => workspaceProblems.some((p) => p.block_merge === true),
[workspaceProblems],
);
const updatedTransforms = useMemo(
() => workspaceTransforms.filter((t) => t.global_id != null),
[workspaceTransforms],
);
const newTransforms = useMemo(
() => workspaceTransforms.filter((t) => t.global_id == null),
[workspaceTransforms],
);
const [selectedTransformId, setSelectedTransformId] = useState<
string | "overview"
>("overview");
const selectedTransform =
selectedTransformId !== "overview"
? workspaceTransforms.find((t) => t.ref_id === selectedTransformId)
: null;
const handleSubmit = async (values: MergeWorkspaceFormValues) => {
try {
await onSubmit(values.commit_message.trim());
onClose();
} catch (error) {
sendErrorToast(t`Failed to merge workspace`);
throw error;
}
};
return (
<Modal
data-testid="review-changes-modal"
onClose={onClose}
opened
title={
<Flex align="center" gap="sm">
<Icon name="git_branch" c="brand" />
<Flex direction="column" gap="xs">
<Title order={3}>{t`Review Changes`}</Title>
<Text size="sm" c="text-medium">
{workspaceName}
</Text>
</Flex>
</Flex>
}
size="xl"
padding={0}
styles={{
header: {
padding: "1.25rem 1.5rem",
borderBottom: "1px solid var(--mb-color-border)",
},
}}
>
<FormProvider
initialValues={initialValues}
validationSchema={validationSchema}
onSubmit={handleSubmit}
>
{({ values, touched, errors }) => (
<Form style={{ display: "flex", flexDirection: "column" }}>
<Flex h={500}>
<Box
pt="md"
w={220}
miw={220}
style={{
borderRight: "1px solid var(--mb-color-border)",
overflowY: "auto",
}}
>
<Flex
align="center"
gap="sm"
px="md"
py="sm"
className={cx(
S.sidebarItem,
selectedTransformId === "overview" && S.sidebarItemActive,
)}
onClick={() => setSelectedTransformId("overview")}
>
<Icon name="document" size={14} c="text-medium" />
<Text>{t`Overview`}</Text>
</Flex>
{newTransforms.length > 0 && (
<>
<Text
px="md"
py="sm"
fz="xs"
fw={700}
tt="uppercase"
lts="0.5px"
c="text-medium"
>
{t`New transforms`}
</Text>
{newTransforms.map((transform) => (
<TransformListItem
key={transform.ref_id}
transform={transform}
isSelected={selectedTransformId === transform.ref_id}
onClick={() => setSelectedTransformId(transform.ref_id)}
workspaceId={workspaceId}
/>
))}
</>
)}
{updatedTransforms.length > 0 && (
<>
<Text
px="md"
py="sm"
fz="xs"
fw={700}
tt="uppercase"
lts="0.5px"
c="text-medium"
>
{t`Modified transforms`}
</Text>
{updatedTransforms.map((transform) => (
<TransformListItem
key={transform.ref_id}
transform={transform}
isSelected={selectedTransformId === transform.ref_id}
onClick={() => setSelectedTransformId(transform.ref_id)}
workspaceId={workspaceId}
/>
))}
</>
)}
</Box>
<Box flex={1} miw={0} style={{ overflow: "auto" }}>
{selectedTransformId === "overview" ? (
<OverviewPanel
commitMessageLength={values.commit_message?.length}
hasCommitMessageError={!!errors.commit_message}
workspaceId={workspaceId}
transformCount={workspaceTransforms.length}
/>
) : selectedTransform ? (
<DiffView
key={selectedTransform.ref_id}
transform={selectedTransform}
workspaceId={workspaceId}
/>
) : (
<Flex align="center" justify="center" h="100%">
<Text c="text-medium">{t`Transform not found`}</Text>
</Flex>
)}
</Box>
</Flex>
<Flex
justify="flex-end"
align="center"
p="md"
style={{ borderTop: "1px solid var(--mb-color-border)" }}
>
<FormSubmitButton
disabled={
!values.commit_message?.trim() ||
(touched.commit_message && !!errors.commit_message) ||
hasBlockingProblems
}
label={t`Merge`}
loading={isLoading}
variant="filled"
/>
</Flex>
</Form>
)}
</FormProvider>
</Modal>
);
};
export { ReviewChangesModal as MergeWorkspaceModal };

View File

@@ -0,0 +1,163 @@
import type { WorkspaceProblem } from "metabase-types/api";
export type ProblemCheckCategory =
| "external-dependencies"
| "internal-dependencies"
| "structural-issues"
| "unused-outputs";
export type GroupedProblems = {
"external-dependencies": WorkspaceProblem[];
"internal-dependencies": WorkspaceProblem[];
"structural-issues": WorkspaceProblem[];
"unused-outputs": WorkspaceProblem[];
};
export type CheckStatus = {
status: "passed" | "failed";
count: number;
problems: WorkspaceProblem[];
};
/**
* Groups problems by category into logical Quality Check categories
*/
export function groupProblemsByCategory(
problems: WorkspaceProblem[],
): GroupedProblems {
const grouped: GroupedProblems = {
"external-dependencies": [],
"internal-dependencies": [],
"structural-issues": [],
"unused-outputs": [],
};
for (const problem of problems) {
if (problem.category === "external-downstream") {
grouped["external-dependencies"].push(problem);
} else if (problem.category === "internal-downstream") {
grouped["internal-dependencies"].push(problem);
} else if (
problem.category === "internal" ||
problem.category === "external"
) {
grouped["structural-issues"].push(problem);
} else if (problem.category === "unused") {
grouped["unused-outputs"].push(problem);
}
}
return grouped;
}
/**
* Calculates the status for a group of problems
* Unused problems are informational and don't affect status (always "passed")
*/
export function getCheckStatus(problems: WorkspaceProblem[]): CheckStatus {
// Filter out unused problems for status calculation - they're informational only
const relevantProblems = problems.filter((p) => p.category !== "unused");
// If all problems are unused, return passed with empty array
// If there are any non-unused problems, return failed with those problems
return {
status: relevantProblems.length === 0 ? "passed" : "failed",
count: relevantProblems.length,
problems: relevantProblems,
};
}
/**
* Calculates status including unused problems (for unused-outputs check)
*/
export function getCheckStatusWithUnused(
problems: WorkspaceProblem[],
): CheckStatus {
return {
status: problems.length === 0 ? "passed" : "passed", // Unused is always passed (informational)
count: problems.length,
problems: problems, // Include all problems for display
};
}
/**
* Formats problem details for display
*/
export function formatProblemDetails(problem: WorkspaceProblem): string {
const parts: string[] = [];
// Add transform reference if available
if ("transform" in problem.data && problem.data.transform) {
const transform = problem.data.transform;
const transformName =
"name" in transform && transform.name
? transform.name
: transform.id.toString();
const transformType =
transform.type === "workspace-transform" ? "workspace" : "external";
parts.push(`${transformType} transform "${transformName}"`);
}
// Add table reference if available
if ("output" in problem.data && problem.data.output) {
const output = problem.data.output;
if ("global" in output && output.global) {
const schema = output.global.schema || "";
const table = output.global.table || "";
if (schema && table) {
parts.push(`table "${schema}.${table}"`);
} else if (table) {
parts.push(`table "${table}"`);
}
}
}
// Add bad-refs for removed-field problems
if ("bad-refs" in problem.data && problem.data["bad-refs"]) {
const badRefs = problem.data["bad-refs"];
if (Array.isArray(badRefs) && badRefs.length > 0) {
const fieldNames = badRefs
.map((ref) => (ref.data?.name ? ref.data.name : null))
.filter(Boolean);
if (fieldNames.length > 0) {
parts.push(`removed fields: ${fieldNames.join(", ")}`);
}
}
}
// Add dependents if available
if ("dependents" in problem.data && problem.data.dependents) {
const dependents = problem.data.dependents;
if (Array.isArray(dependents) && dependents.length > 0) {
const dependentNames = dependents
.map((dep: { name?: string; id?: string | number }) =>
dep.name ? dep.name : dep.id?.toString(),
)
.filter(Boolean);
if (dependentNames.length > 0) {
parts.push(`affects: ${dependentNames.join(", ")}`);
}
}
}
// If we have the description and no specific details, use description
if (parts.length === 0 && problem.description) {
return problem.description;
}
return parts.length > 0
? `${problem.description}. ${parts.join(". ")}`
: problem.description;
}
/**
* Gets user-friendly check name for a category
*/
export function getCheckName(category: ProblemCheckCategory): string {
return category
.split("-")
.map((word, index) =>
index === 0 ? word.charAt(0).toUpperCase() + word.slice(1) : word,
)
.join(" ");
}

View File

@@ -0,0 +1,31 @@
import { RunButton } from "metabase-enterprise/transforms/components/RunButton";
import type {
TransformId,
TransformJobId,
TransformRun,
} from "metabase-types/api";
type WorkspaceRunButtonProps = {
id: TransformId | TransformJobId | undefined;
run: TransformRun | null | undefined;
isDisabled?: boolean;
allowCancellation?: boolean;
onRun: () => void;
onCancel?: () => void;
};
// A workspace-friendly wrapper that shows visual feedback during and after runs
export function WorkspaceRunButton({ run, ...rest }: WorkspaceRunButtonProps) {
// Pass through active/completed run states for visual feedback
// - started/canceling: show in-progress state
// - succeeded/failed: show completion state (controlled by parent)
const activeRun =
run?.status === "started" ||
run?.status === "canceling" ||
run?.status === "succeeded" ||
run?.status === "failed"
? run
: null;
return <RunButton run={activeRun} {...rest} />;
}

View File

@@ -0,0 +1 @@
export { useWorkspaceTransformRun } from "./use-workspace-transform-run";

View File

@@ -0,0 +1,185 @@
import { useMemo, useState } from "react";
import { t } from "ttag";
import { useDispatch } from "metabase/lib/redux";
import { useMetadataToasts } from "metabase/metadata/hooks";
import {
DEFAULT_WORKSPACE_TABLES_QUERY_RESPONSE,
useGetWorkspaceTablesQuery,
useGetWorkspaceTransformQuery,
useRunWorkspaceTransformMutation,
workspaceApi,
} from "metabase-enterprise/api";
import type {
TransformRun,
WorkspaceId,
WorkspaceTransform,
} from "metabase-types/api";
type UseWorkspaceTransformRunOptions = {
workspaceId: WorkspaceId;
transform: WorkspaceTransform;
};
type UseWorkspaceTransformRunResult = {
/** Run object for RunStatus component - always shows last run info */
statusRun: TransformRun | null;
/** Run object for button - only shows feedback during/after user-triggered runs */
buttonRun: TransformRun | null;
/** Whether the run status is still loading */
isRunStatusLoading: boolean;
/** Whether a run is currently in progress */
isRunning: boolean;
/** Trigger a transform run */
handleRun: () => Promise<void>;
/** Output table info */
output: {
db_id: number;
table_id: number;
table_name: string;
schema: string;
} | null;
};
export function useWorkspaceTransformRun({
workspaceId,
transform,
}: UseWorkspaceTransformRunOptions): UseWorkspaceTransformRunResult {
const dispatch = useDispatch();
const { sendErrorToast } = useMetadataToasts();
const [runTransform] = useRunWorkspaceTransformMutation();
const { data: workspaceTables = DEFAULT_WORKSPACE_TABLES_QUERY_RESPONSE } =
useGetWorkspaceTablesQuery(workspaceId);
const [isRunTriggered, setIsRunTriggered] = useState(false);
const [lastRunResult, setLastRunResult] = useState<{
status: "succeeded" | "failed";
end_time: string;
message: string | null;
} | null>(null);
// Fetch workspace transform to get updated last_run_at
const { data: fetchedWorkspaceTransform, isFetching } =
useGetWorkspaceTransformQuery(
{
workspaceId,
transformId: transform.ref_id,
},
{ skip: !transform.ref_id },
);
const lastRunAt =
fetchedWorkspaceTransform?.last_run_at ?? transform.last_run_at;
const lastRunMessage =
fetchedWorkspaceTransform?.last_run_message ?? transform.last_run_message;
const lastRunStatus = lastRunMessage ? "failed" : "succeeded";
const statusRun: TransformRun | null = useMemo(
() =>
lastRunResult
? {
id: -1,
status: lastRunResult.status,
start_time: lastRunResult.end_time,
end_time: lastRunResult.end_time,
message: lastRunResult.message,
run_method: "manual",
}
: lastRunAt
? {
id: -1,
status: lastRunStatus,
start_time: lastRunAt,
end_time: lastRunAt,
message: lastRunMessage,
run_method: "manual",
}
: null,
[lastRunResult, lastRunAt, lastRunMessage, lastRunStatus],
);
const output = useMemo(() => {
const table = workspaceTables.outputs.find(
(table) => table.isolated.transform_id === transform.ref_id,
);
if (!table || statusRun?.status !== "succeeded") {
return null;
}
return {
db_id: table?.db_id,
table_id: table?.isolated.table_id,
table_name: table?.isolated.table,
schema: table?.isolated.schema,
};
}, [workspaceTables, transform.ref_id, statusRun]);
// Run object for button - only shows feedback during/after user-triggered runs
const buttonRun: TransformRun | null = isRunTriggered
? {
id: -1,
status: "started",
start_time: new Date().toISOString(),
end_time: null,
message: null,
run_method: "manual",
}
: lastRunResult
? {
id: -1,
status: lastRunResult.status,
start_time: lastRunResult.end_time,
end_time: lastRunResult.end_time,
message: lastRunResult.message,
run_method: "manual",
}
: null;
const handleRun = async () => {
try {
setIsRunTriggered(true);
setLastRunResult(null);
const result = await runTransform({
workspaceId,
transformId: transform.ref_id,
}).unwrap();
// Transform completed - store the result
const endTime = result.end_time ?? new Date().toISOString();
setLastRunResult({
status: result.status,
end_time: endTime,
message: result.message ?? null,
});
setIsRunTriggered(false);
// Invalidate caches to refetch updated data
dispatch(
workspaceApi.util.invalidateTags([
{ type: "workspace", id: workspaceId },
{ type: "workspace-transform", id: transform.ref_id },
{ type: "workspace-tables", id: workspaceId },
]),
);
if (result.status === "failed") {
sendErrorToast(t`Transform run failed`);
}
} catch (error) {
setIsRunTriggered(false);
setLastRunResult(null);
sendErrorToast(t`Failed to run transform`);
console.error("Failed to run transform", error);
}
};
return {
statusRun,
buttonRun,
isRunStatusLoading: statusRun == null && isFetching,
isRunning: isRunTriggered,
handleRun,
output,
};
}

View File

@@ -0,0 +1,57 @@
import { useEffect, useMemo } from "react";
import { replace } from "react-router-redux";
import { t } from "ttag";
import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
import { useDispatch } from "metabase/lib/redux";
import * as Urls from "metabase/lib/urls";
import { Box, Text, Title } from "metabase/ui";
import { useGetWorkspacesQuery } from "metabase-enterprise/api";
export function WorkspaceListPage() {
const dispatch = useDispatch();
const {
data: workspacesData,
error,
isLoading,
isFetching,
} = useGetWorkspacesQuery();
const workspaces = useMemo(
() => workspacesData?.items ?? [],
[workspacesData],
);
const firstWorkspaceId = workspaces[0]?.id;
useEffect(() => {
if (error || firstWorkspaceId == null) {
return;
}
dispatch(replace(Urls.dataStudioWorkspace(firstWorkspaceId)));
}, [dispatch, error, firstWorkspaceId]);
if (error) {
return <LoadingAndErrorWrapper error={error} loading={false} />;
}
if (firstWorkspaceId != null) {
return null;
}
if (isLoading || isFetching) {
return (
<LoadingAndErrorWrapper error={error} loading={isLoading || isFetching} />
);
}
return (
<Box data-testid="workspaces-page" p="lg">
<Title size="lg">{t`No active workspaces`}</Title>
<Text c="text-secondary">
{t`Create a new one from the left panel or by editing an existing transform.`}
</Text>
</Box>
);
}

View File

@@ -0,0 +1,89 @@
import { t } from "ttag";
import { ActionIcon, Icon, Menu } from "metabase/ui";
import {
getInitialNativeSource,
getInitialPythonSource,
} from "metabase-enterprise/transforms/pages/NewTransformPage/utils";
import type {
DatabaseId,
TransformSource,
WorkspaceId,
} from "metabase-types/api";
import { createMockTransform } from "metabase-types/api/mocks/transform";
import { useWorkspace } from "./WorkspaceProvider";
type TransformType = "sql" | "python";
interface Props {
databaseId: DatabaseId;
workspaceId: WorkspaceId;
disabled?: boolean;
}
export const AddTransformMenu = ({
databaseId,
workspaceId,
disabled,
}: Props) => {
const { addUnsavedTransform } = useWorkspace();
const getSource = (type: TransformType): TransformSource => {
if (type === "sql") {
const source = getInitialNativeSource();
return {
...source,
query: { ...source.query, database: databaseId },
};
}
return {
...getInitialPythonSource(),
"source-database": databaseId,
};
};
return (
<>
<Menu position="bottom-end">
<Menu.Target>
<ActionIcon
size="2rem"
p="0"
ml="auto"
aria-label={t`Add transform`}
disabled={disabled}
>
<Icon name="add" size={16} aria-hidden tooltip={t`Add transform`} />
</ActionIcon>
</Menu.Target>
<Menu.Dropdown>
<Menu.Item
leftSection={<Icon name="sql" />}
onClick={() => {
const mockTransform = createMockTransform({
source: getSource("sql"),
workspace_id: workspaceId,
});
addUnsavedTransform(mockTransform);
}}
>
{t`SQL Transform`}
</Menu.Item>
<Menu.Item
leftSection={<Icon name="code_block" />}
onClick={() => {
const mockTransform = createMockTransform({
source: getSource("python"),
workspace_id: workspaceId,
});
addUnsavedTransform(mockTransform);
}}
>
{t`Python Script`}
</Menu.Item>
</Menu.Dropdown>
</Menu>
</>
);
};

View File

@@ -0,0 +1,179 @@
import { useCallback } from "react";
import { t } from "ttag";
import EmptyState from "metabase/common/components/EmptyState";
import { Stack, Text } from "metabase/ui";
import {
useLazyGetTransformQuery,
useLazyGetWorkspaceTransformQuery,
} from "metabase-enterprise/api";
import type {
ExternalTransform,
Transform,
WorkspaceId,
WorkspaceTransform,
} from "metabase-types/api";
import { useWorkspace } from "../WorkspaceProvider";
import { TransformListItem } from "./TransformListItem";
import { TransformListItemMenu } from "./TransformListItemMenu";
type CodeTabProps = {
activeTransformId?: number | string;
availableTransforms: ExternalTransform[];
workspaceId: WorkspaceId;
workspaceTransforms: WorkspaceTransform[];
onTransformClick: (transform: Transform | WorkspaceTransform) => void;
};
export const CodeTab = ({
activeTransformId,
availableTransforms,
workspaceId,
workspaceTransforms,
onTransformClick,
}: CodeTabProps) => {
const { editedTransforms, hasTransformEdits } = useWorkspace();
const [fetchWorkspaceTransform] = useLazyGetWorkspaceTransformQuery();
const [fetchTransform] = useLazyGetTransformQuery();
const normalizeTransformId = useCallback(
(transform: Transform | WorkspaceTransform) =>
"ref_id" in transform ? transform.ref_id : transform.id,
[],
);
const handleTransformClick = useCallback(
(transform: Transform | WorkspaceTransform) => {
const transformId = normalizeTransformId(transform);
const edited = editedTransforms.get(transformId);
const transformToOpen =
edited != null ? { ...transform, ...edited } : transform;
onTransformClick(transformToOpen);
},
[editedTransforms, normalizeTransformId, onTransformClick],
);
const handleExternalTransformClick = useCallback(
async (externalTransform: ExternalTransform) => {
if (externalTransform.checkout_disabled) {
return;
}
const { data: transform } = await fetchTransform(
externalTransform.id,
true,
);
if (transform) {
handleTransformClick(transform);
}
},
[fetchTransform, handleTransformClick],
);
const handleWorkspaceTransformClick = useCallback(
async (workspaceTransform: WorkspaceTransform) => {
if (
typeof workspaceTransform.id === "number" &&
workspaceTransform.id < 0
) {
return handleTransformClick(workspaceTransform);
}
const { data: transform } = await fetchWorkspaceTransform(
{
workspaceId,
transformId: workspaceTransform.ref_id,
},
true,
);
if (transform) {
handleTransformClick(transform);
}
},
[fetchWorkspaceTransform, workspaceId, handleTransformClick],
);
return (
<Stack h="100%" gap={0}>
<Stack
data-testid="workspace-transforms"
gap="xs"
pb="sm"
style={{
borderBottom: "1px solid var(--mb-color-border)",
}}
>
<Stack gap={0}>
<Text fw={600}>{t`Workspace transforms`}</Text>
{workspaceTransforms.map((transform) => {
const isEdited = hasTransformEdits(transform);
const isActive =
typeof activeTransformId === "string" &&
activeTransformId === transform.ref_id;
return (
<TransformListItem
key={transform.ref_id}
name={transform.name}
icon="pivot_table"
fw={600}
isActive={isActive}
isEdited={isEdited}
menu={
<TransformListItemMenu
transform={transform}
workspaceId={workspaceId}
/>
}
onClick={() => {
handleWorkspaceTransformClick(transform);
}}
/>
);
})}
</Stack>
{workspaceTransforms.length === 0 && (
<EmptyState message={t`Workspace is empty`} />
)}
</Stack>
<Stack data-testid="mainland-transforms" py="sm" gap="xs">
<Text fw={600} mt="sm">{t`Available transforms`}</Text>
{availableTransforms.map((transform) => {
const tooltipLabel =
transform.checkout_disabled === "mbql"
? t`This transform cannot be edited in a workspace because it uses MBQL.`
: transform.checkout_disabled === "card-reference"
? t`This transform cannot be edited in a workspace because it references other questions.`
: transform.checkout_disabled
? t`This transform cannot be edited in a workspace.`
: undefined;
return (
<TransformListItem
key={transform.id}
name={transform.name}
isActive={activeTransformId === transform.id}
isEdited={editedTransforms.has(transform.id)}
onClick={() => {
handleExternalTransformClick(transform);
}}
opacity={transform.checkout_disabled ? 0.5 : 1}
tooltipLabel={tooltipLabel}
/>
);
})}
{availableTransforms.length === 0 && (
<EmptyState message={t`No available transforms`} />
)}
</Stack>
</Stack>
);
};

View File

@@ -0,0 +1,33 @@
.root {
display: flex;
align-items: center;
gap: var(--mantine-spacing-sm);
padding: var(--mantine-spacing-xs) var(--mantine-spacing-sm);
border-radius: var(--mantine-radius-sm);
cursor: default;
flex-wrap: nowrap;
&:hover .menu {
visibility: visible;
}
&:hover {
background-color: var(--mb-color-bg-light);
}
}
.label {
display: flex;
align-items: center;
gap: var(--mantine-spacing-sm);
flex: 1;
min-width: 0;
}
.cursorPointer {
cursor: pointer;
}
.menu {
visibility: hidden;
}

View File

@@ -0,0 +1,62 @@
import cx from "classnames";
import type { ReactNode } from "react";
import {
Box,
type BoxProps,
Icon,
type IconName,
Text,
Tooltip,
} from "metabase/ui";
import { StatusDot } from "../components/StatusDot/StatusDot";
import S from "./TransformListItem.module.css";
type TransformListItemProps = {
name: string;
icon?: IconName;
isActive?: boolean;
isEdited?: boolean;
tooltipLabel?: string;
menu?: ReactNode;
onClick?: () => void;
} & BoxProps;
export const TransformListItem = ({
name,
icon = "database",
isActive,
isEdited,
tooltipLabel,
menu,
onClick,
...props
}: TransformListItemProps) => {
const label = (
<Box className={cx(S.label, { [S.cursorPointer]: !tooltipLabel })}>
<Icon name={icon} size={14} c="brand" />
<Text c={isActive ? "brand" : "text-dark"} truncate>
{name}
</Text>
{isEdited && <StatusDot status="changed" />}
</Box>
);
return (
<Box className={S.root} onClick={onClick} pl="sm" {...props}>
{tooltipLabel ? (
<Tooltip label={tooltipLabel} position="top">
{label}
</Tooltip>
) : (
label
)}
<Box className={S.menu} flex="0 0 auto">
{menu}
</Box>
</Box>
);
};

View File

@@ -0,0 +1,74 @@
import { t } from "ttag";
import { useMetadataToasts } from "metabase/metadata/hooks";
import { ActionIcon, Icon, Menu } from "metabase/ui";
import { useDeleteWorkspaceTransformMutation } from "metabase-enterprise/api";
import type {
Transform,
WorkspaceId,
WorkspaceTransformItem,
} from "metabase-types/api";
import { useWorkspace } from "../WorkspaceProvider";
interface Props {
transform: WorkspaceTransformItem | Transform;
workspaceId: WorkspaceId;
}
export function TransformListItemMenu({ transform, workspaceId }: Props) {
const [deleteWorkspaceTransform] = useDeleteWorkspaceTransformMutation();
const {
removeEditedTransform,
removeOpenedTransform,
removeUnsavedTransform,
} = useWorkspace();
const { sendErrorToast, sendSuccessToast } = useMetadataToasts();
const handleRemove = async () => {
// Handle unsaved transforms (negative IDs) locally
if ("id" in transform && transform.id < 0) {
removeUnsavedTransform(transform.id);
return;
}
if (!("ref_id" in transform)) {
return;
}
try {
await deleteWorkspaceTransform({
workspaceId,
transformId: transform.ref_id,
}).unwrap();
sendSuccessToast(t`Transform removed from the workspace`);
removeEditedTransform(transform.ref_id);
removeOpenedTransform(transform.ref_id);
} catch (error) {
console.error("Failed to remove transform:", error);
sendErrorToast(t`Failed to remove transform from the workspace`);
}
};
return (
<Menu position="bottom-end" withinPortal>
<Menu.Target>
<ActionIcon
onClick={(event) => event.stopPropagation()}
size="sm"
variant="subtle"
>
<Icon name="ellipsis" />
</ActionIcon>
</Menu.Target>
<Menu.Dropdown onClick={(event) => event.stopPropagation()}>
<Menu.Item leftSection={<Icon name="trash" />} onClick={handleRemove}>
{t`Remove`}
</Menu.Item>
</Menu.Dropdown>
</Menu>
);
}

View File

@@ -0,0 +1,89 @@
import { useMemo } from "react";
import { t } from "ttag";
import { createMockMetadata } from "__support__/metadata";
import { skipToken, useGetTableQuery } from "metabase/api";
import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
import CS from "metabase/css/core/index.css";
import { Box, Stack, Text } from "metabase/ui";
import Visualization from "metabase/visualizations/components/Visualization";
import { useGetWorkspaceTransformQuery } from "metabase-enterprise/api";
import type { DatabaseId, TableId } from "metabase-types/api";
import { useTablePreview } from "./useTablePreview";
interface DataTabProps {
workspaceId: number;
databaseId: DatabaseId | null;
tableId: TableId | null;
transformId?: string;
}
export function DataTab({
workspaceId,
databaseId,
tableId,
transformId,
}: DataTabProps) {
const { data: table } = useGetTableQuery(
tableId ? { id: tableId } : skipToken,
);
const { data: transform } = useGetWorkspaceTransformQuery(
transformId ? { workspaceId, transformId } : skipToken,
);
const metadata = useMemo(
() =>
createMockMetadata({
tables: table
? [
{
...table,
visibility_type: null,
},
]
: [],
}),
[table],
);
const { rawSeries, isFetching, error } = useTablePreview({
databaseId,
tableId,
metadata,
last_transform_run_time: transform?.last_run_at,
});
if (!databaseId || !tableId) {
return (
<Stack h="100%" align="center" justify="center">
<Text c="text-medium">{t`Select a table to view its data`}</Text>
</Stack>
);
}
if (isFetching) {
return <LoadingAndErrorWrapper loading />;
}
if (error) {
return (
<Stack h="100%" align="center" justify="center">
<Text c="error">{t`Error loading data`}</Text>
</Stack>
);
}
if (!rawSeries) {
return (
<Stack h="100%" align="center" justify="center">
<Text c="text-medium">{t`No data available`}</Text>
</Stack>
);
}
return (
<Box h="100%" className={CS.relative}>
<Visualization queryBuilderMode="dataset" rawSeries={rawSeries} />
</Box>
);
}

View File

@@ -0,0 +1,105 @@
import { t } from "ttag";
import { Stack, Text } from "metabase/ui";
import type {
TableId,
Transform,
WorkspaceTablesResponse,
WorkspaceTransformItem,
} from "metabase-types/api";
import { type OpenTable, useWorkspace } from "../WorkspaceProvider";
import { TableListItem } from "./TableListItem";
type DataTabSidebarProps = {
tables: WorkspaceTablesResponse;
workspaceTransforms: WorkspaceTransformItem[];
dbTransforms: Transform[];
selectedTableId?: TableId | null;
runningTransforms?: Set<string>;
onTransformClick?: (transform: WorkspaceTransformItem) => void;
onTableSelect?: (table: OpenTable) => void;
onRunTransform?: (transform: WorkspaceTransformItem) => void;
};
export const DataTabSidebar = ({
tables,
workspaceTransforms,
dbTransforms,
selectedTableId,
runningTransforms,
onTransformClick,
onTableSelect,
onRunTransform,
}: DataTabSidebarProps) => {
const { hasTransformEdits } = useWorkspace();
return (
<Stack h="100%" gap="sm">
<Stack
gap="xs"
pb="sm"
style={{ borderBottom: "1px solid var(--mb-color-border)" }}
>
<Text fw={600}>{t`Data active in this workspace`}</Text>
{tables.outputs.length > 0 && (
<Text size="sm" fw={600} c="text-tertiary">{t`Output tables`}</Text>
)}
<Stack gap={0}>
{tables.outputs.map((table, index: number) => {
const workspaceTransform = workspaceTransforms.find(
(t) => t["ref_id"] === table.isolated.transform_id,
);
const originalTransform = workspaceTransform?.global_id
? dbTransforms.find((t) => t.id === workspaceTransform.global_id)
: undefined;
const hasChanges = originalTransform
? hasTransformEdits(originalTransform)
: false;
const tableId = table.isolated.table_id;
return (
<TableListItem
key={`output-${index}`}
name={table.global.table}
schema={table.global.schema}
icon="pivot_table"
type="output"
hasChanges={hasChanges}
transform={workspaceTransform}
tableId={tableId ?? undefined}
isSelected={tableId === selectedTableId}
isRunning={
workspaceTransform
? runningTransforms?.has(workspaceTransform.ref_id)
: false
}
onTransformClick={onTransformClick}
onTableClick={onTableSelect}
onRunTransform={onRunTransform}
/>
);
})}
</Stack>
{tables.inputs.length > 0 && (
<Text size="sm" fw={600} c="text-tertiary">{t`Input tables`}</Text>
)}
<Stack gap={0}>
{tables.inputs.map((table, index) => (
<TableListItem
key={`input-${index}`}
name={table.table}
schema={table.schema}
icon="table"
type="input"
isSelected={table.table_id === selectedTableId}
tableId={table.table_id ?? undefined}
onTransformClick={onTransformClick}
onTableClick={onTableSelect}
/>
))}
</Stack>
</Stack>
</Stack>
);
};

View File

@@ -0,0 +1,34 @@
.root {
display: flex;
align-items: center;
gap: var(--mantine-spacing-sm);
padding: var(--mantine-spacing-xs) var(--mantine-spacing-sm);
border-radius: var(--mantine-radius-sm);
flex-wrap: nowrap;
&:hover .menu {
visibility: visible;
}
&:hover {
background-color: var(--mb-color-bg-light);
}
}
.clickable {
cursor: pointer;
}
.selected {
background-color: var(--mb-color-bg-medium);
}
.name {
flex: 1;
min-width: 0;
}
.actionIcon {
&:hover {
background-color: var(--mb-color-bg-medium) !important;
}
}

View File

@@ -0,0 +1,126 @@
import cx from "classnames";
import { t } from "ttag";
import {
ActionIcon,
Box,
Icon,
type IconName,
Loader,
Text,
Tooltip,
} from "metabase/ui";
import type { TableId, WorkspaceTransformItem } from "metabase-types/api";
import type { OpenTable } from "../WorkspaceProvider";
import { StatusDot } from "../components/StatusDot/StatusDot";
import S from "./TableListItem.module.css";
type TableListItemProps = {
name: string;
schema?: string | null;
icon?: IconName;
type: "input" | "output";
hasChanges?: boolean;
transform?: WorkspaceTransformItem;
tableId?: TableId;
isSelected?: boolean;
isRunning?: boolean;
onTransformClick?: (transform: WorkspaceTransformItem) => void;
onTableClick?: (table: OpenTable) => void;
onRunTransform?: (transform: WorkspaceTransformItem) => void;
};
export const TableListItem = ({
name,
schema,
icon = "table",
type,
hasChanges = false,
transform,
tableId,
isSelected = false,
isRunning = false,
onTransformClick,
onTableClick,
onRunTransform,
}: TableListItemProps) => {
const displayName = schema ? `${schema}.${name}` : name;
const handleTransformClick = (e: React.MouseEvent) => {
e.stopPropagation();
if (transform && onTransformClick) {
onTransformClick(transform);
}
};
const handleTableClick = () => {
if (isRunning) {
return;
}
if (tableId && onTableClick) {
onTableClick({ tableId, name, schema, transformId: transform?.ref_id });
} else if (type === "output" && transform && onRunTransform) {
onRunTransform(transform);
}
};
const hasResults = type === "output" && !!tableId;
const canRunTransform =
type === "output" && !hasResults && transform && onRunTransform;
const isClickable = (tableId && onTableClick) || canRunTransform;
const displayTooltip = type !== "input" && !hasResults && !isRunning;
return (
<Box
className={cx(S.root, {
[S.selected]: isSelected,
[S.clickable]: isClickable,
})}
aria-label={displayName}
onClick={isClickable ? handleTableClick : undefined}
>
{isRunning ? (
<Loader size="sm" />
) : (
<Icon
name={icon}
size={14}
c={type === "input" ? "saturated-green" : "saturated-yellow"}
/>
)}
<Tooltip
label={
isRunning
? t`Running transform...`
: t`Run transform to see the results`
}
position="top"
disabled={!displayTooltip && !isRunning}
>
<Text
className={S.name}
c={displayTooltip ? "text-light" : "text-dark"}
truncate
>
{displayName}
</Text>
</Tooltip>
{hasChanges && <StatusDot status="changed" />}
{type === "output" && transform && (
<Tooltip label={t`Open transform`} position="top">
<ActionIcon
className={S.actionIcon}
ml="auto"
size="sm"
variant="subtle"
onClick={handleTransformClick}
>
<Icon name="code_block" size={14} c="text-medium" />
</ActionIcon>
</Tooltip>
)}
</Box>
);
};

View File

@@ -0,0 +1,4 @@
export { DataTab } from "./DataTab";
export { DataTabSidebar } from "./DataTabSidebar";
export { useTablePreview as useTableQuestion } from "./useTablePreview";
export type { OpenTable } from "../WorkspaceProvider";

View File

@@ -0,0 +1,91 @@
import { useMemo } from "react";
import _ from "underscore";
import { skipToken, useGetAdhocQueryQuery } from "metabase/api";
import { getErrorMessage } from "metabase/api/utils";
import * as Lib from "metabase-lib";
import type Metadata from "metabase-lib/v1/metadata/Metadata";
import type { DatabaseId, RawSeries, TableId } from "metabase-types/api";
import { createMockCard } from "metabase-types/api/mocks";
interface UseTableQuestionProps {
databaseId: DatabaseId | null;
tableId: TableId | null;
metadata: Metadata;
last_transform_run_time?: string | null;
}
interface UseTablePreviewResult {
rawSeries?: RawSeries;
isFetching: boolean;
isLoading: boolean;
isError: boolean;
error?: string;
}
export function useTablePreview({
databaseId,
tableId,
metadata,
last_transform_run_time,
}: UseTableQuestionProps): UseTablePreviewResult {
const metadataProvider =
Object.keys(metadata.tables).length > 0
? Lib.metadataProvider(databaseId, metadata)
: null;
const table =
metadataProvider && tableId
? Lib.tableOrCardMetadata(metadataProvider, tableId)
: null;
const query =
table && metadataProvider
? Lib.queryFromTableOrCardMetadata(metadataProvider, table)
: null;
const { data, ...rest } = useGetAdhocQueryQuery(
query == null
? skipToken
: {
...Lib.toJsQuery(query),
ignore_error: true,
_refetchDeps: last_transform_run_time,
},
);
const base = {
...rest,
error: rest.error ? getErrorMessage(rest.error) : undefined,
rawSeries: undefined,
};
const rawSeries = useMemo(
() =>
query
? [
{
card: createMockCard({
dataset_query: Lib.toJsQuery(query),
display: "table",
visualization_settings: {},
}),
data: {
...data?.data,
rows: _.uniq(data?.data?.rows ?? []),
},
},
]
: ([] as RawSeries),
[data?.data, query],
);
if (data?.status === "failed") {
return { ...base, isError: true, error: getErrorMessage(data) };
}
if (query == null || !data?.data || data.data.cols.length === 0) {
return base;
}
return { ...base, rawSeries };
}

View File

@@ -0,0 +1,29 @@
import { PLUGIN_METABOT } from "metabase/plugins";
import { useRegisterMetabotTransformContext } from "metabase-enterprise/transforms/hooks/use-register-transform-metabot-context";
import type { DraftTransformSource, Transform } from "metabase-types/api";
type MetabotTabProps = {
transform?: Transform;
source?: DraftTransformSource;
};
const MetabotContextRegistration = ({ transform, source }: MetabotTabProps) => {
useRegisterMetabotTransformContext(transform, source);
return null;
};
export const MetabotTab = ({ transform, source }: MetabotTabProps) => {
const MetabotProvider = PLUGIN_METABOT.getMetabotProvider();
const MetabotChat = PLUGIN_METABOT.MetabotChat;
const metabotConfig = {
// Show top-level mention menu for transforms and databases.
suggestionModels: ["transform", "database"],
};
return (
<MetabotProvider>
<MetabotContextRegistration transform={transform} source={source} />
<MetabotChat config={metabotConfig} />
</MetabotProvider>
);
};

View File

@@ -0,0 +1,50 @@
import { useMemo } from "react";
import { t } from "ttag";
import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
import CS from "metabase/css/core/index.css";
import { Box, Stack, Text } from "metabase/ui";
import Visualization from "metabase/visualizations/components/Visualization";
import type { Dataset, RawSeries } from "metabase-types/api";
import { createMockCard } from "metabase-types/api/mocks";
interface PreviewTabProps {
dataset: Dataset | null;
isLoading?: boolean;
}
export function PreviewTab({ dataset, isLoading }: PreviewTabProps) {
const rawSeries: RawSeries = useMemo(() => {
if (!dataset?.data || dataset.data.cols.length === 0) {
return [];
}
return [
{
card: createMockCard({
display: "table",
visualization_settings: {},
}),
data: dataset.data,
},
];
}, [dataset?.data]);
if (isLoading) {
return <LoadingAndErrorWrapper loading />;
}
if (!dataset?.data || dataset.data.cols.length === 0) {
return (
<Stack h="100%" align="center" justify="center">
<Text c="text-medium">{t`No data available`}</Text>
</Stack>
);
}
return (
<Box h="100%" className={CS.relative}>
<Visualization queryBuilderMode="dataset" rawSeries={rawSeries} />
</Box>
);
}

View File

@@ -0,0 +1 @@
export { PreviewTab } from "./PreviewTab";

View File

@@ -0,0 +1,61 @@
import { t } from "ttag";
import { Button } from "metabase/ui";
import { useUpdateWorkspaceTransformMutation } from "metabase-enterprise/api";
import { isSameSource } from "metabase-enterprise/transforms/utils";
import type { DatabaseId, WorkspaceTransform } from "metabase-types/api";
import { useWorkspace } from "./WorkspaceProvider";
interface Props {
databaseId: DatabaseId;
editedTransform: WorkspaceTransform;
transform: WorkspaceTransform;
workspaceId: number;
isArchived: boolean;
}
export const SaveTransformButton = ({
databaseId,
editedTransform,
transform,
workspaceId,
isArchived,
}: Props) => {
const [updateTransform] = useUpdateWorkspaceTransformMutation();
const { updateTransformState } = useWorkspace();
const hasSourceChanged = !isSameSource(
editedTransform.source,
transform.source,
);
const hasTargetNameChanged =
transform.target.name !== editedTransform.target.name;
const hasChanges = hasSourceChanged || hasTargetNameChanged;
const handleClick = async () => {
const updated = await updateTransform({
workspaceId,
transformId: transform.id,
source: editedTransform.source,
name: editedTransform.name,
target: {
type: "table",
name: editedTransform.target.name,
schema: transform.target.schema,
database: databaseId,
},
}).unwrap();
updateTransformState(updated);
};
return (
<Button
disabled={!hasChanges || isArchived}
size="sm"
variant="filled"
onClick={handleClick}
>{t`Save`}</Button>
);
};

View File

@@ -0,0 +1,106 @@
import { useEffect, useMemo, useState } from "react";
import { match } from "ts-pattern";
import { t } from "ttag";
import LoadingAndGenericErrorWrapper from "metabase/common/components/LoadingAndGenericErrorWrapper";
import { Group, Icon, Loader, Stack, Tooltip } from "metabase/ui";
import { useGetWorkspaceLogQuery } from "metabase-enterprise/api";
import type { WorkspaceId, WorkspaceLogStatus } from "metabase-types/api";
interface SetupTabProps {
workspaceId: WorkspaceId;
}
const LOGS_POLLING_INTERVAL = 1000;
export const SetupLog = ({ workspaceId }: SetupTabProps) => {
const [shouldPoll, setShouldPoll] = useState(true);
const {
data: workspace,
error,
isLoading,
} = useGetWorkspaceLogQuery(workspaceId, {
pollingInterval: LOGS_POLLING_INTERVAL,
refetchOnMountOrArgChange: true,
skip: !shouldPoll,
});
useEffect(() => {
if (
workspace?.status === "ready" ||
workspace?.status === "archived" ||
workspace?.status === "uninitialized"
) {
setShouldPoll(false);
}
}, [workspace?.status]);
const logs = useMemo(() => workspace?.logs ?? [], [workspace]);
if (error || isLoading) {
return <LoadingAndGenericErrorWrapper error={error} loading={isLoading} />;
}
return (
<Stack
bd="1px solid var(--mb-color-border)"
bg="background-info"
bdrs="md"
gap="sm"
p="md"
>
{logs.length === 0 && workspace?.status === "pending" && (
<Loader size="xs" />
)}
{logs.map((log) => {
return (
<Group gap="xs" key={log.id}>
<Tooltip disabled={!log.message} label={log.message}>
<Group>
<LogIcon status={log.status} />
</Group>
</Tooltip>
<span>
{log.description}
{log.status === "started" && "…"}
</span>
</Group>
);
})}
{workspace?.status === "ready" && (
<Group gap="xs">
<LogIcon status="success" />
{t`Workspace ready!`}
</Group>
)}
{workspace?.status === "archived" && (
<Group gap="xs">
<Icon c="text-light" name="archive" />
{t`Workspace is archived`}
</Group>
)}
</Stack>
);
};
function LogIcon({ status }: { status: WorkspaceLogStatus | null }) {
return match(status)
.with("failure", () => {
return <Icon c="warning" name="warning_triangle_filled" />;
})
.with("started", () => {
return <Loader size="xs" />;
})
.with("success", () => {
return <Icon c="success" name="check" />;
})
.with(null, () => {
return <Icon name="chevronright" />;
})
.exhaustive();
}

View File

@@ -0,0 +1,104 @@
import { useCallback } from "react";
import { t } from "ttag";
import { useMetadataToasts } from "metabase/metadata/hooks";
import { Select, Stack, Text, Title } from "metabase/ui";
import {
useGetWorkspaceAllowedDatabasesQuery,
useUpdateWorkspaceMutation,
} from "metabase-enterprise/api/workspace";
import type { Workspace } from "metabase-types/api";
import { isWorkspaceUninitialized } from "../../utils";
import { SetupLog } from "./SetupLog";
interface SetupTabProps {
databaseId?: number;
workspace: Workspace;
}
export const SetupTab = ({ databaseId, workspace }: SetupTabProps) => {
const { data: allowedDatabases, isLoading } =
useGetWorkspaceAllowedDatabasesQuery();
const [updateWorkspace, { isLoading: isUpdating }] =
useUpdateWorkspaceMutation();
const { sendErrorToast, sendSuccessToast } = useMetadataToasts();
const workspaceId = workspace.id;
const handleDatabaseChange = useCallback(
async (value: string) => {
const newDatabaseId = Number(value);
if (newDatabaseId !== databaseId) {
try {
await updateWorkspace({
id: workspaceId,
database_id: newDatabaseId,
}).unwrap();
sendSuccessToast(t`Successfully updated workspace database`);
} catch (error) {
console.error("Failed to update workspace database:", error);
sendErrorToast(t`Failed to update workspace database`);
}
}
},
[
databaseId,
workspaceId,
updateWorkspace,
sendErrorToast,
sendSuccessToast,
],
);
const databaseOptions =
allowedDatabases?.databases
?.filter((db) => db.supported)
?.map((db) => ({
value: db.id.toString(),
label: db.name,
})) ?? [];
return (
<Stack gap="lg">
<Stack gap="xs">
<Title order={3}>{t`Workspace Setup`}</Title>
<Text c="text-medium">
{t`Configure your data warehouse connection for this workspace`}
</Text>
</Stack>
<Stack gap="xs">
<Text fw="bold">{t`Data warehouse`}</Text>
{isLoading ? (
<Text>{t`Loading databases...`}</Text>
) : (
<Select
data={databaseOptions}
value={databaseId?.toString() ?? ""}
onChange={handleDatabaseChange}
placeholder={t`Select a database`}
disabled={!isWorkspaceUninitialized(workspace)}
maw="20rem"
/>
)}
<Text c="text-light" size="sm">
{t`Data warehouse selection is locked after adding transforms to the workspace`}
</Text>
</Stack>
{!isWorkspaceUninitialized(workspace) && (
<Stack gap="xs">
<Text fw="bold">{t`Setup log`}</Text>
<SetupLog
key={workspaceId} // avoid showing status of other workspaces (forces RTK query hook remount)
workspaceId={workspaceId}
/>
</Stack>
)}
</Stack>
);
};

Some files were not shown because too many files have changed in this diff Show More