Compare commits

...

169 Commits

Author SHA1 Message Date
Stas Gavrylov
303e05ec59 merge main branch 2025-12-04 17:25:58 +01:00
Stas Gavrylov
a67169c5e5 Cleanup. Add tables refetching on transform run 2025-12-04 17:23:11 +01:00
lbrdnk
14e7b758f4 input tables adj 2025-12-04 17:21:35 +01:00
lbrdnk
a0feade8ae /tables no graph, alt impl 2025-12-04 17:21:35 +01:00
Stas Gavrylov
3509629d84 remove questionWithMetadata 2025-12-04 17:21:35 +01:00
Stas Gavrylov
bccfd459c5 wip4 2025-12-04 17:21:35 +01:00
Stas Gavrylov
f733db75b1 wip3 2025-12-04 17:21:35 +01:00
Stas Gavrylov
4bc044c577 wip2 2025-12-04 17:21:33 +01:00
Stas Gavrylov
2ca514359c wip 2025-12-04 17:20:47 +01:00
Stas Gavrylov
6a29d09d17 wip 2025-12-04 17:20:47 +01:00
Stas Gavrylov
15c9599bfa Refactor edits detection logic 2025-12-04 17:20:44 +01:00
Stas Gavrylov
d1ac164c32 Add Data tab with a list of input and output tables 2025-12-04 17:19:51 +01:00
Kamil Mielnik
7484249328 Fix - Changing transform output table always shows an error
- Fixes GDGT-1344
- Simplifies useTransformValidation
2025-12-04 22:00:23 +07:00
Oleksandr Solovyov
d034244735 expose upstream transform details to fe 2025-12-04 15:58:51 +02:00
Ngoc Khuat
3b837f7593 Isolation v2 (#66374) 2025-12-04 20:49:19 +07:00
Ngoc Khuat
36419de6e7 swapped pool supports mutli pools of the same db (#66498) 2025-12-04 15:06:43 +02:00
Kamil Mielnik
b488c1b082 Fix state management issue (changing output table resets source tables and code inputs)
- Fixes GDGT-1328
2025-12-04 19:40:19 +07:00
Kamil Mielnik
869cfcc024 Improve types 2025-12-04 19:39:55 +07:00
Kamil Mielnik
81a7572e0a Fix endpoint usage 2025-12-04 19:11:52 +07:00
Kamil Mielnik
2ff73cd864 Update validate-target endpoint mapping 2025-12-04 04:07:37 -08:00
Uladzimir Havenchyk
3ebdecc2d2 add change target modal to workspace transform 2025-12-04 08:19:44 +03:00
Oleksandr Solovyov
f0bfd6898f transform filtering (#66411) 2025-12-03 17:30:19 +02:00
Oleksandr Solovyov
1b6528c9f1 don't import unexported stuff 2025-12-03 17:28:35 +02:00
Oleksandr Solovyov
f8f2e235f6 very sorry 2025-12-03 17:15:32 +02:00
Oleksandr Solovyov
9086aa10e9 more useful validate-target 2025-12-03 17:14:19 +02:00
Uladzimir Havenchyk
cfa8c389b2 add transform edit menu 2025-12-02 23:45:47 +03:00
Uladzimir Havenchyk
c670809704 remove unused files 2025-12-02 23:45:47 +03:00
lbrdnk
1f78793830 Add remove add remove test 2025-12-02 19:50:41 +01:00
lbrdnk
4e342e8481 Tables docstring 2025-12-02 19:50:41 +01:00
lbrdnk
b61455bb53 /tables, empty handling 2025-12-02 19:50:41 +01:00
lbrdnk
18c743403e /tables endpoint tests 2025-12-02 19:50:40 +01:00
Stas Gavrylov
b1fd062915 Add ability to rename workspace 2025-12-02 16:55:14 +01:00
Chris Truter
15b446ab04 Remove inline config that's breaking cljfmt 2025-12-02 17:39:23 +02:00
Oleksandr Solovyov
0f61a79a6b make merge work if there were no remapped tables (#66467) 2025-12-02 17:30:51 +02:00
lbrdnk
9ebbf353f6 unblock fe ignoring brokne graph 2025-12-02 16:12:26 +01:00
Stas Gavrylov
be25ff92a1 Fix docs 2025-12-02 15:38:18 +01:00
Kamil Mielnik
11ef81fdee Autofocus 2025-12-02 21:33:04 +07:00
Kamil Mielnik
ce08daf26f Use form in new workspace modal 2025-12-02 21:32:29 +07:00
Kamil Mielnik
e2b5e9dd9d Empty state 2025-12-02 21:29:52 +07:00
lbrdnk
49bbcf7467 Avoid calling delete with nil ids 2025-12-02 15:20:25 +01:00
Kamil Mielnik
757cca20db Remove transform from workspace
- Closes GDGT-1319
2025-12-02 21:19:02 +07:00
Oleksandr Solovyov
b196d10ad3 fix error reporting so that tests are passing 2025-12-02 15:52:24 +02:00
Kamil Mielnik
3e92c2d2ad Disable "Merge" button when there are no transforms in the workspace
- Closes GDGT-1326
2025-12-02 20:30:45 +07:00
lbrdnk
f1d495dfde whitespace 2025-12-02 14:30:31 +01:00
lbrdnk
a14b9c1b3b Add support for transform deletion 2025-12-02 14:28:33 +01:00
Kamil Mielnik
6cffef70f5 Fix alignment 2025-12-02 20:15:51 +07:00
Kamil Mielnik
a5a8fa1f1e Fix crash 2025-12-02 20:14:47 +07:00
Kamil Mielnik
6c7102aa37 Fix app-wide transforms filtering 2025-12-02 20:14:47 +07:00
Oleksandr Solovyov
809e5b6724 api for workspace name change and test fix 2025-12-02 15:11:57 +02:00
Stas Gavrylov
5f75951717 Remove tabs centered positioning 2025-12-02 14:10:23 +01:00
Stas Gavrylov
8ff78faef0 Add scrollIntoView for tabs 2025-12-02 14:09:24 +01:00
Stas Gavrylov
2bd68fc928 Add output table name input validation 2025-12-02 14:04:27 +01:00
Kamil Mielnik
288f07c709 Disable transform editor when transform is not a part of the workspace 2025-12-02 20:00:53 +07:00
Ngoc Khuat
1f199c1dfb Workspaces swapping db details (#66324) 2025-12-02 19:50:40 +07:00
Stas Gavrylov
51efac3d49 Fix workspace transforms origin 2025-12-02 13:35:48 +01:00
Kamil Mielnik
a9ccb8def6 Extract CheckOutTransformButton & SaveTransformButton 2025-12-02 19:31:31 +07:00
Stas Gavrylov
b2dacc7a92 Add transform creation within current workspace 2025-12-02 13:22:33 +01:00
Stas Gavrylov
d769903e13 Refactor table name validation to use Yup schema extension 2025-12-02 13:22:32 +01:00
Stas Gavrylov
167af6a806 Extend CreateTransformModal with external schemas and async table name validation. Integrate to WorkspacePage 2025-12-02 13:22:32 +01:00
Oleksandr Solovyov
e2ad85b627 upstream_id is optional 2025-12-02 14:21:44 +02:00
Chris Truter
c1c1926751 Fix some tests 2025-12-02 10:05:05 +02:00
Chris Truter
40dbb2dd5e WIP 2025-12-02 09:57:13 +02:00
Chris Truter
f206ac8d9b Disallow transforms with card dependencies in workspaces 2025-12-02 09:57:11 +02:00
lbrdnk
c93f52c810 Workspaces GET /tables endpoint (#66421)
* /tables endpoint

* Update keys

* Adjust test

* Update test

* remarks

* Avoid using :outputs or :transforms from the graph structure
2025-12-01 22:28:41 +01:00
lbrdnk
fac1eda9fa Workspaces dependent python transforms id adjustments (#66399)
* Python transitive transforms

* wording
2025-12-01 22:27:12 +01:00
Uladzimir Havenchyk
5baa9d4709 allow merging workspaces when all transforms saved 2025-12-01 22:30:45 +03:00
Uladzimir Havenchyk
78272586f8 redirect to the transform list after merge of a workspace 2025-12-01 21:47:19 +03:00
Ngoc Khuat
86926f69d3 return global transforms only by default and add exclude_workspace_id filter 2025-12-01 22:58:47 +07:00
Kamil Mielnik
0ee7e2fa48 Target table name input 2025-12-01 22:12:15 +07:00
Ngoc Khuat
c3f7c0e3f8 hydrate full transform contents 2025-12-01 21:00:11 +07:00
Chris Truter
1774564beb API to check whether transform target is OK 2025-12-01 15:46:44 +02:00
Chris Truter
557ce7c3c6 Fix linter warnings 2025-11-30 14:54:11 +02:00
Chris Truter
a60fa0c2cb Fix whitespace 2025-11-30 14:50:06 +02:00
Chris Truter
39e4f68e31 Add linter (overkill?) and fix for bad claude migration yaml 2025-11-30 14:43:29 +02:00
Uladzimir Havenchyk
072c06e2bd add a skeleton for merge functionality 2025-11-29 01:15:32 +03:00
Chris Truter
e781690132 Fix assertion to actually check something 2025-11-28 16:49:44 +02:00
Kamil Mielnik
68b19966fc Run transform button 2025-11-28 21:24:41 +07:00
Kamil Mielnik
8b5db401e8 Save button 2025-11-28 21:21:29 +07:00
Oleksandr Solovyov
946fc08c43 complain for wrong args to api 2025-11-28 15:20:10 +02:00
Stas Gavrylov
f5cbb8cc48 Update edited transforms display logic 2025-11-28 13:17:00 +01:00
Oleksandr Solovyov
44d6075e8c do not ask for workspace transform to provide db/schema for target 2025-11-28 14:12:23 +02:00
Chris Truter
fa2ae8e884 Fix broken SQL error 2025-11-28 14:09:51 +02:00
Stas Gavrylov
36e3b6ec3a Update WS transforms styling 2025-11-28 12:43:56 +01:00
Oleksandr Solovyov
ba9a841466 fix graph queries 2025-11-28 13:32:36 +02:00
Oleksandr Solovyov
5ebc738f29 less obtuse names 2025-11-28 13:18:22 +02:00
Stas Gavrylov
045e9a224f Refactor Setup tab 2025-11-28 12:00:34 +01:00
Stas Gavrylov
a6e22daac1 Add onChange for TransformEditor. Refactoring 2025-11-28 11:48:08 +01:00
Kamil Mielnik
6bdd48ffe3 Check if transform is in the workspace 2025-11-28 17:37:16 +07:00
Kamil Mielnik
5bb0f1f560 Style 2025-11-28 17:36:54 +07:00
Kamil Mielnik
2d263d19e3 Save button adds transform into the workspace 2025-11-28 17:30:12 +07:00
Kamil Mielnik
5e98f44b80 Add updateWorkspaceContents API mapping 2025-11-28 17:24:46 +07:00
Kamil Mielnik
da6555f5cf Stub handlers 2025-11-28 17:24:46 +07:00
Kamil Mielnik
937f3f7b03 Add TransformTab 2025-11-28 17:24:46 +07:00
Stas Gavrylov
71d065f0a9 Fix Python sidebar 2025-11-28 11:23:39 +01:00
Oleksandr Solovyov
b30071a69a adding entities and fresh transforms to workspace (#66364) 2025-11-28 12:14:40 +02:00
Stas Gavrylov
407fc2a5ab Update Code tab styling 2025-11-28 11:13:16 +01:00
Stas Gavrylov
de113f72ce Merge branch 'workspaces-master' into workspaces-2025-11 2025-11-28 11:09:08 +01:00
Stas Gavrylov
6199e0056f Merge master 2025-11-28 10:51:57 +01:00
lbrdnk
fd6b176446 Dependecy support for python transforms (#66239)
* Python transform dependencies support

* Upstream deps test

* Pass the transform
2025-11-28 10:16:24 +01:00
lbrdnk
42d1202c18 Support for multiple interdependent transforms in workspaces (#66348)
* Interdependent native Xs not executed test

WIP

Fix X mirroring as per huddle

Test: use atom for cleanup

transform-node->data

Use new mirroring and remap code

typo

typo

Make isolation naming functions public

Fix adding workspace namespace, not tested yet

Fix source-for-dup-transform

Dirty interedependent native transforms test

Instead of writing over graph structure, wrap it in context

Cleanup

Cleanup test, not final

Remove redundant test

Consolidate schema set into single update

Remove juxt in favor of select-fn->fn

nits

* Fix stuff after merging feat branch

graph :dependencies now hold transforms not tables

this change adjusts the rest of mirroring to work with that

* Better test cleanup

* hack to avoid failing when provisioning ws with no entities to checkout
2025-11-28 08:31:36 +01:00
Uladzimir Havenchyk
d1a4656cb8 move archive action to workspace item 2025-11-27 23:13:17 +03:00
Uladzimir Havenchyk
cc934ea766 fix layout 2025-11-27 20:32:00 +03:00
Stas Gavrylov
8bf117b047 Fix tabs header wrap 2025-11-27 17:31:55 +01:00
Uladzimir Havenchyk
caea0b31cf move routes out of transforms 2025-11-27 19:25:15 +03:00
Stas Gavrylov
1d6dd0d16d Add tabs icons 2025-11-27 16:51:41 +01:00
Stas Gavrylov
dca47362f8 Disable DB selection in Python editor 2025-11-27 16:23:30 +01:00
Stas Gavrylov
2f8b666797 Add transforms filtering by DB type and display of source DB name 2025-11-27 16:09:33 +01:00
Stas Gavrylov
f3aef4f4ad Some types refactoring. Make transforms names Ellipsified 2025-11-27 15:52:20 +01:00
Stas Gavrylov
5ac6249eb3 Filter transforms. Add tabs layout improvements 2025-11-27 15:35:15 +01:00
Stas Gavrylov
8acbee8000 tabs improvements 2025-11-27 15:21:41 +01:00
Kamil Mielnik
bfd2ad8f0f Better Metabot isolation 2025-11-27 21:13:49 +07:00
Stas Gavrylov
a982db6bf5 Fix for resetting active transform 2025-11-27 15:13:39 +01:00
Stas Gavrylov
3e6d266a33 Add tabs for transforms 2025-11-27 15:11:37 +01:00
Stas Gavrylov
e197cf97af Minor tabs styling updates 2025-11-27 14:47:41 +01:00
Kamil Mielnik
97d89cc855 Make tabs controlled, open transform tab when opening a transform 2025-11-27 20:44:05 +07:00
Kamil Mielnik
955cc0b4be Render Metabot tab only when plugin is enabled 2025-11-27 20:39:28 +07:00
Kamil Mielnik
1b4424393d Change data studio sidebar width 2025-11-27 20:37:29 +07:00
Kamil Mielnik
3337769411 Add Metabot tab 2025-11-27 20:27:26 +07:00
Stas Gavrylov
0b07779984 Initial version of transforms editor integrated to workspaces 2025-11-27 13:06:04 +01:00
Chris Truter
2e54d68fb2 Put mirrored transforms in the WS collection (#66358)
In anticipation of transforms moving into collection in master, let's get our code ready for that.
2025-11-27 14:01:47 +02:00
Chris Truter
e67c3e625d Defend against table name collissions 2025-11-27 13:32:19 +02:00
Chris Truter
325fe07ab1 Actually database is on the target also 2025-11-27 13:20:24 +02:00
Chris Truter
dc53fec17e Fixes to dag + workaround for deps gotcha 2025-11-27 13:04:17 +02:00
Chris Truter
17c3ed6635 Preconditions and checksum overrides for updated migrations 2025-11-27 13:04:17 +02:00
Chris Truter
f661f0c21e Fix module test 2025-11-27 13:04:17 +02:00
Chris Truter
89d1394fb7 Fix unused linter for stub API 2025-11-27 13:04:17 +02:00
Stas Gavrylov
0cac006357 Fix transforms list rendering 2025-11-27 11:07:46 +01:00
Chris Truter
c9a5ea4dd1 Better multi tx test 2025-11-27 10:30:20 +02:00
Chris Truter
79f7d83c4d Add a less trivial path-induced-subgraph test 2025-11-27 09:39:52 +02:00
Uladzimir Havenchyk
56a1ac2862 add archive method 2025-11-26 23:55:39 +03:00
Uladzimir Havenchyk
8d0acd1436 basic ui 2025-11-26 18:48:18 +03:00
Uladzimir Havenchyk
b6628af72f fix merge conflict 2025-11-26 12:37:34 +03:00
Uladzimir Havenchyk
0169970a8b Merge remote-tracking branch 'origin/workspaces-master' into workspaces-2025-11 2025-11-26 12:32:40 +03:00
Chris Truter
9cc27dfa59 Mode cosmetics 2025-11-25 17:15:55 +02:00
Oleksandr Solovyov
9221d59b7e wrong default 2025-11-25 16:56:40 +02:00
Oleksandr Solovyov
22047bb890 claude fixes tests (#66244) 2025-11-25 16:55:45 +02:00
Stas Gavrylov
772216505c Merge branch 'workspaces-list-transforms' into workspaces-2025-11 2025-11-25 12:57:50 +01:00
Stas Gavrylov
3d7541f3c7 Display list of all transforms and workspace transforms 2025-11-25 12:57:31 +01:00
Oleksandr Solovyov
5a59eef925 oops, do not hydrate for the list 2025-11-25 13:52:13 +02:00
Chris Truter
dc05f59514 Stub Workspaces API (#66124)
Co-authored-by: Oleksandr Solovyov <o@solovyov.net>
2025-11-25 13:47:10 +02:00
Stas Gavrylov
f64ab4ebab Adjust workspace page layout. Extract route 2025-11-25 12:45:17 +01:00
Kamil Mielnik
19497a7470 Fix redirect after creating a workspace from a transform (use imperative code instead of a React effect) 2025-11-25 16:54:08 +07:00
Kamil Mielnik
823690bc76 Fix skip tokens 2025-11-25 16:49:24 +07:00
Kamil Mielnik
82930d7f02 Fix workspaces API types and RTK query tags 2025-11-25 16:49:15 +07:00
Chris Truter
fa9b90beb1 Fix cross rollback test, hopefully 2025-11-21 18:38:19 +02:00
Chris Truter
d4d0c64400 Basic Merge functionality (#66137) 2025-11-21 18:22:44 +02:00
Chris Truter
cb70117c05 Fixes for Workspace merge (#66136) 2025-11-21 18:20:59 +02:00
Oleksandr Solovyov
3d9c45e126 Basic workspace merging (#66118) 2025-11-21 18:18:23 +02:00
Chris Truter
516df0c70e Bug fix for output table remapping 2025-11-21 11:05:51 +02:00
Chris Truter
4cd2aa7c3e Add list and show page for workspaces (#66112) 2025-11-21 10:44:18 +02:00
Chris Truter
e6943a7389 Max UX more fancy (links between mirrors) (#66101) 2025-11-21 10:43:33 +02:00
Chris Truter
a43c90f357 More idempotent migration fixes 2025-11-21 00:09:30 +02:00
Chris Truter
91a42b4fac Add constraint that each downstream entity has a unique mapping 2025-11-21 00:09:20 +02:00
Chris Truter
34689d9a05 Ignore checksums, for luck 2025-11-20 18:49:03 +02:00
Chris Truter
528b264f87 Fix migration lints, hopefully without breaking pr env 2025-11-20 18:47:01 +02:00
Chris Truter
0580e2dc34 Fix migration ordering 2025-11-20 18:41:06 +02:00
Chris Truter
91c985eca9 Adding missing indexes and cascade behavior 2025-11-20 18:06:43 +02:00
Chris Truter
3bb34fa1c9 Fix small lints and sanity tests
Fix deadcode lint
Fix serialization tests
Fix revision description test (workspace_id column doesn't change)
2025-11-20 18:06:15 +02:00
Chris Truter
f539b334dd Support multiple transforms (mostly) (#66079) 2025-11-20 17:56:43 +02:00
Chris Truter
9cfe59b1a2 Whoops, need plurals 2025-11-20 15:26:51 +02:00
Chris Truter
fb59bf9325 Graph Analysis for Workspace dependencies (#66069) 2025-11-20 15:15:48 +02:00
Chris Truter
2b2ff2dae5 Use join tables to track workspace entity mappings (#66062)
First class joins are kinda nice:

1.  Detection of relationships and navigation back-and-forth in the FE becomes easy. 
2. Database constraints might catch bugs.
3. Clean-up of obsolete mappings is trivial.
4. Debugging and testing is potentially more convenient too.

Maintenance on them will be low, we just need to handle creating them, and 

I haven't covered `metabase_field` _yet_, as that would be more complex. Waiting to confirm they're really necessary, and if so, whether having their mapping in JSON is sufficient.
2025-11-20 12:53:15 +02:00
Chris Truter
0dd7d78c07 Crude toposorted dag calculation 2025-11-20 10:25:51 +02:00
lbrdnk
33daf1d822 Mirror single transform (#65982)
* Mirror single transform

Creates dwh namespace
Creates mirror target table
Creates mirror transform artifact
with adjusted
target
name

Abusing toucan models
sending those in entity-info

Most of the code incrementally
updates entity-info
with checkout progress
and we should continue with that
with more reasonable way
(i.e. not toucan models but some
subset of relevant keys)

Testing code at the end of
copying ns

Namespaces will change
into different subsystems probably
for now this is poc

* Assert toplevel, docstring

* Adjust namespaces and cleanup

* drop old file

* Tweak for rich comment, incomplete

* Vibe UX

* Tighten up workspace demo-0 code (#66044)

* Update FE to track API changes. Fix some small, subtle FE and BE issues.

* Critical error

---------

Co-authored-by: Chris Truter <chris@metabase.com>
Co-authored-by: Chris Truter <crisptrutski@users.noreply.github.com>
2025-11-20 08:57:21 +01:00
Oleksandr Solovyov
9faad636cb add namespace to workspace collections 2025-11-19 18:47:16 +02:00
dan sutton
b201dca6cf empty for CI 2025-11-19 09:20:17 -06:00
Chris Truter
f1982e56e3 Fix contents API shape 2025-11-19 12:13:13 +02:00
Chris Truter
439c3f3be5 Add API endpoint to return the contents of a workspace 2025-11-19 12:11:01 +02:00
Oleksandr Solovyov
37449925fa migration and basic model file (#65956)
* migration and basic model file

* basic api
2025-11-19 12:04:35 +02:00
120 changed files with 9717 additions and 483 deletions

View File

@@ -1325,6 +1325,7 @@
metabase.sync.core
metabase.sync.init
metabase.sync.schedules
metabase.sync.sync-metadata.tables
metabase.sync.task.sync-databases
metabase.sync.util}
:uses #{analyze
@@ -2277,6 +2278,7 @@
app-db
config
driver
events
lib
lib-be
query-processor
@@ -2291,6 +2293,23 @@
models
premium-features
upload
util}}
enterprise/workspaces
{:team "Metabot"
:api #{metabase-enterprise.workspaces.api
metabase-enterprise.workspaces.core
metabase-enterprise.workspaces.init}
:uses #{api
api-keys
driver
enterprise/transforms
events
models
queries
request
sync
system
util}}}
;; namespaces matching these patterns (with `re-find`) are excluded from the module linter. Since regex literals

View File

@@ -144,6 +144,18 @@
change-log
#(pos? (compare % "v49.00-000"))))
(defn require-primary-key-exists-has-table-name
"Ensures that all primaryKeyExists preconditions specify a tableName."
[change-log]
(doseq [{:keys [changeSet]} change-log
:when changeSet
:let [s (pr-str (:preConditions changeSet))]
:when (and (str/includes? s ":primaryKeyExists")
(not (str/includes? s ":tableName")))]
(throw (validation-error
(format "Migration '%s' has primaryKeyExists precondition without tableName" (:id changeSet))
{:id (:id changeSet)}))))
(s/def ::changeSet
(s/spec :change-set.strict/change-set))
@@ -163,6 +175,7 @@
(require-no-bare-blob-or-text-types change-log)
(require-no-bare-boolean-types change-log)
(require-no-datetime-type change-log)
(require-primary-key-exists-has-table-name change-log)
(let [{:keys [changeSet]
:as all} (group-by only-key change-log)]
(when-not (set/subset? (set (keys all)) #{:property :objectQuotingStrategy :changeSet})

View File

@@ -42,6 +42,7 @@
[metabase-enterprise.transforms-python.api]
[metabase-enterprise.transforms.api]
[metabase-enterprise.upload-management.api]
[metabase-enterprise.workspaces.api]
[metabase.api.macros :as api.macros]
[metabase.api.routes.common :as api.routes.common]
[metabase.api.util.handlers :as handlers]
@@ -139,7 +140,8 @@
"/transform" (premium-handler metabase-enterprise.transforms.api/routes :transforms)
"/transform-job" (premium-handler metabase-enterprise.transforms.api/transform-job-routes :transforms)
"/transform-tag" (premium-handler metabase-enterprise.transforms.api/transform-tag-routes :transforms)
"/upload-management" (premium-handler metabase-enterprise.upload-management.api/routes :upload-management)})
"/upload-management" (premium-handler metabase-enterprise.upload-management.api/routes :upload-management)
"/workspace" metabase-enterprise.workspaces.api/routes})
;;; ↑↑↑ KEEP THIS SORTED OR ELSE ↑↑↑
(def ^:private routes-map

View File

@@ -24,4 +24,5 @@
[metabase-enterprise.stale.init]
[metabase-enterprise.support-access-grants.init]
[metabase-enterprise.transforms-python.init]
[metabase-enterprise.transforms.init]))
[metabase-enterprise.transforms.init]
[metabase-enterprise.workspaces.init]))

View File

@@ -41,6 +41,12 @@
query-deps
param-card-ids)))
(mu/defn upstream-deps:python-transform :- ::deps.schema/upstream-deps
"Given a Toucan `:model/Transform`, return its upstream dependencies as a map from the kind to a set of IDs."
[{{tables :source-tables} :source :as _py-transform}
:- [:map [:source-tables {:optional true} [:map-of :string :int]]]]
{:table (set (vals tables))})
(mu/defn upstream-deps:transform :- ::deps.schema/upstream-deps
"Given a Transform (in Toucan form), return its upstream dependencies."
[{{:keys [query], source-type :type} :source :as transform} :-
@@ -49,9 +55,15 @@
[:query
[:map [:query ::lib.schema/query]]]
[:python
[:map]]]]]]
(if (= (keyword source-type) :query)
[:map [:source-tables {:optional true} [:map-of :string :int]]]]]]]]
(cond
(= (keyword source-type) :query)
(upstream-deps:query query)
(= (keyword source-type) :python)
(upstream-deps:python-transform transform)
:else
(do (log/warnf "Don't know how to analyze the deps of Transform %d with source type '%s'" (:id transform) source-type)
{})))

View File

@@ -115,4 +115,5 @@
"User"
"UserKeyValue"
"UserParameterValue"
"ViewLog"])
"ViewLog"
"Workspace"])

View File

@@ -4,6 +4,7 @@
[metabase-enterprise.transforms.api.transform-job]
[metabase-enterprise.transforms.api.transform-tag]
[metabase-enterprise.transforms.canceling :as transforms.canceling]
[metabase-enterprise.transforms.execute :as transforms.execute]
[metabase-enterprise.transforms.interface :as transforms.i]
[metabase-enterprise.transforms.models.transform :as transform.model]
[metabase-enterprise.transforms.models.transform-run :as transform-run]
@@ -79,14 +80,32 @@
[402 (deferred-tru "Premium features required for this transform type are not enabled.")]))
(defn get-transforms
"Get a list of transforms."
[& {:keys [last_run_start_time last_run_statuses tag_ids]}]
"Get a list of transforms.
By default, only returns global transforms (workspace_id=null).
When `exclude_workspace_id` is provided, additionally excludes global transforms
that have been mirrored into the specified workspace (i.e., they have a
workspace_mapping_transform entry pointing to a transform in that workspace)."
[& {:keys [last_run_start_time last_run_statuses tag_ids exclude_workspace_id database_id type]}]
(api/check-superuser)
(let [transforms (t2/select :model/Transform {:order-by [[:id :asc]]})]
(let [transforms (if exclude_workspace_id
(t2/select :model/Transform
{:left-join [[:workspace_mapping_transform :wmt]
[:and
[:= :transform.id :wmt.upstream_id]
[:= :wmt.workspace_id exclude_workspace_id]]]
:where [:and
[:= :transform.workspace_id nil]
[:= :wmt.upstream_id nil]]
:order-by [[:id :asc]]})
(t2/select :model/Transform
{:where [:= :workspace_id nil]
:order-by [[:id :asc]]}))]
(into []
(comp (transforms.util/->date-field-filter-xf [:last_run :start_time] last_run_start_time)
(transforms.util/->status-filter-xf [:last_run :status] last_run_statuses)
(transforms.util/->tag-filter-xf [:tag_ids] tag_ids)
(transforms.util/->database-id-filter-xf database_id)
(transforms.util/->type-filter-xf type)
(map #(update % :last_run transforms.util/localize-run-timestamps)))
(t2/hydrate transforms :last_run :transform_tag_ids :creator))))
@@ -94,13 +113,18 @@
;; of the REST API
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-query-params-use-kebab-case]}
(api.macros/defendpoint :get "/"
"Get a list of transforms."
"Get a list of global transforms (workspace_id=null).
When `exclude_workspace_id` is provided, additionally excludes global transforms
that have been mirrored into the specified workspace."
[_route-params
query-params :-
[:map
[:last_run_start_time {:optional true} [:maybe ms/NonBlankString]]
[:last_run_statuses {:optional true} [:maybe (ms/QueryVectorOf [:enum "started" "succeeded" "failed" "timeout"])]]
[:tag_ids {:optional true} [:maybe (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]]]]
[:tag_ids {:optional true} [:maybe (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]]
[:exclude_workspace_id {:optional true} [:maybe ms/PositiveInt]]
[:database_id {:optional true} [:maybe ms/PositiveInt]]
[:type {:optional true} [:maybe (ms/QueryVectorOf [:enum "query" "native" "python"])]]]]
(get-transforms query-params))
(api.macros/defendpoint :post "/"
@@ -264,8 +288,8 @@
_ (check-feature-enabled! transform)
start-promise (promise)]
(u.jvm/in-virtual-thread*
(transforms.i/execute! transform {:start-promise start-promise
:run-method :manual}))
(transforms.execute/execute! transform {:start-promise start-promise
:run-method :manual}))
(when (instance? Throwable @start-promise)
(throw @start-promise))
(let [result @start-promise

View File

@@ -1,6 +1,7 @@
(ns metabase-enterprise.transforms.core
"API namespace for the `metabase-enterprise.transform` module."
(:require
[metabase-enterprise.transforms.models.transform]
[metabase-enterprise.transforms.models.transform-run]
[metabase-enterprise.transforms.settings]
[metabase-enterprise.transforms.util]
@@ -14,4 +15,6 @@
python-transform?
query-transform?]
[metabase-enterprise.transforms.models.transform-run
timeout-run!])
timeout-run!]
[metabase-enterprise.transforms.models.transform
update-transform-tags!])

View File

@@ -1,81 +1,18 @@
(ns metabase-enterprise.transforms.execute
(:require
[metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
[metabase-enterprise.transforms.util :as transforms.util]
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
[metabase.events.core :as events]
[metabase.lib.schema.common :as schema.common]
[metabase.query-processor.compile :as qp.compile]
[metabase.util.log :as log]
[metabase.util.malli.registry :as mr]
[toucan2.core :as t2]))
[metabase-enterprise.transforms.interface :as transforms.i]))
(set! *warn-on-reflection* true)
(mr/def ::transform-details
[:map
[:transform-type [:enum {:decode/normalize schema.common/normalize-keyword} :table :table-incremental]]
[:conn-spec :any]
[:query ::qp.compile/compiled]
[:output-table [:keyword {:decode/normalize schema.common/normalize-keyword}]]])
(defn execute!
"Execute a transform.
(mr/def ::transform-opts
[:map
[:overwrite? :boolean]])
(defn- transform-opts [{:keys [transform-type]}]
(case transform-type
:table {:overwrite? true}
;; once we have more than just append, dispatch on :target-incremental-strategy
:table-incremental {}))
(defn run-mbql-transform!
"Run `transform` and sync its target table.
This is executing synchronously, but supports being kicked off in the background
by delivering the `start-promise` just before the start when the beginning of the execution has been booked
in the database."
([transform] (run-mbql-transform! transform nil))
([{:keys [id source target] :as transform} {:keys [run-method start-promise]}]
(try
(let [db (get-in source [:query :database])
{driver :engine :as database} (t2/select-one :model/Database db)
transform-details {:db-id db
:database database
:transform-id id
:transform-type (keyword (:type target))
:conn-spec (driver/connection-spec driver database)
:query (transforms.util/compile-source transform)
:output-schema (:schema target)
:output-table (transforms.util/qualified-table-name driver target)}
opts (transform-opts transform-details)
features (transforms.util/required-database-features transform)]
(when (transforms.util/db-routing-enabled? database)
(throw (ex-info "Transforms are not supported on databases with DB routing enabled."
{:driver driver, :database database})))
(when-not (every? (fn [feature] (driver.u/supports? (:engine database) feature database)) features)
(throw (ex-info "The database does not support the requested transform target type."
{:driver driver, :database database, :features features})))
;; mark the execution as started and notify any observers
(let [{run-id :id} (transforms.util/try-start-unless-already-running id run-method)]
(when start-promise
(deliver start-promise [:started run-id]))
(log/info "Executing transform" id "with target" (pr-str target))
(transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]]
(transforms.util/run-cancelable-transform!
run-id driver transform-details
(fn [_cancel-chan] (driver/run-transform! driver transform-details opts))))
(transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]
(transforms.util/sync-target! target database)
;; This event must be published only after the sync is complete - the new table needs to be in AppDB.
(events/publish-event! :event/transform-run-complete {:object transform-details}))))
(catch Throwable t
(log/error t "Error executing transform")
(when start-promise
;; if the start-promise has been delivered, this is a no-op,
;; but we assume nobody would catch the exception anyway
(deliver start-promise t))
(throw t)))))
For workspace transforms, execution must go through the workspace-specific endpoint.
Calling this function directly with a workspace transform will throw a 404 error."
[transform opts]
(when (:workspace_id transform)
(throw (ex-info "Workspace transforms must be executed through the workspace API"
{:status-code 404
:transform-id (:id transform)
:workspace-id (:workspace_id transform)})))
(transforms.i/execute! transform opts))

View File

@@ -223,12 +223,14 @@
[_model-name opts]
{:copy [:name :description :entity_id]
:skip [:dependency_analysis_version :source_type]
:transform {:created_at (serdes/date)
:creator_id (serdes/fk :model/User)
:source {:export #(update % :query serdes/export-mbql)
:import #(update % :query serdes/import-mbql)}
:target {:export serdes/export-mbql :import serdes/import-mbql}
:tags (serdes/nested :model/TransformTransformTag :transform_id opts)}})
:transform {:created_at (serdes/date)
:creator_id (serdes/fk :model/User)
:workspace_id (serdes/fk :model/Workspace)
:collection_id (serdes/fk :model/Collection)
:source {:export #(update % :query serdes/export-mbql)
:import #(update % :query serdes/import-mbql)}
:target {:export serdes/export-mbql :import serdes/import-mbql}
:tags (serdes/nested :model/TransformTransformTag :transform_id opts)}})
(defmethod serdes/dependencies "Transform"
[{:keys [source tags]}]

View File

@@ -1,7 +1,15 @@
(ns metabase-enterprise.transforms.query-impl
(:require
[metabase-enterprise.transforms.execute :as transforms.execute]
[metabase-enterprise.transforms.interface :as transforms.i]))
[metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
[metabase-enterprise.transforms.interface :as transforms.i]
[metabase-enterprise.transforms.util :as transforms.util]
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
[metabase.events.core :as events]
[metabase.lib.schema.common :as schema.common]
[metabase.util.log :as log]
[metabase.util.malli.registry :as mr]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -13,5 +21,71 @@
[transform]
(-> transform :source :query :database))
(mr/def ::transform-details
[:map
[:transform-type [:enum {:decode/normalize schema.common/normalize-keyword} :table]]
[:conn-spec :any]
[:query :string]
[:output-table [:keyword {:decode/normalize schema.common/normalize-keyword}]]])
(mr/def ::transform-opts
[:map
[:overwrite? :boolean]])
(defn- transform-opts [{:keys [transform-type]}]
(case transform-type
:table {:overwrite? true}
;; once we have more than just append, dispatch on :target-incremental-strategy
:table-incremental {}))
(defn run-mbql-transform!
"Run `transform` and sync its target table.
This is executing synchronously, but supports being kicked off in the background
by delivering the `start-promise` just before the start when the beginning of the execution has been booked
in the database."
([transform] (run-mbql-transform! transform nil))
([{:keys [id source target] :as transform} {:keys [run-method start-promise]}]
(try
(let [db (get-in source [:query :database])
{driver :engine :as database} (t2/select-one :model/Database db)
transform-details {:db-id db
:database database
:transform-id id
:transform-type (keyword (:type target))
:conn-spec (driver/connection-spec driver database)
:query (transforms.util/compile-source transform)
:output-schema (:schema target)
:output-table (transforms.util/qualified-table-name driver target)}
opts (transform-opts transform-details)
features (transforms.util/required-database-features transform)]
(when (transforms.util/db-routing-enabled? database)
(throw (ex-info "Transforms are not supported on databases with DB routing enabled."
{:driver driver, :database database})))
(when-not (every? (fn [feature] (driver.u/supports? (:engine database) feature database)) features)
(throw (ex-info "The database does not support the requested transform target type."
{:driver driver, :database database, :features features})))
;; mark the execution as started and notify any observers
(let [{run-id :id} (transforms.util/try-start-unless-already-running id run-method)]
(when start-promise
(deliver start-promise [:started run-id]))
(log/info "Executing transform" id "with target" (pr-str target))
(transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]]
(transforms.util/run-cancelable-transform!
run-id driver transform-details
(fn [_cancel-chan] (driver/run-transform! driver transform-details opts))))
(transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]
(transforms.util/sync-target! target database)
;; This event must be published only after the sync is complete - the new table needs to be in AppDB.
(events/publish-event! :event/transform-run-complete {:object transform-details}))))
(catch Throwable t
(log/error t "Error executing transform")
(when start-promise
;; if the start-promise has been delivered, this is a no-op,
;; but we assume nobody would catch the exception anyway
(deliver start-promise t))
(throw t)))))
(defmethod transforms.i/execute! :query [transform opts]
(transforms.execute/run-mbql-transform! transform opts))
(run-mbql-transform! transform opts))

View File

@@ -45,14 +45,14 @@
[:map
[:database {:optional true} :int]
[:type [:= "table"]]
[:schema {:optional true} [:or ms/NonBlankString :nil]]
[:schema {:optional true} [:maybe ms/NonBlankString]]
[:name :string]])
(mr/def ::table-incremental-target
[:map
[:database {:optional true} :int]
[:type [:= "table-incremental"]]
[:schema {:optional true} [:or ms/NonBlankString :nil]]
[:schema {:optional true} [:maybe ms/NonBlankString]]
[:name :string]
[:target-incremental-strategy ::target-incremental-strategy]])

View File

@@ -436,33 +436,52 @@
(defn- matching-timestamp?
[job field-path {:keys [start end]}]
(when-let [field-instant (->instant (get-in job field-path))]
(let [start-instant (some-> start u.date/parse ->instant)
end-instant (some-> end u.date/parse ->instant)]
(and (or (nil? start)
(not (.isBefore field-instant start-instant)))
(or (nil? end)
(.isAfter end-instant field-instant))))))
(let [parse #(-> % u.date/parse ->instant)]
;; logic here is to find when it's not matching and invert this
(not (or (and start (.isBefore field-instant (parse start)))
(and end (.isAfter field-instant (parse end))))))))
(defn ->date-field-filter-xf
"Returns an xform for a date filter."
[field-path filter-value]
(let [range (some-> filter-value (params.dates/date-string->range {:inclusive-end? false}))]
(if range
(filter #(matching-timestamp? % field-path range))
identity)))
(if-let [range (some-> filter-value (params.dates/date-string->range {:inclusive-end? false}))]
(filter #(matching-timestamp? % field-path range))
identity))
(defn ->status-filter-xf
"Returns an xform for a transform run status filter."
[field-path statuses]
(let [statuses (->> statuses (map keyword) set not-empty)]
(if statuses
(filter #(statuses (get-in % field-path)))
identity)))
(if-let [statuses (->> statuses (map keyword) set not-empty)]
(filter #(statuses (get-in % field-path)))
identity))
(defn ->tag-filter-xf
"Returns an xform for a transform tag filter."
[field-path tag-ids]
(let [tag-ids (-> tag-ids set not-empty)]
(if tag-ids
(filter #(some tag-ids (get-in % field-path)))
identity)))
(if-let [tag-ids (-> tag-ids set not-empty)]
(filter #(some tag-ids (get-in % field-path)))
identity))
(defn ->type-filter-xf
"Returns an xform for a transform source type filter."
[types]
(if-let [types (->> types (map keyword) set not-empty)]
(filter (fn [transform]
(let [source-type (-> transform :source :type keyword)
query-type (some-> transform :source :query :type keyword)]
(if (= source-type :python)
(contains? types :python)
(contains? types query-type)))))
identity))
(defn ->database-id-filter-xf
"Returns an xform for a transform database ID filter.
Matches transforms where the source database OR target database matches the given database-id."
[database-id]
(if database-id
(filter (fn [transform]
(let [python-source-db-id (get-in transform [:source :source-database])
target-db-id (get-in transform [:target :database])]
(or (= database-id python-source-db-id)
(= database-id target-db-id)))))
identity))

View File

@@ -12,6 +12,7 @@
[metabase.app-db.core :as app-db]
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
[metabase.events.core :as events]
[metabase.util :as u]
[metabase.util.format :as u.format]
[metabase.util.i18n :as i18n]
@@ -360,6 +361,7 @@
(log/info "Executing Python transform" transform-id "with target" (pr-str target))
(let [start-ms (u/start-timer)
transform-details {:db-id (:id db)
:transform-id transform-id
:transform-type (keyword (:type target))
:conn-spec (driver/connection-spec driver db)
:output-schema (:schema target)
@@ -372,7 +374,8 @@
result (transforms.instrumentation/with-stage-timing [run-id [:computation :python-execution]]
(transforms.util/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn))]
(transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]
(transforms.util/sync-target! target db))
(transforms.util/sync-target! target db)
(events/publish-event! :event/transform-run-complete {:object transform-details}))
{:run_id run-id
:result result}))
(catch Throwable t

View File

@@ -0,0 +1,534 @@
(ns metabase-enterprise.workspaces.api
"`/api/ee/workspace/` routes"
(:require
[honey.sql.helpers :as sql.helpers]
[metabase-enterprise.transforms.util :as transforms.util]
[metabase-enterprise.workspaces.common :as ws.common]
[metabase-enterprise.workspaces.dag :as ws.dag]
[metabase-enterprise.workspaces.promotion :as ws.promotion]
[metabase.api.common :as api]
[metabase.api.macros :as api.macros]
[metabase.api.routes.common :refer [+auth]]
[metabase.driver.util :as driver.u]
[metabase.queries.schema :as queries.schema]
[metabase.request.core :as request]
[metabase.util :as u]
[metabase.util.i18n :refer [deferred-tru]]
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]
[toucan2.core :as t2]))
;;; schemas
(mr/def ::entity-type [:enum :transform])
(mr/def ::entity-grouping [:enum :transforms])
;; Map like {:transforms [1 2 3]}
(mr/def ::entity-map
[:map-of ::entity-grouping [:sequential {:min 1} ms/PositiveInt]])
;; Entities that live within the Workspace
(mr/def ::downstream-entity
[:map
[:id ms/PositiveInt]
[:upstream_id [:maybe ms/PositiveInt]]
[:upstream [:maybe ::ws-transform-target]]
[:type ::entity-type]
[:name :string]])
;; Entity reference used in requests
(mr/def ::entity-reference
[:map
[:type ::entity-type]
[:id ms/PositiveInt]])
;; Graph node for view-graph endpoint
(mr/def ::graph-node
[:map
[:node_id :string]
[:id ms/PositiveInt]
[:type [:enum :table :transform]]
[:title :string]
[:dependents [:map-of [:enum :table :transform] ms/PositiveInt]]])
;; Graph edge for view-graph endpoint
(mr/def ::graph-edge
[:map
[:from :string]
[:to :string]])
;; Error: entity that needs to be checked out
(mr/def ::unchecked-out-entity
[:map
[:type ::entity-type]
[:id ms/PositiveInt]
[:name :string]])
;; Error: entity that cannot be cloned
(mr/def ::uncloneable-entity
[:map
[:type ::entity-type]
[:id ms/PositiveInt]
[:name :string]
[:error :string]])
;; Error response for graph-not-closed
(mr/def ::graph-not-closed-error
[:map
[:error [:= :graph-not-closed]]
[:message :string]
[:entities [:sequential ::unchecked-out-entity]]])
;; Error response for uncloneable entities
(mr/def ::uncloneable-error
[:map
[:error [:= :contains-uncloneable-entities]]
[:message :string]
[:entities [:sequential ::uncloneable-entity]]])
(def ^:private CreateWorkspace
[:map
[:name {:optional true} [:string {:min 1}]]
[:database_id {:optional true} :int]
[:upstream {:optional true} ::entity-map]])
(def ^:private ModifyEntities
[:map {:closed true}
[:add {:optional true} ::entity-map]
[:remove {:optional true} ::entity-map]])
(def ^:private Workspace
[:map
[:id ms/PositiveInt]
[:name :string]
[:collection_id :int]
[:database_id :int]
[:created_at :any]
[:updated_at :any]
[:archived_at [:maybe :any]]])
(def ^:private FullWorkspace
[:and Workspace
[:map
[:contents [:map-of ::entity-grouping [:sequential ::downstream-entity]]]]])
(def ^:private ExecuteResult
"Schema for workspace execution result"
[:map
[:succeeded ::entity-map]
[:failed ::entity-map]
[:not_run ::entity-map]])
(def ^:private GraphResult
"Schema for workspace graph visualization"
[:map
[:nodes [:sequential ::graph-node]]
[:edges [:sequential ::graph-edge]]])
;; Transform-related schemas (adapted from transforms/api.clj)
(mr/def ::transform-source
[:multi {:dispatch (comp keyword :type)}
[:query
[:map
[:type [:= "query"]]
[:query ::queries.schema/query]]]
[:python
[:map {:closed true}
[:source-database {:optional true} :int]
[:source-tables [:map-of :string :int]]
[:type [:= "python"]]
[:body :string]]]])
(mr/def ::ws-transform-target
[:map
[:type [:enum "table"]]
[:name :string]])
(mr/def ::run-trigger
[:enum "none" "global-schedule"])
(defn- check-transforms-enabled!
[db-id]
(let [database (api/check-400 (t2/select-one :model/Database db-id)
(deferred-tru "The target database cannot be found."))]
(api/check (transforms.util/check-feature-enabled nil)
[402 (deferred-tru "Premium features required for transforms are not enabled.")])
(api/check-400 (not (:is_sample database))
(deferred-tru "Cannot run transforms on the sample database."))
(api/check-400 (not (:is_audit database))
(deferred-tru "Cannot run transforms on audit databases."))
(api/check-400 (driver.u/supports? (:engine database) :transforms/table database)
(deferred-tru "The database does not support the requested transform target type."))
(api/check-400 (not (transforms.util/db-routing-enabled? database))
(deferred-tru "Transforms are not supported on databases with DB routing enabled."))))
(defn- ws->response [ws]
(select-keys ws
[:id :name :collection_id :database_id :created_at :updated_at :archived_at :contents]))
;;; routes
(api.macros/defendpoint :get "/" :- [:map [:items [:sequential Workspace]]]
"Get a list of all workspaces"
[_route-params
_query-params]
{:items (->> (t2/select :model/Workspace :archived_at [:is nil]
(cond-> {:order-by [[:created_at :desc]]}
(request/limit) (sql.helpers/limit (request/limit))
(request/offset) (sql.helpers/offset (request/offset))))
(mapv ws->response))
:limit (request/limit)
:offset (request/offset)})
;;;; /tables start
(defn- workspace-input-tables
[workspace-id]
(reduce
(fn [acc {:keys [x_id t_id]}]
(cond-> acc
(pos-int? t_id) (update x_id (fnil conj #{}) t_id)))
{}
(t2/query {:with [[:ws_xs {:select [:x.*]
:from [[(t2/table-name :model/Transform) :x]]
:where [:= [:inline workspace-id] :x.workspace_id]}]
[:ws_xs_deps {:select [[:x.id :x_id]
[:d.to_entity_id :t_id]]
:from [[:ws_xs :x]]
:left-join [[(t2/table-name :model/Dependency) :d]
[:and
[:= :d.from_entity_type "transform"]
[:= :d.from_entity_id :x.id]
[:= :d.to_entity_type "table"]]]}]]
:select [:xd.x_id :xd.t_id]
:from [[:ws_xs_deps :xd]]
:where [:not [:in :xd.t_id
[;; TODO: Revisit this with refactor next week.
;; set of upstream AND downstream so this works even after anticipated remapping changes.
{:union-all [{:select [:wmt.upstream_id]
:from [[(t2/table-name :model/WorkspaceMappingTable) :wmt]]}
{:select [:wmt.downstream_id]
:from [[(t2/table-name :model/WorkspaceMappingTable) :wmt]]}]}]]]})))
(defn- workspace-output-tables
[workspace-id]
(reduce
(fn [acc {:keys [x_id t_id]}]
(cond-> acc
(pos-int? t_id) (update x_id (fnil conj #{}) t_id)))
{}
(t2/query {:with [[:ws_xs {:select [:x.*]
:from [[(t2/table-name :model/Transform) :x]]
:where [:= [:inline workspace-id] :x.workspace_id]}]
[:ws_xs_deps {:select [[:x.id :x_id]
[:d.from_entity_id :t_id]]
:from [[:ws_xs :x]]
:left-join [[(t2/table-name :model/Dependency) :d]
[:and
[:= :d.from_entity_type "table"]
[:= :d.to_entity_id :x.id]
[:= :d.to_entity_type "transform"]]]}]]
:select [:xd.x_id :xd.t_id]
:from [[:ws_xs_deps :xd]]})))
(api.macros/defendpoint :get "/:id/tables" :- [:map
[:inputs [:sequential
[:map
[:schema [:maybe :string]]
[:table [:maybe :string]]]]]
[:outputs [:sequential
[:map
[:global [:map
[:schema [:maybe :string]]
[:table [:maybe :string]]]]
[:workspace [:map
[:transform-id :int]
[:table-id :int]
[:schema [:maybe :string]]
[:table [:maybe :string]]]]]]]]
"Get workspace tables"
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params]
(let [workspace (-> (api/check-404 (t2/select-one :model/Workspace :id id))
(t2/hydrate :contents))]
{:inputs
(let [ids (into #{}
(mapcat (fn [[_x-id t-ids]]
t-ids))
(workspace-input-tables (:id workspace)))]
(if-not (seq ids)
[]
(t2/select-fn-vec identity [:model/Table :id :schema [:name :table]]
:id [:in ids])))
:outputs
(let [ids (workspace-output-tables (:id workspace))]
(into []
(keep (fn [[x-id t-ids]]
(when (seq t-ids)
(let [ws-x (t2/select-one :model/Transform :id x-id)
orig-x (when-some [orig-id (:upstream_id (t2/select-one :model/WorkspaceMappingTransform :downstream_id (:id ws-x)))]
(t2/select-one :model/Transform :id orig-id))]
{:global {:schema (-> orig-x :target :schema)
:table (-> orig-x :target :name)}
:workspace {:transform-id x-id
:table-id (first t-ids)
:schema (-> ws-x :target :schema)
:table (-> ws-x :target :name)}}))))
ids))}))
;;;; /tables end
(api.macros/defendpoint :get "/:id" :- FullWorkspace
"Get a single workspace by ID"
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params]
(-> (api/check-404 (t2/select-one :model/Workspace :id id))
(t2/hydrate :contents)
ws->response))
(api.macros/defendpoint :post "/" :- Workspace
"Create a new workspace
Potential payload:
{:name \"a\" :database_id 2 :upstream {:transforms [1 2 3]}}"
[_route-params
_query-params
{:keys [_database_id upstream] :as body} :- CreateWorkspace]
;; TODO (Sanya) Oops, I forgot that this is optional, and can get inferred later. I broke a bunch of tests.
;; Validation logic should all move to common in any case.
;; TODO (Sanya) Oops, there are tests using databases that fail this as well, update the tests first.
#_(when database_id
(check-transforms-enabled! database_id))
(when-let [transform-ids (seq (get upstream :transforms []))]
(ws.common/check-transforms-not-in-workspace! transform-ids)
(ws.common/check-no-card-dependencies! transform-ids))
(-> (ws.common/create-workspace! api/*current-user-id* body)
ws->response))
(api.macros/defendpoint :post "/:id/name" :- Workspace
"Update a workspace's name"
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params
{:keys [name]} :- [:map [:name [:string {:min 1}]]]]
(u/prog1 (api/check-404 (t2/select-one :model/Workspace :id id))
(api/check-400 (nil? (:archived_at <>)) "Cannot update an archived workspace"))
(t2/update! :model/Workspace id {:name name})
(-> (t2/select-one :model/Workspace :id id)
ws->response))
(api.macros/defendpoint :post "/:id/archive" :- Workspace
"Archive a workspace. Deletes the isolated schema and tables, but preserves mirrored entities."
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params
_body-params]
(let [ws (api/check-404 (t2/select-one :model/Workspace :id id))]
(api/check-400 (nil? (:archived_at ws)) "You cannot archive an archived workspace")
(t2/update! :model/Workspace id {:archived_at [:now]})
(-> (t2/select-one :model/Workspace :id id)
ws->response)))
(api.macros/defendpoint :post "/:id/unarchive" :- Workspace
"Restore an archived workspace. Recreates the isolated schema and tables."
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params
_body-params]
(let [ws (api/check-404 (t2/select-one :model/Workspace :id id))]
(api/check-400 (some? (:archived_at ws)) "You cannot unarchive a workspace that is not archived")
(t2/update! :model/Workspace id {:archived_at nil})
(-> (t2/select-one :model/Workspace :id id)
ws->response)))
(api.macros/defendpoint :delete "/:id" :- [:map [:ok [:= true]]]
"Delete a workspace and all its contents, including mirrored entities."
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params]
(api/check-404 (t2/select-one :model/Workspace :id id))
;; TODO (Chris 11/21/25) -- implement actual deletion logic
{:ok true})
(api.macros/defendpoint :post "/:id/execute" :- ExecuteResult
"Execute all transforms in the workspace in dependency order.
Returns which transforms succeeded, failed, and were not run."
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params
_body-params]
(u/prog1 (t2/select-one :model/Workspace :id id)
(api/check-404 <>)
(api/check-400 (nil? (:archived_at <>)) "Cannot execute archived workspace"))
;; TODO (Chris 11/21/25) -- implement execution logic
{:succeeded []
:failed []
:not_run []})
(api.macros/defendpoint :get "/:id/graph" :- GraphResult
"Get the dependency graph for a workspace, for visualization.
Shows tables and transforms the workspace depends on, with edges representing dependencies.
Tables produced by transforms in the workspace are not shown; instead, dependencies appear
directly between transforms."
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params]
(api/check-404 (t2/select-one :model/Workspace :id id))
;; TODO (Chris 11/21/25) -- implement graph generation logic
{:nodes []
:edges []})
(api.macros/defendpoint :post "/:id/contents"
:- [:map [:contents [:map-of ::entity-grouping [:sequential ::downstream-entity]]]]
"Add upstream entities to workspace by mirroring them into the workspace's isolated environment.
The entities and their dependencies will be mirrored into the workspace.
Returns the workspace's updated contents.
Removal is not implemented yet."
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params
body :- ModifyEntities]
(let [workspace (api/check-404 (t2/select-one :model/Workspace :id id))
upstream (:add body)
to-remove (:remove body)]
(api/check-400 (nil? (:archived_at workspace)) "Cannot add entities to an archived workspace")
(when-let [transform-ids (seq (get upstream :transforms []))]
(ws.common/check-transforms-not-in-workspace! transform-ids)
(ws.common/check-no-card-dependencies! transform-ids))
(let [existing-upstream-ids (t2/select-fn-set :upstream_id :model/WorkspaceMappingTransform
:workspace_id id)
combined-upstream (update upstream :transforms #(into (set existing-upstream-ids) %))
graph (ws.dag/path-induced-subgraph combined-upstream)
table-ids (seq (keep :id (concat (:inputs graph) (:outputs graph))))
db-ids (when table-ids (t2/select-fn-set :db_id :model/Table :id [:in table-ids]))
db-id (first db-ids)]
(when db-id
(api/check-400 (= db-id (:database_id workspace)) "All entities must belong to the workspace's database")))
;; Add new entities
(when (not-empty upstream)
(ws.common/add-entities! workspace upstream))
;; Remove existing entities
(when (not-empty to-remove)
(ws.common/remove-entities! workspace to-remove))
{:contents (:contents (t2/hydrate (t2/select-one :model/Workspace :id id) :contents))}))
(api.macros/defendpoint :post "/:id/validate-target"
"Validate the name of a transform target."
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params
{:keys [db_id target]} :- [:map
[:db_id {:optional true} ms/PositiveInt]
[:target [:map
[:type [:= "table"]]
[:schema {:optional true} [:or ms/NonBlankString :nil]]
[:name :string]]]]]
(cond
(transforms.util/target-table-exists? {:target (merge {:database db_id} target)
:source {:type :python}})
{:status 403 :body (deferred-tru "A table with that name already exists.")}
(some #(= (:name (:target %)) (-> target :name)) (t2/select :model/Transform :workspace_id id))
{:status 403 :body (deferred-tru "Another transform in this workspace already targets that table.")}
:else
{:status 200 :body "OK"}))
(api.macros/defendpoint :post "/:id/transform"
"Create a new transform directly within a workspace.
This creates a transform that exists only in the workspace's isolated schema.
The transform is not mirrored from an existing transform, but created from scratch."
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params
body :- [:map
[:name :string]
[:description {:optional true} [:maybe :string]]
[:source ::transform-source]
[:target ::ws-transform-target]
[:run_trigger {:optional true} ::run-trigger]
[:tag_ids {:optional true} [:sequential ms/PositiveInt]]]]
(api/check (transforms.util/check-feature-enabled body)
[402 (deferred-tru "Premium features required for this transform type are not enabled.")])
(let [workspace (u/prog1 (api/check-404 (t2/select-one :model/Workspace :id id))
(api/check-400 (nil? (:archived_at <>)) "Cannot create transforms in an archived workspace"))
target-name (get-in body [:target :name])
existing-target (some #(= (:name (:target %)) target-name) (t2/select :model/Transform :workspace_id id))
body (update body :target
assoc :database_id (:database_id workspace) :schema (:schema workspace))]
(api/check-400 (not existing-target)
(deferred-tru "Another transform in this workspace already targets that table."))
(ws.common/create-transform! workspace body api/*current-user-id*)))
(api.macros/defendpoint :get "/mapping/transform/:id/downstream"
:- [:map
[:transforms [:sequential
[:map
[:id ms/PositiveInt]
[:name :string]
[:workspace [:map
[:id ms/PositiveInt]
[:name :string]]]]]]]
"Get all downstream transforms for a transform that is not in a workspace.
Returns the transforms that were mirrored from this upstream transform, with workspace info."
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params]
(let [mappings (t2/select :model/WorkspaceMappingTransform :upstream_id id)
_ (when (empty? mappings) (api/check-404 (t2/exists? :model/Workspace id)))
tid->wid (u/for-map [m mappings]
[(:downstream_id m) (:workspace_id m)])
transform-ids (map :downstream_id mappings)
workspace-ids (map :workspace_id mappings)
transforms (when (seq transform-ids)
(t2/select [:model/Transform :id :name] :id [:in transform-ids] {:order-by [:created_at]}))
workspaces-by-id (when (seq workspace-ids)
(u/index-by :id (t2/select [:model/Workspace :id :name] :id [:in workspace-ids])))]
{:transforms (for [transform transforms]
(assoc transform :workspace (get workspaces-by-id (tid->wid (:id transform)))))}))
(api.macros/defendpoint :post "/:id/merge"
:- [:or
[:map
[:promoted [:sequential [:map [:id ms/PositiveInt] [:name :string]]]]
[:errors [:maybe [:sequential [:map [:id ms/PositiveInt] [:name :string] [:error :string]]]]]
[:workspace [:map [:id ms/PositiveInt] [:name :string]]]
[:archived_at [:maybe :any]]]
;; error message from check-404 or check-400
:string]
"Promote workspace transforms back to main Metabase and archive the workspace.
This will:
1. Update original transforms with workspace versions
2. Re-execute transforms in the original schema
3. Archive the workspace and clean up isolated resources
Returns a report of promoted transforms and any errors."
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
_query-params
_body-params]
(let [ws (u/prog1 (t2/select-one :model/Workspace :id id)
(api/check-404 <>)
(api/check-400 (nil? (:archived_at <>)) "Cannot promote an already archived workspace"))
{:keys [promoted
errors]} (ws.promotion/promote-transforms! ws)]
(u/prog1
{:promoted (vec promoted)
:errors errors
:workspace {:id id :name (:name ws)}
:archived_at (when-not (seq errors)
(t2/update! :model/Workspace :id id {:archived_at [:now]})
(t2/select-one-fn :archived_at [:model/Workspace :archived_at] :id id))}
(when-not (seq errors)
;; Most of the APIs and the FE are not respecting when a Workspace is archived yet.
(t2/delete! :model/Workspace id)))))
(def ^{:arglists '([request respond raise])} routes
"`/api/ee/workspace/` routes."
(api.macros/ns-handler *ns* api/+check-superuser +auth))

View File

@@ -0,0 +1,307 @@
(ns metabase-enterprise.workspaces.common
(:require
[clojure.string :as str]
[metabase-enterprise.transforms.core :as transforms]
[metabase-enterprise.workspaces.dag :as ws.dag]
[metabase-enterprise.workspaces.driver.common :as driver.common]
[metabase-enterprise.workspaces.isolation :as ws.isolation]
[metabase-enterprise.workspaces.mirroring :as ws.mirroring]
[metabase.api-keys.core :as api-key]
[metabase.api.common :as api]
[metabase.events.core :as events]
[metabase.util :as u]
[metabase.util.i18n :refer [tru]]
[toucan2.core :as t2]))
(defn check-no-card-dependencies!
"Check that transforms don't depend on cards. Throws 400 if they do."
[transform-ids]
(when-let [card-ids (seq (ws.dag/card-dependencies transform-ids))]
(api/check-400 false
(format "Cannot add transforms that depend on saved questions (cards). Found dependencies on card IDs: %s"
(pr-str (vec card-ids))))))
(defn check-transforms-not-in-workspace!
"Check that none of the transforms already belong to a workspace. Throws 400 if any do."
[transform-ids]
(when (seq (t2/select [:model/Transform :id :workspace_id]
:id [:in transform-ids]
:workspace_id [:not= nil]))
(throw (ex-info (tru "Cannot add transforms that belong to another workspace")
{:status-code 400}))))
(defn- extract-suffix-number
"Extract the numeric suffix from a workspace name like 'Foo (3)', or nil if no valid suffix."
[ws-name base-name]
(when (= (str/replace ws-name #"\s*\(\d+\)$" "") base-name)
(when-let [[_ n] (re-find #"\((\d+)\)$" ws-name)]
(parse-long n))))
(defn- generate-unique-workspace-name
"Generate a unique workspace name by appending (N) if the name already exists.
Strategy:
1. If the base name doesn't exist, use it as-is
2. If it exists, strip any existing (N) suffix and find the highest N among similar names
3. Return the base name with (N+1) appended
Note: This function has a potential race condition between checking existence and insertion.
Callers should handle unique constraint violations by retrying."
[base-name]
(when (str/blank? base-name)
(throw (ex-info "Workspace name cannot be empty" {:status-code 400})))
(if-not (t2/exists? :model/Workspace :name base-name)
base-name
(let [stripped-name (str/replace base-name #"\s*\(\d+\)$" "")
existing (t2/select-fn-set :name :model/Workspace
:name [:like (str stripped-name " (%)")])
numbers (keep #(extract-suffix-number % stripped-name) existing)
next-num (inc (apply max 0 numbers))]
(str stripped-name " (" next-num ")"))))
;; TODO (Chris 2025-11-20) Just subsume the rest of this up into ws.dag/path-induced-subgraph
(defn- build-graph
"Thin wrapper around the dag module, that should probably be absorbed by it."
[upstream]
(if (not-any? seq (vals upstream))
{:db_id nil
:transforms []
:inputs []
:outputs []}
(let [graph (ws.dag/path-induced-subgraph upstream)
db-ids (when-let [table-ids (seq (keep :id (concat (:inputs graph) (:outputs graph))))]
(t2/select-fn-set :db_id :model/Table :id [:in table-ids]))
_ (assert (<= (count db-ids) 1) "All inputs and outputs must belong to the same database.")]
;; One reason this is here, is that I don't want the DAG module to have the single-DWH assumption.
(assoc graph :db_id (first db-ids)))))
;; TODO: Generate new metabase user for the workspace
;; TODO: Should we move this to model as per the diagram?
(defn- create-workspace-container!
"Create the workspace and its related collection, user, and api key."
[creator-id database-id workspace-name]
;; TODO (Chris 2025-11-19) Unsure API key name is unique, and remove this (insecure) workaround.
(let [api-key (let [key-name (format "API key for Workspace %s" workspace-name)]
(or (t2/select-one :model/ApiKey :name key-name)
(api-key/create-api-key-with-new-user! {:key-name key-name})))
;; TODO (Chris 2025-11-19) Associate the api-key user with the workspace as well.
ws (t2/insert-returning-instance! :model/Workspace
{:name workspace-name
:creator_id creator-id
:database_id database-id
:api_key_id (:id api-key)
:execution_user (:user_id api-key)})
coll (t2/insert-returning-instance! :model/Collection
{:name (format "Collection for Workspace %s" workspace-name)
:namespace "workspace"
:workspace_id (:id ws)})
schema (driver.common/isolation-namespace-name ws)
ws (assoc ws
:collection_id (:id coll)
:schema schema)]
;; Set the backlink from the workspace to the collection inside it and set the schema.
(t2/update! :model/Workspace (:id ws) {:collection_id (:id coll)
:schema schema})
;; TODO (Sanya 2025-11-18) - not sure how to transfer this api key to agent
#_(log/infof "Generated API key for workspace: %s" (u.secret/expose (:unmasked_key api-key)))
ws))
(defn- unique-constraint-violation?
"Check if an exception is due to a unique constraint violation."
[e]
(let [msg (str (ex-message e) (some-> (ex-cause e) ex-message))]
(or (str/includes? msg "unique")
(str/includes? msg "duplicate")
(str/includes? msg "UNIQUE"))))
;; TODO (Chris 2025-11-20) We have not added a uniqueness constraint to the db, we should either do that, or remove
;; the whole retry song and dance.
(defn- create-workspace-with-unique-name!
"Create a workspace, retrying with a new unique name if there's a constraint violation."
[creator-id db-id database ws-name graph max-retries]
(loop [attempt 1]
(let [unique-name (generate-unique-workspace-name ws-name)
result (try
{:workspace (create-workspace-container! creator-id db-id unique-name)}
(catch Exception e
(if (and (< attempt max-retries) (unique-constraint-violation? e))
{:retry true}
(throw e))))]
(if (:retry result)
(recur (inc attempt))
(let [{:keys [schema
database_details]} (ws.isolation/ensure-database-isolation! (:workspace result) database)
workspace (assoc (:workspace result)
:schema schema
:database_details database_details)
;; temp hack to avoid running mirror and failing if there are no checkouts
graph (if (not-empty (:check-outs graph))
(let [graph (ws.mirroring/mirror-entities! workspace database graph)
input-tables (when-let [table-ids (->> (:inputs graph)
(filter #(= :table (:type %)))
(map :id)
seq)]
(t2/select :model/Table :id [:in table-ids]))]
(ws.isolation/grant-read-access-to-tables! database (:user database_details) input-tables)
graph)
{})]
(t2/update! :model/Workspace {:id (:id workspace)}
{:graph graph
:schema schema
:database_details database_details})
(assoc workspace :graph graph))))))
;; TODO internal: test!
(defn create-workspace!
"Create workspace"
[creator-id {ws-name-maybe :name
maybe-db-id :database_id
upstream :upstream}]
;; TODO put this in the malli schema for a request
(assert (or maybe-db-id (some seq (vals upstream))) "Must provide a database_id unless initial entities are given.")
(let [ws-name (or ws-name-maybe (str (random-uuid)))
graph (build-graph upstream)
db-id (or (:db_id graph) maybe-db-id)
_ (when (and maybe-db-id (:db_id graph))
(assert (= maybe-db-id (:db_id graph))
"The database_id provided must match that of the upstream entities."))
_ (assert db-id "Was not given and could not infer a database_id for the workspace.")
database (api/check-500 (t2/select-one :model/Database :id db-id))]
(create-workspace-with-unique-name! creator-id db-id database ws-name graph 5)))
(defn- rebuild-workspace-graph!
"Rebuild the workspace graph from scratch based on all upstream transforms.
Returns the new graph."
[workspace-id]
(let [all-upstream-ids (t2/select-fn-vec :upstream_id :model/WorkspaceMappingTransform
:workspace_id workspace-id)
graph (build-graph {:transforms all-upstream-ids})
;; Add from-scratch transforms (those created directly in workspace, not mirrored)
;; These have workspace_id set but no entry in WorkspaceMappingTransform
mirrored-ids (t2/select-fn-set :downstream_id :model/WorkspaceMappingTransform
:workspace_id workspace-id)
from-scratch-txs (->> (t2/select [:model/Transform :id] :workspace_id workspace-id
{:where (if (seq mirrored-ids)
[:not [:in :id mirrored-ids]]
true)})
(mapv #(assoc % :type :transform)))
graph (update graph :transforms into from-scratch-txs)]
(t2/update! :model/Workspace workspace-id {:graph graph})
graph))
(defn add-entities!
"Add upstream entities to an existing workspace by mirroring them.
Returns the workspace with updated graph."
[workspace upstream]
(let [new-graph (ws.dag/path-induced-subgraph upstream)
database (t2/select-one :model/Database (:database_id workspace))]
(t2/with-transaction [_]
(ws.mirroring/mirror-entities! workspace database new-graph)
(let [graph (rebuild-workspace-graph! (:id workspace))]
(assoc workspace :graph graph)))))
(defn create-transform!
"Create a new transform directly within a workspace.
Returns the created transform with hydrated tag_ids and creator."
[workspace body creator-id]
(u/prog1
(t2/with-transaction [_]
(let [workspace-id (:id workspace)
workspace-db-id (:database_id workspace)
database (t2/select-one :model/Database workspace-db-id)
tag-ids (:tag_ids body)
body (assoc-in body [:target :database] workspace-db-id)
transform (t2/insert-returning-instance!
:model/Transform
(assoc (select-keys body [:name :description :source :target :run_trigger])
:creator_id creator-id
:workspace_id workspace-id))]
(when (seq tag-ids)
(transforms/update-transform-tags! (:id transform) tag-ids))
;; Create isolated output table for the new transform
(let [target (:target transform)
new-graph {:db_id workspace-db-id
:transforms [transform]
:inputs []
:outputs [{:id nil
:schema (:schema target)
:name (:name target)}]
:check-outs []}]
(ws.isolation/create-isolated-output-tables! workspace database new-graph))
;; Rebuild the full graph from scratch
(rebuild-workspace-graph! workspace-id)
(t2/hydrate transform :transform_tag_ids :creator)))
(events/publish-event! :event/transform-create {:object <> :user-id creator-id})))
(defn- mirror-table-to-delete-where
[database-id targets]
(let [s+t (mapv (fn [[schema name]]
[:and
[:= [:inline schema] :schema]
[:= [:inline name] :name]])
targets)]
(into [:and
[:= [:inline database-id] :db_id]
(into [:or] s+t)])))
(defn remove-entities!
"Remove multiple entities from the workspace."
[workspace entities]
;; count does not have to be 1
(assert (= 1 (count entities))
"Single kv")
(assert (= :transforms (-> entities keys first))
"For transforms")
(assert (every? pos-int? (:transforms entities))
"With seq of ids")
;; drop transform, drop its target table, no checking of dependencies
(let [mirror-transforms-ids (set (:transforms entities))
mirror-transforms-data (t2/select :model/Transform :id [:in mirror-transforms-ids])
database (t2/select-one :model/Database :id (:database_id workspace))
targets (mapv (fn [{:keys [target]}]
(assert (= "table" (:type target)))
[(:schema target) (:name target)])
mirror-transforms-data)
_ (assert (< 0 (count targets)))
tables-where-clause (mirror-table-to-delete-where (:database_id workspace) targets)
tables-data (t2/select :model/Table {:where tables-where-clause})
tables-ids (into #{} (map :id) tables-data)]
(assert (every? pos-int? tables-ids))
(when (seq targets)
(ws.isolation/drop-isolated-tables! database targets))
(when (seq tables-ids)
(t2/delete! :model/Table :id [:in tables-ids]))
(when (seq mirror-transforms-ids)
(t2/delete! :model/Transform :id [:in mirror-transforms-ids]))))
#_:clj-kondo/ignore
(comment
(defn- clean-up-ws!* [& [ws-id]]
;; Pass nil to clean up all workspaces
(let [ws-clause (or ws-id [:not= nil])]
(t2/delete! :model/Collection :workspace_id ws-clause)
(t2/delete! :model/Transform :workspace_id ws-clause)
(doseq [[db_id schema] (t2/select-fn-set (juxt :db_id :schema) :model/Table :workspace_id [:not= nil])]
(t2/delete! :model/Table :schema schema)
(let [db (t2/select-one :model/Database db_id)
driver (metabase.driver.util/database->driver db)
make-spec (requiring-resolve 'metabase.driver.sql-jdbc.connection/connection-details->spec)
jdbc-spec (make-spec driver (:details db))]
(clojure.java.jdbc/execute! jdbc-spec [(str "DROP SCHEMA \"" schema "\" CASCADE")])))
(t2/delete! :model/Workspace :id ws-clause)))
(def upstream-id (t2/select-one-pk :model/Transform :workspace_id nil {:order-by [:id]}))
(def upstream-ids (t2/select-pks-vec :model/Transform :workspace_id nil {:order-by [:id]}))
;; Ensure output table exists
(#'metabase-enterprise.transforms.execute/run-mbql-transform! (t2/select-one :model/Transform upstream-id))
(let [admin-id (t2/select-one-pk :model/User :is_superuser true {:order-by [:id]})]
(binding [api/*current-user-id* admin-id]
(create-workspace! admin-id {:name "Workplace Workspace", :upstream {:transforms upstream-ids #_[upstream-id]}})))
(clean-up-ws!*))

View File

@@ -0,0 +1,105 @@
(ns metabase-enterprise.workspaces.context
(:require
[metabase.util.i18n :refer [tru]]
[toucan2.core :as t2]))
;; TODO (lbrdnk 2025-11-26): Following should have its own module
;;;; Fetching the graph data
(defn- fetch-graph-transforms
[transform-ids]
(if (empty? transform-ids)
{}
(let [id->transform (t2/select-fn->fn :id identity :model/Transform :id [:in transform-ids])]
(when (not= (count transform-ids) (count id->transform))
(throw (ex-info (tru "Unable to fetch all transforms within the graph")
{:status-code 500
:transform-ids transform-ids
:missing-transform-ids (remove id->transform transform-ids)})))
id->transform)))
(defn- fetch-graph-tables
[table-ids]
(when (seq table-ids)
(let [id->table (into {}
(t2/select-fn->fn :id identity [:model/Table :id :schema :name]
:id [:in table-ids]))]
(assert (= (count table-ids) (count id->table)))
id->table)))
(defn- fetch-graph-resources
[graph]
;; TODO: keep -> map when we have guards ensuring outputs have id?
;; figure out the details
{:id->output-table (fetch-graph-tables (keep :id (:outputs graph)))
:id->input-table (fetch-graph-tables (keep :id (:inputs graph)))
:id->transform (fetch-graph-transforms (map :id (:transforms graph)))})
;;;; Public
(defn ->context-with-resources
"TODO (lbrdnk): Add docstring."
[graph]
{:data (fetch-graph-resources graph)
:graph graph})
;;;; Misc
(defn transform-node->data
"TODO (lbrdnk): Add docstring."
[ctx transform-node]
;; TODO: :var
(get-in ctx [:data :id->transform (:id transform-node)]))
(defn direct-dependencies
"Get direct dependencies of a node"
[ctx node]
(get-in ctx [:graph :dependencies node]))
(defn has-dependencies?
"TODO (lbrdnk): Add docstring."
[ctx node]
(boolean (seq (direct-dependencies ctx node))))
(defn- transform-node->remapped-target-rf
[ctx acc transform-dep-node]
(let [transform-data (transform-node->data ctx transform-dep-node)
{:keys [schema name]} (:target transform-data)
k [schema name]
v (get-in ctx [:src-schema+table->dst->schema+table k])]
(assoc acc k v)))
(defn transform-node->src->dst-remapping
"TODO (lbrdnk): Add docstring."
[ctx transform-node]
(let [transform-deps-nodes (direct-dependencies ctx transform-node)]
(assert (every? (comp #{:transform} :type) transform-deps-nodes))
(reduce
(partial transform-node->remapped-target-rf ctx)
{}
transform-deps-nodes)))
;; for when :dependencies hold table-nodes, now we go with transform nodes
#_(defn schema+table->schema+table-outputs-remapping
[ctx transform-node]
(not-empty
(let [deps (direct-dependencies ctx transform-node)
deps-ids (map :id deps)
src-output-id->dst-output (:src-output-id->dst-output ctx)
src-id->schema+table (-> (get-in ctx [:data :id->output-table])
(update-vals (juxt :schema :name)))]
(-> (select-keys src-output-id->dst-output deps-ids)
(update-keys src-id->schema+table)
(update-vals (juxt :schema :name))))))
(defn transform-node->py-tables-remaps
"Remaps for python transform node"
[ctx transform-node]
(let [{:keys [source]} (transform-node->data ctx transform-node)
all-mappings (:src-output-id->dst-output ctx)
transform-table-ids (-> source :source-tables vals set)]
(into {}
(keep (fn [src-id]
(when-some [dst-id (get-in all-mappings [src-id :id])]
[src-id dst-id])))
transform-table-ids)))

View File

@@ -0,0 +1,8 @@
(ns metabase-enterprise.workspaces.core
(:require
[metabase-enterprise.workspaces.isolation]
[potemkin :as p]))
(p/import-vars
[metabase-enterprise.workspaces.isolation
with-workspace-isolation])

View File

@@ -0,0 +1,261 @@
(ns metabase-enterprise.workspaces.dag
(:require
[clojure.set :as set]
[flatland.ordered.map :as ordered-map]
[toucan2.core :as t2]))
;;;; Internal helpers
(def ^:private group->db-type
"Mapping from our entity type grouping keywords to the dependency table's type strings."
{:transforms "transform"
#_#_:models "card"})
(defn- rows->entity-set
"Convert query result rows to a set of {:id :type} maps."
[rows]
(into #{}
(map (fn [{:keys [entity_type entity_id]}]
{:id entity_id :type (keyword entity_type)}))
rows))
(defn- entities-map->tuples
"Convert a map of {type ids} to a sequence of [db-type id] tuples.
`entities-by-type` is a map like {:transform [1 2 3]}."
[entities-by-type]
(assert (every? group->db-type (keys entities-by-type)) "Not all entity types are supported")
(into []
(mapcat (fn [[entity-type ids]]
(let [dep-type (group->db-type entity-type)]
(map (fn [id] [dep-type id]) ids))))
entities-by-type))
(defn- tuples->entity-set
"Convert a sequence of [type id] tuples to a set of {:id :type} maps."
[tuples]
(into #{}
(map (fn [[entity-type entity-id]]
{:id entity-id :type (keyword entity-type)}))
tuples))
(defn- tuples->starting-cte
"Build a UNION ALL of SELECT statements for the starting tuples."
[tuples]
{:union-all (mapv (fn [[t id]]
{:select [[[:inline t] :entity_type]
[[:inline id] :entity_id]]})
tuples)})
;;;; Toposort
(defn- toposort-visit [node child->parents visited result]
(cond
(visited node) [visited result]
:else (let [parents (child->parents node [])
[visited' result'] (reduce (fn [[v r] p]
(toposort-visit p child->parents v r))
[(conj visited node) result]
parents)]
[visited' (conj result' node)])))
(defn- toposort-dfs [child->parents]
;; TODO (Chris 2025-11-20): Detect cycles and throw an error. (In practice inputs will never be cyclic, but still.)
(let [all-nodes (set (keys child->parents))]
(loop [visited #{}
result []
remaining all-nodes]
(if (empty? remaining)
result
(let [node (first remaining)
[visited' result'] (toposort-visit node child->parents visited result)]
(recur visited' result' (disj remaining node)))))))
;;;; Transitive closure queries
(defn- transitive-closure
"Compute transitive closure in the given direction.
`direction` is :upstream (follow dependencies) or :downstream (follow dependents)."
[entities-by-type direction]
(let [starting-tuples (entities-map->tuples entities-by-type)
[cte-name match-type match-id result-type result-id]
(case direction
:upstream [:upstream, :d.from_entity_type, :d.from_entity_id, :d.to_entity_type, :d.to_entity_id]
:downstream [:downstream, :d.to_entity_type, :d.to_entity_id, :d.from_entity_type, :d.from_entity_id])]
(when (seq starting-tuples)
(rows->entity-set
(t2/query {:with-recursive
[[:starting (tuples->starting-cte starting-tuples)]
[cte-name {:union-all
[{:select [:entity_type :entity_id]
:from [:starting]}
{:select [[result-type :entity_type]
[result-id :entity_id]]
:from [[:dependency :d]]
:join [[cte-name :r]
[:and
[:= match-type :r.entity_type]
[:= match-id :r.entity_id]]]}]}]]
:select [:entity_type :entity_id]
:from [cte-name]})))))
(defn- upstream-entities
"Given a map of entity types to IDs, find all upstream entities (dependencies).
Returns a set of {:id :type} maps."
[entities-by-type]
(transitive-closure entities-by-type :upstream))
(defn- downstream-entities
"Given a map of entity types to IDs, find all downstream entities (dependents).
Returns a set of {:id :type} maps."
[entities-by-type]
(transitive-closure entities-by-type :downstream))
;;;; Dependency fetching
(defn- entity-dependencies
"Fetch all dependencies between the given entities within the dependency table.
Returns a map from {:id :type} to a vector of {:id :type} dependencies."
[entities]
(if (empty? entities)
{}
(let [conditions (mapv (fn [{:keys [id type]}]
[:and
[:= :from_entity_type (name type)]
[:= :from_entity_id id]])
entities)
rows (t2/query {:select [:from_entity_type :from_entity_id
:to_entity_type :to_entity_id]
:from [:dependency]
:where (into [:or] conditions)})]
(reduce (fn [acc {:keys [from_entity_type from_entity_id to_entity_type to_entity_id]}]
(let [from-entity {:id from_entity_id, :type (keyword from_entity_type)}
to-entity {:id to_entity_id, :type (keyword to_entity_type)}]
(if (contains? entities to-entity)
(update acc from-entity (fnil conj []) to-entity)
acc)))
{}
rows))))
(defn- toposort-entities
"Toposort entities based on their dependencies.
Returns entities in topological order (dependencies before dependents)."
[entity-set dependencies-map]
(let [child->parents (into {}
(map (fn [entity]
[entity (vec (get dependencies-map entity []))]))
entity-set)]
(toposort-dfs child->parents)))
(defn- table? [entity] (= :table (:type entity)))
(defn- transform? [entity] (= :transform (:type entity)))
(defn- fetch-dependent-tables
"Fetch tables that are the output targets of the given transforms.
Returns tables with their IDs if they exist in the database, or with :id nil if they don't exist yet.
This supports workspace checkout for transforms that haven't been executed yet."
[entities]
(when (seq entities)
(let [transform-ids (map :id (filter transform? entities))
transforms (when (seq transform-ids)
(t2/select [:model/Transform :source :target] :id [:in transform-ids]))]
(vec
(for [{:keys [target]} transforms, :when target]
(case (:type target)
"table"
;; Note: id will be nil if the table has not been created yet
{:id (t2/select-one-pk :model/Table
:db_id (:database target)
:schema (:schema target)
:name (:name target))
:schema (:schema target)
:name (:name target)
:type :table}
(throw (ex-info "Unsupported target type" {:target target}))))))))
(defn- toposort-key-fn [ordering]
(let [index-map (zipmap ordering (range))
index-of #(get index-map % Integer/MAX_VALUE)]
(fn [e]
[(index-of e) (:type e) (:id e)])))
;;;; Card dependency detection
;; TODO this may be overkill:
;; 1. We currently care only that there is at-least-one card being depended on.
;; 2. We know for transforms that we can only have any card dependencies if there is a first-order card dependency.
;;
;; ie. we could just do an existence check on direct ancestors.
;;
;; If we only disallow MBQL cards however, we would still need to walk transitive dependencies in
;; case we hit one further up. We could still short-circuit however.
(defn card-dependencies
"Find all card IDs that the given transforms transitively depend on.
Queries the dependency table for upstream cards.
Returns a set of card IDs, or empty set if no card dependencies."
[transform-ids]
(when (seq transform-ids)
(let [starting-tuples (mapv (fn [id] ["transform" id]) transform-ids)
rows (t2/query {:with-recursive
[[:starting (tuples->starting-cte starting-tuples)]
[:upstream {:union-all
[{:select [:entity_type :entity_id]
:from [:starting]}
{:select [[:d.to_entity_type :entity_type]
[:d.to_entity_id :entity_id]]
:from [[:dependency :d]]
:join [[:upstream :r]
[:and
[:= :d.from_entity_type :r.entity_type]
[:= :d.from_entity_id :r.entity_id]]]}]}]]
:select-distinct [:entity_id]
:from [:upstream]
:where [:= :entity_type "card"]})]
(into #{} (map :entity_id) rows))))
;;;; Public API
(defn path-induced-subgraph
"Given a map of entity types to IDs, compute the path-induced subgraph.
`entities-by-type` is a map like {:transform [1 2 3]}.
Returns a map with:
- :check-outs - the entities we want to make editable within the workspace, in topological order
- :inputs - external tables that entities in the subgraph depend on, ordered by id
- :outputs - output tables generated by entities in the subgraph, in topological order
- :transforms - transforms in the subgraph, in topological order
- :dependencies - map from each subgraph entity to its dependencies within subgraph, in topological order"
[entities-by-type]
(let [starting-set (tuples->entity-set (entities-map->tuples entities-by-type))
upstream (or (upstream-entities entities-by-type) #{})
downstream (or (downstream-entities entities-by-type) #{})
subgraph (set/intersection upstream downstream)
transforms (filterv transform? subgraph)
output-tables (or (fetch-dependent-tables transforms) [])
inputs (->> upstream
(filter table?)
(remove subgraph)
(sort-by :id)
vec)
deps-map (entity-dependencies subgraph)
sorted (toposort-entities subgraph deps-map)
;; Right now, this doesn't quite yield the comparator we want, since we have a partial ordering, and the
;; toposort is rather arbitrary in this regard. It also excludes the upstream and downstream dependencies.
;; What we'd prefer is to have the following:
;; 1. Dependencies come before their dependents.
;; 2. For independent nodes, order by type (tables before transforms), then by ID.
sort-key-fn (toposort-key-fn sorted)]
{:check-outs (vec (sort-by sort-key-fn starting-set))
;; By definition, these are all tables, and all external to the subgraph, so ID is the only thing to sort by.
;; Ideally, they'd be ordered by their own dependency relationships too, but for now this is good enough.
:inputs (vec (sort-by :id inputs))
:outputs (vec (sort-by sort-key-fn output-tables))
:transforms (vec (sort-by sort-key-fn transforms))
:dependencies (into (ordered-map/ordered-map)
(keep (fn [entity]
(when-not (table? entity)
[entity (vec (map #(if (table? %)
(first (get deps-map % []))
%)
(get deps-map entity [])))])))
sorted)}))

View File

@@ -0,0 +1,37 @@
(ns metabase-enterprise.workspaces.driver.common
"Common utilities for workspace isolation drivers."
(:require
[clojure.string :as str]
[metabase.system.core :as system]))
;;;; Naming
;; re-using https://github.com/metabase/metabase/pull/61887/commits/c92e4a9cc451c61a13fef19ed9d6107873b17f07
;; (original ws isolation code)
(defn- instance-uuid-slug
"Create a slug from the site UUID, taking the first character of each section."
[site-uuid-string]
(apply str (map first (str/split site-uuid-string #"-"))))
(defn isolation-namespace-name
"Generate namespace/database name for workspace isolation following mb__isolation_<slug>_<workspace-id> pattern.
Uses 'namespace' as the generic term that maps to 'schema' in Postgres, 'database' in ClickHouse, etc."
[workspace]
(assert (some? (:id workspace)) "Workspace must have an :id")
(let [instance-slug (instance-uuid-slug (str (system/site-uuid)))
clean-workspace-id (str/replace (str (:id workspace)) #"[^a-zA-Z0-9]" "_")]
(format "mb__isolation_%s_%s" instance-slug clean-workspace-id)))
(defn isolated-table-name
"Generate name for a table mirroring transform target table in the isolated database namespace."
[{:keys [schema name] :as _source-table}]
;; TODO: This naming scheme is not guaranteed to give a unique name, even if it's likely
;; to always be unique in practice. Consider adding a hash suffix or conflict detection
;; before merging to master.
(format "%s__%s" schema name))
(defn isolation-user-name
"Generate username for workspace isolation."
[workspace]
(let [instance-slug (instance-uuid-slug (str (system/site-uuid)))]
(format "mb_isolation_%s_%s" instance-slug (:id workspace))))

View File

@@ -0,0 +1,84 @@
(ns metabase-enterprise.workspaces.driver.h2
"H2-specific implementations for workspace isolation.
H2 differences from PostgreSQL:
- H2 doesn't support CREATE USER with PASSWORD syntax like Postgres
- H2 uses CREATE USER userName PASSWORD 'password'
- H2 GRANT syntax: GRANT { SELECT | ALL } ON { SCHEMA schemaName | tableName } TO userName
- H2 supports CREATE TABLE ... AS SELECT syntax for table duplication"
(:require
[clojure.java.jdbc :as jdbc]
[metabase-enterprise.workspaces.driver.common :as driver.common]
[metabase-enterprise.workspaces.isolation :as isolation]
[metabase-enterprise.workspaces.sync :as ws.sync]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.util :as driver.u]))
(defmethod isolation/grant-read-access-to-tables! :h2
[database username tables]
(let [driver (driver.u/database->driver database)
jdbc-spec (sql-jdbc.conn/connection-details->spec driver (:details database))
schemas (distinct (map :schema tables))]
;; H2 uses GRANT SELECT ON SCHEMA schemaName TO userName
(doseq [schema schemas]
(jdbc/execute! jdbc-spec [(format "GRANT SELECT ON SCHEMA \"%s\" TO \"%s\"" schema username)]))
;; Also grant on individual tables for more fine-grained access
(doseq [table tables]
(jdbc/execute! jdbc-spec [(format "GRANT SELECT ON \"%s\".\"%s\" TO \"%s\""
(:schema table) (:name table) username)]))))
(defmethod isolation/init-workspace-database-isolation! :h2
[database workspace]
(let [driver (driver.u/database->driver database)
schema-name (driver.common/isolation-namespace-name workspace)
;; H2 uses simpler user creation syntax
read-user {:user (driver.common/isolation-user-name workspace)
:password (str (random-uuid))}
jdbc-spec (sql-jdbc.conn/connection-details->spec driver (:details database))]
(doseq [sql [(format "CREATE SCHEMA IF NOT EXISTS \"%s\"" schema-name)
;; H2 syntax: CREATE USER userName PASSWORD 'password'
(format "CREATE USER IF NOT EXISTS \"%s\" PASSWORD '%s'"
(:user read-user) (:password read-user))
;; Grant access on the isolation schema
(format "GRANT ALL ON SCHEMA \"%s\" TO \"%s\"" schema-name (:user read-user))]]
(jdbc/execute! jdbc-spec [sql]))
{:schema schema-name
:database_details read-user}))
(defmethod isolation/duplicate-output-table! :h2
[database workspace output]
(let [details (:details database)
driver* (driver.u/database->driver database)
jdbc-spec (sql-jdbc.conn/connection-details->spec driver* details)
source-schema (:schema output)
source-table (:name output)
isolated-schema (:schema workspace)
isolated-table (driver.common/isolated-table-name output)]
(assert (every? some? [source-schema source-table isolated-schema isolated-table])
"All table identifiers must be present")
;; H2 supports CREATE TABLE AS SELECT syntax
;; Using LIMIT 0 instead of WITH NO DATA (which is Postgres-specific)
(doseq [sql [(format "CREATE TABLE \"%s\".\"%s\" AS SELECT * FROM \"%s\".\"%s\" WHERE 1=0"
isolated-schema
isolated-table
source-schema
source-table)
;; Grant ownership/access to the isolation user
(format "GRANT ALL ON \"%s\".\"%s\" TO \"%s\""
isolated-schema
isolated-table
(-> workspace :database_details :user))]]
(jdbc/execute! jdbc-spec [sql]))
(let [table-metadata (ws.sync/sync-transform-mirror! database isolated-schema isolated-table)]
(select-keys table-metadata [:id :schema :name]))))
(defmethod isolation/drop-isolated-tables! :h2
[database s+t-tuples]
(let [driver (driver.u/database->driver database)
jdbc-spec (sql-jdbc.conn/connection-details->spec driver (:details database))]
(run!
(fn [[schema-name table-name]]
(jdbc/execute! jdbc-spec
[(format "DROP TABLE IF EXISTS \"%s\".\"%s\""
schema-name table-name)]))
s+t-tuples)))

View File

@@ -0,0 +1,6 @@
(ns metabase-enterprise.workspaces.driver.init
"Load all workspace driver implementations.
Require this namespace to ensure all driver methods are registered."
(:require
[metabase-enterprise.workspaces.driver.h2]
[metabase-enterprise.workspaces.driver.postgres]))

View File

@@ -0,0 +1,70 @@
(ns metabase-enterprise.workspaces.driver.postgres
"Postgres-specific implementations for workspace isolation."
(:require
[clojure.java.jdbc :as jdbc]
[metabase-enterprise.workspaces.driver.common :as driver.common]
[metabase-enterprise.workspaces.isolation :as isolation]
[metabase-enterprise.workspaces.sync :as ws.sync]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.util :as driver.u]))
(defmethod isolation/grant-read-access-to-tables! :postgres
[database username tables]
(let [driver (driver.u/database->driver database)
jdbc-spec (sql-jdbc.conn/connection-details->spec driver (:details database))
schemas (distinct (map :schema tables))]
(doseq [schema schemas]
(jdbc/execute! jdbc-spec [(format "GRANT USAGE ON SCHEMA %s TO %s" schema username)]))
(doseq [table tables]
(jdbc/execute! jdbc-spec [(format "GRANT SELECT ON TABLE %s.%s TO %s" (:schema table) (:name table) username)]))))
(defmethod isolation/init-workspace-database-isolation! :postgres
[database workspace]
(let [driver (driver.u/database->driver database)
schema-name (driver.common/isolation-namespace-name workspace)
read-user {:user (driver.common/isolation-user-name workspace)
:password (str (random-uuid))}
jdbc-spec (sql-jdbc.conn/connection-details->spec driver (:details database))]
(doseq [sql [(format "CREATE SCHEMA %s" schema-name)
(format "CREATE USER %s WITH PASSWORD '%s'" (:user read-user) (:password read-user))
;; grant all access on the destination schema
(format "GRANT USAGE ON SCHEMA %s TO %s" schema-name (:user read-user))
;; need to be able insert and dropping, rename tables from this chema
(format "GRANT ALL PRIVILEGES ON SCHEMA %s TO %s" schema-name (:user read-user))]]
(jdbc/execute! jdbc-spec [sql]))
{:schema schema-name
:database_details read-user}))
(defmethod isolation/duplicate-output-table! :postgres
[database workspace output]
(let [details (:details database)
driver* (driver.u/database->driver database)
jdbc-spec (sql-jdbc.conn/connection-details->spec driver* details)
source-schema (:schema output)
source-table (:name output)
isolated-schema (:schema workspace)
isolated-table (driver.common/isolated-table-name output)]
(assert (every? some? [source-schema source-table isolated-schema isolated-table]) "Figured out table")
;; TODO: execute the following only if the transform was previously executed and its table exists.
(doseq [sql [(format (str "CREATE TABLE \"%s\".\"%s\""
" AS SELECT * FROM \"%s\".\"%s\""
"WITH NO DATA")
isolated-schema
isolated-table
source-schema
source-table)
(format "ALTER TABLE \"%s\".\"%s\" OWNER TO %s" isolated-schema isolated-table (-> workspace :database_details :user))]]
(jdbc/execute! jdbc-spec [sql]))
(let [table-metadata (ws.sync/sync-transform-mirror! database isolated-schema isolated-table)]
(select-keys table-metadata [:id :schema :name]))))
(defmethod isolation/drop-isolated-tables! :postgres
[database s+t-tuples]
(let [driver (driver.u/database->driver database)
jdbc-spec (sql-jdbc.conn/connection-details->spec driver (:details database))]
(run!
(fn [[schema-name table-name]]
(jdbc/execute! jdbc-spec
[(format "DROP TABLE IF EXISTS \"%s\".\"%s\""
schema-name table-name)]))
s+t-tuples)))

View File

@@ -0,0 +1,6 @@
(ns metabase-enterprise.workspaces.init
(:require
[metabase-enterprise.workspaces.driver.h2]
[metabase-enterprise.workspaces.driver.postgres]
[metabase-enterprise.workspaces.models.workspace]
[metabase-enterprise.workspaces.models.workspace-mapping]))

View File

@@ -0,0 +1,110 @@
(ns metabase-enterprise.workspaces.isolation
(:require
[metabase-enterprise.workspaces.driver.common :as driver.common]
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
[toucan2.core :as t2]))
;;;; Driver multimethods
;; Implementations are in metabase-enterprise.workspaces.driver.{postgres,h2}
(defn dispatch-on-engine
"Take engine from database `db` and dispatch on that."
[database & _args]
(driver.u/database->driver database))
(defmulti grant-read-access-to-tables!
"Grant read access to these tables."
{:added "0.59.0" :arglists '([database username tables])}
#'dispatch-on-engine
:hierarchy #'driver/hierarchy)
(defmulti init-workspace-database-isolation!
"Create database isolation for a workspace. Return the database details."
{:added "0.59.0" :arglists '([database workspace])}
#'dispatch-on-engine
:hierarchy #'driver/hierarchy)
(defmulti duplicate-output-table!
"Create an isolated copy of the given output tables, for a workspace transform to write to.
TODO: Consider removing this method once we have 'remap-on-execute' semantics, where
transforms write directly to the isolated location without needing to duplicate existing tables."
{:added "0.59.0" :arglists '([database workspace output])}
#'dispatch-on-engine
:hierarchy #'driver/hierarchy)
(defmulti drop-isolated-tables!
"Drop isolated tables"
{:added "0.59.0" :arglists '([database s+t-tuples])}
#'dispatch-on-engine)
;;;; Public API
(defn create-isolated-output-tables!
"Create new _isolated tables_ to correspond to the outputs of the upstream graph.
Decorate the graph outputs with the mapping to the new tables.
For outputs where the table doesn't exist yet (id=nil), includes them in the graph with the intended
isolated table location, but skips actual table duplication."
;; TODO: Should be part of mirroring!
[workspace database ctx]
(let [graph (:graph ctx)
existing-output-ids (keep :id (:outputs graph))
;; TODO (Chris 2025-11-20) Avoid querying again here, let's have this data passed down as part of the graph
table-by-id (when (seq existing-output-ids)
(into {}
(map (juxt :id identity))
(t2/select [:model/Table :id :name :schema] :id [:in existing-output-ids])))
outputs (for [upstream-output (:outputs graph)]
(if (:id upstream-output)
;; Table exists, duplicate it
(let [hydrated-output (merge upstream-output (get table-by-id (:id upstream-output)))
isolated-table (duplicate-output-table! database workspace hydrated-output)]
(t2/insert! :model/WorkspaceMappingTable
{:upstream_id (:id upstream-output)
:downstream_id (:id isolated-table)
:workspace_id (:id workspace)})
(assoc hydrated-output :mapping isolated-table))
;; Table doesn't exist yet, provide the intended isolated location
(let [isolated-schema (driver.common/isolation-namespace-name workspace)
isolated-name (driver.common/isolated-table-name upstream-output)]
(assoc upstream-output :mapping {:id nil
:schema isolated-schema
:name isolated-name}))))
src-output-id->dst-output
(into {}
(comp (filter :id)
(map (fn [{:keys [id mapping]}]
[id mapping])))
outputs)
src-schema+table->dst->schema+table
(into {}
(map (fn [{:keys [mapping] :as output}]
(let [src-schema (:schema output)
src-table (:name output)
dst-schema (:schema mapping)
dst-table (:name mapping)]
[[src-schema src-table]
[dst-schema dst-table]])))
outputs)]
(-> ctx
(update :graph assoc :outputs (vec outputs))
(assoc :src-output-id->dst-output src-output-id->dst-output)
(assoc :src-schema+table->dst->schema+table src-schema+table->dst->schema+table))))
(defn ensure-database-isolation!
"Wrapper around the driver method, to make migrations easier in future."
[workspace database]
(init-workspace-database-isolation! database workspace))
(defn do-with-workspace-isolation
"Impl of* with-workspace-isolation*."
[workspace thunk]
(driver/with-swapped-connection-details (:database_id workspace)
(:database_details workspace)
(thunk)))
(defmacro with-workspace-isolation
"Execute body with necessary isolation."
[workspace & body]
`(do-with-workspace-isolation ~workspace (fn [] ~@body)))

View File

@@ -0,0 +1,48 @@
(ns metabase-enterprise.workspaces.mirroring
(:require
[metabase-enterprise.workspaces.context :as ws.ctx]
[metabase-enterprise.workspaces.isolation :as ws.isolation]
[metabase-enterprise.workspaces.remap :as ws.remap]
[toucan2.core :as t2]))
(def ^:private transform-mirrored-keys
#{:description :name :source :target :source_type})
(defn- mirror-data
[transform-node ctx workspace-id]
(merge {:workspace_id workspace-id}
(select-keys (ws.ctx/transform-node->data ctx transform-node)
transform-mirrored-keys)
(ws.remap/remap-for-transform transform-node ctx)))
;; TODO (lbrdnk 2025-11-26): Make this logic pure, move inserts upstream into single insert.
(defn- mirror-transform!
"Create the mirroring transform and transform mapping rows for a single transform."
[workspace ctx transform-node]
(let [mirror-data (mirror-data transform-node ctx (:id workspace))
mirror (t2/insert-returning-instance! :model/Transform mirror-data)
;; TODO (lbrdnk 2025-11-26): Strong gut feeling we want "mapping" represented differently.
;; Motivation: Transforms _created_ in the workspace have no "original" map in graph
;; where they could live. Having some transforms in "mapping" and some
;; elsewhere feels dirty.
graph-node (assoc transform-node :mapping (select-keys mirror [:id :name]))]
(t2/insert! :model/WorkspaceMappingTransform
{:upstream_id (:id transform-node)
:downstream_id (:id mirror)
:workspace_id (:id workspace)})
(update-in ctx [:graph :transforms] #(conj (or % []) graph-node))))
(defn- mirror-transforms!
[workspace ctx]
(reduce #(mirror-transform! workspace %1 %2)
(update ctx :graph dissoc :transforms)
(-> ctx :graph :transforms)))
(defn mirror-entities!
"TODO (lbrdnk): Add docstring."
[workspace database graph]
(let [ctx (ws.ctx/->context-with-resources graph)]
(->> ctx
(ws.isolation/create-isolated-output-tables! workspace database)
(mirror-transforms! workspace)
:graph)))

View File

@@ -0,0 +1,47 @@
(ns metabase-enterprise.workspaces.models.workspace
(:require
[metabase.models.interface :as mi]
[metabase.util.json :as json]
[methodical.core :as methodical]
[toucan2.core :as t2]))
(def ^:private type->grouping "How models are grouped in lists"
;; do not forget to update `entities` if you add stuff here
{:transform :transforms})
(methodical/defmethod t2/table-name :model/Workspace [_model] :workspace)
(t2/deftransforms :model/Workspace
{:graph mi/transform-json
:database_details mi/transform-encrypted-json})
(doto :model/Workspace
(derive :metabase/model)
(derive :hook/timestamped?))
(defn- entities [entity-type ids]
(->> (case entity-type
:transform
(t2/select [:model/Transform
:id :name :description :source :target :source_type :creator_id :collection_id
:created_at :updated_at :wmt.workspace_id :wmt.upstream_id
[:upstream.target :upstream]]
:transform.workspace_id [:in ids]
{:left-join [[:workspace_mapping_transform :wmt] [:= :wmt.downstream_id :transform.id]
[:transform :upstream] [:= :upstream.id :wmt.upstream_id]]}))
(mapv #(-> %
(update :upstream json/decode+kw)
(assoc :type entity-type)))))
(defn- ws-contents [ids]
(fn []
(->> (mapcat #(entities % ids) (keys type->grouping))
(reduce (fn [acc e]
(update-in acc [(:workspace_id e) (type->grouping (:type e))] (fnil conj []) e))
{}))))
(methodical/defmethod t2/batched-hydrate [:model/Workspace :contents]
"Batch hydrate `Workspace` contents"
[_model k wses]
(let [ids (mapv :id wses)]
(mi/instances-with-hydrated-data wses k (ws-contents ids) :id {:default {}})))

View File

@@ -0,0 +1,18 @@
(ns metabase-enterprise.workspaces.models.workspace-mapping
(:require
[methodical.core :as methodical]
[toucan2.core :as t2]))
(methodical/defmethod t2/table-name :model/WorkspaceMappingTable [_model] :workspace_mapping_table)
(methodical/defmethod t2/primary-keys :model/WorkspaceMappingTable [_model] [:upstream_id :downstream_id])
(doto :model/WorkspaceMappingTable
(derive :metabase/model))
(methodical/defmethod t2/table-name :model/WorkspaceMappingTransform [_model] :workspace_mapping_transform)
(methodical/defmethod t2/primary-keys :model/WorkspaceMappingTransform [_model] [:upstream_id :downstream_id])
(doto :model/WorkspaceMappingTransform
(derive :metabase/model))

View File

@@ -0,0 +1,114 @@
(ns metabase-enterprise.workspaces.promotion
"Functionality for promoting workspace transforms back to main Metabase."
(:require
[metabase.util :as u]
[metabase.util.log :as log]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
(defn- find-upstream-xf
"Find the original transform that was mirrored into this workspace.
Uses the workspace_mapping_transform table to find the upstream transform."
[xf workspace-id]
(or (when-let [mapping (t2/select-one :model/WorkspaceMappingTransform
:downstream_id (:id xf)
:workspace_id workspace-id)]
(t2/select-one :model/Transform :id (:upstream_id mapping)))
(throw (ex-info "Original transform not found" {:transform-id (:id xf)
:workspace-id workspace-id}))))
(defn- promote-single-transform!
"Promote a single workspace transform to replace its original.
Returns a map with :id, :name, and :status."
[{:keys [target] :as downstream-xf} upstream-xf new->old-s+n]
(try
;; TODO assert database is what we expect
(assert (= "table" (:type target)) "Only table targets are supported for promotion.")
(let [upstream-id (:id upstream-xf)
actual (or (new->old-s+n (select-keys target [:schema :name]))
(select-keys (:target upstream-xf) [:schema :name]))
_ (assert (:name actual) "Unable to find the upstream table to sub for the isolated one.")
remapped-xf (-> downstream-xf
;; TODO: do query remapping here, if the transform is dependent on an isolated table.
;; ... with the current FE this is not possible! :-D
(update :target merge actual))
;; TODO revisit whether there are any other fields we want
updates (select-keys remapped-xf [:name :description :source :target])]
(log/infof "Promoting transform %d -> %d: %s" (:id downstream-xf) upstream-id (:name upstream-xf))
(t2/update! :model/Transform upstream-id updates)
(assoc updates :id upstream-id))
(catch Throwable e
(log/errorf e "Failed to promote transform %s" (:name downstream-xf))
{:id (:id downstream-xf)
:name (:name downstream-xf)
:error (ex-message e)})))
(defn- transform-ids [graph]
(->> (:transforms graph) (keep :mapping) (mapv :id)))
(defn promote-transforms!
"Promote all transforms in a workspace back to main Metabase.
This will:
1. Get all workspace transforms
2. Order them by dependencies (using the workspace graph)
3. For each transform:
- Find the original transform
- Update it with the workspace version's definition
- Re-execute it in the original schema
Returns a map with:
- :promoted - sequence of {:id, :name} maps
- :errors - any transforms that failed to promote"
[ws]
(log/infof "Starting promotion of transforms from workspace %s" (:id ws))
(let [ids (transform-ids (:graph ws))
xs (if (seq ids)
(t2/select :model/Transform :id [:in ids])
;; fallback for when there is no graph
(t2/select :model/Transform :workspace_id (:id ws)))
_ (log/infof "Found %d workspace transforms to promote" (count xs))
new->old-s+n (u/for-map [{:keys [old_schema old_name new_schema new_name]}
(t2/query {:select [[:t1.schema :old_schema]
[:t1.name :old_name]
[:t2.schema :new_schema]
[:t2.name :new_name]]
:from [[(t2/table-name :model/WorkspaceMappingTable) :m]
[(t2/table-name :model/Table) :t1]
[(t2/table-name :model/Table) :t2]]
:where [:and
[:= :m.workspace_id (:id ws)]
[:= :t1.id :m.upstream_id]
[:= :t2.id :m.downstream_id]]})]
[{:schema new_schema :name new_name}
{:schema old_schema :name old_name}])
results (for [xf xs
:let [upstream-xf (find-upstream-xf xf (:id ws))]]
(promote-single-transform! xf upstream-xf new->old-s+n))
{promoted false
errors true} (group-by #(contains? % :error) results)]
(log/infof "Promotion complete. %d succeeded, %d failed"
(count promoted)
(count errors))
{:promoted promoted
:errors errors}))
(comment
(let [ws {:id 836}]
(t2/query {:select [[:t1.schema :old_schema]
[:t1.name :old_name]
[:t2.schema :new_schema]
[:t2.name :new_name]]
:from [[(t2/table-name :model/WorkspaceMappingTable) :m]
[(t2/table-name :model/Table) :t1]
[(t2/table-name :model/Table) :t2]]
:where [:and
[:= :m.workspace_id (:id ws)]
[:= :t1.id :m.upstream_id]
[:= :t2.id :m.downstream_id]]}))
(promote-transforms! (t2/select-one :model/Workspace :id 836)))

View File

@@ -0,0 +1,95 @@
(ns metabase-enterprise.workspaces.remap
(:require
[macaw.core :as macaw]
[metabase-enterprise.workspaces.context :as ws.ctx]))
;;;; Transform source remapping
(defn- remapped-transform-source-dispatch
[transform-node graph]
(let [{source-type :source_type} (ws.ctx/transform-node->data graph transform-node)]
source-type))
(defmulti remapped-transform-source
"todo"
{:added "0.59.0" :arglists '([transform-node graph])}
#'remapped-transform-source-dispatch)
;;;; Transform native source remaping impl.
(defn- mappings-for-macaw-rf
[acc [[src-schema src-table]
[dst-schema dst-table]]]
(assert (or (not (contains? (:schemas acc) src-schema))
(= dst-schema (get-in acc [:schemas src-schema])))
"Mapping mismatch")
(-> acc
(assoc-in [:schemas src-schema] dst-schema)
(assoc-in [:tables {:schema src-schema :table src-table}] dst-table)))
(defn- macaw-renames
[src-schema+table->dst-schema+table]
(reduce mappings-for-macaw-rf
{}
src-schema+table->dst-schema+table))
(defn- native-mappings
[transform-node ctx]
(ws.ctx/transform-node->src->dst-remapping ctx transform-node)
;; for when :dependencies hold table-nodes, now we go with transform nodes
#_(ws.ctx/schema+table->schema+table-outputs-remapping ctx transform-node))
(defn- remap-sql-tables
[sql-str macaw-mappings]
(macaw/replace-names sql-str macaw-mappings))
(def ^:private query-sql-path
[:query :stages 0 :native])
(defmethod remapped-transform-source :native
[transform-node ctx]
(let [mappings (native-mappings transform-node ctx)
renames (macaw-renames mappings)
transform-data (ws.ctx/transform-node->data ctx transform-node)
{:keys [source]} transform-data
native-str (get-in source query-sql-path)
remapped-str (remap-sql-tables native-str renames)]
(assoc-in source query-sql-path remapped-str)))
;;;; Python
(defmethod remapped-transform-source :python
[transform-node ctx]
(let [{:keys [source]} (ws.ctx/transform-node->data ctx transform-node)
remaps (ws.ctx/transform-node->py-tables-remaps ctx transform-node)
val-remapper (fn [id] (get remaps id id))]
(update source :source-tables update-vals val-remapper)))
;;;; Public
(defn- source-for-mirrored-transform
[transform-node ctx]
(let [transform-data (ws.ctx/transform-node->data ctx transform-node)
{source :source} transform-data]
(if-not (ws.ctx/has-dependencies? ctx transform-node)
source
(remapped-transform-source transform-node ctx))))
(defn target-for-mirrored-transform
"This should happen to every transform"
[transform-node ctx]
(let [data (ws.ctx/transform-node->data ctx transform-node)
target (:target data)
[dst-schema dst-table] (get (:src-schema+table->dst->schema+table ctx)
[(:schema target) (:name target)])]
(-> target
(assoc :schema dst-schema)
(assoc :name dst-table))))
(defn remap-for-transform
"Return _transform data_ to be used to create transform duplicate."
[transform-node ctx]
(let [dst-target (target-for-mirrored-transform transform-node ctx)
dst-source (source-for-mirrored-transform transform-node ctx)]
{:target dst-target
:source dst-source}))

View File

@@ -0,0 +1,26 @@
(ns metabase-enterprise.workspaces.sync
;; TODO (Chris 2025-11-20) -- Decide whether to mirror metadata manually, and drop sync dependency
(:require
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
^{:clj-kondo/ignore [:metabase/modules]}
[metabase.sync.core :as sync]
^{:clj-kondo/ignore [:metabase/modules]}
[metabase.sync.sync-metadata.tables :as sync-tables]))
;; this ns will be probably moved
;; will figure out whether that is a good idea soonish
;; this may be tricky!!!
(defn sync-transform-mirror!
"TODO"
[database schema-name table-name]
(let [driver (driver.u/database->driver database)
table {:name table-name
:schema schema-name}]
(if (driver/table-exists? driver database table)
(let [created (sync-tables/create-or-reactivate-table! database table)]
(doto created sync/sync-table!))
(throw (ex-info "Can not find transform mirror table"
{:schema schema-name
:table table-name})))))

View File

@@ -169,6 +169,20 @@
:table #{checkins-id venues-id}}
(calculation/upstream-deps:transform transform)))))))
(deftest ^:parallel upstream-deps-python-transform-test
(mt/with-premium-features #{:transforms}
(let [products-id (mt/id :products)
orders-id (mt/id :orders)]
(mt/with-temp [:model/Transform transform {:name "Test Transform"
:source {:type :python
:source-tables {"PRODUCTS" products-id
"ORDERS" orders-id}
:body "..."}
:target {:schema "PUBLIC"
:name "test_output"}}]
(is (= {:table #{products-id orders-id}}
(calculation/upstream-deps:transform transform)))))))
(deftest ^:parallel upstream-deps-card-native-with-parameter-source-test
(let [mp (mt/metadata-provider)
products-id (mt/id :products)

View File

@@ -96,6 +96,8 @@
:model/TransformJobRun
:model/TransformJobTransformTag
:model/TransformTransformTag
:model/WorkspaceMappingTable
:model/WorkspaceMappingTransform
:model/PythonLibrary
:model/Undo
:model/User
@@ -106,7 +108,8 @@
:model/ConnectionImpersonation
:model/CloudMigration
:model/Comment
:model/CommentReaction})
:model/CommentReaction
:model/Workspace})
(deftest ^:parallel comprehensive-entity-id-test
(let [entity-id-models (->> (v2.entity-ids/toucan-models)

View File

@@ -1,6 +1,7 @@
(ns ^:mb/driver-tests metabase-enterprise.transforms.api-test
"Tests for /api/transform endpoints."
(:require
[clojure.string :as str]
[clojure.test :refer :all]
[medley.core :as m]
[metabase-enterprise.transforms.query-test-util :as query-test-util]
@@ -269,6 +270,229 @@
(is (=? [{:id t2-id}]
(mt/user-http-request :crowberto :get 200 "ee/transform" :tag_ids [tag2-id])))))))))
(deftest filter-transforms-by-workspace-test
(testing "GET /api/ee/transform filters out workspace transforms by default"
(mt/with-premium-features #{:transforms :workspaces}
(mt/with-temp [:model/Transform {global-id :id} {}
:model/Workspace {ws-id :id} {:name "Test Workspace"}
:model/Transform {ws-transform-id :id} {:workspace_id ws-id}]
(testing "by default, only global transforms (workspace_id=null) are returned"
(let [transforms (mt/user-http-request :crowberto :get 200 "ee/transform")
ids (set (map :id transforms))]
(is (contains? ids global-id))
(is (not (contains? ids ws-transform-id)))))))))
(deftest exclude-workspace-id-filter-test
(testing "GET /api/ee/transform with exclude_workspace_id excludes mirrored transforms"
(mt/with-premium-features #{:transforms :workspaces}
(mt/with-temp [;; Global transforms (workspace_id = null)
:model/Transform {global1-id :id} {:name "Global Transform 1"}
:model/Transform {global2-id :id} {:name "Global Transform 2"}
:model/Transform {global3-id :id} {:name "Global Transform 3"}
;; Workspace
:model/Workspace {ws-id :id} {:name "Test Workspace"}
;; Workspace transforms (mirrored from global1 and global2)
:model/Transform {ws-xf1-id :id} {:name "Workspace Transform 1"
:workspace_id ws-id}
:model/Transform {ws-xf2-id :id} {:name "Workspace Transform 2"
:workspace_id ws-id}
;; Mappings: global1 -> ws-xf1, global2 -> ws-xf2
:model/WorkspaceMappingTransform _ {:upstream_id global1-id
:downstream_id ws-xf1-id
:workspace_id ws-id}
:model/WorkspaceMappingTransform _ {:upstream_id global2-id
:downstream_id ws-xf2-id
:workspace_id ws-id}]
(testing "without exclude_workspace_id, returns all global transforms"
(let [transforms (mt/user-http-request :crowberto :get 200 "ee/transform")
ids (set (map :id transforms))]
(is (contains? ids global1-id))
(is (contains? ids global2-id))
(is (contains? ids global3-id))
;; workspace transforms should NOT be returned (they have workspace_id set)
(is (not (contains? ids ws-xf1-id)))
(is (not (contains? ids ws-xf2-id)))))
(testing "with exclude_workspace_id, excludes global transforms mirrored into that workspace"
(let [transforms (mt/user-http-request :crowberto :get 200 "ee/transform"
:exclude_workspace_id ws-id)
ids (set (map :id transforms))]
;; global1 and global2 are mirrored into the workspace, so they should be excluded
(is (not (contains? ids global1-id)))
(is (not (contains? ids global2-id)))
;; global3 is NOT mirrored, so it should be returned
(is (contains? ids global3-id))
;; workspace transforms should still NOT be returned
(is (not (contains? ids ws-xf1-id)))
(is (not (contains? ids ws-xf2-id))))))))
(testing "exclude_workspace_id only excludes transforms mirrored into that specific workspace"
(mt/with-premium-features #{:transforms :workspaces}
(mt/with-temp [;; Global transforms
:model/Transform {global1-id :id} {:name "Global Transform 1"}
:model/Transform {global2-id :id} {:name "Global Transform 2"}
;; Two workspaces
:model/Workspace {ws1-id :id} {:name "Workspace 1"}
:model/Workspace {ws2-id :id} {:name "Workspace 2"}
;; global1 mirrored only into ws1
:model/Transform {ws1-xf-id :id} {:name "WS1 Transform"
:workspace_id ws1-id}
:model/WorkspaceMappingTransform _ {:upstream_id global1-id
:downstream_id ws1-xf-id
:workspace_id ws1-id}
;; global2 mirrored only into ws2
:model/Transform {ws2-xf-id :id} {:name "WS2 Transform"
:workspace_id ws2-id}
:model/WorkspaceMappingTransform _ {:upstream_id global2-id
:downstream_id ws2-xf-id
:workspace_id ws2-id}]
(testing "exclude_workspace_id=ws1 excludes only global1"
(let [transforms (mt/user-http-request :crowberto :get 200 "ee/transform"
:exclude_workspace_id ws1-id)
ids (set (map :id transforms))]
(is (not (contains? ids global1-id)))
(is (contains? ids global2-id))))
(testing "exclude_workspace_id=ws2 excludes only global2"
(let [transforms (mt/user-http-request :crowberto :get 200 "ee/transform"
:exclude_workspace_id ws2-id)
ids (set (map :id transforms))]
(is (contains? ids global1-id))
(is (not (contains? ids global2-id)))))))))
(deftest filter-transforms-by-type-test
(testing "should be able to filter transforms by source type"
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/with-premium-features #{:transforms :transforms-python}
(mt/with-temp [:model/Transform {native-transform-id :id}
{:name "Native Transform"
:source {:type "query"
:query {:database (mt/id)
:type "native"
:native {:query "SELECT 1"
:template-tags {}}}}
:target {:type "table"
:name (str "test_native_" (u/generate-nano-id))}}
:model/Transform {python-transform-id :id}
{:name "Python Transform"
:source {:type "python"
:body "print('hello')"
:source-tables {}}
:target {:type "table"
:name (str "test_python_" (u/generate-nano-id))
:database (mt/id)}}]
(testing "filter by native type"
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :type ["native"])]
(is (some #(= native-transform-id (:id %)) results))
(is (not (some #(= python-transform-id (:id %)) results)))))
(testing "filter by python type"
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :type ["python"])]
(is (some #(= python-transform-id (:id %)) results))
(is (not (some #(= native-transform-id (:id %)) results)))))
(testing "filter by both types returns all"
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :type ["native" "python"])]
(is (some #(= native-transform-id (:id %)) results))
(is (some #(= python-transform-id (:id %)) results)))))))))
(deftest filter-transforms-by-database-id-test
(testing "should be able to filter transforms by database_id"
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/with-premium-features #{:transforms}
(mt/with-temp [:model/Database {other-db-id :id} {:engine :h2 :details {}}
:model/Transform {transform1-id :id}
{:name "Transform for main DB"
:source {:type "query"
:query {:database (mt/id)
:type "native"
:native {:query "SELECT 1"
:template-tags {}}}}
:target {:type "table"
:name (str "test_main_" (u/generate-nano-id))}}
:model/Transform {transform2-id :id}
{:name "Transform for other DB"
:source {:type "query"
:query {:database other-db-id
:type "native"
:native {:query "SELECT 2"
:template-tags {}}}}
:target {:type "table"
:name (str "test_other_" (u/generate-nano-id))}}]
(testing "filter by main database id"
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :database_id (mt/id))]
(is (some #(= transform1-id (:id %)) results))
(is (not (some #(= transform2-id (:id %)) results)))))
(testing "filter by other database id"
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :database_id other-db-id)]
(is (some #(= transform2-id (:id %)) results))
(is (not (some #(= transform1-id (:id %)) results)))))
(testing "filter by non-existent database returns empty for our transforms"
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :database_id 999999)]
(is (not (some #(= transform1-id (:id %)) results)))
(is (not (some #(= transform2-id (:id %)) results))))))))))
(deftest filter-transforms-by-database-id-target-test
(testing "should match transforms where target database matches"
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/with-premium-features #{:transforms :transforms-python}
(mt/with-temp [:model/Database {target-db-id :id} {:engine :h2 :details {}}
:model/Transform {transform-id :id}
{:name "Python Transform with target DB"
:source {:type "python"
:body "print('hello')"
:source-tables {}}
:target {:type "table"
:name (str "test_target_" (u/generate-nano-id))
:database target-db-id}}]
(testing "filter by target database id"
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :database_id target-db-id)]
(is (some #(= transform-id (:id %)) results)))))))))
(deftest filter-transforms-combined-test
(testing "should be able to combine database_id and type filters"
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/with-premium-features #{:transforms :transforms-python}
(mt/with-temp [:model/Database {other-db-id :id} {:engine :h2 :details {}}
:model/Transform {query-main-id :id}
{:name "Query on main DB"
:source {:type "query"
:query {:database (mt/id)
:type "native"
:native {:query "SELECT 1"
:template-tags {}}}}
:target {:type "table"
:name (str "test_qm_" (u/generate-nano-id))}}
:model/Transform {python-main-id :id}
{:name "Python on main DB"
:source {:type "python"
:body "print('hello')"
:source-tables {}}
:target {:type "table"
:name (str "test_pm_" (u/generate-nano-id))
:database (mt/id)}}
:model/Transform {query-other-id :id}
{:name "Query on other DB"
:source {:type "query"
:query {:database other-db-id
:type "native"
:native {:query "SELECT 2"
:template-tags {}}}}
:target {:type "table"
:name (str "test_qo_" (u/generate-nano-id))}}]
(testing "filter by main database and query type"
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform"
:database_id (mt/id)
:type ["query"])]
(is (some #(= query-main-id (:id %)) results))
(is (not (some #(= python-main-id (:id %)) results)))
(is (not (some #(= query-other-id (:id %)) results)))))
(testing "filter by main database and python type"
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform"
:database_id (mt/id)
:type ["python"])]
(is (some #(= python-main-id (:id %)) results))
(is (not (some #(= query-main-id (:id %)) results)))
(is (not (some #(= query-other-id (:id %)) results))))))))))
(deftest get-transforms-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/with-premium-features #{:transforms}

View File

@@ -108,3 +108,81 @@
(catch Exception _e
;; Ignore cleanup errors
nil)))))))))
;;; ------------------------------------------------------------
;;; Filter xf tests
;;; ------------------------------------------------------------
(deftest ^:parallel type-filter-xf-test
(testing "->type-filter-xf filters transforms by source type"
;; Note: :query :type "query" = MBQL, :query :type "native" = native SQL
(let [mbql-transform {:id 1 :source {:type "query" :query {:type "query"}}}
native-transform {:id 2 :source {:type "query" :query {:type "native"}}}
python-transform {:id 3 :source {:type "python"}}
transforms [mbql-transform native-transform python-transform]]
(are [x y] (= x (into [] (transforms.util/->type-filter-xf y) transforms))
transforms nil
transforms []
[mbql-transform] ["query"]
[native-transform] ["native"]
[python-transform] ["python"]
[mbql-transform native-transform] ["query" "native"]
transforms ["query" "native" "python"]
[] ["whatever"]))))
(deftest ^:parallel database-id-filter-xf-test
(testing "->database-id-filter-xf filters transforms by database ID"
(let [db1-query-x {:id 1
:name "Query on DB1"
:source {:type "query" :query {:database 10}}
:target {:type "table" :name "t1" :database 1}}
db2-target-x {:id 2
:name "Python targeting DB4"
:source {:type "python" :body "" :source-tables {}}
:target {:type "table" :name "t4" :database 2}}
transforms [db1-query-x db2-target-x]]
(are [x y] (= x (into [] (transforms.util/->database-id-filter-xf y) transforms))
transforms nil
[db1-query-x] 1
[] 10 ; source does not match
[db2-target-x] 2
[] 999))))
(deftest ^:parallel matching-timestamp?-test
(testing "matching-timestamp? checks if a timestamp falls within a date range [start, end)"
(let [matching-timestamp? #'transforms.util/matching-timestamp?
field-path [:start_time]
range-jan-feb {:start "2024-01-01T00:00:00Z" :end "2024-02-01T00:00:00Z"}
range-start-only {:start "2024-01-01T00:00:00Z" :end nil}
range-end-only {:start nil :end "2024-02-01T00:00:00Z"}]
(testing "with both start and end bounds"
(are [expected timestamp]
(= expected (matching-timestamp? {:start_time timestamp} field-path range-jan-feb))
nil nil ; missing field returns nil
true "2024-01-15T12:00:00Z" ; timestamp in middle of range
false "2023-12-15T12:00:00Z" ; timestamp before range
false "2024-02-15T12:00:00Z" ; timestamp after range
true "2024-01-01T00:00:00Z" ; start boundary is inclusive
true "2024-02-01T00:00:00Z")) ; end boundary is inclusive too 🤷
(testing "with only start bound"
(are [expected timestamp]
(= expected (matching-timestamp? {:start_time timestamp} field-path range-start-only))
true "2024-01-15T12:00:00Z" ; timestamp after start
true "2024-02-15T12:00:00Z" ; any timestamp after start
false "2023-12-15T12:00:00Z")) ; timestamp before start
(testing "with only end bound"
(are [expected timestamp]
(= expected (matching-timestamp? {:start_time timestamp} field-path range-end-only))
true "2024-01-15T12:00:00Z" ; timestamp before end
true "2023-12-15T12:00:00Z" ; any timestamp before end
false "2024-02-15T12:00:00Z")) ; timestamp after end
(testing "returns nil when field value is missing"
(are [job] (nil? (matching-timestamp? job field-path range-jan-feb))
{}
{:other "value"})))))

View File

@@ -0,0 +1,468 @@
(ns ^:mb/driver-tests metabase-enterprise.workspaces.api-test
(:require
[clojure.string :as str]
[clojure.test :refer :all]
[metabase-enterprise.transforms.interface :as transforms.i]
[metabase-enterprise.transforms.test-dataset :as transforms-dataset]
[metabase-enterprise.transforms.test-util :refer [with-transform-cleanup!]]
[metabase.lib.core :as lib]
[metabase.search.test-util :as search.tu]
[metabase.test :as mt]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
(defn ws-url [id path]
(str "ee/workspace/" id path))
(deftest workspace-endpoints-require-superuser-test
(mt/with-temp [:model/Workspace workspace {:name "Private Workspace"}]
(testing "GET /api/ee/workspace requires superuser"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :get 403 "ee/workspace"))))
(testing "GET /api/ee/workspace/:id requires superuser"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :get 403 (str "ee/workspace/" (:id workspace))))))
(testing "POST /api/ee/workspace requires superuser"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :post 403 "ee/workspace"
{:name "Unauthorized Workspace"})))))
(mt/with-temp [:model/Workspace workspace {:name "Put Workspace"}]
(testing "PUT /api/ee/workspace/:id requires superuser"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :put 403 (str "ee/workspace/" (:id workspace))
{:name "Updated"})))))
(mt/with-temp [:model/Workspace workspace {:name "Delete Workspace"}]
(testing "DELETE /api/ee/workspace/:id requires superuser"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :delete 403 (str "ee/workspace/" (:id workspace)))))))
(mt/with-temp [:model/Workspace workspace {:name "Promote Workspace"}]
(testing "POST /api/ee/workspace/:id/promote requires superuser"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :post 403 (str "ee/workspace/" (:id workspace) "/merge")))))))
(deftest workspace-crud-flow-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/with-premium-features [:workspaces :dependencies :transforms]
(mt/with-model-cleanup [:model/Collection :model/Workspace :model/Transform]
(let [tx-id (->> (t2/select :model/Transform :workspace_id nil)
(filter #(= (mt/id) (get-in % [:target :database])))
(sort-by :id >)
first
:id)
workspace-name (str "Workspace " (random-uuid))
;; Create workspace with or without upstream transforms depending on availability
created (mt/user-http-request :crowberto :post 200 "ee/workspace"
(cond-> {:name workspace-name
:database_id (mt/id)}
tx-id (assoc :upstream {:transforms [tx-id]})))
workspace-id (:id created)
collection-id (:collection_id created)]
(is (=? {:id int?
:collection_id int?
:name workspace-name}
created))
(is (t2/exists? :model/Workspace :id workspace-id :collection_id collection-id))
(is (t2/exists? :model/Collection :id collection-id :workspace_id workspace-id))
(testing "workspace appears in list response"
(let [{:keys [items]} (mt/user-http-request :crowberto :get 200 "ee/workspace")]
(is (some #(= workspace-id (:id %)) items))))
(testing "workspace can be fetched individually"
(let [response (mt/user-http-request :crowberto :get 200 (str "ee/workspace/" workspace-id))]
(is (= workspace-id (:id response)))))
#_(testing "workspace can be archived"
(let [updated (mt/user-http-request :crowberto :post 200 (str "ee/workspace/" workspace-id "/archive"))]
(is (some? (:archived_at updated))))))))))
(deftest ^:parallel promote-workspace-test
(testing "POST /api/ee/workspace/:id/promote requires superuser"
(mt/with-temp [:model/Workspace workspace {:name "Promote Test"}]
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :post 403 (str "ee/workspace/" (:id workspace) "/merge"))))))
(testing "Cannot promote an already archived workspace"
(mt/with-temp [:model/Workspace workspace {:name "Archived Workspace"
:archived_at (java.time.OffsetDateTime/now)}]
(is (= "Cannot promote an already archived workspace"
(mt/user-http-request :crowberto :post 400 (str "ee/workspace/" (:id workspace) "/merge")))))))
(deftest merge-workspace-with-transform-test
(search.tu/with-index-disabled
(testing "POST /api/ee/workspace/:id/merge promotes transforms and archives workspace"
(mt/with-premium-features #{:workspaces :dependencies}
(mt/with-temp [:model/Table _table {:schema "public" :name "merge_test_table"}
:model/Transform x1 {:name "Upstream Transform"
:description "Original description"
:target {:type "table"
:database 1
:schema "public"
:name "merge_test_table"}}]
(mt/with-model-cleanup [:model/Workspace :model/Transform :model/WorkspaceMappingTransform]
(let [{ws-id :id} (mt/user-http-request :crowberto :post 200 "ee/workspace"
{:name "Merge test"
:database_id (mt/id)
:upstream {:transforms [(:id x1)]}})]
(testing "We've got our workspace with transform to merge"
(is (int? ws-id))
;; (sanya) TODO: maybe switch to using transform APIs once we get our own
(let [x2-id (t2/select-one-fn :downstream_id :model/WorkspaceMappingTransform :upstream_id (:id x1))]
(t2/update! :model/Transform :id x2-id {:description "Modified in workspace"})))
(testing "returns promoted transforms"
(is (=? {:promoted [{:id (:id x1)}]
:workspace {:id ws-id :name "Merge test"}}
(mt/user-http-request :crowberto :post 200
(str "ee/workspace/" ws-id "/merge")))))
(testing "original transform was updated with workspace version"
(is (= "Modified in workspace"
(t2/select-one-fn :description :model/Transform :id (:id x1)))))
(testing "workspace was deleted after successful merge"
(is (nil? (t2/select-one :model/Workspace :id ws-id)))))))))))
(deftest create-workspace-transform-permissions-test
(testing "POST /api/ee/workspace/:id/transform requires superuser"
(mt/with-temp [:model/Workspace workspace {:name "Transform Test"}]
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :post 403
(str "ee/workspace/" (:id workspace) "/transform")
{:name "Should Fail"
:source {:type "query"
:query {}}
:target {:type "table"
:name "should_fail"}}))))))
(deftest create-workspace-transform-archived-test
(testing "Cannot create transform in archived workspace"
(mt/with-premium-features [:transforms]
(mt/with-temp [:model/Workspace workspace {:name "Archived"
:archived_at (java.time.OffsetDateTime/now)}]
(is (= "Cannot create transforms in an archived workspace"
(mt/user-http-request :crowberto :post 400
(str "ee/workspace/" (:id workspace) "/transform")
{:name "Should Fail"
:source {:type "query"
:query (mt/mbql-query venues)}
:target {:type "table"
:name "should_fail"}})))))))
(deftest add-entities-to-workspace-test
(testing "Add entities to workspace"
(search.tu/with-index-disabled
(mt/with-premium-features [:workspaces :dependencies :transforms]
(with-transform-cleanup! [orig-name "ws_tables_test"]
(mt/with-temp [:model/Transform {x1-id :id} {:target {:type "table"
:database (mt/id)
:schema "public"
:name orig-name}}
:model/Transform {x2-id :id} {:target {:type "table"
:database (mt/id)
:schema "public"
:name orig-name}}]
(mt/with-model-cleanup [:model/Collection :model/Workspace :model/Transform :model/WorkspaceMappingTransform]
(let [workspace-id (:id (mt/user-http-request :crowberto :post 200 "ee/workspace"
{:name "Add Entities Test"
:database_id (mt/id)
:upstream {:transforms [x1-id]}}))]
(is (int? workspace-id))
(testing "Can add new entities to workspace"
(is (=? {:contents {:transforms #(>= (count %) 2)}}
(mt/user-http-request :crowberto :post 200
(str "ee/workspace/" workspace-id "/contents")
{:add {:transforms [x2-id]}}))))
(testing "Adding duplicate entity is a noop"
(is (=? {:contents {:transforms #(>= (count %) 2)}}
(mt/user-http-request :crowberto :post 200
(str "ee/workspace/" workspace-id "/contents")
{:add {:transforms [x1-id]}}))))
(testing "Cannot add entities to archived workspace"
(t2/update! :model/Workspace workspace-id {:archived_at (java.time.OffsetDateTime/now)})
(is (= "Cannot add entities to an archived workspace"
(mt/user-http-request :crowberto :post 400
(str "ee/workspace/" workspace-id "/contents")
{:add {:transforms [x2-id]}}))))))))))))
(deftest add-entities-requires-superuser-test
(testing "POST /api/ee/workspace/:id/add requires superuser"
(mt/with-temp [:model/Workspace workspace {:name "Permission Test"}]
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :post 403
(str "ee/workspace/" (:id workspace) "/contents")
{:add {:transforms [1]}}))))))
(deftest add-entities-no-nested-branching-test
(testing "Cannot add transforms that belong to another workspace (no nested branching)"
(mt/with-temp [:model/Workspace workspace-1 {:name "Workspace 1"}
:model/Workspace workspace-2 {:name "Workspace 2"}
:model/Transform transform {:name "Downstream Transform"
:workspace_id (:id workspace-1)}]
(is (= "Cannot add transforms that belong to another workspace"
(mt/user-http-request :crowberto :post 400
(str "ee/workspace/" (:id workspace-2) "/contents")
{:add {:transforms [(:id transform)]}}))))))
(deftest create-workspace-transform-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/with-premium-features #{:workspaces :transforms}
(mt/dataset transforms-dataset/transforms-test
(mt/with-model-cleanup [:model/Workspace :model/Transform :model/Collection]
(let [workspace-id (:id (mt/user-http-request :crowberto :post 200 "ee/workspace"
{:name "Test Workspace"
:database_id (mt/id)}))]
(with-transform-cleanup! [table-name "workspace_transform_test"]
(is (=? {:id pos-int?
:workspace_id workspace-id
:creator_id (mt/user->id :crowberto)
:target {:database (mt/id)}}
(mt/user-http-request :crowberto :post 200
(str "ee/workspace/" workspace-id "/transform")
{:name "Workspace Transform"
:source {:type "query"
:query (mt/mbql-query transforms_products)}
:target {:type "table"
:name table-name}}))))))))))
(deftest tables-endpoint-empty-ws-test
(mt/test-driver
:postgres
(mt/with-model-cleanup [:model/Workspace]
(let [user-id (mt/user->id :crowberto)
ws (mt/user-http-request :crowberto :post 200 "ee/workspace/"
{:name "My test ws"
:creator_id user-id
:database_id (mt/id)})]
(is (= {:inputs []
:outputs []}
(mt/user-http-request :crowberto :get 200
(str "ee/workspace/" (:id ws) "/tables"))))))))
(deftest tables-endpoint-transform-not-run-test
(mt/test-driver
:postgres
(mt/with-model-cleanup [:model/Workspace :model/Transform :model/Collection]
(let [mp (mt/metadata-provider)
query (lib/native-query mp "select * from orders limit 10;")
orig-schema "public"]
(with-transform-cleanup! [orig-name "ws_tables_test"]
(mt/with-temp [:model/Transform x1 {:source_type "native"
:name "My X1"
:source {:type "query"
:query query}
:target {:type "table"
:database (mt/id)
:schema "public"
:name orig-name}}]
(let [;; create the workspace
workspace (mt/user-http-request :crowberto :post 200 "ee/workspace"
{:name "Test Workspace"
:database_id (mt/id)})
;; add the transform
_ (mt/user-http-request :crowberto :post 200
(str "ee/workspace/" (:id workspace) "/contents")
{:add {:transforms [(:id x1)]}})
;; get the tables
tables-result (mt/user-http-request :crowberto :get 200
(str "ee/workspace/" (:id workspace) "/tables"))]
(testing "/tables returns expected results"
(is (= {:inputs [{:id (mt/id :orders) :schema orig-schema :table "orders"}]
:outputs []}
tables-result))))))))))
(deftest tables-endpoint-test
(mt/test-driver
:postgres
(mt/with-model-cleanup [:model/Workspace :model/Transform :model/Collection]
(let [mp (mt/metadata-provider)
query (lib/native-query mp "select * from orders limit 10;")
orig-schema "public"]
(with-transform-cleanup! [orig-name "ws_tables_test"]
(mt/with-temp [:model/Transform x1 {:source_type "native"
:name "My X1"
:source {:type "query"
:query query}
:target {:type "table"
:database (mt/id)
:schema "public"
:name orig-name}}]
;; create the target table
(transforms.i/execute! x1 {:run-method :manual})
(let [;; create the workspace
workspace (mt/user-http-request :crowberto :post 200 "ee/workspace"
{:name "Test Workspace"
:database_id (mt/id)})
;; add the transform
_ (mt/user-http-request :crowberto :post 200
(str "ee/workspace/" (:id workspace) "/contents")
{:add {:transforms [(:id x1)]}})
;; get the tables
tables-result (mt/user-http-request :crowberto :get 200
(str "ee/workspace/" (:id workspace) "/tables"))
mirror-transform (t2/select-one :model/Transform :workspace_id (:id workspace))
mirror-table (t2/select-one :model/Table
:schema (-> mirror-transform :target :schema)
:name (-> mirror-transform :target :name))]
(testing "/tables returns expected results"
(is (=? {:inputs [{:id (mt/id :orders) :schema orig-schema :table "orders"}]
:outputs
[{:global {:schema orig-schema :table orig-name}
:workspace {:transform-id (:id mirror-transform)
:table-id (:id mirror-table)}}]}
tables-result))))))))))
;;;; Card dependency rejection tests
(defn- query-with-source-card
"Create a pMBQL query that uses a card as its source."
[card-id]
{:lib/type :mbql/query
:database (mt/id)
:stages [{:lib/type :mbql.stage/mbql
:source-card card-id}]})
(defn- create-transform-with-card-source!
"Create a transform whose source query depends on a card.
The after-insert hook triggers dependency calculation automatically."
[card]
(t2/insert-returning-instance! :model/Transform
{:name "Transform depending on card"
:source {:type :query
:query (query-with-source-card (:id card))}
:target {:type "table"
:database (mt/id)
:schema "public"
:name "card_dep_output"}}))
(deftest create-workspace-rejects-card-dependencies-test
(testing "Cannot create workspace with transforms that depend on cards"
(mt/with-premium-features #{:workspaces :dependencies :transforms}
(mt/with-model-cleanup [:model/Dependency :model/Transform]
(mt/with-temp [:model/Card card {:name "Test Card"
:database_id (mt/id)
:dataset_query (mt/mbql-query venues)}]
(let [tx (create-transform-with-card-source! card)
response (mt/user-http-request :crowberto :post 400 "ee/workspace"
{:name "Card Dep Workspace"
:database_id (mt/id)
:upstream {:transforms [(:id tx)]}})]
(is (re-find #"Cannot add transforms that depend on saved questions" response))))))))
(deftest create-workspace-rejects-transitive-card-dependencies-test
(testing "Cannot create workspace with transforms that transitively depend on cards"
(mt/with-premium-features #{:workspaces :dependencies :transforms}
(mt/with-model-cleanup [:model/Dependency :model/Transform]
(mt/with-temp [:model/Card card {:name "Base Card"
:database_id (mt/id)
:dataset_query (mt/mbql-query venues)}]
;; tx1 depends on card
(let [tx1 (create-transform-with-card-source! card)
;; tx2 depends on tx1 (via a manually created dependency - simulating transform chain)
tx2 (t2/insert-returning-instance! :model/Transform
{:name "Transform 2 - depends on tx1"
:source {:type :query
:query {:database (mt/id)
:type :native
:native {:query "SELECT 1"}}}
:target {:type "table"
:database (mt/id)
:schema "public"
:name "tx2_output"}})]
;; Create dependency: tx2 depends on tx1
(t2/insert! :model/Dependency
{:from_entity_type "transform"
:from_entity_id (:id tx2)
:to_entity_type "transform"
:to_entity_id (:id tx1)})
;; Try to create workspace with tx2 (which transitively depends on card via tx1)
(let [response (mt/user-http-request :crowberto :post 400 "ee/workspace"
{:name "Transitive Card Dep Workspace"
:database_id (mt/id)
:upstream {:transforms [(:id tx2)]}})]
(is (re-find #"Cannot add transforms that depend on saved questions" response)))))))))
(deftest rename-workspace-test
(search.tu/with-index-disabled
(testing "POST /api/ee/workspace/:id/name updates the workspace name"
(mt/with-model-cleanup [:model/Collection :model/Workspace]
(let [workspace (mt/user-http-request :crowberto :post 200 "ee/workspace"
{:name "Original Name"
:database_id (mt/id)})
response (mt/user-http-request :crowberto :post 200
(str "ee/workspace/" (:id workspace) "/name")
{:name "Updated Name"})]
(is (= "Updated Name"
(:name response)
(t2/select-one-fn :name :model/Workspace :id (:id workspace))))))))
(testing "Requires superuser"
(mt/with-temp [:model/Workspace workspace {:name "Permission Test"}]
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :post 403
(str "ee/workspace/" (:id workspace) "/name")
{:name "Should Fail"})))))
(testing "Cannot rename an archived workspace"
(mt/with-temp [:model/Workspace workspace {:name "Archived"
:archived_at (java.time.OffsetDateTime/now)}]
(is (= "Cannot update an archived workspace"
(mt/user-http-request :crowberto :post 400
(str "ee/workspace/" (:id workspace) "/name")
{:name "Should Fail"}))))))
(deftest add-entities-rejects-card-dependencies-test
(testing "Cannot add transforms with card dependencies to existing workspace"
(mt/with-premium-features #{:workspaces :dependencies :transforms}
(mt/with-model-cleanup [:model/Workspace :model/Dependency :model/Collection :model/Transform]
(mt/with-temp [:model/Card card {:name "Test Card"
:database_id (mt/id)
:dataset_query (mt/mbql-query venues)}]
(let [tx (create-transform-with-card-source! card)
;; Create a workspace without the card-dependent transform
workspace-id (:id (mt/user-http-request :crowberto :post 200 "ee/workspace"
{:name "Empty Workspace"
:database_id (mt/id)}))
response (mt/user-http-request :crowberto :post 400 (ws-url workspace-id "/contents")
{:add {:transforms [(:id tx)]}})]
(is (re-find #"Cannot add transforms that depend on saved questions" response))))))))
(deftest validate-target-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/with-premium-features #{:transforms}
(let [table (t2/select-one :model/Table :active true)]
(mt/with-temp [:model/Workspace {ws-id :id} {:name "test"}
:model/Transform _x1 {:workspace_id ws-id
:target {:database (:db_id table)
:type "table"
:schema (:schema table)
:name (str "q_" (:name table))}}]
(testing "Unique"
(is (= "OK"
(mt/with-log-level [metabase.driver.sql-jdbc.sync.describe-table :fatal]
(mt/user-http-request :crowberto :post 200 (ws-url ws-id "/validate-target")
{:db_id (mt/id)
:target {:type "table"
:schema "public"
:name (str/replace (str (random-uuid)) "-" "_")}})))))
(testing "Conflict outside of workspace"
(is (= "A table with that name already exists."
(mt/user-http-request :crowberto :post 403 (ws-url ws-id "/validate-target")
{:db_id (:db_id table)
:target {:type "table"
:schema (:schema table)
:name (:name table)}}))))
(testing "Conflict inside of workspace"
(let [table (t2/select-one :model/Table :active true)
tn (str (random-uuid))]
(is (= "Another transform in this workspace already targets that table."
(mt/with-log-level [metabase.driver.sql-jdbc.sync.describe-table :fatal]
(mt/user-http-request :crowberto :post 403 (ws-url ws-id "/validate-target")
{:db_id (:db_id table)
:target {:type "table"
:schema (:schema table)
:name (str "q_" (:name table))}})))))))))))

View File

@@ -0,0 +1,422 @@
(ns ^:mb/driver-tests metabase-enterprise.workspaces.common-test
(:require
[clojure.java.jdbc :as jdbc]
[clojure.test :refer [deftest testing is]]
[medley.core :as m]
[metabase-enterprise.transforms.interface :as transforms.i]
[metabase-enterprise.transforms.test-util :as transforms.tu]
[metabase-enterprise.workspaces.common :as ws.common]
[metabase.driver :as driver]
[metabase.driver.sql-jdbc.connection :as driver.conn]
[metabase.lib.core :as lib]
[metabase.test :as mt]
[toucan2.core :as t2]))
(defn- setup!
[schema]
(let [spec (driver.conn/connection-details->spec :postgres (:details (mt/db)))]
(jdbc/execute! spec (format "DROP SCHEMA IF EXISTS %s CASCADE" schema))
(jdbc/execute! spec (format "CREATE SCHEMA %s" schema))))
(defn teardown!
[ws-id]
(let [[ws-graph ws-schema] (t2/select-one-fn (juxt :graph :schema) :model/Workspace :id ws-id)
spec (driver.conn/connection-details->spec :postgres (:details (mt/db)))
output-ids (-> ws-graph :outputs (->> (map :id)))
dup-tables-ids (into #{}
(map #(-> % :mapping :id))
(-> ws-graph :outputs))]
(when (seq dup-tables-ids)
(t2/delete! :model/Table :id [:in dup-tables-ids]))
(when (seq output-ids)
(t2/delete! :model/Table :id [:in output-ids]))
(when (some? ws-id)
(t2/delete! :model/Workspace :id ws-id))
(when (string? (not-empty ws-schema))
(jdbc/execute! spec (format "DROP SCHEMA IF EXISTS %s CASCADE" ws-schema)))))
(deftest create-workspace-independent-native-transforms-test
(mt/test-driver
:postgres
(let [mp (mt/metadata-provider)
native-base (lib/native-query mp "select * from ORDERS LIMIT 10")
test-schema "test_schema"
new-workspace (atom nil)]
(try
(setup! test-schema)
(mt/with-temp
[:model/Transform x1 {:name "x1"
:source_type :native
:source {:type :query
:query native-base}
:target {:type :table
:database (mt/id)
:schema test-schema
:name "t1"}}
:model/Transform x2 {:name "x2"
:source_type :native
:source {:type :query
:query (assoc-in native-base [:stages 0 :native]
(str "SELECT *\n"
"FROM PRODUCTS\n"
"LIMIT 10\n"))}
:target {:type :table
:database (mt/id)
:schema test-schema
:name "t2"}}
:model/Transform x3 {:name "x3"
:source_type :native
:source {:type :query
:query (assoc-in native-base [:stages 0 :native]
(str "SELECT *\n"
"FROM PEOPLE\n"
"LIMIT 10\n"))}
:target {:type :table
:database (mt/id)
:schema test-schema
:name "t3"}}]
(transforms.i/execute! x1 {:run-method :manual})
(transforms.i/execute! x2 {:run-method :manual})
(transforms.i/execute! x3 {:run-method :manual})
(let [result (mt/with-current-user (mt/user->id :crowberto)
(ws.common/create-workspace! (mt/user->id :crowberto)
{:name "test_ws"
:database_id (mt/id)
:upstream {:transforms (mapv :id [x1 x2 x3])}}))
checkouts (-> result :graph :check-outs)
transforms (-> result :graph :transforms)]
(reset! new-workspace result)
(testing "Name is correct"
(is (= "test_ws" (:name result))))
(testing "Checkouts are as expected"
(is (= [{:type :transform :id (:id x1)}
{:type :transform :id (:id x2)}
{:type :transform :id (:id x3)}]
(sort-by :id checkouts))))
(testing "Transforms look as expected"
(is (= #{{:id (:id x1), :type :transform, :mapping {#_#_:id pos-int?, :name "x1"}}
{:id (:id x2), :type :transform, :mapping {#_#_:id pos-int?, :name "x2"}}
{:id (:id x3), :type :transform, :mapping {#_#_:id pos-int?, :name "x3"}}}
(into #{} (map #(m/update-existing % :mapping dissoc :id)) transforms))))))
;; For development, subject to change
(catch Throwable t
(def ttt t)
(throw t))
(finally
(teardown! (:id @new-workspace)))))))
(deftest mirroring-multiple-interdependent-native-transforms-test
(mt/test-driver
:postgres
(let [mp (mt/metadata-provider)
native-base (lib/native-query mp "select * from ORDERS LIMIT 10")
test-schema "test_schema"
new-workspace (atom nil)]
(try
(setup! test-schema)
(testing "Interdependent native transforms"
(mt/with-temp
[:model/Transform x1 {:name "x1"
:source_type :native
:source {:type :query
:query native-base}
:target {:type :table
:database (mt/id)
:schema test-schema
:name "t1"}}
:model/Transform x2 {:name "x2"
:source_type :native
:source {:type :query
:query (assoc-in native-base [:stages 0 :native]
(str "SELECT *\n"
(format "FROM \"%s\".\"%s\"\n" test-schema "t1")
"LIMIT 10\n"))}
:target {:type :table
:database (mt/id)
:schema test-schema
:name "t2"}}
:model/Transform x3 {:name "x3"
:source_type :native
:source {:type :query
:query (assoc-in native-base [:stages 0 :native]
(str "SELECT *\n"
(format "FROM \"%s\".\"%s\"\n" test-schema "t2")
"LIMIT 10\n"))}
:target {:type :table
:database (mt/id)
:schema test-schema
:name "t3"}}]
(transforms.i/execute! x1 {:run-method :manual})
(transforms.i/execute! x2 {:run-method :manual})
(transforms.i/execute! x3 {:run-method :manual})
(let [result-ws (mt/with-current-user (mt/user->id :crowberto)
(ws.common/create-workspace! (mt/user->id :crowberto)
{:name "test_ws"
:database_id (mt/id)
:upstream {:transforms (mapv :id [x1 x3])}}))
checkouts (-> result-ws :graph :check-outs)
transforms (-> result-ws :graph :transforms)]
(reset! new-workspace result-ws)
(testing "Workspace has expected name"
(is (= "test_ws" (:name result-ws))))
(testing "Checkouts contain only the explicitly checked out transforms"
(is (= #{{:type :transform :id (:id x1)}
{:type :transform :id (:id x3)}}
(set checkouts))))
(testing "Workspace has correct mappings in graph"
(is (= #{{:id (:id x1), :type :transform, :mapping {#_#_:id pos-int?, :name "x1"}}
{:id (:id x2), :type :transform, :mapping {#_#_:id pos-int?, :name "x2"}}
{:id (:id x3), :type :transform, :mapping {#_#_:id pos-int?, :name "x3"}}}
(into #{} (map #(m/update-existing % :mapping dissoc :id)) transforms))))
(let [orig-id->mirror-id (into {}
(map (fn [{:keys [id mapping]}]
[id (:id mapping)]))
transforms)
mirror-id->mirror (t2/select-fn->fn :id identity :model/Transform :id
[:in (vals orig-id->mirror-id)])
x1-mirror (-> (:id x1) orig-id->mirror-id mirror-id->mirror)
x2-mirror (-> (:id x2) orig-id->mirror-id mirror-id->mirror)
x3-mirror (-> (:id x3) orig-id->mirror-id mirror-id->mirror)]
(testing "X1 was properly mirrored"
(testing "name"
(is (= "x1" (:name x1-mirror))))
(testing "target adjusted"
(is (=? {:type "table"
:database (mt/id)
:schema (:schema result-ws)
;; table name = orig schema + __ + orig target name atm
:name (str test-schema "__" (-> x1 :target :name))}
(:target x1-mirror))))
(testing "source not modifed"
(is (=? (:source x1)
(:source x1-mirror)))))
(testing "X2 was properly mirrored"
(testing "name"
(is (= (:name x2) (:name x2-mirror))))
(testing "target"
(is (=? {:type "table"
:database (mt/id)
:schema (:schema result-ws)
:name (str test-schema "__" (-> x2 :target :name))}
(:target x2-mirror))))
(testing "source query referencing target of x1 mirror"
;; TODO (lbrdnk 2025-11-27): Quotes are removed, figure out whether what has
;; some unwanted consequences.
(is (=? {:query {:stages [{:native (format "SELECT *\nFROM %s.%s\nLIMIT 10\n"
(-> x1-mirror :target :schema)
(-> x1-mirror :target :name))}]}}
(:source x2-mirror)))))
(testing "X3 was properly mirrored"
(testing "name"
(is (= (:name x3) (:name x3-mirror))))
(testing "target"
(is (=? {:type "table"
:database (mt/id)
:schema (:schema result-ws)
:name (str test-schema "__" (-> x3 :target :name))}
(:target x3-mirror))))
(testing "source query referencing target of x2 mirror"
(is (=? {:query {:stages [{:native (format "SELECT *\nFROM %s.%s\nLIMIT 10\n"
(-> x2-mirror :target :schema)
(-> x2-mirror :target :name))}]}}
(:source x3-mirror)))))))))
;; For development, subject to change
(catch Throwable t
(def ttt t)
(throw t))
(finally
(teardown! (:id @new-workspace)))))))
(deftest python-transforms-checkout-test
(mt/test-driver
:postgres
(let [target-schema "public"]
(try (transforms.tu/with-transform-cleanup! [t1 "t1"
t2 "t2"]
(mt/with-temp
[:model/Transform x1 {:name "initial"
:source {:type "python"
:source-database (mt/id)
:source-tables {"checkins" (mt/id :checkins)
"venues" (mt/id :venues)}
:body (str "import pandas as pd\n"
"\n"
"def transform(checkins, venues):\n"
" return checkins.set_index('venue_id').join(venues.set_index('id'), lsuffix='_venues')\n")}
:target {:type "table"
:database (mt/id)
:schema target-schema
:name t1}}]
(transforms.i/execute! x1 {:run-method :manual})
(let [t1-table (t2/select-one :model/Table :name t1)]
(mt/with-temp [:model/Transform x2 {:name "dependent"
:source {:type "python"
:source-database (mt/id)
:source-tables {"t1" (:id t1-table)}
:body (str "import pandas as pd\n"
"\n"
"def transform(t1):\n"
" return t1")}
:target {:type "table"
:database (mt/id)
:schema target-schema
:name t2}}]
(transforms.i/execute! x2 {:run-method :manual})
(let [user-id (mt/user->id :crowberto)
result-workspace (mt/with-test-user :crowberto
(ws.common/create-workspace! user-id
{:name "my test workspace x"
:database_id (mt/id)
:upstream {:transforms [(:id x1) (:id x2)]}}))
mirror-initial (t2/select-one :model/Transform :name "initial" :workspace_id (:id result-workspace))
mirror-dependent (t2/select-one :model/Transform :name "dependent" :workspace_id (:id result-workspace))
t1-mirror (t2/select-one :model/Table
:schema (-> mirror-initial :target :schema)
:name (-> mirror-initial :target :name))]
(testing "targets"
(is (=? {:name (str "public__" (-> x1 :target :name))
:schema (:schema result-workspace)}
(:target mirror-initial)))
(is (=? {:name (str "public__" (-> x2 :target :name))
:schema (:schema result-workspace)}
(:target mirror-dependent))))
(testing "sources"
(is (=? {:source-tables {"venues" (mt/id :venues)
"checkins" (mt/id :checkins)}}
(:source mirror-initial)))
(is (=? {:source-tables {"t1" (:id t1-mirror)}}
(:source mirror-dependent)))))))))
(catch Throwable t
(def ttt t)
(throw t))))))
(deftest remove-transforms-test
(mt/test-driver
:postgres
(let [mp (mt/metadata-provider)
q1 (lib/native-query mp "select * from orders limit 10;")
q2 (lib/native-query mp "select * from products limit 10;")
test-schema "my_test_schema"
ws-id (atom nil)]
(try
(setup! test-schema)
(transforms.tu/with-transform-cleanup! [t1 "t1"
t2 "t2"]
(mt/with-temp
[:model/Transform x1 {:source_type :native
:source {:type :query
:query q1}
:target {:type :table
:database (mt/id)
:schema test-schema
:name t1}}
:model/Transform x2 {:source_type :native
:source {:type :query
:query q2}
:target {:type :table
:database (mt/id)
:schema test-schema
:name t2}}]
;; Create the target tables
(transforms.i/execute! x1 {:run-method :manual})
(transforms.i/execute! x2 {:run-method :manual})
;; Add into a workspace
(let [creator-id (mt/user->id :crowberto)
ws (mt/with-test-user :crowberto
(ws.common/create-workspace! creator-id {:name "my test ws xxx"
:database_id (mt/id)
:upstream {:transforms [(:id x1) (:id x2)]}}))
_ (reset! ws-id (:id ws))
mirrored-transforms-ids (t2/select-fn-vec :downstream_id :model/WorkspaceMappingTransform :workspace_id (:id ws))
ws-db (t2/select-one :model/Database :id (:database_id ws))
mirrored-tables-ids (t2/select-fn-vec :downstream_id :model/WorkspaceMappingTable :workspace_id (:id ws))
mirrored-tables (t2/select :model/Table :id [:in mirrored-tables-ids])]
(testing "Sanity"
(testing "Mirrored transforms created"
(is (= 2 (count mirrored-transforms-ids))))
(testing "Mirrored tables created in isolated schema"
(run!
(fn [{:keys [schema name]}]
(is (true? (driver/table-exists? :postgres ws-db {:schema schema :name name}))))
mirrored-tables))
(testing "Mirrored tables created in appdb"
(is (= 2 (count mirrored-tables-ids)))))
(testing "Remove transforms from a workspace"
(mt/with-test-user :crowberto
;; ACT!
(ws.common/remove-entities! ws {:transforms mirrored-transforms-ids}))
(testing "Mirrored transforms deleted"
(is (empty? (t2/select :model/Transform :id [:in mirrored-transforms-ids]))))
(testing "Mirrored tables dropped from the isolated schema"
(run!
(fn [{:keys [schema name]}]
;; driver/table-exists? -- _prints_ exception on non-existent relation, while having correct
;; return value.
;; one way or another -- this test will be rewritten completely. Then we will handle that.
(is (false? (driver/table-exists? :postgres ws-db {:schema schema :name name}))))
mirrored-tables))
(testing "Mirrored tables deleted from the appdb"
(is (empty? (t2/select :model/Table :id [:in mirrored-tables-ids]))))
(testing "Mirrored entites are not present in the mapping tables"
(is (empty? (t2/select :model/WorkspaceMappingTable :workspace_id (:id ws))))
(is (empty? (t2/select :model/WorkspaceMappingTransform :workspace_id (:id ws)))))))))
(catch Throwable t
(def ttt t)
(throw t))
(finally
(teardown! @ws-id))))))
(deftest add-one-by-one-and-remove-test
(mt/test-driver
:postgres
(let [mp (mt/metadata-provider)
q1 (lib/native-query mp "select * from orders limit 10;")
test-schema "my_test_schema"
ws-id (atom nil)]
(try
(setup! test-schema)
(transforms.tu/with-transform-cleanup! [t1 "t1"]
(mt/with-temp
[:model/Transform x1 {:source_type :native
:source {:type :query
:query q1}
:target {:type :table
:database (mt/id)
:schema test-schema
:name t1}}]
;; Create the target tables
(transforms.i/execute! x1 {:run-method :manual})
;; Add into a workspace
(let [creator-id (mt/user->id :crowberto)
;; create empty workspace
ws (mt/user-http-request :crowberto :post 200
"/ee/workspace"
{:name "my test ws xxx"
:creator_id creator-id
:database_id (mt/id)})
_ (reset! ws-id (:id ws))]
(let [add-res (mt/user-http-request :crowberto :post 200
(str "ee/workspace/" (:id ws) "/contents")
{:add {:transforms [(:id x1)]}})]
(testing "Successfully added"
(is (some? (-> add-res :contents :transforms (nth 0)))))
(let [rem-res (mt/user-http-request :crowberto :post 200
(str "ee/workspace/" (:id ws) "/contents")
{:remove {:transforms [(-> add-res :contents :transforms (nth 0) :id)]}})]
(testing "successfully removed"
(is (empty? (-> rem-res :contents :transforms))))))
(let [add-res (mt/user-http-request :crowberto :post 200
(str "ee/workspace/" (:id ws) "/contents")
{:add {:transforms [(:id x1)]}})]
(testing "Successfully added"
(is (some? (-> add-res :contents :transforms (nth 0)))))
(let [rem-res (mt/user-http-request :crowberto :post 200
(str "ee/workspace/" (:id ws) "/contents")
{:remove {:transforms [(-> add-res :contents :transforms (nth 0) :id)]}})]
(testing "successfully removed"
(is (empty? (-> rem-res :contents :transforms)))))))))
(catch Throwable t
(def ttt t)
(throw t))
(finally
(teardown! @ws-id))))))

View File

@@ -0,0 +1,114 @@
(ns metabase-enterprise.workspaces.dag-abstract
"Abstract graph solver using shorthand notation for testing.
Shorthand uses keywords like :x1, :t2, :m3 where:
- :x<n> is a transform
- :t<n> is a table (source/generated)
- :m<n> is a model (card)"
(:require
[clojure.set :as set]
[flatland.ordered.map :as ordered-map]
[metabase-enterprise.workspaces.dag :as ws.dag]
[metabase.util :as u]))
;;;; Shorthand notation helpers
(defn- related [t id]
(keyword (str (name t) (subs (name id) 1))))
(def ^:private x->t (partial related :t))
(defn is?
"Check if id is of type t (e.g., (is? :x :x1) => true)."
[t id]
(= (name t) (str (first (name id)))))
(defn- lexico [id]
(let [n (name id)]
[(parse-long (subs n 1))
({\x 1 \t 2 \m 3} (first n))]))
(defn expand-shorthand
"As shorthand, we can use transforms in the place of their induced tables.
This function expands the graph to insert the induced tables, and replace the corresponding references."
[{:keys [dependencies] :as graph}]
(assoc graph :dependencies
(let [xs (set (filter #(is? :x %) (concat (keys dependencies) (mapcat val dependencies))))]
(into (sorted-map-by #(compare (lexico %1) (lexico %2)))
(merge (zipmap (map x->t xs) (map vector xs))
(into {} (filter (comp #(is? :t %) key)) dependencies)
(update-vals
(into {} (remove (comp #(is? :t %) key)) dependencies)
(fn [deps] (->> deps
(map (fn [id]
(if (is? :x id)
(x->t id)
id)))
(sort-by lexico)
(vec)))))))))
;;;; Graph traversal helpers
(defn- children [dependencies]
(reduce
(fn [acc [child parents]]
(reduce (fn [acc p] (update acc p u/conjv child)) acc parents))
(sorted-map-by #(compare (lexico %1) (lexico %2)))
dependencies))
(defn- ->ts [ids]
(into #{}
(keep (fn [id]
(cond (is? :t id) id
(is? :x id) (x->t id))))
ids))
;;;; Toposort
(defn- toposort-map [child->parents]
(let [child->parents (if (map? child->parents) child->parents (into {} child->parents))]
(into (ordered-map/ordered-map)
(keep (fn [n] (when-let [deps (get child->parents n)] [n deps])))
(#'ws.dag/toposort-dfs child->parents))))
;;;; Abstract path-induced subgraph solver
(def ^:private max-iterations 100)
(defn path-induced-subgraph
"Abstract solver for path-induced subgraph using shorthand notation."
[{:keys [check-outs dependencies] :as _graph}]
(let [->parents dependencies
->children (children dependencies)]
(loop [iteration 1
below check-outs
above check-outs
seen-down? check-outs
seen-up? check-outs
to-descend check-outs
to-ascend check-outs]
(if (and (empty? to-descend) (empty? to-ascend))
(let [nodes (set/intersection above below)
entities (remove #(is? :t %) nodes)
tables (sort-by lexico (filter #(is? :t %) nodes))]
{:check-outs (sort-by lexico check-outs)
:inputs (sort-by lexico (remove nodes (->ts (mapcat ->parents check-outs))))
:tables tables
:transforms (sort-by lexico (filter #(is? :x %) nodes))
:entities (sort-by lexico entities)
:dependencies (apply dissoc
(toposort-map
(for [n nodes]
[n (vec (filter nodes (get dependencies n)))]))
tables)})
(if (> iteration max-iterations)
{:error "max iterations exceeded"}
(let [next-down (into #{} (comp (mapcat ->children) (remove seen-down?)) to-descend)
next-up (into #{} (comp (mapcat ->parents) (remove seen-up?)) to-ascend)]
(recur (inc iteration)
(into below next-down)
(into above next-up)
(into seen-down? next-down)
(into seen-up? next-up)
next-down
next-up)))))))

View File

@@ -0,0 +1,337 @@
(ns metabase-enterprise.workspaces.dag-test
(:require
[clojure.string :as str]
[clojure.test :refer :all]
[flatland.ordered.map :as ordered-map]
[metabase-enterprise.dependencies.models.dependency :as deps]
[metabase-enterprise.workspaces.dag :as ws.dag]
[metabase-enterprise.workspaces.dag-abstract :as dag-abstract]
[metabase.test :as mt]
[metabase.util :as u]
[toucan2.core :as t2]))
;;;; Example graphs for testing
(def ^:private example
{:check-outs #{:x3 :m6 :m10 :m13}
:dependencies {:x3 [:x1 :t2]
:x4 [:x3]
:m6 [:x4 :t5]
:m10 [:t9]
:x11 [:m10]
:m12 [:x11]
:m13 [:x11 :m12]}})
;;;; Test data helpers
(defn- transform? [kw] (= \x (first (name kw))))
(defn- table? [kw] (= \t (first (name kw))))
(defn- kw->id [kw] (parse-long (subs (name kw) 1)))
(defn- create-test-graph!
"Create test transforms and tables from shorthand notation, returning id mappings.
Shorthand uses keywords like :x1, :t2, :m3 where:
- :x<n> is a transform
- :t<n> is a table (source/generated)
- :m<n> is a model (card)
Returns a map from shorthand id to real database id, e.g. {:x1 123, :t2 456}
NOTE: Caller should wrap in mt/with-model-cleanup for proper cleanup."
[{:keys [check-outs dependencies]}]
(let [schema (str/replace (str (random-uuid)) "-" "_")
all-ids (set (concat (keys dependencies)
(mapcat val dependencies)
check-outs))
transforms (filter transform? all-ids)
tables (filter table? all-ids)
table-ids (u/for-map [t tables]
[t (t2/insert-returning-pk! :model/Table
{:db_id (mt/id)
:schema schema
:name (str "test_table_" (kw->id t))
:active true})])
;; Create transforms that reference their input tables
tx-ids (u/for-map [tx transforms]
(let [parent-tables (->> (get dependencies tx []) (filter table?) (map table-ids))
id (t2/insert-returning-pk! :model/Transform
{:name (str "Test " (name tx))
:source {:type :query
:query (if (seq parent-tables)
{:database (mt/id)
:type :query
:query {:source-table (first parent-tables)
:joins (for [pt (rest parent-tables)]
{:source-table pt
:condition [:= 1 1]})}}
{:database (mt/id)
:type :native
:native {:query "SELECT 1"}})}
:target {:type "table"
:database (mt/id)
:schema schema
:name (str "test_table_" (kw->id tx))}})]
[tx id]))]
;; TODO This is a workaround for the dependency between transforms and their output only being inserted on run.
;; We will need to do something about this when we mirror as well - ideally the deps module would "just work"
(doseq [[tx-kw tx-id] tx-ids]
(t2/insert! :model/Dependency
{:from_entity_type "table"
:from_entity_id (table-ids (keyword (str "t" (kw->id tx-kw))))
:to_entity_type "transform"
:to_entity_id tx-id}))
(merge tx-ids table-ids)))
(defn- translate-result
"Translate result from real IDs back to shorthand notation for easier comparison."
[result id-map]
(let [reverse-map (u/for-map [[k v] id-map] [v k])
translate (fn [{:keys [id]}] (get reverse-map id))]
(-> (reduce (fn [m k] (update m k #(set (map translate %))))
result
[:check-outs :inputs :outputs :transforms])
(update :dependencies
(fn [deps]
(into {}
(map (fn [[k v]]
[(translate k) (set (map translate v))]))
deps))))))
;;;; Tests
(deftest path-induced-subgraph-basic-test
(mt/with-premium-features #{:workspaces :dependencies}
(mt/with-temp [:model/Transform tx {:name "Test Transform"}]
(let [table-id (mt/id :orders)]
;; Set up: transform depends on orders table
(deps/replace-dependencies! "transform" (:id tx) {"table" #{table-id}})
(testing "single transform returns correct subgraph"
(let [result (ws.dag/path-induced-subgraph {:transforms [(:id tx)]})]
(is (= [{:id (:id tx) :type :transform}] (:check-outs result)))
(is (= [{:id (:id tx) :type :transform}] (:transforms result)))
;; Input should be the orders table
(is (some #(= table-id (:id %)) (:inputs result)))))))))
(deftest path-induced-subgraph-chain-test
(mt/with-premium-features #{:workspaces :dependencies}
(mt/with-model-cleanup [:model/Transform :model/Dependency]
(mt/with-temp [:model/Transform tx1 {:name "Transform 1"}
:model/Transform tx2 {:name "Transform 2"}
:model/Transform tx3 {:name "Transform 3"}]
(let [table-id (mt/id :orders)]
;; Chain: orders -> tx1 -> tx2 -> tx3
(deps/replace-dependencies! "transform" (:id tx1) {"table" #{table-id}})
(deps/replace-dependencies! "transform" (:id tx2) {"transform" #{(:id tx1)}})
(deps/replace-dependencies! "transform" (:id tx3) {"transform" #{(:id tx2)}})
(testing "middle transform includes upstream and downstream"
(let [result (ws.dag/path-induced-subgraph {:transforms [(:id tx2)]})]
;; tx2 is checked out
(is (= [{:id (:id tx2) :type :transform}] (:check-outs result)))
;; Should include tx1 (upstream) and tx3 (downstream) since tx2 is on path between them
;; Actually no - tx2 is the only thing checked out, so we only get what's on paths through tx2
;; tx1 -> tx2 -> tx3 means tx2 has tx1 upstream and tx3 downstream
;; The intersection is just tx2 itself
(is (= #{{:id (:id tx2) :type :transform}} (set (:transforms result)))))))))))
(deftest path-induced-subgraph-diamond-test
(mt/with-premium-features #{:workspaces :dependencies}
(mt/with-model-cleanup [:model/Transform :model/Dependency]
(testing "diamond dependency pattern"
;; Create: t1 -> x1 -> t2
;; \-> x2 -> t3
;; both x1 and x2 depend on t1
;; if we check out x1 and x2, we should get both
(mt/with-temp [:model/Transform tx1 {:name "Transform 1"}
:model/Transform tx2 {:name "Transform 2"}]
(let [source-table (mt/id :orders)]
(deps/replace-dependencies! "transform" (:id tx1) {"table" #{source-table}})
(deps/replace-dependencies! "transform" (:id tx2) {"table" #{source-table}})
(let [result (ws.dag/path-induced-subgraph {:transforms [(:id tx1) (:id tx2)]})]
(is (= 2 (count (:check-outs result))))
(is (= 2 (count (:transforms result)))))))))))
(deftest upstream-entities-test
(mt/with-premium-features #{:workspaces :dependencies}
(mt/with-model-cleanup [:model/Transform :model/Dependency]
(mt/with-temp [:model/Transform tx1 {:name "Transform 1"}
:model/Transform tx2 {:name "Transform 2"}]
(let [table-id (mt/id :orders)]
(deps/replace-dependencies! "transform" (:id tx1) {"table" #{table-id}})
(deps/replace-dependencies! "transform" (:id tx2) {"transform" #{(:id tx1)}})
(testing "upstream includes transitive dependencies"
(let [upstream (#'ws.dag/upstream-entities {:transforms [(:id tx2)]})]
(is (contains? upstream {:id (:id tx2) :type :transform}))
(is (contains? upstream {:id (:id tx1) :type :transform}))
(is (contains? upstream {:id table-id :type :table})))))))))
(deftest downstream-entities-test
(mt/with-premium-features #{:workspaces :dependencies}
(mt/with-model-cleanup [:model/Transform :model/Dependency]
(mt/with-temp [:model/Transform tx1 {:name "Transform 1"}
:model/Transform tx2 {:name "Transform 2"}]
(deps/replace-dependencies! "transform" (:id tx2) {"transform" #{(:id tx1)}})
(testing "downstream includes transitive dependents"
(let [downstream (#'ws.dag/downstream-entities {:transforms [(:id tx1)]})]
(is (contains? downstream {:id (:id tx1) :type :transform}))
(is (contains? downstream {:id (:id tx2) :type :transform}))))))))
(deftest path-induced-subgraph-shorthand-test
(mt/with-premium-features #{:workspaces :dependencies}
(mt/with-model-cleanup [:model/Table :model/Transform :model/Dependency]
(testing "graph built from shorthand matches abstract solver"
(let [shorthand {:check-outs #{:x1}
:dependencies {:x1 [:t1]}}
id-map (create-test-graph! shorthand)
result (ws.dag/path-induced-subgraph {:transforms [(id-map :x1)]})
translated (translate-result result id-map)]
;; Check-outs should be :x1
(is (= #{:x1} (:check-outs translated)))
;; Transforms should be :x1
(is (= #{:x1} (:transforms translated)))
;; Inputs should be :t1
(is (contains? (:inputs translated) :t1)))))))
(deftest path-induced-subgraph-larger-test
(mt/with-premium-features #{:workspaces :dependencies}
(mt/with-model-cleanup [:model/Table :model/Transform :model/Dependency]
(testing "graph built from shorthand matches abstract solver"
(let [shorthand {:check-outs #{:x2, :x4}
:dependencies {:x1 [:t0]
:x2 [:x1, :t10]
:x3 [:x2, :t8]
:x4 [:x3]
:x5 [:x2, :x4, :t9]}}
id-map (create-test-graph! (dag-abstract/expand-shorthand shorthand))
result (ws.dag/path-induced-subgraph {:transforms (mapv id-map (:check-outs shorthand))})
translated (translate-result result id-map)]
(is (=? {:check-outs #{:x2, :x4}
:transforms #{:x2, :x3, :x4}
:inputs #{:t0, :t1, :t8, :t10}
:outputs #{:t2, :t3, :t4}
;; Dependencies outside the subgraph are not listed
:dependencies {:x2 #{}
:x3 #{:x2}
:x4 #{:x3}}}
translated)))))))
(deftest expand-solver-test
(testing "expand-shorthand inserts interstitial nodes for transform output tables"
(is (= {:check-outs #{:x3 :m6 :m10 :m13}
:dependencies {:t1 [:x1]
:x3 [:t1 :t2]
:t3 [:x3]
:x4 [:t3]
:t4 [:x4]
:m6 [:t4 :t5]
:m10 [:t9]
:x11 [:m10]
:t11 [:x11]
:m12 [:t11]
:m13 [:t11 :m12]}}
(dag-abstract/expand-shorthand example)))))
(deftest abstract-path-induced-subgraph-test
(testing "path-induced-subgraph computes correct result for example graph"
(is (= {:check-outs [:x3 :m6 :m10 :m13]
:inputs [:t1 :t2 :t5 :t9]
:tables [:t3 :t4 :t11]
:transforms [:x3 :x4 :x11]
:entities [:x3 :x4 :m6 :m10 :x11 :m12 :m13]
:dependencies (ordered-map/ordered-map
:m10 []
:x11 [:m10]
:m12 [:t11]
:m13 [:t11 :m12]
:x3 []
:x4 [:t3]
:m6 [:t4])}
(dag-abstract/path-induced-subgraph (dag-abstract/expand-shorthand example))))))
;;;; Card dependency detection tests
(defn- query-with-source-card
"Create a pMBQL query that uses a card as its source."
[card-id]
{:lib/type :mbql/query
:database (mt/id)
:stages [{:lib/type :mbql.stage/mbql
:source-card card-id}]})
(defn- create-transform-with-card-source!
"Create a transform whose source query depends on a card.
The after-insert hook triggers dependency calculation automatically."
[card]
(t2/insert-returning-instance! :model/Transform
{:name "Transform depending on card"
:source {:type :query
:query (query-with-source-card (:id card))}
:target {:type "table"
:database (mt/id)
:schema "public"
:name "card_dep_output"}}))
(deftest transforms-depend-on-cards-no-deps-test
(mt/with-premium-features #{:workspaces :dependencies}
(testing "transform with no card dependencies returns empty set"
(mt/with-temp [:model/Transform tx {:name "Transform without card deps"
:source {:type :query
:query {:database (mt/id)
:type :query
:query {:source-table (mt/id :orders)}}}
:target {:type "table"
:database (mt/id)
:schema "public"
:name "no_card_output"}}]
;; after-insert hook triggers dependency calculation automatically
(is (empty? (ws.dag/card-dependencies [(:id tx)])))))))
(deftest transforms-depend-on-cards-direct-dep-test
(mt/with-premium-features #{:workspaces :dependencies}
(testing "transform with direct card dependency returns card IDs"
(mt/with-model-cleanup [:model/Dependency :model/Transform]
(mt/with-temp [:model/Card card {:name "Test Card"
:database_id (mt/id)
:dataset_query (mt/mbql-query venues)}]
(let [tx (create-transform-with-card-source! card)
result (ws.dag/card-dependencies [(:id tx)])]
(is (seq result))
(is (contains? result (:id card)))))))))
(deftest transforms-depend-on-cards-transitive-dep-test
(mt/with-premium-features #{:workspaces :dependencies}
(testing "transform with transitive card dependency (via another transform) returns card IDs"
(mt/with-model-cleanup [:model/Dependency :model/Transform]
(mt/with-temp [:model/Card card {:name "Base Card"
:database_id (mt/id)
:dataset_query (mt/mbql-query venues)}]
;; tx1 depends on card
(let [tx1 (create-transform-with-card-source! card)
;; tx2 depends on tx1
tx2 (t2/insert-returning-instance! :model/Transform
{:name "Transform 2 - depends on tx1"
:source {:type :query
:query {:database (mt/id)
:type :native
:native {:query "SELECT 1"}}}
:target {:type "table"
:database (mt/id)
:schema "public"
:name "tx2_output"}})]
;; Create dependency: tx2 depends on tx1
(deps/replace-dependencies! "transform" (:id tx2) {"transform" #{(:id tx1)}})
(let [result (ws.dag/card-dependencies [(:id tx2)])]
(is (seq result))
(is (contains? result (:id card))))))))))
(deftest transforms-depend-on-cards-empty-input-test
(mt/with-premium-features #{:workspaces :dependencies}
(testing "empty input returns nil/empty"
(is (empty? (ws.dag/card-dependencies []))))))

View File

@@ -0,0 +1,20 @@
(ns ^:mb/driver-tests metabase-enterprise.workspaces.drivers-test
(:require
[clojure.test :refer :all]
[metabase.driver :as driver]
[metabase.test :as mt]))
(deftest drivers-support-workspaces-has-to-support-db-swapping-test
(mt/test-drivers (mt/normal-drivers-with-feature :workspace)
(let [db-swapped? (fn []
(try
(mt/run-mbql-query venues {:limit 1})
false
(catch Exception _e
true)))]
(testing "db is swapped if being executed inside a macros"
(driver/with-swapped-connection-details (mt/id) {:user "unicorn"}
(is (db-swapped?))))
(testing "sanity check that it's not swapped outside of the macros"
(is (not (db-swapped?)))))))

View File

@@ -0,0 +1,56 @@
(ns ^:mb/driver-tests metabase-enterprise.workspaces.e2e-test
(:require
[clojure.string :as str]
[clojure.test :refer :all]
[metabase-enterprise.transforms.interface :as transforms.i]
[metabase-enterprise.transforms.test-util :as transforms.tu]
[metabase-enterprise.workspaces.common :as ws.common]
[metabase-enterprise.workspaces.core :as workspaces]
[metabase.driver :as driver]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.query-processor.preprocess :as qp.preprocess]
^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]
[metabase.test :as mt]
[toucan2.core :as t2]))
(defn- execute-workspace-transform!
"Execute a transform within workspace isolation context. For testing purposes."
[workspace transform opts]
(workspaces/with-workspace-isolation workspace
(transforms.i/execute! transform opts)))
(defn- mbql->native [query]
(qp.store/with-metadata-provider (mt/id)
(sql.qp/mbql->native driver/*driver* (qp.preprocess/preprocess query))))
(deftest isolation-e2e-test
(mt/test-drivers #_(mt/normal-drivers-with-feature :workspace) #{:postgres #_:h2}
(transforms.tu/with-transform-cleanup! [output-table-name (str "test_table_1_" (mt/random-name))]
(mt/with-model-cleanup [:model/Transform :model/Workspace]
(mt/with-temp [:model/Transform {transform-id :id} {:name "Transform 1"
:source {:type "query"
:query (mt/native-query (mbql->native (mt/mbql-query orders {:limit 1})))}
:target {:type "table"
:name output-table-name}}]
(let [workspace (mt/with-current-user (mt/user->id :crowberto)
(ws.common/create-workspace! (mt/user->id :crowberto)
{:name (mt/random-name)
:database_id (mt/id)
:upstream {:transforms [transform-id]}}))
isolated-transform-id (t2/select-one-fn :downstream_id :model/WorkspaceMappingTransform :workspace_id (:id workspace))
isolated-transform (t2/select-one :model/Transform isolated-transform-id)
executed-transform (execute-workspace-transform! workspace isolated-transform {:run-method :manual})
output-table (-> executed-transform :object :output-table)]
(testing "execute the transform with the original query is fine and the output is in an isolated schema"
(transforms.tu/wait-for-table (name output-table) 1000)
(is (str/starts-with? (namespace output-table) "mb__isolation"))
(is (= 1 (count (transforms.tu/table-rows (name output-table))))))
(testing "changing the query without granting access will fail"
(t2/update! :model/Transform isolated-transform-id {:source {:type "query"
;; TODO we should use native query here
:query (mt/mbql-query venues {:limit 1})}})
(is (thrown-with-msg?
Exception
#"ERROR: permission denied for table.*"
#p (execute-workspace-transform! workspace (t2/select-one :model/Transform isolated-transform-id) {:run-method :manual}))))))))))

View File

@@ -0,0 +1,97 @@
(ns metabase-enterprise.workspaces.promotion-test
(:require
[clojure.test :refer :all]
[metabase-enterprise.workspaces.promotion :as ws.promotion]
[metabase.test :as mt]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
(deftest promote-transforms-test
(mt/with-premium-features #{:workspaces :dependencies}
(mt/with-temp [:model/Table t1 {:schema "public", :name "table_1"}
:model/Table t2 {:schema "public", :name "table_2"}
:model/Transform x1 {:name "Transform 1"
:description "Original description"
:target {:type "table"
:database 1
:schema "public"
:name "table_1"}}
:model/Transform x2 {:name "Transform 2"
:description "Another original"
:target {:type "table"
:database 1
:schema "public"
:name "table_2"}}
:model/Workspace ws {:name "Test Workspace"}
:model/Table wst1 {:schema "isolated__place", :name "public__table_1"}
:model/Table wst2 {:schema "isolated__place", :name "public__table_2"}
:model/WorkspaceMappingTable _ {:upstream_id (:id t1)
:downstream_id (:id wst1)
:workspace_id (:id ws)}
:model/WorkspaceMappingTable _ {:upstream_id (:id t2)
:downstream_id (:id wst2)
:workspace_id (:id ws)}
:model/Transform wsx1 {:name "Transform 1"
:description "Modified description"
:workspace_id (:id ws)
:target {:type "table"
:database 1
:schema "isolated__place"
:name "public__table_1"}}
:model/Transform wsx2 {:name "Transform 2"
:description "Modified description 2"
:workspace_id (:id ws)
:target {:type "table"
:database 1
:schema "isolated__place"
:name "public__table_2"}}
:model/WorkspaceMappingTransform _m1 {:upstream_id (:id x1)
:downstream_id (:id wsx1)
:workspace_id (:id ws)}
:model/WorkspaceMappingTransform _m2 {:upstream_id (:id x2)
:downstream_id (:id wsx2)
:workspace_id (:id ws)}]
(testing "promote-transforms! updates originals and reports success"
(let [result (ws.promotion/promote-transforms! ws)]
(testing "promotion worked"
(is (=? {:promoted [{:id (:id x1)}
{:id (:id x2)}]}
result)))
(testing "original transforms were updated with ws versions"
(is (= (:description wsx1)
(t2/select-one-fn :description :model/Transform :id (:id x1))))
(is (= (:description wsx2)
(t2/select-one-fn :description :model/Transform :id (:id x2))))))))))
(deftest promote-transforms-with-error-test
(mt/with-premium-features #{:workspaces :dependencies}
(mt/with-temp [:model/Table t1 {:schema "public", :name "table_1"}
:model/Transform x1 {:name "Transform 1"}
:model/Workspace ws {:name "Test Workspace"}
:model/Table t2 {:schema "isolated", :name "public__table_1"}
:model/Transform wsx1 {:name "Transform 1"
:workspace_id (:id ws)
:target {:type "table"
:database 1
:schema "isolated"
:name "public__table_1"}}
:model/WorkspaceMappingTable _ {:upstream_id (:id t1)
:downstream_id (:id t2)
:workspace_id (:id ws)}
:model/WorkspaceMappingTransform _m {:upstream_id (:id x1)
:downstream_id (:id wsx1)
:workspace_id (:id ws)}]
(testing "promote-transforms! handles errors gracefully"
(with-redefs [t2/update!
(fn [& _args]
(throw (ex-info "Execution failed" {})))]
(is (=? {:promoted nil
:errors [{:id (:id wsx1)
:error "Execution failed"}]}
(mt/with-log-level [metabase-enterprise.workspaces.promotion :fatal]
(ws.promotion/promote-transforms! ws)))))))))

View File

@@ -21,3 +21,4 @@ export * from "./transform";
export * from "./transform-job";
export * from "./transform-tag";
export * from "./upload-management";
export * from "./workspace";

View File

@@ -25,6 +25,9 @@ import type {
TransformJob,
TransformRun,
TransformTag,
Workspace,
WorkspaceContentItem,
WorkspaceContents,
} from "metabase-types/api";
export const ENTERPRISE_TAG_TYPES = [
@@ -49,6 +52,7 @@ export const ENTERPRISE_TAG_TYPES = [
"remote-sync-branches",
"remote-sync-current-task",
"python-transform-library",
"workspace",
"support-access-grant",
"support-access-grant-current",
"library-collection",
@@ -82,6 +86,32 @@ export function invalidateTags(
return !error ? tags : [];
}
export function provideWorkspacesTags(
workspaces: Workspace[],
): TagDescription<EnterpriseTagType>[] {
return [listTag("workspace"), ...workspaces.flatMap(provideWorkspaceTags)];
}
export function provideWorkspaceTags(
workspace: Workspace,
): TagDescription<EnterpriseTagType>[] {
return [idTag("workspace", workspace.id)];
}
export function provideWorkspaceContentsTags(
workspaceContents: WorkspaceContents,
): TagDescription<EnterpriseTagType>[] {
return provideWorkspaceContentItemsTags(
workspaceContents.contents.transforms,
);
}
export function provideWorkspaceContentItemsTags(
items: WorkspaceContentItem[],
): TagDescription<EnterpriseTagType>[] {
return items.flatMap((transform) => idTag("transform", transform.id));
}
export function provideTransformTags(
transform: Transform,
): TagDescription<EnterpriseTagType>[] {

View File

@@ -0,0 +1,184 @@
import type {
CreateWorkspaceRequest,
CreateWorkspaceTransformRequest,
CreateWorkspaceTransformResponse,
TransformDownstreamMapping,
TransformId,
TransformUpstreamMapping,
ValidateTableNameRequest,
ValidateTableNameResponse,
Workspace,
WorkspaceContents,
WorkspaceId,
WorkspaceListResponse,
WorkspaceMergeResponse,
WorkspaceTablesResponse,
WorkspaceUpdateContentsRequest,
} from "metabase-types/api";
import { EnterpriseApi } from "./api";
import {
idTag,
invalidateTags,
listTag,
provideWorkspaceContentItemsTags,
provideWorkspaceContentsTags,
provideWorkspaceTags,
provideWorkspacesTags,
tag,
} from "./tags";
export const workspaceApi = EnterpriseApi.injectEndpoints({
endpoints: (builder) => ({
getWorkspaces: builder.query<WorkspaceListResponse, void>({
query: () => ({
method: "GET",
url: "/api/ee/workspace",
}),
providesTags: (response) =>
response ? provideWorkspacesTags(response.items) : [],
}),
getWorkspace: builder.query<Workspace, WorkspaceId>({
query: (id) => ({
method: "GET",
url: `/api/ee/workspace/${id}`,
}),
providesTags: (workspace) =>
workspace ? provideWorkspaceTags(workspace) : [],
}),
createWorkspace: builder.mutation<Workspace, CreateWorkspaceRequest>({
query: (body) => ({
method: "POST",
url: "/api/ee/workspace",
body,
}),
invalidatesTags: (_, error) =>
invalidateTags(error, [listTag("workspace"), listTag("transform")]),
}),
getWorkspaceContents: builder.query<WorkspaceContents, WorkspaceId>({
query: (id) => ({
method: "GET",
url: `/api/ee/workspace/${id}`,
}),
transformResponse: (workspace: any) => {
return {
contents: workspace?.contents ?? { transforms: [] },
} as WorkspaceContents;
},
providesTags: (workspaceContents) =>
workspaceContents
? provideWorkspaceContentsTags(workspaceContents)
: [],
}),
updateWorkspaceContents: builder.mutation<
WorkspaceContents,
WorkspaceUpdateContentsRequest
>({
query: ({ id, ...body }) => ({
method: "POST",
url: `/api/ee/workspace/${id}/contents`,
body,
}),
invalidatesTags: (_, error, { id }) =>
invalidateTags(error, [idTag("workspace", id), tag("transform")]),
}),
createWorkspaceTransform: builder.mutation<
CreateWorkspaceTransformResponse,
{ id: WorkspaceId } & CreateWorkspaceTransformRequest
>({
query: ({ id, ...body }) => ({
method: "POST",
url: `/api/ee/workspace/${id}/transform`,
body,
}),
invalidatesTags: (_, error, { id }) =>
invalidateTags(error, [idTag("workspace", id), tag("transform")]),
}),
getTransformUpstreamMapping: builder.query<
TransformUpstreamMapping,
TransformId
>({
query: (id) => ({
method: "GET",
url: `/api/ee/workspace/mapping/transform/${id}/upstream`,
}),
providesTags: (mapping) =>
mapping?.transform
? provideWorkspaceContentItemsTags([mapping.transform])
: [],
}),
getTransformDownstreamMapping: builder.query<
TransformDownstreamMapping,
TransformId
>({
query: (id) => ({
method: "GET",
url: `/api/ee/workspace/mapping/transform/${id}/downstream`,
}),
providesTags: (mapping) =>
mapping ? provideWorkspaceContentItemsTags(mapping.transforms) : [],
}),
mergeWorkspace: builder.mutation<WorkspaceMergeResponse, WorkspaceId>({
query: (id) => ({
method: "POST",
url: `/api/ee/workspace/${id}/merge`,
}),
invalidatesTags: (_, error) =>
invalidateTags(error, [tag("workspace"), tag("transform")]),
}),
archiveWorkspace: builder.mutation<Workspace, WorkspaceId>({
query: (id) => ({
method: "POST",
url: `/api/ee/workspace/${id}/archive`,
}),
invalidatesTags: (_, error) =>
invalidateTags(error, [tag("workspace"), tag("transform")]),
}),
validateTableName: builder.mutation<
ValidateTableNameResponse,
ValidateTableNameRequest
>({
query: ({ id, ...body }) => ({
method: "POST",
url: `/api/ee/workspace/${id}/validate-target`,
body,
}),
}),
updateWorkspaceName: builder.mutation<
Workspace,
{ id: WorkspaceId; name: string }
>({
query: ({ id, ...body }) => ({
method: "POST",
url: `/api/ee/workspace/${id}/name`,
body,
}),
invalidatesTags: (_, error, { id }) =>
invalidateTags(error, [idTag("workspace", id)]),
}),
getWorkspaceTables: builder.query<WorkspaceTablesResponse, WorkspaceId>({
query: (id) => ({
method: "GET",
url: `/api/ee/workspace/${id}/tables`,
}),
providesTags: (_, __, id) => [idTag("workspace", id)],
}),
}),
});
export const {
useGetWorkspacesQuery,
useGetWorkspaceQuery,
useCreateWorkspaceMutation,
useGetWorkspaceContentsQuery,
useUpdateWorkspaceContentsMutation,
useLazyGetWorkspaceContentsQuery,
useCreateWorkspaceTransformMutation,
useGetTransformUpstreamMappingQuery,
useGetTransformDownstreamMappingQuery,
useMergeWorkspaceMutation,
useArchiveWorkspaceMutation,
useValidateTableNameMutation,
useUpdateWorkspaceNameMutation,
useGetWorkspaceTablesQuery,
} = workspaceApi;

View File

@@ -1,5 +1,13 @@
.nav {
border-right: 1px solid var(--mb-color-border);
transition: width 200ms ease;
width: auto;
min-width: 250px;
}
.navCollapsed {
width: 60px;
min-width: 60px;
}
.tab {
@@ -14,9 +22,99 @@
.tab,
.toggle {
color: var(--mb-color-text-primary);
font-weight: 600;
&:hover,
&.selected {
color: var(--mb-color-brand);
}
}
.workspacesSection {
border-top: 1px solid var(--mb-color-border);
padding-top: 0.75rem;
}
.workspacesSectionHeader {
background-color: var(--mb-color-bg-secondary);
color: var(--mb-color-text-primary);
font-weight: 600;
transition: background-color 150ms ease;
&:hover {
background-color: var(--mb-color-brand-light);
color: var(--mb-color-brand);
}
&.selected {
background-color: var(--mb-color-brand-light);
color: var(--mb-color-brand);
}
}
.workspaceItem {
background-color: var(--mb-color-bg-white);
color: var(--mb-color-text-primary);
cursor: pointer;
border: 1px solid var(--mb-color-border);
transition:
background-color 150ms ease,
border-color 150ms ease,
color 150ms ease;
&:hover {
background-color: var(--mb-color-bg-light);
border-color: var(--mb-color-border-dark);
}
&:hover .workspaceMenuButton {
opacity: 1;
}
&.selected {
background-color: var(--mb-color-brand-lighter);
border-color: var(--mb-color-brand);
color: var(--mb-color-brand);
}
&.selected .workspaceMenuButton {
opacity: 1;
}
&:disabled {
opacity: 0.5;
cursor: not-allowed;
}
}
.workspaceMenuButton {
color: var(--mb-color-text-secondary);
opacity: 0;
transition:
opacity 150ms ease,
background-color 150ms ease,
color 150ms ease;
flex-shrink: 0;
&:hover {
background-color: var(--mb-color-bg-medium);
color: var(--mb-color-text-primary);
}
}
.newWorkspaceButton {
background-color: transparent;
color: var(--mb-color-text-primary);
cursor: pointer;
border: none;
transition: color 150ms ease;
&:hover {
color: var(--mb-color-brand);
}
&:disabled {
opacity: 0.5;
cursor: not-allowed;
}
}

View File

@@ -1,10 +1,14 @@
import cx from "classnames";
import { type ReactNode, useMemo, useState } from "react";
import dayjs from "dayjs";
import { type ReactNode, useCallback, useMemo, useState } from "react";
import { push } from "react-router-redux";
import { t } from "ttag";
import { useListDatabasesQuery } from "metabase/api";
import { ForwardRefLink } from "metabase/common/components/Link";
import { useSelector } from "metabase/lib/redux";
import { useDispatch, useSelector } from "metabase/lib/redux";
import * as Urls from "metabase/lib/urls";
import { useMetadataToasts } from "metabase/metadata/hooks";
import {
PLUGIN_DEPENDENCIES,
PLUGIN_FEATURE_LEVEL_PERMISSIONS,
@@ -12,15 +16,27 @@ import {
} from "metabase/plugins";
import { getLocation } from "metabase/selectors/routing";
import {
ActionIcon,
Box,
FixedSizeIcon,
Flex,
Icon,
type IconName,
Menu,
Skeleton,
Stack,
Text,
Tooltip,
UnstyledButton,
} from "metabase/ui";
import {
useArchiveWorkspaceMutation,
useCreateWorkspaceMutation,
useGetWorkspacesQuery,
} from "metabase-enterprise/api";
import { DataStudioContext } from "metabase-enterprise/data-studio/common/contexts/DataStudioContext";
import { CreateWorkspaceModal } from "metabase-enterprise/workspaces/components/CreateWorkspaceModal/CreateWorkspaceModal";
import type { Database, WorkspaceId } from "metabase-types/api";
import S from "./DataStudioLayout.module.css";
@@ -31,6 +47,7 @@ type DataStudioLayoutProps = {
export function DataStudioLayout({ children }: DataStudioLayoutProps) {
const [isSidebarOpened, setIsSidebarOpened] = useState(false);
const [isSidebarAvailable, setIsSidebarAvailable] = useState(false);
const [isNavExpanded, setIsNavExpanded] = useState(true);
const contextValue = useMemo(
() => ({
isSidebarOpened,
@@ -48,6 +65,8 @@ export function DataStudioLayout({ children }: DataStudioLayoutProps) {
isSidebarOpened={isSidebarOpened}
isSidebarAvailable={isSidebarAvailable}
onSidebarToggle={setIsSidebarOpened}
isNavExpanded={isNavExpanded}
onNavToggle={setIsNavExpanded}
/>
<Box h="100%" flex={1} miw={0}>
{children}
@@ -61,12 +80,16 @@ type DataStudioNavProps = {
isSidebarOpened: boolean;
isSidebarAvailable: boolean;
onSidebarToggle: (isOpened: boolean) => void;
isNavExpanded: boolean;
onNavToggle: (isExpanded: boolean) => void;
};
function DataStudioNav({
isSidebarOpened,
isSidebarAvailable,
onSidebarToggle,
isNavExpanded,
onNavToggle,
}: DataStudioNavProps) {
const { pathname } = useSelector(getLocation);
const canAccessDataModel = useSelector(
@@ -76,42 +99,62 @@ function DataStudioNav({
PLUGIN_TRANSFORMS.canAccessTransforms,
);
const isDataTab = pathname.startsWith(Urls.dataStudioData());
const isTransformsTab = pathname.startsWith(Urls.transformList());
const isWorkspacePage = pathname.startsWith(Urls.dataStudioWorkspaceList());
const isTransformsTab =
pathname.startsWith(Urls.transformList()) && !isWorkspacePage;
const isModelingTab = pathname.startsWith(Urls.dataStudioModeling());
const isDependenciesTab = pathname.startsWith(Urls.dependencyGraph());
return (
<Stack className={S.nav} h="100%" p="0.75rem" justify="space-between">
<Stack
className={cx(S.nav, { [S.navCollapsed]: !isNavExpanded })}
h="100%"
p="0.75rem"
justify="space-between"
>
<Stack gap="0.75rem">
{canAccessDataModel && (
<DataStudioTab
label={t`Data`}
icon="database"
to={Urls.dataStudioData()}
isSelected={isDataTab}
/>
)}
{canAccessTransforms && (
<DataStudioTab
label={t`Transforms`}
icon="transform"
to={Urls.transformList()}
isSelected={isTransformsTab}
/>
)}
<DataStudioTab
label={t`Modeling`}
icon="model"
to={Urls.dataStudioModeling()}
isSelected={isModelingTab}
<DataStudioNavToggle
isNavExpanded={isNavExpanded}
onNavToggle={onNavToggle}
/>
{PLUGIN_DEPENDENCIES.isEnabled && (
<Stack gap="0.75rem">
{canAccessDataModel && (
<DataStudioTab
label={t`Data`}
icon="database"
to={Urls.dataStudioData()}
isSelected={isDataTab}
isExpanded={isNavExpanded}
/>
)}
{canAccessTransforms && (
<DataStudioTab
label={t`Transforms`}
icon="transform"
to={Urls.transformList()}
isSelected={isTransformsTab}
isExpanded={isNavExpanded}
/>
)}
<DataStudioTab
label={t`Dependency graph`}
icon="schema"
to={Urls.dependencyGraph()}
isSelected={isDependenciesTab}
label={t`Modeling`}
icon="model"
to={Urls.dataStudioModeling()}
isSelected={isModelingTab}
isExpanded={isNavExpanded}
/>
{PLUGIN_DEPENDENCIES.isEnabled && (
<DataStudioTab
label={t`Dependency graph`}
icon="schema"
to={Urls.dependencyGraph()}
isSelected={isDependenciesTab}
isExpanded={isNavExpanded}
/>
)}
</Stack>
{canAccessTransforms && (
<WorkspacesSection isExpanded={isNavExpanded} />
)}
</Stack>
{isSidebarAvailable && (
@@ -129,23 +172,237 @@ type DataStudioTabProps = {
icon: IconName;
to: string;
isSelected: boolean;
isExpanded: boolean;
};
const TOOLTIP_OPEN_DELAY = 1000;
function DataStudioTab({ label, icon, to, isSelected }: DataStudioTabProps) {
function DataStudioTab({
label,
icon,
to,
isSelected,
isExpanded,
}: DataStudioTabProps) {
const content = (
<Box
className={cx(S.tab, { [S.selected]: isSelected })}
component={ForwardRefLink}
to={to}
p="0.75rem"
bdrs="md"
>
<Flex
align="center"
gap="sm"
justify={isExpanded ? "flex-start" : "center"}
>
<FixedSizeIcon name={icon} display="block" />
{isExpanded && (
<Text size="sm" fw={600}>
{label}
</Text>
)}
</Flex>
</Box>
);
if (!isExpanded) {
return (
<Tooltip label={label} position="right" openDelay={TOOLTIP_OPEN_DELAY}>
{content}
</Tooltip>
);
}
return content;
}
type WorkspacesSectionProps = {
isExpanded: boolean;
};
function WorkspacesSection({
isExpanded: isNavExpanded,
}: WorkspacesSectionProps) {
const dispatch = useDispatch();
const [isWorkspacesExpanded, setIsWorkspacesExpanded] = useState(true);
const [isCreateModalOpen, setIsCreateModalOpen] = useState(false);
const { pathname } = useSelector(getLocation);
const { data: workspacesData, isLoading } = useGetWorkspacesQuery();
const { data: databaseData, isLoading: isLoadingDatabases } =
useListDatabasesQuery({ include_analytics: true });
const [createWorkspace, { isLoading: isCreating }] =
useCreateWorkspaceMutation();
const workspaces = useMemo(
() =>
[...(workspacesData?.items ?? [])].sort(
(a, b) =>
new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime(),
),
[workspacesData],
);
const databaseOptions = useMemo(
() =>
(databaseData?.data ?? []).map((db: Database) => ({
value: String(db.id),
label: db.name,
})),
[databaseData],
);
const handleOpenWorkspace = useCallback(
(workspaceId: number) => {
dispatch(push(Urls.dataStudioWorkspace(workspaceId)));
},
[dispatch],
);
const handleOpenCreateModal = useCallback(() => {
setIsCreateModalOpen(true);
}, []);
const handleCloseCreateModal = useCallback(() => {
setIsCreateModalOpen(false);
}, []);
const { sendErrorToast, sendSuccessToast } = useMetadataToasts();
const handleCreateWorkspace = useCallback(
async ({ name, databaseId }: { name: string; databaseId: string }) => {
try {
const workspace = await createWorkspace({
name,
database_id: Number(databaseId),
upstream: {},
}).unwrap();
handleCloseCreateModal();
handleOpenWorkspace(workspace.id);
} catch (error) {
sendErrorToast(t`Failed to create workspace`);
}
},
[
createWorkspace,
handleCloseCreateModal,
handleOpenWorkspace,
sendErrorToast,
],
);
const [archiveWorkspace] = useArchiveWorkspaceMutation();
const handleWorkspaceArchive = async (id: WorkspaceId) => {
try {
await archiveWorkspace(id).unwrap();
sendSuccessToast(t`Workspace archived successfully`);
dispatch(push(Urls.dataStudioWorkspaceList()));
} catch (error) {
sendErrorToast(t`Failed to archive workspace`);
}
};
const isWorkspaceListPage = pathname === Urls.dataStudioWorkspaceList();
if (!isNavExpanded) {
// In collapsed mode, show only an icon button with tooltip
return (
<Tooltip
label={t`Workspaces`}
position="right"
openDelay={TOOLTIP_OPEN_DELAY}
>
<UnstyledButton
className={cx(S.tab, { [S.selected]: isWorkspaceListPage })}
component={ForwardRefLink}
to={Urls.dataStudioWorkspaceList()}
p="0.75rem"
bdrs="md"
>
<Flex align="center" justify="center">
<FixedSizeIcon name="git_branch" display="block" />
</Flex>
</UnstyledButton>
</Tooltip>
);
}
return (
<Tooltip label={label} position="right" openDelay={TOOLTIP_OPEN_DELAY}>
<Box
className={cx(S.tab, { [S.selected]: isSelected })}
component={ForwardRefLink}
to={to}
<Stack gap="0.5rem" className={S.workspacesSection}>
<UnstyledButton
className={cx(S.workspacesSectionHeader, {
[S.selected]: isWorkspaceListPage,
})}
onClick={() => setIsWorkspacesExpanded(!isWorkspacesExpanded)}
p="0.75rem"
bdrs="md"
>
<FixedSizeIcon name={icon} display="block" />
</Box>
</Tooltip>
<Flex align="center" justify="space-between" w="100%">
<Text size="sm" fw={600}>
{t`Workspaces`}
</Text>
<Icon
name={isWorkspacesExpanded ? "chevrondown" : "chevronright"}
size={16}
/>
</Flex>
</UnstyledButton>
{isWorkspacesExpanded && (
<>
<UnstyledButton
className={S.newWorkspaceButton}
onClick={handleOpenCreateModal}
disabled={isLoadingDatabases}
p="0.5rem"
bdrs="md"
>
<Flex align="center" gap="xs">
<Icon name="add" size={16} />
<Text size="sm" fw={500}>
{t`New workspace`}
</Text>
</Flex>
</UnstyledButton>
<Stack gap="0.75rem" style={{ overflowY: "auto", maxHeight: "50vh" }}>
{isLoading ? (
<>
<Skeleton height={80} radius="md" />
<Skeleton height={80} radius="md" />
</>
) : workspaces.length === 0 ? (
<Text c="text-secondary" size="xs" px="0.5rem">
{t`No workspaces yet`}
</Text>
) : (
workspaces.map((workspace) => {
const isSelected =
pathname === Urls.dataStudioWorkspace(workspace.id);
return (
<WorkspaceItem
key={workspace.id}
workspace={workspace}
isSelected={isSelected}
onOpen={handleOpenWorkspace}
archiveWorkspace={handleWorkspaceArchive}
/>
);
})
)}
</Stack>
</>
)}
<CreateWorkspaceModal
opened={isCreateModalOpen}
onClose={handleCloseCreateModal}
onSubmit={handleCreateWorkspace}
databaseOptions={databaseOptions}
isSubmitting={isCreating}
/>
</Stack>
);
}
@@ -161,6 +418,7 @@ function DataStudioSidebarToggle({
return (
<Tooltip
label={isSidebarOpened ? t`Close sidebar` : t`Open sidebar`}
position="right"
openDelay={TOOLTIP_OPEN_DELAY}
>
<UnstyledButton
@@ -176,3 +434,100 @@ function DataStudioSidebarToggle({
</Tooltip>
);
}
type DataStudioNavToggleProps = {
isNavExpanded: boolean;
onNavToggle: (isExpanded: boolean) => void;
};
function DataStudioNavToggle({
isNavExpanded,
onNavToggle,
}: DataStudioNavToggleProps) {
return (
<Tooltip
label={isNavExpanded ? t`Collapse navigation` : t`Expand navigation`}
position="right"
openDelay={TOOLTIP_OPEN_DELAY}
>
<UnstyledButton
className={S.toggle}
p="0.75rem"
bdrs="md"
onClick={() => onNavToggle(!isNavExpanded)}
>
<FixedSizeIcon name={isNavExpanded ? "chevronleft" : "chevronright"} />
</UnstyledButton>
</Tooltip>
);
}
interface WorkspaceItemProps {
workspace: { id: number; name: string; updated_at: string };
isSelected: boolean;
onOpen: (workspaceId: WorkspaceId) => void;
archiveWorkspace: (workspaceId: WorkspaceId) => Promise<void>;
}
function WorkspaceItem({
workspace,
isSelected,
onOpen,
archiveWorkspace,
}: WorkspaceItemProps) {
const timeAgo = dayjs(workspace.updated_at).fromNow();
const handleArchive = () => {
archiveWorkspace(workspace.id);
};
const handleDelete = () => {
archiveWorkspace(workspace.id);
};
return (
<UnstyledButton
className={cx(S.workspaceItem, { [S.selected]: isSelected })}
onClick={() => onOpen(workspace.id)}
p="0.75rem"
bdrs="md"
>
<Flex align="flex-start" justify="space-between" gap="xs">
<Stack gap="xs" style={{ flex: 1, minWidth: 0 }}>
<Text size="sm" fw={600} truncate>
{workspace.name}
</Text>
<Text size="xs" c="text-secondary" truncate>
{t`Updated ${timeAgo}`}
</Text>
</Stack>
<Menu position="right" withinPortal>
<Menu.Target>
<ActionIcon
className={S.workspaceMenuButton}
onClick={(e: React.MouseEvent) => e.stopPropagation()}
size="sm"
variant="subtle"
>
<Icon name="ellipsis" size={16} />
</ActionIcon>
</Menu.Target>
<Menu.Dropdown onClick={(e: React.MouseEvent) => e.stopPropagation()}>
<Menu.Item
leftSection={<Icon name="archive" />}
onClick={handleArchive}
>
{t`Archive`}
</Menu.Item>
<Menu.Divider />
<Menu.Item
leftSection={<Icon name="trash" />}
onClick={handleDelete}
>
{t`Move to trash`}
</Menu.Item>
</Menu.Dropdown>
</Menu>
</Flex>
</UnstyledButton>
);
}

View File

@@ -0,0 +1,22 @@
import { t } from "ttag";
import { usePageTitle } from "metabase/hooks/use-page-title";
import { SectionLayout, SectionTitle } from "../../components/SectionLayout";
type WorkspacesSectionLayoutProps = {
location: Location;
children?: React.ReactNode;
};
export function WorkspacesSectionLayout({
children,
}: WorkspacesSectionLayoutProps) {
usePageTitle(t`Workspaces`);
return (
<SectionLayout title={<SectionTitle title={t`Workspaces`} />}>
{children}
</SectionLayout>
);
}

View File

@@ -0,0 +1 @@
export * from "./WorkspacesSectionLayout";

View File

@@ -15,12 +15,14 @@ import { DataStudioLayout } from "./app/pages/DataStudioLayout";
import { DependenciesSectionLayout } from "./app/pages/DependenciesSectionLayout";
import { ModelingSectionLayout } from "./app/pages/ModelingSectionLayout";
import { TransformsSectionLayout } from "./app/pages/TransformsSectionLayout";
import { WorkspacesSectionLayout } from "./app/pages/WorkspacesSectionLayout";
import { getDataStudioMetadataRoutes } from "./data-model/routes";
import { getDataStudioGlossaryRoutes } from "./glossary/routes";
import { getDataStudioMetricRoutes } from "./metrics/routes";
import { getDataStudioModelingRoutes } from "./modeling/routes";
import { getDataStudioModelRoutes } from "./models/routes";
import { getDataStudioSnippetRoutes } from "./snippets/routes";
import { getDataStudioWorkspaceRoutes } from "./workspaces/routes";
export function getDataStudioRoutes(
store: Store<State>,
@@ -55,6 +57,9 @@ export function getDataStudioRoutes(
{getDataStudioSnippetRoutes()}
{getDataStudioGlossaryRoutes()}
</Route>
<Route path="workspaces" component={WorkspacesSectionLayout}>
{getDataStudioWorkspaceRoutes()}
</Route>
{PLUGIN_DEPENDENCIES.isEnabled && (
<Route path="dependencies" component={DependenciesSectionLayout}>
{PLUGIN_DEPENDENCIES.getDataStudioDependencyRoutes()}

View File

@@ -0,0 +1,48 @@
import { t } from "ttag";
import { ForwardRefLink } from "metabase/common/components/Link";
import * as Urls from "metabase/lib/urls";
import { Anchor, Box, Card, Group, Stack, Text, Title } from "metabase/ui";
import { useGetWorkspacesQuery } from "metabase-enterprise/api";
export function WorkspaceListPage() {
const { data: workspacesData, isLoading } = useGetWorkspacesQuery();
if (isLoading) {
return (
<Box p="lg">
<Text>{t`Loading...`}</Text>
</Box>
);
}
const workspaces = workspacesData?.items ?? [];
return (
<Box p="lg">
<Title order={2} mb="lg">{t`Workspaces`}</Title>
{workspaces.length === 0 ? (
<Text c="text-secondary">{t`No workspaces yet`}</Text>
) : (
<Stack gap="md">
{workspaces.map((workspace) => (
<Card key={workspace.id} p="md" shadow="none" withBorder>
<Group justify="space-between">
<Anchor
component={ForwardRefLink}
to={Urls.dataStudioWorkspace(workspace.id)}
fw={500}
>
{workspace.name}
</Anchor>
<Text c="text-secondary" size="sm">
{t`Created ${new Date(workspace.created_at).toLocaleDateString()}`}
</Text>
</Group>
</Card>
))}
</Stack>
)}
</Box>
);
}

View File

@@ -0,0 +1,210 @@
import { useCallback, useMemo, useState } from "react";
import { t } from "ttag";
import * as Yup from "yup";
import { skipToken, useListDatabaseSchemasQuery } from "metabase/api";
import { getErrorMessage } from "metabase/api/utils";
import { ActionIcon, Icon, Menu } from "metabase/ui";
import {
useCreateWorkspaceTransformMutation,
useValidateTableNameMutation,
} from "metabase-enterprise/api";
import {
CreateTransformModal,
type NewTransformValues,
} from "metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal";
import {
getInitialNativeSource,
getInitialPythonSource,
} from "metabase-enterprise/transforms/pages/NewTransformPage/utils";
import type {
CreateWorkspaceTransformRequest,
DatabaseId,
Transform,
TransformSource,
TransformTarget,
WorkspaceId,
} from "metabase-types/api";
type TransformType = "sql" | "python";
type AddTransformMenuProps = {
databaseId: number;
workspaceId: number;
onCreate: (transform: Transform) => void;
};
export const AddTransformMenu = ({
databaseId,
workspaceId,
onCreate,
}: AddTransformMenuProps) => {
const [modalType, setModalType] = useState<TransformType | null>(null);
const [createWorkspaceTransform] = useCreateWorkspaceTransformMutation();
const { data: fetchedSchemas = [] } = useListDatabaseSchemasQuery(
databaseId ? { id: databaseId, include_hidden: false } : skipToken,
);
const allowedSchemas = useMemo(
() =>
fetchedSchemas.filter((schema) => !schema.startsWith("mb__isolation")),
[fetchedSchemas],
);
const getSource = useCallback(
(type: TransformType): TransformSource => {
if (type === "sql") {
const source = getInitialNativeSource();
return {
...source,
query: { ...source.query, database: databaseId },
};
}
return {
...getInitialPythonSource(),
"source-database": databaseId,
};
},
[databaseId],
);
const handleClose = () => setModalType(null);
const handleSubmit = useCallback(
async (values: NewTransformValues): Promise<Transform> => {
const source = getSource(modalType!);
const request: CreateWorkspaceTransformRequest & { id: WorkspaceId } =
values.incremental
? {
id: workspaceId,
name: values.name,
description: null,
source,
target: {
type: "table-incremental" as const,
name: values.targetName,
schema: values.targetSchema,
database: databaseId,
"target-incremental-strategy": {
type: "append" as const,
},
},
}
: {
id: workspaceId,
name: values.name,
description: null,
source,
target: {
type: "table" as const,
name: values.targetName,
schema: values.targetSchema,
database: databaseId,
},
};
const transform = await createWorkspaceTransform(request).unwrap();
onCreate(transform);
handleClose();
return transform;
},
[
databaseId,
workspaceId,
modalType,
createWorkspaceTransform,
onCreate,
getSource,
],
);
const validationSchemaExtension = useTransformValidation({
databaseId,
workspaceId,
});
return (
<>
<Menu position="bottom-end">
<Menu.Target>
<ActionIcon size="2rem" p="0" ml="auto" aria-label={t`Add transform`}>
<Icon name="add" size={16} aria-hidden />
</ActionIcon>
</Menu.Target>
<Menu.Dropdown>
<Menu.Item
leftSection={<Icon name="sql" />}
onClick={() => setModalType("sql")}
>
{t`SQL Transform`}
</Menu.Item>
<Menu.Item
leftSection={<Icon name="code_block" />}
onClick={() => setModalType("python")}
>
{t`Python Transform`}
</Menu.Item>
</Menu.Dropdown>
</Menu>
{modalType && (
<CreateTransformModal
source={getSource(modalType)}
defaultValues={{ name: t`New transform` }}
onClose={handleClose}
schemas={allowedSchemas}
showIncrementalSettings={false}
validationSchemaExtension={validationSchemaExtension}
handleSubmit={handleSubmit}
/>
)}
</>
);
};
export const useTransformValidation = ({
databaseId,
target,
workspaceId,
}: {
databaseId: DatabaseId;
target?: TransformTarget;
workspaceId: WorkspaceId;
}) => {
const [validateTableName] = useValidateTableNameMutation();
const yupSchema = useMemo(
() => ({
targetName: Yup.string()
.required("Target table name is required")
.test(async (value, context) => {
if (!value) {
return context.createError({
message: "Target table name is required",
});
}
const schema = context.parent.targetSchema;
if (target && target.name === value && target.schema === schema) {
return true;
}
try {
const message = await validateTableName({
id: workspaceId,
db_id: databaseId,
target: { type: "table", name: value, schema },
}).unwrap();
return message === "OK" ? true : context.createError({ message });
} catch (error) {
const message = getErrorMessage(error);
return context.createError({ message });
}
}),
}),
[databaseId, target, workspaceId, validateTableName],
);
return yupSchema;
};

View File

@@ -0,0 +1,67 @@
import { t } from "ttag";
import { Button } from "metabase/ui";
import {
useLazyGetTransformQuery,
useUpdateWorkspaceContentsMutation,
} from "metabase-enterprise/api";
import type { Transform, TransformId, WorkspaceId } from "metabase-types/api";
import { useWorkspace } from "./WorkspaceProvider";
interface Props {
transform: Transform;
workspaceId: WorkspaceId;
onOpenTransform: (transformId: TransformId) => void;
}
export const CheckOutTransformButton = ({
transform,
workspaceId,
onOpenTransform,
}: Props) => {
const [getTransform] = useLazyGetTransformQuery();
const [updateWorkspaceContents] = useUpdateWorkspaceContentsMutation();
const {
addOpenedTransform,
removeOpenedTransform,
setActiveTransform,
removeEditedTransform,
} = useWorkspace();
const handleClick = async () => {
const response = await updateWorkspaceContents({
id: workspaceId,
add: {
transforms: [transform.id],
},
});
const newTransform = response.data?.contents.transforms.find(
(t) => t.upstream_id === transform.id,
);
const newTransformId = newTransform?.id;
if (newTransformId != null) {
// TODO: remove when backend adds contents hydration to previous request
const newTransform = await getTransform(newTransformId).unwrap();
if (newTransform) {
removeEditedTransform(transform.id);
addOpenedTransform(newTransform);
removeOpenedTransform(transform.id);
setActiveTransform(newTransform);
onOpenTransform(newTransform.id);
}
}
};
return (
<Button
size="sm"
variant="filled"
onClick={handleClick}
>{t`Check out`}</Button>
);
};

View File

@@ -0,0 +1,103 @@
import { useMemo } from "react";
import { t } from "ttag";
import EmptyState from "metabase/common/components/EmptyState";
import { Stack, Text } from "metabase/ui";
import type { Transform, WorkspaceId } from "metabase-types/api";
import { useWorkspace } from "../WorkspaceProvider";
import { TransformListItem } from "./TransformListItem";
import { TransformListItemMenu } from "./TransformListItemMenu";
type CodeTabProps = {
activeTransformId?: number;
transforms: Transform[];
workspaceId: WorkspaceId;
workspaceTransforms: Transform[];
onTransformClick: (transform: Transform) => void;
};
export const CodeTab = ({
activeTransformId,
transforms,
workspaceId,
workspaceTransforms,
onTransformClick,
}: CodeTabProps) => {
const { editedTransforms, hasTransformEdits } = useWorkspace();
const workspaceTransformsUpstreamIds = useMemo(
() => new Set(workspaceTransforms.map((t) => t.upstream_id)),
[workspaceTransforms],
);
const availableTransforms = useMemo(() => {
return transforms.filter((transform) => {
return (
transform.workspace_id == null &&
!workspaceTransformsUpstreamIds.has(transform.id)
);
});
}, [workspaceTransformsUpstreamIds, transforms]);
const handleTransformClick = (transform: Transform) => {
const edited = editedTransforms.get(transform.id);
const transformToOpen = edited
? ({ ...transform, ...edited } as Transform)
: transform;
onTransformClick(transformToOpen);
};
return (
<Stack h="100%" gap={0}>
<Stack
gap="xs"
pb="sm"
style={{
borderBottom: "1px solid var(--mb-color-border)",
}}
>
<Stack gap={0}>
<Text fw={600}>{t`Workspace Transforms`}</Text>
{workspaceTransforms.map((transform) => {
const isEdited = hasTransformEdits(transform);
return (
<TransformListItem
key={transform.id}
name={transform.name}
icon="pivot_table"
fw={600}
isActive={activeTransformId === transform.id}
isEdited={isEdited}
menu={
<TransformListItemMenu
transform={transform}
workspaceId={workspaceId}
/>
}
onClick={() => handleTransformClick(transform)}
/>
);
})}
</Stack>
{workspaceTransforms.length === 0 && (
<EmptyState message={t`Workspace is empty`} />
)}
</Stack>
<Stack py="sm" gap="xs">
{availableTransforms.map((transform) => (
<TransformListItem
key={transform.id}
name={transform.name}
isActive={activeTransformId === transform.id}
isEdited={editedTransforms.has(transform.id)}
onClick={() => handleTransformClick(transform)}
/>
))}
</Stack>
</Stack>
);
};

View File

@@ -0,0 +1,26 @@
.root {
display: flex;
align-items: center;
gap: var(--mantine-spacing-sm);
padding: var(--mantine-spacing-xs) var(--mantine-spacing-sm);
border-radius: var(--mantine-radius-sm);
cursor: pointer;
flex-wrap: nowrap;
&:hover .menu {
visibility: visible;
}
&:hover {
background-color: var(--mb-color-bg-light);
}
}
.name {
flex: 1;
min-width: 0;
}
.menu {
visibility: hidden;
}

View File

@@ -0,0 +1,40 @@
import type { ReactNode } from "react";
import { Box, type BoxProps, Icon, type IconName, Text } from "metabase/ui";
import { StatusDot } from "../components/StatusDot/StatusDot";
import S from "./TransformListItem.module.css";
type TransformListItemProps = {
name: string;
icon?: IconName;
isActive?: boolean;
isEdited?: boolean;
menu?: ReactNode;
onClick?: () => void;
} & BoxProps;
export const TransformListItem = ({
name,
icon = "database",
isActive,
isEdited,
menu,
onClick,
...props
}: TransformListItemProps) => {
return (
<Box className={S.root} onClick={onClick} {...props}>
<Icon name={icon} size={14} c="brand" />
<Text className={S.name} c={isActive ? "brand" : "text-dark"} truncate>
{name}
</Text>
{isEdited && <StatusDot status="changed" />}
<Box className={S.menu} flex="0 0 auto">
{menu}
</Box>
</Box>
);
};

View File

@@ -0,0 +1,56 @@
import { t } from "ttag";
import { useMetadataToasts } from "metabase/metadata/hooks";
import { ActionIcon, Icon, Menu } from "metabase/ui";
import { useUpdateWorkspaceContentsMutation } from "metabase-enterprise/api";
import type { Transform, WorkspaceId } from "metabase-types/api";
import { useWorkspace } from "../WorkspaceProvider";
interface Props {
transform: Transform;
workspaceId: WorkspaceId;
}
export function TransformListItemMenu({ transform, workspaceId }: Props) {
const [updateWorkspaceContents] = useUpdateWorkspaceContentsMutation();
const { removeEditedTransform, removeOpenedTransform } = useWorkspace();
const { sendErrorToast, sendSuccessToast } = useMetadataToasts();
const handleRemove = async () => {
const response = await updateWorkspaceContents({
id: workspaceId,
remove: {
transforms: [transform.id],
},
});
if (response.error) {
sendErrorToast(t`Failed to remove transform from the workspace`);
} else {
sendSuccessToast(t`Transform removed from the workspace`);
removeEditedTransform(transform.id);
removeOpenedTransform(transform.id);
}
};
return (
<Menu position="bottom-end" withinPortal>
<Menu.Target>
<ActionIcon
onClick={(event) => event.stopPropagation()}
size="sm"
variant="subtle"
>
<Icon name="ellipsis" />
</ActionIcon>
</Menu.Target>
<Menu.Dropdown onClick={(event) => event.stopPropagation()}>
<Menu.Item leftSection={<Icon name="trash" />} onClick={handleRemove}>
{t`Remove`}
</Menu.Item>
</Menu.Dropdown>
</Menu>
);
}

View File

@@ -0,0 +1,55 @@
import { t } from "ttag";
import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
import CS from "metabase/css/core/index.css";
import { Box, Stack, Text } from "metabase/ui";
import Visualization from "metabase/visualizations/components/Visualization";
import type { DatabaseId, TableId } from "metabase-types/api";
import { useTablePreview } from "./useTablePreview";
interface DataTabProps {
databaseId: DatabaseId | null;
tableId: TableId | null;
}
export function DataTab({ databaseId, tableId }: DataTabProps) {
const { rawSeries, isFetching, error } = useTablePreview({
databaseId,
tableId,
});
if (!databaseId || !tableId) {
return (
<Stack h="100%" align="center" justify="center">
<Text c="text-medium">{t`Select a table to view its data`}</Text>
</Stack>
);
}
if (isFetching) {
return <LoadingAndErrorWrapper loading />;
}
if (error) {
return (
<Stack h="100%" align="center" justify="center">
<Text c="error">{t`Error loading data`}</Text>
</Stack>
);
}
if (!rawSeries) {
return (
<Stack h="100%" align="center" justify="center">
<Text c="text-medium">{t`No data available`}</Text>
</Stack>
);
}
return (
<Box h="100%" className={CS.relative}>
<Visualization queryBuilderMode="dataset" rawSeries={rawSeries} />
</Box>
);
}

View File

@@ -0,0 +1,92 @@
import { t } from "ttag";
import { Stack, Text } from "metabase/ui";
import type {
TableId,
Transform,
WorkspaceTablesResponse,
WorkspaceTransformItem,
} from "metabase-types/api";
import { type OpenTable, useWorkspace } from "../WorkspaceProvider";
import { TableListItem } from "./TableListItem";
type DataTabSidebarProps = {
tables: WorkspaceTablesResponse;
workspaceTransforms: WorkspaceTransformItem[];
dbTransforms: Transform[];
selectedTableId?: TableId | null;
onTransformClick?: (transform: WorkspaceTransformItem) => void;
onTableSelect?: (table: OpenTable) => void;
};
export const DataTabSidebar = ({
tables,
workspaceTransforms,
dbTransforms,
selectedTableId,
onTransformClick,
onTableSelect,
}: DataTabSidebarProps) => {
const { hasTransformEdits } = useWorkspace();
return (
<Stack h="100%" gap="sm">
<Stack
gap="xs"
pb="sm"
style={{ borderBottom: "1px solid var(--mb-color-border)" }}
>
<Text fw={600}>{t`Data active in this workspace`}</Text>
<Stack gap={0}>
{tables.outputs.map((table, index: number) => {
const workspaceTransform = table.workspace
? workspaceTransforms.find(
(t) => t.id === table.workspace?.["transform-id"],
)
: undefined;
const originalTransform = workspaceTransform
? dbTransforms.find(
(t) => t.id === workspaceTransform.upstream_id,
)
: undefined;
const hasChanges = originalTransform
? hasTransformEdits(originalTransform)
: false;
const tableId = table.workspace?.["table-id"];
return (
<TableListItem
key={`output-${index}`}
name={table.workspace?.table || ""}
schema={table.workspace?.schema}
icon="pivot_table"
type="output"
hasChanges={hasChanges}
transform={workspaceTransform}
tableId={tableId}
isSelected={tableId === selectedTableId}
onTransformClick={onTransformClick}
onTableClick={onTableSelect}
/>
);
})}
</Stack>
<Stack gap={0}>
{tables.inputs.map((table, index) => (
<TableListItem
key={`input-${index}`}
name={table.table || ""}
schema={table.schema}
icon="table"
type="input"
tableId={table.id}
onTransformClick={onTransformClick}
onTableClick={onTableSelect}
/>
))}
</Stack>
</Stack>
</Stack>
);
};

View File

@@ -0,0 +1,34 @@
.root {
display: flex;
align-items: center;
gap: var(--mantine-spacing-sm);
padding: var(--mantine-spacing-xs) var(--mantine-spacing-sm);
border-radius: var(--mantine-radius-sm);
flex-wrap: nowrap;
&:hover .menu {
visibility: visible;
}
&:hover {
background-color: var(--mb-color-bg-light);
}
}
.clickable {
cursor: pointer;
}
.selected {
background-color: var(--mb-color-bg-medium);
}
.name {
flex: 1;
min-width: 0;
}
.actionIcon {
&:hover {
background-color: var(--mb-color-bg-medium) !important;
}
}

View File

@@ -0,0 +1,93 @@
import cx from "classnames";
import { t } from "ttag";
import {
ActionIcon,
Box,
Icon,
type IconName,
Text,
Tooltip,
} from "metabase/ui";
import type { TableId, WorkspaceTransformItem } from "metabase-types/api";
import type { OpenTable } from "../WorkspaceProvider";
import { StatusDot } from "../components/StatusDot/StatusDot";
import S from "./TableListItem.module.css";
type TableListItemProps = {
name: string;
schema?: string | null;
icon?: IconName;
type: "input" | "output";
hasChanges?: boolean;
transform?: WorkspaceTransformItem;
tableId?: TableId;
isSelected?: boolean;
onTransformClick?: (transform: WorkspaceTransformItem) => void;
onTableClick?: (table: OpenTable) => void;
};
export const TableListItem = ({
name,
schema,
icon = "table",
type,
hasChanges = false,
transform,
tableId,
isSelected = false,
onTransformClick,
onTableClick,
}: TableListItemProps) => {
const displayName = schema ? `${schema}.${name}` : name;
const handleTransformClick = (e: React.MouseEvent) => {
e.stopPropagation();
if (transform && onTransformClick) {
onTransformClick(transform);
}
};
const handleTableClick = () => {
if (tableId && onTableClick) {
onTableClick({ tableId, name, schema });
}
};
const isClickable = tableId && onTableClick;
return (
<Box
className={cx(S.root, {
[S.selected]: isSelected,
[S.clickable]: isClickable,
})}
onClick={isClickable ? handleTableClick : undefined}
>
<Icon
name={icon}
size={14}
c={type === "input" ? "saturated-green" : "saturated-yellow"}
/>
<Text className={S.name} c="text-dark" truncate>
{displayName}
</Text>
{hasChanges && <StatusDot status="changed" />}
{type === "output" && transform && (
<Tooltip label={t`Open transform`} position="top">
<ActionIcon
className={S.actionIcon}
ml="auto"
size="sm"
variant="subtle"
onClick={handleTransformClick}
>
<Icon name="code_block" size={14} c="text-medium" />
</ActionIcon>
</Tooltip>
)}
</Box>
);
};

View File

@@ -0,0 +1,4 @@
export { DataTab } from "./DataTab";
export { DataTabSidebar } from "./DataTabSidebar";
export { useTablePreview as useTableQuestion } from "./useTablePreview";
export type { OpenTable } from "../WorkspaceProvider";

View File

@@ -0,0 +1,101 @@
import { useMemo } from "react";
import _ from "underscore";
import {
skipToken,
useGetAdhocQueryQuery,
useGetTableQueryMetadataQuery,
} from "metabase/api";
import { getErrorMessage } from "metabase/api/utils";
import { useSelector } from "metabase/lib/redux";
import { getTableMetadataQuery } from "metabase/metadata/pages/shared";
import { getMetadata } from "metabase/selectors/metadata";
import * as Lib from "metabase-lib";
import type { DatabaseId, RawSeries, TableId } from "metabase-types/api";
import { createMockCard } from "metabase-types/api/mocks";
interface UseTableQuestionProps {
databaseId: DatabaseId | null;
tableId: TableId | null;
}
interface UseTablePreviewResult {
rawSeries?: RawSeries;
isFetching: boolean;
isLoading: boolean;
isError: boolean;
error?: string;
}
export function useTablePreview({
databaseId,
tableId,
}: UseTableQuestionProps): UseTablePreviewResult {
const metadata = useSelector(getMetadata);
// Need to fetch this to make sure metadata is populated with proper table information on
// initial page load.
useGetTableQueryMetadataQuery(
tableId && tableId in metadata.tables
? skipToken
: getTableMetadataQuery(tableId),
);
const metadataProvider =
Object.keys(metadata.tables).length > 0
? Lib.metadataProvider(databaseId, metadata)
: null;
const table =
metadataProvider && tableId
? Lib.tableOrCardMetadata(metadataProvider, tableId)
: null;
const query =
table && metadataProvider
? Lib.queryFromTableOrCardMetadata(metadataProvider, table)
: null;
const { data, refetch, ...rest } = useGetAdhocQueryQuery(
query == null
? skipToken
: {
...Lib.toJsQuery(query),
ignore_error: true,
},
);
const base = {
...rest,
error: rest.error ? getErrorMessage(rest.error) : undefined,
rawSeries: undefined,
};
const rawSeries = useMemo(
() =>
query
? [
{
card: createMockCard({
dataset_query: Lib.toJsQuery(query),
display: "table",
visualization_settings: {},
}),
data: {
...data?.data,
rows: _.uniq(data?.data?.rows ?? []),
},
},
]
: ([] as RawSeries),
[data?.data, query],
);
if (data?.status === "failed") {
return { ...base, isError: true, error: getErrorMessage(data) };
}
if (query == null || !data?.data || data.data.cols.length === 0) {
return base;
}
return { ...base, rawSeries };
}

View File

@@ -0,0 +1,12 @@
import { PLUGIN_METABOT } from "metabase/plugins";
export const MetabotTab = () => {
const MetabotProvider = PLUGIN_METABOT.getMetabotProvider();
const MetabotChat = PLUGIN_METABOT.MetabotChat;
return (
<MetabotProvider>
<MetabotChat />
</MetabotProvider>
);
};

View File

@@ -0,0 +1,58 @@
import { t } from "ttag";
import { Button } from "metabase/ui";
import { useUpdateTransformMutation } from "metabase-enterprise/api";
import { isSameSource } from "metabase-enterprise/transforms/utils";
import type { DatabaseId, Transform } from "metabase-types/api";
import { type EditedTransform, useWorkspace } from "./WorkspaceProvider";
interface Props {
databaseId: DatabaseId;
editedTransform: EditedTransform;
transform: Transform;
}
export const SaveTransformButton = ({
databaseId,
editedTransform,
transform,
}: Props) => {
const [updateTransform] = useUpdateTransformMutation();
const { setActiveTransform, removeEditedTransform } = useWorkspace();
const hasSourceChanged = !isSameSource(
editedTransform.source,
transform.source,
);
const hasTargetNameChanged =
transform.target.name !== editedTransform.target.name;
const hasChanges = hasSourceChanged || hasTargetNameChanged;
const handleClick = async () => {
const response = await updateTransform({
id: transform.id,
source: editedTransform.source,
name: editedTransform.name,
target: {
type: "table",
name: editedTransform.target.name,
schema: transform.target.schema,
database: databaseId,
},
});
removeEditedTransform(transform.id);
setActiveTransform(response.data);
};
return (
<Button
disabled={!hasChanges}
size="sm"
variant="filled"
onClick={handleClick}
>{t`Save`}</Button>
);
};

View File

@@ -0,0 +1,30 @@
import { t } from "ttag";
import { Stack, Text, Title } from "metabase/ui";
interface SetupTabProps {
databaseName?: string;
}
export const SetupTab = ({ databaseName }: SetupTabProps) => {
return (
<Stack gap="lg">
<Stack gap="xs">
<Title order={3}>{t`Workspace Setup`}</Title>
<Text c="text-medium">
{t`Configure your data warehouse connection for this workspace`}
</Text>
</Stack>
<Stack gap="xs">
<Text fw={600}>{t`Data Warehouse`}</Text>
<Text>
<code>{databaseName}</code>
</Text>
<Text c="text-light" size="sm">
{t`Data warehouse selection is locked after editing or timeout`}
</Text>
</Stack>
</Stack>
);
};

View File

@@ -0,0 +1,58 @@
import { useMemo, useState } from "react";
import _ from "underscore";
import {
TransformQueryPageEditor,
type TransformQueryPageEditorUiState,
} from "metabase-enterprise/transforms/pages/TransformQueryPage/TransformQueryPage";
import type { DraftTransformSource } from "metabase-types/api";
type TransformEditorProps = {
disabled: boolean;
source: DraftTransformSource;
onChange: (source: DraftTransformSource) => void;
};
export function TransformEditor({
disabled,
source,
onChange,
}: TransformEditorProps) {
const [uiState, setUiState] = useState(getInitialUiStateForTransform);
const uiOptions = useMemo(
() => ({
canChangeDatabase: false,
readOnly: disabled,
}),
[disabled],
);
const handleSourceChange = (source: DraftTransformSource) => {
onChange?.(source);
};
return (
<TransformQueryPageEditor
source={source}
isDirty={false}
uiState={uiState}
setUiState={setUiState}
uiOptions={uiOptions}
databases={[]}
setSourceAndRejectProposed={handleSourceChange}
acceptProposed={_.noop}
rejectProposed={_.noop}
/>
);
}
function getInitialUiStateForTransform() {
return {
lastRunResult: null,
lastRunQuery: null,
selectionRange: [],
modalSnippet: null,
sidebarType: null,
modalType: null,
} as TransformQueryPageEditorUiState;
}

View File

@@ -0,0 +1,272 @@
import { useDisclosure } from "@mantine/hooks";
import { useCallback, useMemo } from "react";
import { t } from "ttag";
import * as Yup from "yup";
import { Form, FormProvider, FormTextInput } from "metabase/forms";
import { useDispatch } from "metabase/lib/redux";
import { useMetadataToasts } from "metabase/metadata/hooks";
import { Box, Button, Group, Icon, Stack, Text, rem } from "metabase/ui";
import { useRunTransformMutation, workspaceApi } from "metabase-enterprise/api";
import { UpdateTargetModal } from "metabase-enterprise/transforms/pages/TransformTargetPage/TargetSection/UpdateTargetModal";
import {
isSameSource,
isTransformRunning,
} from "metabase-enterprise/transforms/utils";
import type {
DatabaseId,
DraftTransformSource,
Transform,
TransformId,
WorkspaceId,
} from "metabase-types/api";
import { useTransformValidation } from "./AddTransformMenu";
import { CheckOutTransformButton } from "./CheckOutTransformButton";
import { SaveTransformButton } from "./SaveTransformButton";
import { TransformEditor } from "./TransformEditor";
import { type EditedTransform, useWorkspace } from "./WorkspaceProvider";
interface Props {
databaseId: DatabaseId;
editedTransform: EditedTransform;
transform: Transform;
workspaceId: WorkspaceId;
onChange: (patch: Partial<EditedTransform>) => void;
onOpenTransform: (transformId: TransformId) => void;
}
export const TransformTab = ({
databaseId,
editedTransform,
transform,
workspaceId,
onChange,
onOpenTransform,
}: Props) => {
const { updateTransformState } = useWorkspace();
const { sendSuccessToast } = useMetadataToasts();
const [
isChangeTargetModalOpen,
{ open: openChangeTargetModal, close: closeChangeTargetModal },
] = useDisclosure();
const dispatch = useDispatch();
const hasSourceChanged = !isSameSource(
editedTransform.source,
transform.source,
);
const hasTargetNameChanged =
transform.target.name !== editedTransform.target.name;
const hasChanges = hasSourceChanged || hasTargetNameChanged;
const isSaved = transform.workspace_id === workspaceId;
const [runTransform] = useRunTransformMutation();
const handleRun = async () => {
try {
await runTransform(transform.id).unwrap();
// Invalidate the workspace tables cache since transform execution
// may affect the list of workspace tables.
if (transform.workspace_id) {
dispatch(
workspaceApi.util.invalidateTags([
{ type: "workspace", id: transform.workspace_id },
]),
);
}
} catch (error) {
console.error("Failed to run transform", error);
}
};
const handleSourceChange = (source: DraftTransformSource) => {
onChange({ source });
};
const handleNameChange = useCallback(
(name: string) => {
onChange({
target: {
type: editedTransform.target.type,
name,
},
});
},
[onChange, editedTransform.target.type],
);
const validationSchemaExtension = useTransformValidation({
databaseId,
target: transform.target,
workspaceId,
});
const validationSchema = useMemo(
() =>
Yup.object({
targetName:
validationSchemaExtension?.targetName ||
Yup.string().required("Target table name is required"),
targetSchema: Yup.string().nullable(),
}),
[validationSchemaExtension],
);
const initialValues = useMemo(
() => ({
targetName: editedTransform.target.name,
targetSchema: transform.target.schema || null,
}),
[editedTransform.target.name, transform.target.schema],
);
const handleFormSubmit = useCallback(
(values: typeof initialValues) => {
handleNameChange(values.targetName);
},
[handleNameChange],
);
const handleFieldChange = useCallback(
(field: string, value: string) => {
if (field === "targetName") {
handleNameChange(value);
}
},
[handleNameChange],
);
const handleTargetUpdate = useCallback(
(updatedTransform: Transform) => {
const hasNameChanged = editedTransform.name !== transform.name;
const hasSourceChanged = !isSameSource(
editedTransform.source,
transform.source,
);
const editedTransformToKeep =
hasNameChanged || hasSourceChanged
? {
name: editedTransform.name,
source: editedTransform.source,
target: {
type: updatedTransform.target.type,
name: updatedTransform.target.name,
},
}
: null;
updateTransformState(updatedTransform, editedTransformToKeep);
sendSuccessToast(t`Transform target updated`);
closeChangeTargetModal();
},
[
closeChangeTargetModal,
editedTransform.name,
editedTransform.source,
transform.name,
transform.source,
updateTransformState,
sendSuccessToast,
],
);
const isRunning = isTransformRunning(transform);
return (
<Stack gap={0} h="100%">
<Group
flex="0 0 auto"
justify="space-between"
mih={rem(73)} // avoid CLS when showing/hiding output table input
p="md"
style={{ borderBottom: "1px solid var(--mb-color-border)" }}
>
<Group>
{isSaved && (
<FormProvider
key={transform.id}
initialValues={initialValues}
validationSchema={validationSchema}
onSubmit={handleFormSubmit}
>
<Form>
<Group>
<Text
c="text-dark"
component="label"
fw="bold"
>{t`Output table`}</Text>
<FormTextInput
name="targetName"
miw={rem(300)}
onChange={(e) =>
handleFieldChange("targetName", e.target.value)
}
/>
</Group>
</Form>
</FormProvider>
)}
</Group>
<Group>
{isSaved && (
<Button
disabled={hasChanges}
leftSection={<Icon name="play" />}
size="sm"
onClick={handleRun}
>{t`Run`}</Button>
)}
{isSaved && (
<Button
leftSection={<Icon name="pencil_lines" />}
size="sm"
disabled={isRunning}
onClick={openChangeTargetModal}
>{t`Change target`}</Button>
)}
{isSaved && (
<SaveTransformButton
databaseId={databaseId}
editedTransform={editedTransform}
transform={transform}
/>
)}
{!isSaved && (
<CheckOutTransformButton
transform={transform}
workspaceId={workspaceId}
onOpenTransform={onOpenTransform}
/>
)}
</Group>
</Group>
{editedTransform && (
<Box flex="1">
<TransformEditor
disabled={!isSaved}
source={editedTransform.source}
onChange={handleSourceChange}
/>
</Box>
)}
{isChangeTargetModalOpen && (
<UpdateTargetModal
transform={transform}
onUpdate={handleTargetUpdate}
onClose={closeChangeTargetModal}
/>
)}
</Stack>
);
};

View File

@@ -0,0 +1,20 @@
.tabsPanel {
display: flex;
gap: 0.5rem;
flex-wrap: nowrap;
overflow: auto;
scrollbar-width: none;
-ms-overflow-style: none;
&::-webkit-scrollbar {
display: none;
}
&::before {
display: none;
}
& > button {
border-radius: 0;
}
}

View File

@@ -0,0 +1,516 @@
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { push } from "react-router-redux";
import { t } from "ttag";
import { useListDatabasesQuery } from "metabase/api";
import { useDispatch } from "metabase/lib/redux";
import { checkNotNull } from "metabase/lib/types";
import * as Urls from "metabase/lib/urls";
import { useMetadataToasts } from "metabase/metadata/hooks";
import { PLUGIN_METABOT } from "metabase/plugins";
import {
ActionIcon,
Box,
Button,
Flex,
Group,
Icon,
Stack,
Tabs,
Text,
} from "metabase/ui";
import {
useGetWorkspaceQuery,
useGetWorkspaceTablesQuery,
useListTransformsQuery,
useMergeWorkspaceMutation,
useUpdateWorkspaceNameMutation,
} from "metabase-enterprise/api";
import { PaneHeaderInput } from "metabase-enterprise/data-studio/common/components/PaneHeader";
import { NAME_MAX_LENGTH } from "metabase-enterprise/transforms/constants";
import type { Transform } from "metabase-types/api";
import { AddTransformMenu } from "./AddTransformMenu";
import { CodeTab } from "./CodeTab/CodeTab";
import { DataTab, DataTabSidebar } from "./DataTab";
import { MetabotTab } from "./MetabotTab";
import { SetupTab } from "./SetupTab";
import { TransformTab } from "./TransformTab";
import styles from "./WorkspacePage.module.css";
import {
type EditedTransform,
type OpenTable,
WorkspaceProvider,
useWorkspace,
} from "./WorkspaceProvider";
type WorkspacePageProps = {
params: {
workspaceId: string;
};
};
function WorkspacePageContent({ params }: WorkspacePageProps) {
const id = Number(params.workspaceId);
const dispatch = useDispatch();
const { sendErrorToast } = useMetadataToasts();
const isMetabotAvailable = PLUGIN_METABOT.isEnabled();
const [tab, setTab] = useState<string>("setup");
const { data: databases = { data: [] } } = useListDatabasesQuery({});
const { data: allTransforms = [] } = useListTransformsQuery({});
const { data: workspace, isLoading: isLoadingWorkspace } =
useGetWorkspaceQuery(id);
const { data: workspaceTables = { inputs: [], outputs: [] } } =
useGetWorkspaceTablesQuery(id);
const [mergeWorkspace, { isLoading: isMerging }] =
useMergeWorkspaceMutation();
const [updateWorkspaceName] = useUpdateWorkspaceNameMutation();
const sourceDb = databases?.data.find(
(db) => db.id === workspace?.database_id,
);
const dbTransforms = useMemo(
() =>
allTransforms.filter((t) => {
// TODO: @uladzimirdev add guards
if (t.source_type === "python") {
return (
"source-database" in t.source &&
t.source["source-database"] === sourceDb?.id
);
}
if (t.source_type === "native") {
return (
"query" in t.source && t.source.query.database === sourceDb?.id
);
}
return false;
}),
[allTransforms, sourceDb],
);
const {
openedTransforms,
activeTransform,
activeEditedTransform,
setActiveTransform,
addOpenedTransform,
removeOpenedTransform,
patchEditedTransform,
hasUnsavedChanges,
openedTables,
activeTable,
setActiveTable,
addOpenedTable,
removeOpenedTable,
} = useWorkspace();
const workspaceTransforms = useMemo(
() => workspace?.contents?.transforms ?? [],
[workspace],
);
useEffect(() => {
if (activeTransform) {
setTab(String(activeTransform.id));
} else if (activeTable) {
setTab(`table-${activeTable.tableId}`);
} else {
setTab("setup");
}
}, [id, activeTransform, activeTable]);
const tabsListRef = useRef<HTMLDivElement>(null);
useEffect(() => {
if (tabsListRef.current && tab) {
const activeTabElement = tabsListRef.current.querySelector(
`[data-active="true"]`,
) as HTMLElement;
if (activeTabElement) {
activeTabElement.scrollIntoView({
behavior: "smooth",
block: "nearest",
});
}
}
}, [tab]);
const handleTransformChange = useCallback(
(patch: Partial<EditedTransform>) => {
patchEditedTransform(checkNotNull(activeTransform).id, patch);
},
[activeTransform, patchEditedTransform],
);
const handleCloseClick = useCallback(
(event: React.MouseEvent, transform: Transform, index: number) => {
event.stopPropagation();
const isActive = activeTransform?.id === transform.id;
const remaining = openedTransforms.filter(
(item) => item.id !== transform.id,
);
removeOpenedTransform(transform.id);
if (!isActive) {
return;
}
const fallback = remaining[index - 1] ?? remaining[index] ?? undefined;
setActiveTransform(fallback);
if (fallback) {
setTab(String(fallback.id));
} else {
setTab("setup");
}
},
[
activeTransform,
removeOpenedTransform,
setActiveTransform,
setTab,
openedTransforms,
],
);
const handleMergeWorkspace = useCallback(async () => {
try {
const response = await mergeWorkspace(id).unwrap();
if (response.errors && response.errors.length > 0) {
sendErrorToast(
t`Failed to merge workspace: ${response.errors.map((e) => e.error).join(", ")}`,
);
return;
}
dispatch(push(Urls.transformList()));
} catch (error) {
sendErrorToast(t`Failed to merge workspace`);
}
}, [id, mergeWorkspace, sendErrorToast, dispatch]);
const handleWorkspaceNameChange = useCallback(
async (newName: string) => {
if (!workspace || newName.trim() === workspace.name.trim()) {
return;
}
try {
await updateWorkspaceName({ id, name: newName.trim() }).unwrap();
} catch (error) {
sendErrorToast(t`Failed to update workspace name`);
}
},
[workspace, id, updateWorkspaceName, sendErrorToast],
);
const handleTableSelect = useCallback(
(table: OpenTable) => {
addOpenedTable(table);
setActiveTable(table);
setActiveTransform(undefined);
},
[addOpenedTable, setActiveTable, setActiveTransform],
);
const handleTableClose = useCallback(
(event: React.MouseEvent, table: OpenTable, index: number) => {
event.stopPropagation();
const isActive = activeTable?.tableId === table.tableId;
const remaining = openedTables.filter(
(item) => item.tableId !== table.tableId,
);
removeOpenedTable(table.tableId);
if (!isActive) {
return;
}
if (index > 0 && remaining.length > 0) {
setActiveTable(remaining[index - 1]);
setTab(`table-${remaining[index - 1].tableId}`);
} else if (remaining.length > 0) {
setActiveTable(remaining[0]);
setTab(`table-${remaining[0].tableId}`);
} else if (openedTransforms.length > 0) {
setActiveTransform(openedTransforms[openedTransforms.length - 1]);
} else {
setActiveTable(undefined);
setTab("setup");
}
},
[
activeTable,
openedTables,
openedTransforms,
removeOpenedTable,
setActiveTable,
setActiveTransform,
setTab,
],
);
if (isLoadingWorkspace) {
return (
<Box p="lg">
<Text>{t`Loading...`}</Text>
</Box>
);
}
if (!workspace) {
return (
<Box p="lg">
<Text c="text-dark">{t`Workspace not found`}</Text>
</Box>
);
}
return (
<Stack h="100%" gap={0}>
<Group
px="md"
py="sm"
style={{ borderBottom: "1px solid var(--mb-color-border)" }}
justify="space-between"
>
<Flex gap="xs" align="center">
<Icon name="git_branch" size="1rem" aria-hidden />
<PaneHeaderInput
initialValue={workspace.name}
placeholder={t`Workspace name`}
maxLength={NAME_MAX_LENGTH}
onChange={handleWorkspaceNameChange}
/>
</Flex>
<Button
variant="filled"
onClick={handleMergeWorkspace}
loading={isMerging}
disabled={hasUnsavedChanges() || workspaceTransforms.length === 0}
size="xs"
>
{t`Merge`}
</Button>
</Group>
<Group align="flex-start" gap={0} flex="1 1 auto" wrap="nowrap">
<Box
w="70%"
h="100%"
style={{ borderRight: "1px solid var(--mb-color-border)" }}
pos="relative"
>
<Tabs
defaultValue="setup"
display="flex"
h="100%"
style={{ flexDirection: "column" }}
value={tab}
onChange={(tab) => {
if (tab) {
setTab(tab);
}
if (
tab === "setup" ||
(tab === "metabot" && (activeTransform || activeTable))
) {
setActiveTransform(undefined);
setActiveTable(undefined);
}
}}
>
<Flex
wrap="nowrap"
flex="0 0 auto"
px="md"
style={{ borderBottom: "1px solid var(--mb-color-border)" }}
>
<Tabs.List ref={tabsListRef} className={styles.tabsPanel}>
<Tabs.Tab value="setup">
<Group gap="xs" wrap="nowrap">
<Icon name="database" aria-hidden />
{t`Setup`}
</Group>
</Tabs.Tab>
{isMetabotAvailable && (
<Tabs.Tab value="metabot">
<Group gap="xs" wrap="nowrap">
<Icon name="message_circle" aria-hidden />
{t`Agent Chat`}
</Group>
</Tabs.Tab>
)}
{openedTables.map((table, index) => (
<Tabs.Tab
key={`table-${table.tableId}`}
value={`table-${table.tableId}`}
onClick={() => {
setActiveTable(table);
setActiveTransform(undefined);
}}
>
<Group gap="xs" wrap="nowrap">
<Icon name="table" aria-hidden />
{table.schema
? `${table.schema}.${table.name}`
: table.name}
<ActionIcon size="1rem" p="0" ml="xs">
<Icon
name="close"
size={10}
aria-hidden
onClick={(event) =>
handleTableClose(event, table, index)
}
/>
</ActionIcon>
</Group>
</Tabs.Tab>
))}
{openedTransforms.map((transform, index) => (
<Tabs.Tab
key={transform.id}
value={String(transform.id)}
onClick={() => {
setActiveTransform(transform);
}}
>
<Group gap="xs" wrap="nowrap">
<Icon name="pivot_table" aria-hidden />
{transform.name}
<ActionIcon size="1rem" p="0" ml="xs">
<Icon
name="close"
size={10}
aria-hidden
onClick={(event) =>
handleCloseClick(event, transform, index)
}
/>
</ActionIcon>
</Group>
</Tabs.Tab>
))}
</Tabs.List>
</Flex>
<Box flex={1} mih={0}>
<Tabs.Panel value="setup" h="100%" p="md">
<SetupTab databaseName={sourceDb?.name} />
</Tabs.Panel>
{isMetabotAvailable && (
<Tabs.Panel value="metabot" h="100%">
<MetabotTab />
</Tabs.Panel>
)}
<Tabs.Panel value={`table-${activeTable?.tableId}`} h="100%">
{!activeTable ? null : (
<DataTab
databaseId={workspace?.database_id ?? null}
tableId={activeTable.tableId}
/>
)}
</Tabs.Panel>
<Tabs.Panel value={String(activeTransform?.id)} h="100%">
{openedTransforms.length === 0 ||
!activeTransform ||
!activeEditedTransform ? (
<Text c="text-medium">
{t`Select a transform on the right.`}
</Text>
) : (
<TransformTab
databaseId={checkNotNull(workspace.database_id)}
transform={activeTransform}
editedTransform={activeEditedTransform}
workspaceId={id}
onChange={handleTransformChange}
onOpenTransform={(transformId) =>
setTab(String(transformId))
}
/>
)}
</Tabs.Panel>
</Box>
</Tabs>
</Box>
<Box style={{ flex: "1 0 auto", width: "30%" }}>
<Tabs defaultValue="code">
<Flex
px="md"
align="center"
style={{ borderBottom: "1px solid var(--mb-color-border)" }}
>
<Tabs.List className={styles.tabsPanel}>
<Tabs.Tab value="code">{t`Code`}</Tabs.Tab>
<Tabs.Tab value="data">{t`Data`}</Tabs.Tab>
</Tabs.List>
{sourceDb && (
<AddTransformMenu
databaseId={sourceDb.id}
workspaceId={id}
onCreate={(transform) => {
addOpenedTransform(transform);
setActiveTransform(transform);
}}
/>
)}
</Flex>
<Tabs.Panel value="code" p="md">
<CodeTab
activeTransformId={activeTransform?.id}
transforms={dbTransforms}
workspaceId={workspace.id}
workspaceTransforms={workspaceTransforms}
onTransformClick={(transform) => {
setTab(String(transform.id));
addOpenedTransform(transform);
setActiveTransform(transform);
setActiveTable(undefined);
}}
/>
</Tabs.Panel>
<Tabs.Panel value="data" p="md">
<DataTabSidebar
tables={workspaceTables}
workspaceTransforms={workspaceTransforms}
dbTransforms={dbTransforms}
selectedTableId={activeTable?.tableId}
onTransformClick={(transform) => {
setTab(String(transform.id));
addOpenedTransform(transform);
setActiveTransform(transform);
setActiveTable(undefined);
}}
onTableSelect={handleTableSelect}
/>
</Tabs.Panel>
</Tabs>
</Box>
</Group>
</Stack>
);
}
export const WorkspacePage = ({ params }: WorkspacePageProps) => {
const workspaceId = Number(params.workspaceId);
return (
<WorkspaceProvider workspaceId={workspaceId}>
<WorkspacePageContent params={params} />
</WorkspaceProvider>
);
};

View File

@@ -0,0 +1,417 @@
import type { ReactNode } from "react";
import {
createContext,
useCallback,
useContext,
useMemo,
useState,
} from "react";
import { checkNotNull } from "metabase/lib/types";
import { isSameSource } from "metabase-enterprise/transforms/utils";
import type {
DraftTransformSource,
TableId,
Transform,
TransformTargetType,
} from "metabase-types/api";
export interface OpenTable {
tableId: TableId;
name: string;
schema?: string | null;
}
export interface EditedTransform {
name: string;
source: DraftTransformSource;
target: {
name: string;
type: TransformTargetType;
};
}
export interface WorkspaceContextValue {
openedTransforms: Transform[];
activeTransform: Transform | undefined;
activeEditedTransform: EditedTransform | undefined;
setActiveTransform: (transform: Transform | undefined) => void;
addOpenedTransform: (transform: Transform) => void;
removeOpenedTransform: (transformId: number) => void;
editedTransforms: Map<number, EditedTransform>;
patchEditedTransform: (
transformId: number,
patch: Partial<EditedTransform>,
) => void;
removeEditedTransform: (transformId: number) => void;
runTransforms: Set<number>;
updateTransformState: (
transform: Transform,
editedTransform?: EditedTransform | null,
) => void;
hasUnsavedChanges: () => boolean;
hasTransformEdits: (originalTransform: Transform) => boolean;
openedTables: OpenTable[];
activeTable: OpenTable | undefined;
setActiveTable: (table: OpenTable | undefined) => void;
addOpenedTable: (table: OpenTable) => void;
removeOpenedTable: (tableId: TableId) => void;
}
const WorkspaceContext = createContext<WorkspaceContextValue | undefined>(
undefined,
);
interface WorkspaceState {
openedTransforms: Transform[];
activeTransform: Transform | undefined;
editedTransforms: Map<number, EditedTransform>;
runTransforms: Set<number>;
openedTables: OpenTable[];
activeTable: OpenTable | undefined;
}
interface WorkspaceProviderProps {
children: ReactNode;
workspaceId: number;
}
const createEmptyWorkspaceState = (): WorkspaceState => ({
openedTransforms: [],
activeTransform: undefined,
editedTransforms: new Map(),
runTransforms: new Set(),
openedTables: [],
activeTable: undefined,
});
export const WorkspaceProvider = ({
children,
workspaceId,
}: WorkspaceProviderProps) => {
const [workspaceStates, setWorkspaceStates] = useState<
Map<number, WorkspaceState>
>(new Map());
const currentState = useMemo(() => {
const existing = workspaceStates.get(workspaceId);
if (existing) {
return existing;
}
const newState = createEmptyWorkspaceState();
setWorkspaceStates((prev) => new Map(prev).set(workspaceId, newState));
return newState;
}, [workspaceId, workspaceStates]);
const {
openedTransforms,
activeTransform,
editedTransforms,
runTransforms,
openedTables,
activeTable,
} = currentState;
const updateWorkspaceState = useCallback(
(updater: (state: WorkspaceState) => WorkspaceState) => {
setWorkspaceStates((prev) => {
const currentState =
prev.get(workspaceId) ?? createEmptyWorkspaceState();
const newState = updater(currentState);
return new Map(prev).set(workspaceId, newState);
});
},
[workspaceId],
);
const setActiveTransform = useCallback(
(transform: Transform | undefined) => {
updateWorkspaceState((state) => ({
...state,
activeTransform: transform,
}));
},
[updateWorkspaceState],
);
const addOpenedTransform = useCallback(
(transform: Transform) => {
updateWorkspaceState((state) => {
const exists = state.openedTransforms.some(
(item) => item.id === transform.id,
);
if (exists) {
return state;
}
return {
...state,
openedTransforms: [...state.openedTransforms, transform],
};
});
},
[updateWorkspaceState],
);
const removeOpenedTransform = useCallback(
(transformId: number) => {
updateWorkspaceState((state) => {
const filteredTransforms = state.openedTransforms.filter(
(item) => item.id !== transformId,
);
// If the removed transform was active, update to a neighboring transform
let newActiveTransform = state.activeTransform;
if (state.activeTransform?.id === transformId) {
const currentIndex = state.openedTransforms.findIndex(
(item) => item.id === transformId,
);
// Prefer the transform before (previous), otherwise use the one after (next)
if (currentIndex > 0) {
// Use the previous transform
newActiveTransform = state.openedTransforms[currentIndex - 1];
} else if (filteredTransforms.length > 0) {
// Use the next transform (which is now at index 0 after filtering)
newActiveTransform = filteredTransforms[0];
} else {
// No more transforms, set to undefined
newActiveTransform = undefined;
}
}
return {
...state,
openedTransforms: filteredTransforms,
activeTransform: newActiveTransform,
};
});
},
[updateWorkspaceState],
);
const patchEditedTransform = useCallback(
(transformId: number, patch: Partial<EditedTransform>) => {
updateWorkspaceState((state) => {
const activeTransform = checkNotNull(state.activeTransform);
const currentTransform =
state.editedTransforms.get(transformId) ?? activeTransform;
const newEditedTransform = {
name: patch.name ? patch.name : currentTransform.name,
source: patch.source ? patch.source : currentTransform.source,
target: patch.target ? patch.target : currentTransform.target,
};
const hasChanges =
!isSameSource(newEditedTransform.source, activeTransform.source) ||
newEditedTransform.name !== activeTransform.name ||
newEditedTransform.target.name !== activeTransform.target.name;
const newEditedTransforms = new Map(state.editedTransforms);
if (hasChanges) {
newEditedTransforms.set(transformId, newEditedTransform);
} else {
newEditedTransforms.delete(transformId);
}
const newRunTransforms = new Set(state.runTransforms);
newRunTransforms.delete(transformId);
return {
...state,
editedTransforms: newEditedTransforms,
runTransforms: newRunTransforms,
};
});
},
[updateWorkspaceState],
);
const removeEditedTransform = useCallback(
(transformId: number) => {
updateWorkspaceState((state) => {
const newEditedTransforms = new Map(state.editedTransforms);
newEditedTransforms.delete(transformId);
return {
...state,
editedTransforms: newEditedTransforms,
};
});
},
[updateWorkspaceState],
);
const updateTransformState = useCallback(
(transform: Transform, editedTransform?: EditedTransform | null) => {
updateWorkspaceState((state) => {
const openedTransforms = state.openedTransforms.map((item) =>
item.id === transform.id ? transform : item,
);
const newEditedTransforms = new Map(state.editedTransforms);
if (editedTransform == null) {
newEditedTransforms.delete(transform.id);
} else {
newEditedTransforms.set(transform.id, editedTransform);
}
const newRunTransforms = new Set(state.runTransforms);
newRunTransforms.delete(transform.id);
return {
...state,
openedTransforms,
activeTransform:
state.activeTransform?.id === transform.id
? transform
: state.activeTransform,
editedTransforms: newEditedTransforms,
runTransforms: newRunTransforms,
};
});
},
[updateWorkspaceState],
);
const hasUnsavedChanges = useCallback(() => {
return editedTransforms.size > 0;
}, [editedTransforms]);
const hasTransformEdits = useCallback(
(originalTransform: Transform) => {
const edited = editedTransforms.get(originalTransform.id);
return (
edited != null &&
(!isSameSource(edited.source, originalTransform.source) ||
edited.name !== originalTransform.name ||
edited.target.name !== originalTransform.target.name)
);
},
[editedTransforms],
);
const setActiveTable = useCallback(
(table: OpenTable | undefined) => {
updateWorkspaceState((state) => ({
...state,
activeTable: table,
}));
},
[updateWorkspaceState],
);
const addOpenedTable = useCallback(
(table: OpenTable) => {
updateWorkspaceState((state) => {
const exists = state.openedTables.some(
(item) => item.tableId === table.tableId,
);
if (exists) {
return {
...state,
activeTable: table,
};
}
return {
...state,
openedTables: [...state.openedTables, table],
activeTable: table,
};
});
},
[updateWorkspaceState],
);
const removeOpenedTable = useCallback(
(tableId: TableId) => {
updateWorkspaceState((state) => {
const filteredTables = state.openedTables.filter(
(item) => item.tableId !== tableId,
);
let newActiveTable = state.activeTable;
if (state.activeTable?.tableId === tableId) {
const currentIndex = state.openedTables.findIndex(
(item) => item.tableId === tableId,
);
if (currentIndex > 0) {
newActiveTable = state.openedTables[currentIndex - 1];
} else if (filteredTables.length > 0) {
newActiveTable = filteredTables[0];
} else {
newActiveTable = undefined;
}
}
return {
...state,
openedTables: filteredTables,
activeTable: newActiveTable,
};
});
},
[updateWorkspaceState],
);
const activeEditedTransform = activeTransform
? (editedTransforms.get(activeTransform.id) ?? activeTransform)
: activeTransform;
const value = useMemo(
() => ({
openedTransforms,
activeTransform,
activeEditedTransform,
setActiveTransform,
addOpenedTransform,
removeOpenedTransform,
editedTransforms,
patchEditedTransform,
removeEditedTransform,
runTransforms,
updateTransformState,
hasUnsavedChanges,
hasTransformEdits,
openedTables,
activeTable,
setActiveTable,
addOpenedTable,
removeOpenedTable,
}),
[
openedTransforms,
activeTransform,
activeEditedTransform,
editedTransforms,
runTransforms,
setActiveTransform,
addOpenedTransform,
removeOpenedTransform,
patchEditedTransform,
removeEditedTransform,
updateTransformState,
hasUnsavedChanges,
hasTransformEdits,
openedTables,
activeTable,
setActiveTable,
addOpenedTable,
removeOpenedTable,
],
);
return (
<WorkspaceContext.Provider value={value}>
{children}
</WorkspaceContext.Provider>
);
};
export const useWorkspace = (): WorkspaceContextValue => {
const context = useContext(WorkspaceContext);
if (!context) {
throw new Error("useWorkspace must be used within a WorkspaceProvider");
}
return context;
};

View File

@@ -0,0 +1,7 @@
.statusDot {
width: 0.5rem;
height: 0.5rem;
border-radius: 50%;
background-color: var(--mb-color-warning);
flex-shrink: 0;
}

View File

@@ -0,0 +1,15 @@
import { Box } from "metabase/ui";
import S from "./StatusDot.module.css";
type StatusDotProps = {
status?: "changed";
};
export const StatusDot = ({ status }: StatusDotProps) => {
if (status !== "changed") {
return null;
}
return <Box className={S.statusDot} />;
};

View File

@@ -0,0 +1,13 @@
import { IndexRoute, Route } from "react-router";
import { WorkspaceListPage } from "./pages/WorkspaceListPage/WorkspaceListPage";
import { WorkspacePage } from "./pages/WorkspacePage/WorkspacePage";
export function getDataStudioWorkspaceRoutes() {
return (
<>
<IndexRoute component={WorkspaceListPage} />
<Route path=":workspaceId" component={WorkspacePage} />
</>
);
}

View File

@@ -3,6 +3,7 @@ import { tinykeys } from "tinykeys";
import ErrorBoundary from "metabase/ErrorBoundary";
import { useSelector } from "metabase/lib/redux";
import { Sidebar } from "metabase/nav/containers/MainNavbar/MainNavbar.styled";
import { getUser } from "metabase/selectors/user";
import type { SuggestionModel } from "metabase-enterprise/rich_text_editing/tiptap/extensions/shared/types";
@@ -55,7 +56,14 @@ export const MetabotAuthenticated = ({ hide, config }: MetabotProps) => {
return (
<ErrorBoundary errorComponent={() => null}>
<MetabotChat config={config} />
<Sidebar
isOpen={visible}
side="right"
width="30rem"
aria-hidden={!visible}
>
<MetabotChat config={config} />
</Sidebar>
</ErrorBoundary>
);
};

View File

@@ -3,7 +3,6 @@ import { useCallback, useMemo } from "react";
import { t } from "ttag";
import EmptyDashboardBot from "assets/img/dashboard-empty.svg?component";
import { Sidebar } from "metabase/nav/containers/MainNavbar/MainNavbar.styled";
import {
ActionIcon,
Box,
@@ -38,11 +37,11 @@ const defaultConfig: MetabotConfig = {
],
};
export const MetabotChat = ({
config = defaultConfig,
}: {
interface Props {
config?: MetabotConfig;
}) => {
}
export const MetabotChat = ({ config = defaultConfig }: Props) => {
const metabot = useMetabotAgent();
const { handleSubmitInput, handleRetryMessage, handleResetInput } =
useMetabotChatHandlers(config);
@@ -77,137 +76,127 @@ export const MetabotChat = ({
}, [handleSubmitInput, metabot.prompt]);
return (
<Sidebar
isOpen={metabot.visible}
side="right"
width="30rem"
aria-hidden={!metabot.visible}
>
<Box className={Styles.container} data-testid="metabot-chat">
{/* header */}
<Box ref={headerRef} className={Styles.header}>
<Flex align-items="center">
<Text lh={1} fz="sm" c="text-secondary">
{metabot.profileOverride
? t`Using profile: ${metabot.profileOverride}`
: t`Metabot isn't perfect. Double-check results.`}
</Text>
</Flex>
<Box className={Styles.container} data-testid="metabot-chat">
{/* header */}
<Box ref={headerRef} className={Styles.header}>
<Flex align-items="center">
<Text lh={1} fz="sm" c="text-secondary">
{metabot.profileOverride
? t`Using profile: ${metabot.profileOverride}`
: t`Metabot isn't perfect. Double-check results.`}
</Text>
</Flex>
<Flex gap="sm">
<Tooltip label={t`Clear conversation`} position="bottom">
<ActionIcon
onClick={() => metabot.resetConversation()}
data-testid="metabot-reset-chat"
>
<Icon c="text-primary" name="revert" />
</ActionIcon>
</Tooltip>
{!config.preventClose && (
<ActionIcon
onClick={handleClose}
data-testid="metabot-close-chat"
>
<Icon c="text-primary" name="close" />
</ActionIcon>
)}
</Flex>
</Box>
{/* chat messages */}
<Box
ref={scrollContainerRef}
className={Styles.messagesContainer}
data-testid="metabot-chat-messages"
>
{!hasMessages && !metabot.isDoingScience && (
<>
{/* empty state */}
<Flex
h="100%"
gap="md"
direction="column"
align="center"
justify="center"
data-testid="metabot-empty-chat-info"
>
<Box component={EmptyDashboardBot} w="6rem" />
<Text c="text-light" maw="12rem" ta="center">
{config.emptyText ??
t`I can help you explore your metrics and models.`}
</Text>
</Flex>
{!config.hideSuggestedPrompts && (
<Stack
gap="sm"
className={Styles.promptSuggestionsContainer}
data-testid="metabot-prompt-suggestions"
>
<>
{suggestedPrompts.map(({ prompt }, index) => (
<Box key={index}>
<Button
fz="sm"
size="xs"
onClick={() => handleSubmitInput(prompt)}
className={Styles.promptSuggestionButton}
>
{prompt}
</Button>
</Box>
))}
</>
</Stack>
)}
</>
)}
{(hasMessages || metabot.isDoingScience) && (
<Box
className={Styles.messages}
data-testid="metabot-chat-inner-messages"
<Flex gap="sm">
<Tooltip label={t`Clear conversation`} position="bottom">
<ActionIcon
onClick={() => metabot.resetConversation()}
data-testid="metabot-reset-chat"
>
{/* conversation messages */}
<Messages
messages={metabot.messages}
errorMessages={metabot.errorMessages}
onRetryMessage={handleRetryMessage}
isDoingScience={metabot.isDoingScience}
showFeedbackButtons
/>
{/* loading */}
{metabot.isDoingScience && (
<MetabotThinking toolCalls={metabot.activeToolCalls} />
)}
{/* filler - height gets set via ref mutation */}
<div ref={fillerRef} data-testid="metabot-message-filler" />
{/* long convo warning */}
{metabot.isLongConversation && <MetabotResetLongChatButton />}
</Box>
<Icon c="text-primary" name="revert" />
</ActionIcon>
</Tooltip>
{!config.preventClose && (
<ActionIcon onClick={handleClose} data-testid="metabot-close-chat">
<Icon c="text-primary" name="close" />
</ActionIcon>
)}
</Box>
<Box className={Styles.textInputContainer}>
<Paper
className={cx(
Styles.inputContainer,
metabot.isDoingScience && Styles.inputContainerLoading,
)}
>
<MetabotChatEditor
ref={metabot.promptInputRef}
value={metabot.prompt}
autoFocus
isResponding={metabot.isDoingScience}
placeholder={t`Tell me to do something, or ask a question`}
onChange={handleEditorChange}
onSubmit={handleEditorSubmit}
onStop={metabot.cancelRequest}
suggestionModels={config.suggestionModels}
/>
</Paper>
</Box>
</Flex>
</Box>
</Sidebar>
{/* chat messages */}
<Box
ref={scrollContainerRef}
className={Styles.messagesContainer}
data-testid="metabot-chat-messages"
>
{!hasMessages && !metabot.isDoingScience && (
<>
{/* empty state */}
<Flex
h="100%"
gap="md"
direction="column"
align="center"
justify="center"
data-testid="metabot-empty-chat-info"
>
<Box component={EmptyDashboardBot} w="6rem" />
<Text c="text-light" maw="12rem" ta="center">
{config.emptyText ??
t`I can help you explore your metrics and models.`}
</Text>
</Flex>
{!config.hideSuggestedPrompts && (
<Stack
gap="sm"
className={Styles.promptSuggestionsContainer}
data-testid="metabot-prompt-suggestions"
>
<>
{suggestedPrompts.map(({ prompt }, index) => (
<Box key={index}>
<Button
fz="sm"
size="xs"
onClick={() => handleSubmitInput(prompt)}
className={Styles.promptSuggestionButton}
>
{prompt}
</Button>
</Box>
))}
</>
</Stack>
)}
</>
)}
{(hasMessages || metabot.isDoingScience) && (
<Box
className={Styles.messages}
data-testid="metabot-chat-inner-messages"
>
{/* conversation messages */}
<Messages
messages={metabot.messages}
errorMessages={metabot.errorMessages}
onRetryMessage={handleRetryMessage}
isDoingScience={metabot.isDoingScience}
showFeedbackButtons
/>
{/* loading */}
{metabot.isDoingScience && (
<MetabotThinking toolCalls={metabot.activeToolCalls} />
)}
{/* filler - height gets set via ref mutation */}
<div ref={fillerRef} data-testid="metabot-message-filler" />
{/* long convo warning */}
{metabot.isLongConversation && <MetabotResetLongChatButton />}
</Box>
)}
</Box>
<Box className={Styles.textInputContainer}>
<Paper
className={cx(
Styles.inputContainer,
metabot.isDoingScience && Styles.inputContainerLoading,
)}
>
<MetabotChatEditor
ref={metabot.promptInputRef}
value={metabot.prompt}
autoFocus
isResponding={metabot.isDoingScience}
placeholder={t`Tell me to do something, or ask a question`}
onChange={handleEditorChange}
onSubmit={handleEditorSubmit}
onStop={metabot.cancelRequest}
suggestionModels={config.suggestionModels}
/>
</Paper>
</Box>
</Box>
);
};

View File

@@ -12,6 +12,7 @@ import { Metabot } from "./components/Metabot";
import { MetabotAdminPage } from "./components/MetabotAdmin/MetabotAdminPage";
import { MetabotTrialPage } from "./components/MetabotAdmin/MetabotTrialPage";
import { MetabotAppBarButton } from "./components/MetabotAppBarButton";
import { MetabotChat } from "./components/MetabotChat";
import { MetabotDataStudioButton } from "./components/MetabotDataStudioButton";
import { getMetabotQuickLinks } from "./components/MetabotQuickLinks";
import { MetabotToggleButton } from "./components/MetabotToggleButton";
@@ -38,6 +39,7 @@ export function initializePlugin() {
if (hasPremiumFeature("metabot_v3")) {
PLUGIN_METABOT.isEnabled = () => true;
PLUGIN_METABOT.Metabot = Metabot;
PLUGIN_METABOT.MetabotChat = MetabotChat;
PLUGIN_METABOT.getMetabotRoutes = getMetabotQuickLinks;
PLUGIN_METABOT.getAdminPaths = () => [

View File

@@ -30,6 +30,8 @@ import {
type PythonDataPickerProps = {
database?: DatabaseId;
disabled?: boolean;
canChangeDatabase?: boolean;
tables: PythonTransformTableAliases;
onChange: (
database: number,
@@ -40,8 +42,10 @@ type PythonDataPickerProps = {
export function PythonDataPicker({
database,
disabled,
tables,
onChange,
canChangeDatabase = true,
}: PythonDataPickerProps) {
const [tableSelections, setTableSelections] = useState<TableSelection[]>(
getInitialTableSelections(tables),
@@ -150,22 +154,25 @@ export function PythonDataPicker({
className={S.dataPicker}
data-testid="python-data-picker"
>
<Box>
<Text fw="bold">{t`Source database`}</Text>
<Text size="sm" c="text-light" mb="sm">
{t`Select the database that contains your source data.`}
</Text>
{canChangeDatabase && (
<Box>
<Text fw="bold">{t`Source database`}</Text>
<Text size="sm" c="text-light" mb="sm">
{t`Select the database that contains your source data.`}
</Text>
<DatabaseDataSelector
className={S.databaseSelector}
selectedDatabaseId={database}
setDatabaseFn={handleDatabaseChange}
databases={databases?.data ?? []}
databaseIsDisabled={(database: Database) =>
!hasFeature(database, "transforms/python")
}
/>
</Box>
<DatabaseDataSelector
className={S.databaseSelector}
readOnly={disabled}
selectedDatabaseId={database}
setDatabaseFn={handleDatabaseChange}
databases={databases?.data ?? []}
databaseIsDisabled={(database: Database) =>
!hasFeature(database, "transforms/python")
}
/>
</Box>
)}
{database && (
<Box>
<Text fw="bold">{t`Pick tables and alias them`}</Text>
@@ -185,12 +192,12 @@ export function PythonDataPicker({
handleSelectionChange(index, selection)
}
onRemove={() => handleRemoveTable(index)}
disabled={isLoadingTables}
disabled={disabled || isLoadingTables}
/>
))}
<AddTableButton
onClick={handleAddTable}
disabled={availableTables.length === 0}
disabled={disabled || availableTables.length === 0}
/>
</Stack>
</Box>

View File

@@ -98,6 +98,7 @@ export function TableSelector({
<Tooltip label={t`Remove this table`}>
<ActionIcon
disabled={disabled}
onClick={onRemove}
pr="sm"
aria-label={t`Remove this table`}

View File

@@ -24,6 +24,7 @@ import { ResizableBoxHandle } from "./ResizableBoxHandle";
import { hasImport, insertImport, removeImport } from "./utils";
type PythonEditorBodyProps = {
disabled?: boolean;
source: string;
proposedSource?: string;
isRunnable: boolean;
@@ -41,6 +42,7 @@ type PythonEditorBodyProps = {
const EDITOR_HEIGHT = 400;
export function PythonEditorBody({
disabled,
source,
proposedSource,
onChange,
@@ -57,6 +59,7 @@ export function PythonEditorBody({
<MaybeResizableBox resizable={withDebugger}>
<Flex h="100%" align="end" bg="bg-light" pos="relative">
<PythonEditor
readOnly={disabled}
value={source}
proposedValue={proposedSource}
onChange={onChange}
@@ -106,7 +109,11 @@ export function PythonEditorBody({
</Box>
)}
</Stack>
<SharedLibraryActions source={source} onChange={onChange} />
<SharedLibraryActions
disabled={disabled}
source={source}
onChange={onChange}
/>
</Flex>
</MaybeResizableBox>
);
@@ -140,24 +147,32 @@ function MaybeResizableBox({
}
function SharedLibraryActions({
disabled,
source,
onChange,
}: {
disabled?: boolean;
source: string;
onChange: (source: string) => void;
}) {
return (
<Group className={S.libraryActions} p="md" gap="sm">
<SharedLibraryImportButton source={source} onChange={onChange} />
<SharedLibraryImportButton
disabled={disabled}
source={source}
onChange={onChange}
/>
<SharedLibraryEditLink />
</Group>
);
}
function SharedLibraryImportButton({
disabled,
source,
onChange,
}: {
disabled?: boolean;
source: string;
onChange: (source: string) => void;
}) {
@@ -173,7 +188,11 @@ function SharedLibraryImportButton({
return (
<Tooltip label={label}>
<ActionIcon aria-label={label} onClick={handleToggleSharedLib}>
<ActionIcon
aria-label={label}
disabled={disabled}
onClick={handleToggleSharedLib}
>
<Icon name="reference" c="text-dark" />
</ActionIcon>
</Tooltip>

View File

@@ -2,7 +2,11 @@ import { useHotkeys } from "@mantine/hooks";
import type { PythonTransformEditorProps } from "metabase/plugins";
import { Flex, Stack } from "metabase/ui";
import type { PythonTransformTableAliases, Table } from "metabase-types/api";
import type {
DatabaseId,
PythonTransformTableAliases,
Table,
} from "metabase-types/api";
import { PythonDataPicker } from "./PythonDataPicker";
import { PythonEditorBody } from "./PythonEditorBody";
@@ -17,6 +21,7 @@ import {
export function PythonTransformEditor({
source,
proposedSource,
uiOptions,
isDirty,
onChangeSource,
onAcceptProposed,
@@ -34,7 +39,7 @@ export function PythonTransformEditor({
};
const handleDataChange = (
database: number,
database: DatabaseId,
sourceTables: PythonTransformTableAliases,
tableInfo: Table[],
) => {
@@ -71,12 +76,15 @@ export function PythonTransformEditor({
return (
<Flex h="100%" w="100%">
<PythonDataPicker
disabled={uiOptions?.readOnly}
database={source["source-database"]}
canChangeDatabase={uiOptions?.canChangeDatabase}
tables={source["source-tables"]}
onChange={handleDataChange}
/>
<Stack w="100%" h="100%" gap={0}>
<PythonEditorBody
disabled={uiOptions?.readOnly}
isRunnable={isPythonTransformSource(source)}
isRunning={isRunning}
isDirty={isDirty}

View File

@@ -3,6 +3,7 @@ import { useMemo } from "react";
import { useSelector } from "metabase/lib/redux";
import {
QueryEditor,
type QueryEditorUiOptions,
type QueryEditorUiState,
} from "metabase/querying/editor/components/QueryEditor";
import { getMetadata } from "metabase/selectors/metadata";
@@ -11,9 +12,10 @@ import type { Database, QueryTransformSource } from "metabase-types/api";
import { getEditorOptions } from "./utils";
type TransformEditorProps = {
export type TransformEditorProps = {
source: QueryTransformSource;
uiState: QueryEditorUiState;
uiOptions?: QueryEditorUiOptions;
proposedSource: QueryTransformSource | undefined;
databases: Database[];
onChangeSource: (source: QueryTransformSource) => void;
@@ -27,6 +29,7 @@ export function TransformEditor({
proposedSource,
databases,
uiState,
uiOptions,
onChangeSource,
onChangeUiState,
onAcceptProposed,
@@ -44,7 +47,10 @@ export function TransformEditor({
: undefined,
[proposedSource, metadata],
);
const uiOptions = useMemo(() => getEditorOptions(databases), [databases]);
const mergedUiOptions = useMemo(
() => ({ ...getEditorOptions(databases), ...uiOptions }),
[databases, uiOptions],
);
const handleQueryChange = (query: Lib.Query) => {
const newSource: QueryTransformSource = {
@@ -60,7 +66,7 @@ export function TransformEditor({
<QueryEditor
query={query}
uiState={uiState}
uiOptions={uiOptions}
uiOptions={mergedUiOptions}
proposedQuery={proposedQuery}
onChangeQuery={handleQueryChange}
onChangeUiState={onChangeUiState}

View File

@@ -0,0 +1,232 @@
import dayjs from "dayjs";
import { useMemo, useState } from "react";
import { t } from "ttag";
import { useListDatabasesQuery } from "metabase/api";
import { useMetadataToasts } from "metabase/metadata/hooks";
import {
Box,
Button,
Flex,
Icon,
Loader,
Menu,
ScrollArea,
Stack,
Text,
} from "metabase/ui";
import {
useCreateWorkspaceMutation,
useGetTransformDownstreamMappingQuery,
useGetWorkspacesQuery,
useUpdateWorkspaceContentsMutation,
} from "metabase-enterprise/api";
import { CreateWorkspaceModal } from "metabase-enterprise/workspaces/components/CreateWorkspaceModal/CreateWorkspaceModal";
import type { Transform, Workspace } from "metabase-types/api";
type EditTransformMenuProps = {
transform: Transform;
};
export function EditTransformMenu({ transform }: EditTransformMenuProps) {
const { sendErrorToast } = useMetadataToasts();
const sourceDatabaseId = getTransformDatabaseId(transform);
const { data: workspacesData, isLoading: isLoadingWorkspaces } =
useGetWorkspacesQuery();
const [updateWorkspaceContents, { isLoading: isUpdatingWorkspace }] =
useUpdateWorkspaceContentsMutation();
const [createWorkspace, { isLoading: isCreatingWorkspace }] =
useCreateWorkspaceMutation();
const { data: downstreamMapping } = useGetTransformDownstreamMappingQuery(
transform.id,
);
const [isCreateModalOpen, setIsCreateModalOpen] = useState(false);
const [addedWorkspaceIds, setAddedWorkspaceIds] = useState<Set<number>>(
() => new Set(),
);
const { data: databasesData } = useListDatabasesQuery({
include_analytics: true,
});
const workspaces = useMemo(
() => workspacesData?.items ?? [],
[workspacesData],
);
const existingWorkspaceIds = useMemo(() => {
const ids = new Set<number>();
if (transform.workspace_id != null) {
ids.add(transform.workspace_id);
}
downstreamMapping?.transforms?.forEach((item) => {
if (item.workspace?.id != null) {
ids.add(item.workspace.id);
}
});
addedWorkspaceIds.forEach((id) => ids.add(id));
return ids;
}, [transform.workspace_id, downstreamMapping, addedWorkspaceIds]);
const matchingWorkspaces = useMemo(
() =>
workspaces
.filter((workspace) => !existingWorkspaceIds.has(workspace.id))
.filter((workspace) =>
sourceDatabaseId == null
? true
: workspace.database_id === sourceDatabaseId,
),
[workspaces, sourceDatabaseId, existingWorkspaceIds],
);
const isBusy = isUpdatingWorkspace || isCreatingWorkspace;
const emptyMessage =
workspaces.length === 0 || sourceDatabaseId == null
? t`No workspaces yet`
: t`No workspaces for this database yet`;
const databaseOptions = useMemo(
() =>
(databasesData?.data ?? []).map((db) => ({
value: String(db.id),
label: db.name,
})),
[databasesData],
);
const handleOpenCreateModal = () => {
setIsCreateModalOpen(true);
};
const handleCloseCreateModal = () => {
setIsCreateModalOpen(false);
};
const handleWorkspaceSelect = async (workspace: Workspace) => {
try {
await updateWorkspaceContents({
id: workspace.id,
add: { transforms: [transform.id] },
}).unwrap();
setAddedWorkspaceIds((prev) => new Set(prev).add(workspace.id));
} catch (error) {
sendErrorToast(t`Failed to add transform to the workspace`);
}
};
const handleCreateWorkspace = async ({
name,
databaseId,
}: {
name: string;
databaseId: string;
}) => {
try {
const workspace = await createWorkspace({
name: name.trim() || t`New workspace`,
database_id: Number(databaseId),
upstream: { transforms: [transform.id] },
}).unwrap();
setAddedWorkspaceIds((prev) => new Set(prev).add(workspace.id));
handleCloseCreateModal();
} catch (error) {
sendErrorToast(t`Failed to create workspace`);
}
};
return (
<Menu position="bottom-end" width={280}>
<Menu.Target>
<Button
size="sm"
variant="filled"
leftSection={<Icon name="pencil" />}
rightSection={<Icon name="chevrondown" />}
loading={isBusy}
>
{t`Edit Transform`}
</Button>
</Menu.Target>
<Menu.Dropdown>
<Menu.Label>{t`Add to workspace`}</Menu.Label>
<Menu.Item
disabled={isBusy}
leftSection={<Icon name="add" />}
onClick={handleOpenCreateModal}
>
<Stack gap={0} align="flex-start">
<Text fw={600}>{t`New workspace`}</Text>
<Text size="sm" c="text-light">
{t`Create a new workspace`}
</Text>
</Stack>
</Menu.Item>
<Menu.Divider />
{isLoadingWorkspaces ? (
<Flex justify="center" align="center" py="md">
<Loader size="sm" />
</Flex>
) : matchingWorkspaces.length === 0 ? (
<Box px="md" py="sm">
<Text size="sm" c="text-light">
{emptyMessage}
</Text>
</Box>
) : (
<ScrollArea.Autosize mah={320} type="scroll">
<Stack gap={0}>
{matchingWorkspaces.map((workspace) => (
<Menu.Item
key={workspace.id}
leftSection={<Icon name="sparkles" />}
onClick={() => handleWorkspaceSelect(workspace)}
disabled={isBusy}
>
<Stack gap={2} align="flex-start">
<Text fw={600}>{workspace.name}</Text>
<Text size="sm" c="text-light">
{formatWorkspaceDate(workspace.created_at)}
</Text>
</Stack>
</Menu.Item>
))}
</Stack>
</ScrollArea.Autosize>
)}
</Menu.Dropdown>
<CreateWorkspaceModal
opened={isCreateModalOpen}
onClose={handleCloseCreateModal}
onSubmit={handleCreateWorkspace}
databaseOptions={databaseOptions}
isSubmitting={isBusy}
defaultDatabaseId={
sourceDatabaseId != null ? String(sourceDatabaseId) : null
}
/>
</Menu>
);
}
function getTransformDatabaseId(transform: Transform) {
if (
transform.source_type === "python" &&
"source-database" in transform.source
) {
return transform.source["source-database"];
}
if ("query" in transform.source) {
return transform.source.query.database;
}
return undefined;
}
function formatWorkspaceDate(createdAt: string) {
return dayjs(createdAt).format("DD/MM/YYYY");
}

View File

@@ -0,0 +1 @@
export * from "./EditTransformMenu";

View File

@@ -32,30 +32,40 @@ import { trackTransformCreated } from "../../../analytics";
import { SchemaFormSelect } from "./../../../components/SchemaFormSelect";
function getValidationSchema() {
return Yup.object({
name: Yup.string().required(Errors.required),
targetName: Yup.string().required(Errors.required),
targetSchema: Yup.string().nullable().defined(),
incremental: Yup.boolean().required(),
// For native queries, use checkpointFilter (plain string)
checkpointFilter: Yup.string().nullable(),
// For MBQL/Python queries, use checkpointFilterUniqueKey (prefixed format)
checkpointFilterUniqueKey: Yup.string().nullable(),
sourceStrategy: Yup.mixed<"checkpoint">().oneOf(["checkpoint"]).required(),
targetStrategy: Yup.mixed<"append">().oneOf(["append"]).required(),
});
}
const DEFAULT_VALIDATION_SCHEMA = Yup.object({
name: Yup.string().required(Errors.required),
targetName: Yup.string().required(Errors.required),
targetSchema: Yup.string().nullable().defined(),
incremental: Yup.boolean().required(),
// For native queries, use checkpointFilter (plain string)
checkpointFilter: Yup.string().nullable(),
// For MBQL/Python queries, use checkpointFilterUniqueKey (prefixed format)
checkpointFilterUniqueKey: Yup.string().nullable(),
sourceStrategy: Yup.mixed<"checkpoint">().oneOf(["checkpoint"]).required(),
targetStrategy: Yup.mixed<"append">().oneOf(["append"]).required(),
});
export type NewTransformValues = Yup.InferType<
ReturnType<typeof getValidationSchema>
>;
export type NewTransformValues = Yup.InferType<typeof DEFAULT_VALIDATION_SCHEMA>;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
export type ValidationSchemaExtension = Record<string, Yup.AnySchema>;
function getValidationSchema(extension?: ValidationSchemaExtension) {
if (!extension) {
return DEFAULT_VALIDATION_SCHEMA;
}
return DEFAULT_VALIDATION_SCHEMA.shape(extension);
}
type CreateTransformModalProps = {
source: TransformSource;
defaultValues: Partial<NewTransformValues>;
onCreate: (transform: Transform) => void;
onCreate?: (transform: Transform) => void;
onClose: () => void;
schemas?: string[] | null;
showIncrementalSettings?: boolean;
validationSchemaExtension?: ValidationSchemaExtension;
handleSubmit?: (values: NewTransformValues) => Promise<Transform>;
};
export function CreateTransformModal({
@@ -63,6 +73,10 @@ export function CreateTransformModal({
defaultValues,
onCreate,
onClose,
schemas,
showIncrementalSettings = true,
validationSchemaExtension,
handleSubmit,
}: CreateTransformModalProps) {
return (
<Modal title={t`Save your transform`} opened padding="xl" onClose={onClose}>
@@ -71,6 +85,10 @@ export function CreateTransformModal({
defaultValues={defaultValues}
onCreate={onCreate}
onClose={onClose}
schemas={schemas}
showIncrementalSettings={showIncrementalSettings}
validationSchemaExtension={validationSchemaExtension}
handleSubmit={handleSubmit}
/>
</Modal>
);
@@ -79,8 +97,12 @@ export function CreateTransformModal({
type CreateTransformFormProps = {
source: TransformSource;
defaultValues: Partial<NewTransformValues>;
onCreate: (transform: Transform) => void;
onCreate?: (transform: Transform) => void;
onClose: () => void;
schemas?: string[] | null;
showIncrementalSettings?: boolean;
validationSchemaExtension?: ValidationSchemaExtension;
handleSubmit?: (values: NewTransformValues) => Promise<Transform>;
};
function CreateTransformForm({
@@ -88,11 +110,18 @@ function CreateTransformForm({
defaultValues,
onCreate,
onClose,
schemas: schemasProp,
showIncrementalSettings = true,
validationSchemaExtension,
handleSubmit,
}: CreateTransformFormProps) {
const [sendToast] = useToast();
const databaseId =
source.type === "query" ? source.query.database : source["source-database"];
const shouldFetchSchemas = schemasProp === undefined;
const showSchemaField = schemasProp !== null;
const {
data: database,
isLoading: isDatabaseLoading,
@@ -100,15 +129,22 @@ function CreateTransformForm({
} = useGetDatabaseQuery(databaseId ? { id: databaseId } : skipToken);
const {
data: schemas = [],
data: fetchedSchemas = [],
isLoading: isSchemasLoading,
error: schemasError,
} = useListDatabaseSchemasQuery(
databaseId ? { id: databaseId, include_hidden: true } : skipToken,
shouldFetchSchemas && databaseId
? { id: databaseId, include_hidden: true }
: skipToken,
);
const isLoading = isDatabaseLoading || isSchemasLoading;
const error = databaseError ?? schemasError;
const schemas = useMemo(
() => schemasProp ?? fetchedSchemas ?? [],
[schemasProp, fetchedSchemas],
);
const isLoading =
isDatabaseLoading || (shouldFetchSchemas && isSchemasLoading);
const error = databaseError ?? (shouldFetchSchemas ? schemasError : null);
const [createTransform] = useCreateTransformMutation();
const supportsSchemas = database && hasFeature(database, "schemas");
@@ -118,13 +154,16 @@ function CreateTransformForm({
[schemas, defaultValues],
);
const validationSchema = useMemo(() => getValidationSchema(), []);
const validationSchema = useMemo(
() => getValidationSchema(validationSchemaExtension),
[validationSchemaExtension],
);
if (isLoading || error != null) {
return <LoadingAndErrorWrapper loading={isLoading} error={error} />;
}
const handleSubmit = async (values: NewTransformValues) => {
const defaultHandleSubmit = async (values: NewTransformValues) => {
if (!databaseId) {
throw new Error("Database ID is required");
}
@@ -132,7 +171,7 @@ function CreateTransformForm({
try {
const transform = await createTransform(request).unwrap();
trackTransformCreated({ transformId: transform.id });
onCreate(transform);
onCreate?.(transform);
} catch (error) {
sendToast({
message: getErrorMessage(error, t`Failed to create transform`),
@@ -145,7 +184,7 @@ function CreateTransformForm({
<FormProvider
initialValues={initialValues}
validationSchema={validationSchema}
onSubmit={handleSubmit}
onSubmit={handleSubmit || defaultHandleSubmit}
>
<Form>
<Stack gap="lg" mt="sm">
@@ -155,7 +194,7 @@ function CreateTransformForm({
placeholder={t`My Great Transform`}
data-autofocus
/>
{supportsSchemas && (
{showSchemaField && supportsSchemas && (
<SchemaFormSelect
name="targetSchema"
label={t`Schema`}
@@ -167,7 +206,9 @@ function CreateTransformForm({
label={t`Table name`}
placeholder={t`descriptive_name`}
/>
<IncrementalTransformSettings source={source} />
{showIncrementalSettings && (
<IncrementalTransformSettings source={source} />
)}
<Group>
<Box flex={1}>
<FormErrorMessage />
@@ -182,7 +223,7 @@ function CreateTransformForm({
}
function getInitialValues(
schemas: string[],
schemas: string[] | null,
defaultValues: Partial<NewTransformValues>,
): NewTransformValues {
return {

View File

@@ -102,6 +102,7 @@ function NewTransformPageBody({
useRegisterMetabotTransformContext(undefined, source);
const validationResult = useMemo(() => {
return { isValid: true };
return source.type === "query"
? getValidationResult(Lib.fromJsQueryAndMetadata(metadata, source.query))
: PLUGIN_TRANSFORMS_PYTHON.getPythonSourceValidationResult(source);

View File

@@ -15,17 +15,25 @@ import {
} from "metabase/plugins";
import { getInitialUiState } from "metabase/querying/editor/components/QueryEditor";
import { getMetadata } from "metabase/selectors/metadata";
import { Stack } from "metabase/ui";
import { Group, Stack } from "metabase/ui";
import {
useGetTransformQuery,
useUpdateTransformMutation,
} from "metabase-enterprise/api";
import { PaneHeaderActions } from "metabase-enterprise/data-studio/common/components/PaneHeader";
import * as Lib from "metabase-lib";
import type { Database, Transform } from "metabase-types/api";
import type {
Database,
DraftTransformSource,
Transform,
} from "metabase-types/api";
import { TransformEditor } from "../../components/TransformEditor";
import {
TransformEditor,
type TransformEditorProps,
} from "../../components/TransformEditor";
import { TransformHeader } from "../../components/TransformHeader";
import { EditTransformMenu } from "../../components/TransformHeader/EditTransformMenu";
import { useRegisterMetabotTransformContext } from "../../hooks/use-register-transform-metabot-context";
import { useSourceState } from "../../hooks/use-source-state";
import { getValidationResult, isNotDraftSource } from "../../utils";
@@ -51,6 +59,8 @@ export function TransformQueryPage({ params, route }: TransformQueryPageProps) {
isLoading: isLoadingDatabases,
error: databasesError,
} = useListDatabasesQuery({ include_analytics: true });
const showEditWorkspaceMenu =
transform?.source_type === "python" || transform?.source_type === "native";
const isLoading = isLoadingTransform || isLoadingDatabases;
const error = transformError || databasesError;
@@ -67,6 +77,7 @@ export function TransformQueryPage({ params, route }: TransformQueryPageProps) {
transform={transform}
databases={databases.data}
route={route}
showEditWorkspaceMenu={showEditWorkspaceMenu}
/>
);
}
@@ -75,12 +86,14 @@ type TransformQueryPageBodyProps = {
transform: Transform;
databases: Database[];
route: Route;
showEditWorkspaceMenu: boolean;
};
function TransformQueryPageBody({
transform,
databases,
route,
showEditWorkspaceMenu,
}: TransformQueryPageBodyProps) {
const {
source,
@@ -160,42 +173,33 @@ function TransformQueryPageBody({
<TransformHeader
transform={transform}
actions={
<PaneHeaderActions
errorMessage={validationResult.errorMessage}
isValid={validationResult.isValid}
isDirty={isDirty}
isSaving={isSaving || isCheckingDependencies}
onSave={handleSave}
onCancel={handleCancel}
/>
<Group gap="sm">
{showEditWorkspaceMenu && (
<EditTransformMenu transform={transform} />
)}
<PaneHeaderActions
errorMessage={validationResult.errorMessage}
isValid={validationResult.isValid}
isDirty={isDirty}
isSaving={isSaving || isCheckingDependencies}
onSave={handleSave}
onCancel={handleCancel}
/>
</Group>
}
hasMenu={!isDirty}
/>
{source.type === "python" ? (
<PLUGIN_TRANSFORMS_PYTHON.TransformEditor
source={source}
proposedSource={
proposedSource?.type === "python" ? proposedSource : undefined
}
isDirty={isDirty}
onChangeSource={setSourceAndRejectProposed}
onAcceptProposed={acceptProposed}
onRejectProposed={rejectProposed}
/>
) : (
<TransformEditor
source={source}
proposedSource={
proposedSource?.type === "query" ? proposedSource : undefined
}
uiState={uiState}
databases={databases}
onChangeSource={setSourceAndRejectProposed}
onChangeUiState={setUiState}
onAcceptProposed={acceptProposed}
onRejectProposed={rejectProposed}
/>
)}
<TransformQueryPageEditor
source={source}
proposedSource={proposedSource}
isDirty={isDirty}
uiState={uiState}
databases={databases}
setSourceAndRejectProposed={setSourceAndRejectProposed}
setUiState={setUiState}
acceptProposed={acceptProposed}
rejectProposed={rejectProposed}
/>
</Stack>
{isConfirmationShown && checkData != null && (
<PLUGIN_DEPENDENCIES.CheckDependenciesModal
@@ -213,3 +217,62 @@ function TransformQueryPageBody({
</>
);
}
export type TransformQueryPageEditorUiState = ReturnType<
typeof getInitialUiState
>;
export type TransformQueryPageEditorProps = {
source: DraftTransformSource;
proposedSource?: DraftTransformSource;
isDirty: boolean;
uiState: TransformQueryPageEditorUiState;
databases: Database[];
setSourceAndRejectProposed: (source: DraftTransformSource) => void;
setUiState: (uiState: TransformQueryPageEditorUiState) => void;
acceptProposed: () => void;
rejectProposed: () => void;
uiOptions?: TransformEditorProps["uiOptions"];
};
export function TransformQueryPageEditor({
source,
proposedSource,
isDirty,
uiState,
uiOptions,
databases,
setSourceAndRejectProposed,
setUiState,
acceptProposed,
rejectProposed,
}: TransformQueryPageEditorProps) {
return source.type === "python" ? (
<PLUGIN_TRANSFORMS_PYTHON.TransformEditor
source={source}
uiOptions={uiOptions}
proposedSource={
proposedSource?.type === "python" ? proposedSource : undefined
}
isDirty={isDirty}
onChangeSource={setSourceAndRejectProposed}
onAcceptProposed={acceptProposed}
onRejectProposed={rejectProposed}
/>
) : (
<TransformEditor
source={source}
proposedSource={
proposedSource?.type === "query" ? proposedSource : undefined
}
uiState={uiState}
uiOptions={uiOptions}
databases={databases}
onChangeSource={setSourceAndRejectProposed}
onChangeUiState={setUiState}
onAcceptProposed={acceptProposed}
onRejectProposed={rejectProposed}
/>
);
}

View File

@@ -36,7 +36,7 @@ import type { Transform, UpdateTransformRequest } from "metabase-types/api";
type UpdateTargetModalProps = {
transform: Transform;
onUpdate: () => void;
onUpdate: (transform?: Transform) => void;
onClose: () => void;
};
@@ -74,7 +74,7 @@ const EDIT_TRANSFORM_SCHEMA = Yup.object({
type UpdateTargetFormProps = {
transform: Transform;
onUpdate: () => void;
onUpdate: (transform?: Transform) => void;
onClose: () => void;
};
@@ -119,10 +119,10 @@ function UpdateTargetForm({
if (shouldDeleteTarget) {
await deleteTransformTarget(transform.id).unwrap();
}
await updateTransform(
const updatedTransform = await updateTransform(
getUpdateRequest(transform, values, databaseId),
).unwrap();
onUpdate();
onUpdate(updatedTransform);
};
return (

View File

@@ -29,6 +29,7 @@ export function getDataStudioTransformRoutes() {
<Route path="runs" component={TransformTopNavLayout}>
<IndexRoute component={RunListPage} />
</Route>
<Route component={TransformSidebarLayout}>
<IndexRoute component={TransformEmptyPage} />
<Route path="jobs" component={JobEmptyPage} />

View File

@@ -0,0 +1,105 @@
import { type FormEvent, useEffect, useMemo, useState } from "react";
import { t } from "ttag";
import { Button, Flex, Modal, Select, Stack, TextInput } from "metabase/ui";
type DatabaseOption = {
value: string;
label: string;
};
export type CreateWorkspaceModalProps = {
opened: boolean;
onClose: () => void;
onSubmit: (workspace: { name: string; databaseId: string }) => void;
databaseOptions: DatabaseOption[];
isSubmitting?: boolean;
defaultName?: string;
defaultDatabaseId?: string | null;
};
export function CreateWorkspaceModal({
opened,
onClose,
onSubmit,
databaseOptions,
isSubmitting = false,
defaultName,
defaultDatabaseId = null,
}: CreateWorkspaceModalProps) {
const initialName = useMemo(
() => defaultName ?? t`New workspace`,
[defaultName],
);
const [workspaceName, setWorkspaceName] = useState(initialName);
const [selectedDatabaseId, setSelectedDatabaseId] = useState<string | null>(
defaultDatabaseId,
);
useEffect(() => {
if (!opened) {
return;
}
setWorkspaceName(initialName);
setSelectedDatabaseId(defaultDatabaseId);
}, [opened, initialName, defaultDatabaseId]);
const handleSubmit = (event: FormEvent) => {
event.preventDefault();
if (!selectedDatabaseId || !workspaceName.trim() || isSubmitting) {
return;
}
onSubmit({
name: workspaceName.trim(),
databaseId: selectedDatabaseId,
});
};
return (
<Modal
opened={opened}
onClose={onClose}
title={t`Create new workspace`}
withinPortal
>
<form onSubmit={handleSubmit}>
<Stack gap="md">
<TextInput
autoFocus
label={t`Workspace name`}
placeholder={t`Enter workspace name`}
value={workspaceName}
onChange={(e) => setWorkspaceName(e.target.value)}
required
/>
<Select
label={t`Database`}
placeholder={t`Select a database`}
data={databaseOptions}
value={selectedDatabaseId}
onChange={setSelectedDatabaseId}
required
searchable
/>
<Flex gap="sm" justify="flex-end">
<Button variant="subtle" onClick={onClose}>
{t`Cancel`}
</Button>
<Button
variant="filled"
disabled={
!selectedDatabaseId || !workspaceName.trim() || isSubmitting
}
type="submit"
loading={isSubmitting}
>
{t`Create`}
</Button>
</Flex>
</Stack>
</form>
</Modal>
);
}

View File

@@ -60,6 +60,7 @@ export * from "./util";
export * from "./visualization";
export * from "./visualization-settings";
export * from "./visualizer";
export * from "./workspace";
// ISO8601 timestamp
export type ISO8601Time = string;

View File

@@ -73,6 +73,7 @@ export function createMockTransform(opts?: Partial<Transform>): Transform {
target: createMockTransformTarget(),
created_at: "2000-01-01T00:00:00Z",
updated_at: "2000-01-01T00:00:00Z",
workspace_id: null,
...opts,
};
}

View File

@@ -14,9 +14,11 @@ export type Transform = {
name: string;
description: string | null;
source: TransformSource;
source_type: "native" | "python" | "mbql";
target: TransformTarget;
created_at: string;
updated_at: string;
workspace_id: number | null;
// hydrated fields
tag_ids?: TransformTagId[];

View File

@@ -0,0 +1,131 @@
import type { CollectionId } from "./collection";
import type { DatabaseId } from "./database";
import type {
Transform,
TransformId,
TransformSource,
TransformTagId,
TransformTarget,
} from "./transform";
export type WorkspaceId = number;
export type Workspace = {
id: WorkspaceId;
name: string;
collection_id: CollectionId | null;
database_id: DatabaseId | null;
created_at: string;
updated_at: string;
archived_at?: string | null;
contents?: WorkspaceContents["contents"];
};
export type WorkspaceItem = {
id: WorkspaceId;
name: string;
};
export type CreateWorkspaceRequest = {
name: string;
database_id?: DatabaseId;
upstream: {
transforms?: TransformId[];
};
};
export type WorkspaceListResponse = {
items: Workspace[];
};
export type WorkspaceContentItem = WorkspaceTransformItem;
export type WorkspaceTransformItem = {
type: "transform";
id: TransformId;
name: string;
upstream_id: TransformId;
workspace_id: WorkspaceId;
};
export type WorkspaceContents = {
contents: {
transforms: Transform[];
};
};
export type TransformUpstreamMapping = {
transform: WorkspaceTransformItem | null;
};
export type DownstreamTransformInfo = {
id: TransformId;
name: string;
workspace: WorkspaceItem;
};
export type TransformDownstreamMapping = {
transforms: DownstreamTransformInfo[];
};
export type WorkspaceMergeResponse = {
promoted: WorkspaceContentItem[];
errors?: (WorkspaceContentItem & { error: string })[];
workspace: WorkspaceItem;
archived_at: string | null;
};
export type WorkspaceUpdateContentsRequest = {
id: WorkspaceId;
add?: {
transforms?: TransformId[];
};
remove?: {
transforms?: TransformId[];
};
};
export type ValidateTableNameRequest = {
id: WorkspaceId;
db_id: DatabaseId;
target: {
type: "table";
name: string;
schema: string | null;
};
};
export type ValidateTableNameResponse =
| "OK"
| "A table with that name already exists";
export type CreateWorkspaceTransformRequest = {
name: string;
description?: string | null;
source: TransformSource;
target: TransformTarget;
tag_ids?: TransformTagId[];
};
export type CreateWorkspaceTransformResponse = Transform;
export type WorkspaceTable = {
id?: number;
schema: string | null;
table: string | null;
};
export type WorkspaceOutputTable = {
global?: WorkspaceTable;
workspace?: {
"transform-id": number;
"table-id": number;
schema: string | null;
table: string | null;
};
};
export type WorkspaceTablesResponse = {
inputs: WorkspaceTable[];
outputs: WorkspaceOutputTable[];
};

View File

@@ -106,3 +106,11 @@ export function dataStudioSnippetDependencies(snippetId: NativeQuerySnippetId) {
export function newDataStudioSnippet() {
return `${dataStudioModeling()}/snippets/new`;
}
export function dataStudioWorkspaceList() {
return `${ROOT_URL}/workspaces`;
}
export function dataStudioWorkspace(workspaceId: number) {
return `${dataStudioWorkspaceList()}/${workspaceId}`;
}

View File

@@ -46,7 +46,7 @@ export function getUrl(baseUrl: string, params: ParsedRouteParams): string {
}
export function getTableMetadataQuery(
tableId: TableId | undefined,
tableId: TableId | undefined | null,
): GetTableQueryMetadataRequest | typeof skipToken {
if (tableId == null) {
return skipToken;

View File

@@ -54,6 +54,9 @@ type PluginMetabotType = {
hide?: boolean;
config?: PluginMetabotConfig;
}) => React.ReactElement | null;
MetabotChat: (props: {
config?: PluginMetabotConfig;
}) => React.ReactElement | null;
defaultMetabotContextValue: MetabotContext;
MetabotContext: React.Context<MetabotContext>;
getMetabotProvider: () => ComponentType<{ children: React.ReactNode }>;
@@ -100,6 +103,8 @@ const getDefaultPluginMetabot = (): PluginMetabotType => ({
isEnabled: () => false,
Metabot: (_props: { hide?: boolean; config?: PluginMetabotConfig }) =>
null as React.ReactElement | null,
MetabotChat: (_props: { config?: PluginMetabotConfig }) =>
null as React.ReactElement | null,
defaultMetabotContextValue,
MetabotContext: React.createContext(defaultMetabotContextValue),
getMetabotProvider: () => {

View File

@@ -36,9 +36,15 @@ export type TransformsPlugin = {
TransformPicker: ComponentType<TransformPickerProps>;
};
export type PythonTransformEditorUiOptions = {
canChangeDatabase?: boolean;
readOnly?: boolean;
};
export type PythonTransformEditorProps = {
source: PythonTransformSourceDraft;
proposedSource?: PythonTransformSourceDraft;
uiOptions?: PythonTransformEditorUiOptions;
isDirty: boolean;
onChangeSource: (source: PythonTransformSourceDraft) => void;
onAcceptProposed: () => void;

View File

@@ -89,6 +89,7 @@ export function QueryEditor({
modalSnippet={uiState.modalSnippet}
nativeEditorSelectedText={selectedText}
readOnly={uiOptions?.readOnly}
canChangeDatabase={uiOptions?.canChangeDatabase}
isNative={isNative}
isRunnable={isRunnable}
isRunning={isRunning}

View File

@@ -45,6 +45,7 @@ type QueryEditorBodyProps = {
| null;
nativeEditorSelectedText?: string | null;
readOnly?: boolean;
canChangeDatabase?: boolean;
isNative: boolean;
isRunnable: boolean;
isRunning: boolean;
@@ -74,6 +75,7 @@ export function QueryEditorBody({
modalSnippet,
nativeEditorSelectedText,
readOnly,
canChangeDatabase,
isNative,
isRunnable,
isRunning,
@@ -136,6 +138,7 @@ export function QueryEditorBody({
isInitiallyOpen
isNativeEditorOpen
readOnly={readOnly}
canChangeDatabase={canChangeDatabase}
hasParametersList={false}
isRunnable={isRunnable}
isRunning={isRunning}

Some files were not shown because too many files have changed in this diff Show More