Compare commits
457 Commits
persistent
...
test-fixes
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8b790caf20 | ||
|
|
d7eb9eb86b | ||
|
|
57dfb6dc2a | ||
|
|
ae2200f1c2 | ||
|
|
94c8c5287f | ||
|
|
14b7975613 | ||
|
|
08a80d7047 | ||
|
|
2a53ccb93f | ||
|
|
34527055d6 | ||
|
|
ce4f7e4b09 | ||
|
|
37de244380 | ||
|
|
77b3a47cea | ||
|
|
5264a10205 | ||
|
|
74a2bbaf9f | ||
|
|
f53d7917e3 | ||
|
|
39f775516b | ||
|
|
c7f42e866a | ||
|
|
bf75ac26b2 | ||
|
|
dd4581f2e5 | ||
|
|
ae2071b875 | ||
|
|
85371a7298 | ||
|
|
6dc85458ec | ||
|
|
098b1fc826 | ||
|
|
c740caa5bb | ||
|
|
b7556bf964 | ||
|
|
06efa792ed | ||
|
|
85ce13c5b3 | ||
|
|
a1e6c01458 | ||
|
|
1ffecf3d9e | ||
|
|
c4a847322f | ||
|
|
a7a38fd4ca | ||
|
|
486bc64112 | ||
|
|
3101e32edd | ||
|
|
b51c20c9b0 | ||
|
|
8cdf482f1f | ||
|
|
5d173e849d | ||
|
|
aabd4344fb | ||
|
|
f94822749b | ||
|
|
d3a30cc4ab | ||
|
|
c8958bea54 | ||
|
|
dd579af20c | ||
|
|
0d57823e39 | ||
|
|
d0b96d16ed | ||
|
|
3bea2940a6 | ||
|
|
7b0f4cfb9e | ||
|
|
925d8d76ee | ||
|
|
f1a8e6a623 | ||
|
|
8a9b4fbd05 | ||
|
|
1deee13b0c | ||
|
|
b89c4b6431 | ||
|
|
a0f9335ec0 | ||
|
|
8d717aded4 | ||
|
|
2c7848edb7 | ||
|
|
edeac448d2 | ||
|
|
290b486e1a | ||
|
|
67345120b6 | ||
|
|
24ee602364 | ||
|
|
7bdffa72a6 | ||
|
|
4b228cd956 | ||
|
|
7da7ca3839 | ||
|
|
9e3af41bda | ||
|
|
b2ca4d492a | ||
|
|
c0636ba772 | ||
|
|
506fcd0970 | ||
|
|
4a8d9e2085 | ||
|
|
1188beea7d | ||
|
|
538318a38d | ||
|
|
bd0387cd60 | ||
|
|
0471190c74 | ||
|
|
3407b668a3 | ||
|
|
ec826befeb | ||
|
|
092bf6d679 | ||
|
|
16a258250c | ||
|
|
8c9e5886ab | ||
|
|
d161b663ae | ||
|
|
95a86c4acc | ||
|
|
51b02e259b | ||
|
|
8ef14ebcf1 | ||
|
|
d7e0b28be5 | ||
|
|
75231939b5 | ||
|
|
5f91a003bb | ||
|
|
278fc0c46c | ||
|
|
c7086253b6 | ||
|
|
c83ea462d2 | ||
|
|
306d80de92 | ||
|
|
a345fb257d | ||
|
|
2f65a02744 | ||
|
|
b4a9ece8d8 | ||
|
|
75b3a5b509 | ||
|
|
18db799cd0 | ||
|
|
ea1b32a7c1 | ||
|
|
fe8f2dbe37 | ||
|
|
1c9f3d4fe1 | ||
|
|
667563b11b | ||
|
|
d697be5665 | ||
|
|
c1030a16f7 | ||
|
|
b5e5bd328a | ||
|
|
777f3a47da | ||
|
|
9042477db1 | ||
|
|
52f602f0c2 | ||
|
|
b1f42b9497 | ||
|
|
704bbd280e | ||
|
|
5e370f937c | ||
|
|
662fc910b6 | ||
|
|
4448f190fa | ||
|
|
00f58316a7 | ||
|
|
d956fa61d5 | ||
|
|
ca6372be38 | ||
|
|
5e226fee6e | ||
|
|
89e92ee71f | ||
|
|
a111938611 | ||
|
|
fc95d79d04 | ||
|
|
d8d32ecb32 | ||
|
|
e4b1d46e78 | ||
|
|
f8a7f0f2d8 | ||
|
|
ca9b70efbe | ||
|
|
c1841b41cb | ||
|
|
070009d6aa | ||
|
|
d7078d30a1 | ||
|
|
a22f42d04e | ||
|
|
c50d16407f | ||
|
|
c46e93e25f | ||
|
|
8de115c0d6 | ||
|
|
e18f364ab2 | ||
|
|
73a024c794 | ||
|
|
58da27dc1a | ||
|
|
dee89e8f31 | ||
|
|
934d5dfd3f | ||
|
|
47b00082ff | ||
|
|
328b9a8988 | ||
|
|
d16d5c77eb | ||
|
|
5d62ce224d | ||
|
|
4d8df063ae | ||
|
|
e70210ede1 | ||
|
|
30baf69e2c | ||
|
|
9070d8592f | ||
|
|
57709669e1 | ||
|
|
17df352531 | ||
|
|
ff524bc410 | ||
|
|
544cacc4a5 | ||
|
|
4603d73aa5 | ||
|
|
16952f5d4c | ||
|
|
0cb4318906 | ||
|
|
836dbe993f | ||
|
|
3b5d5a5e6b | ||
|
|
ace6b58330 | ||
|
|
c2525e9fdb | ||
|
|
da7385a275 | ||
|
|
e09bba85ef | ||
|
|
31549a9745 | ||
|
|
3e26069789 | ||
|
|
0124c006a7 | ||
|
|
3143e4bddb | ||
|
|
1aefab5285 | ||
|
|
31bb4085dd | ||
|
|
7fb2dc1b2a | ||
|
|
ba5860f72d | ||
|
|
a839e0948b | ||
|
|
750fce2617 | ||
|
|
455bf8f6ed | ||
|
|
290e5dc5d4 | ||
|
|
e255be5b45 | ||
|
|
a1bb788f46 | ||
|
|
bb05aceda2 | ||
|
|
1d56cce877 | ||
|
|
591db0554f | ||
|
|
13b26368cd | ||
|
|
70486e25d5 | ||
|
|
5fd6d2477d | ||
|
|
c6b0a63cd0 | ||
|
|
f644c57b7b | ||
|
|
a27baf8ea1 | ||
|
|
cd2a7f6519 | ||
|
|
e0930a4019 | ||
|
|
203c115810 | ||
|
|
2885b08b35 | ||
|
|
a30a4c9652 | ||
|
|
e9682a4907 | ||
|
|
85eb1b5076 | ||
|
|
1b310378c4 | ||
|
|
01d481c618 | ||
|
|
cc21ddb5d0 | ||
|
|
9d740a86aa | ||
|
|
86c5570385 | ||
|
|
c63accec35 | ||
|
|
74a5d9cdd8 | ||
|
|
019c7fdd8b | ||
|
|
46f3302c7d | ||
|
|
8dd28bdc68 | ||
|
|
ed3df65de7 | ||
|
|
4646272bc0 | ||
|
|
aa334a09ec | ||
|
|
ed70fe2e6c | ||
|
|
8d90453a16 | ||
|
|
5a3c32cbf0 | ||
|
|
633d8b80b2 | ||
|
|
3c7116b01c | ||
|
|
8d3ffad856 | ||
|
|
57159b6017 | ||
|
|
afecaddbd3 | ||
|
|
496b420f34 | ||
|
|
6b1da0b2b5 | ||
|
|
d763760ddc | ||
|
|
2c9d550ba2 | ||
|
|
58fbea310e | ||
|
|
264dde2418 | ||
|
|
8df148bac2 | ||
|
|
c994dc7fc8 | ||
|
|
daf6357fc6 | ||
|
|
0c58d96ace | ||
|
|
af2bcec859 | ||
|
|
8e488b7527 | ||
|
|
964e9c12cf | ||
|
|
fa2033d129 | ||
|
|
495c5c63de | ||
|
|
a0b3e19657 | ||
|
|
76ccbe1622 | ||
|
|
7907a6ccb4 | ||
|
|
f2c159652c | ||
|
|
909331f485 | ||
|
|
7855a5b795 | ||
|
|
6ba64985da | ||
|
|
69d5973321 | ||
|
|
48597d14b4 | ||
|
|
5fd46596cb | ||
|
|
6d5fec7122 | ||
|
|
783e2254d6 | ||
|
|
0ab3735bf8 | ||
|
|
631800884f | ||
|
|
4e992e8909 | ||
|
|
a3e14d9641 | ||
|
|
6af8f6e242 | ||
|
|
05c94b0ce3 | ||
|
|
7bb0bbaa8c | ||
|
|
a6fb7c76c3 | ||
|
|
e1a8b7e90b | ||
|
|
705dd79908 | ||
|
|
ba739ee291 | ||
|
|
40ee74bb0b | ||
|
|
4d8a03d04a | ||
|
|
2fff970042 | ||
|
|
d8f39ea54f | ||
|
|
2d69e365bb | ||
|
|
24141d20d0 | ||
|
|
42c3fbe7e3 | ||
|
|
e1a77204d0 | ||
|
|
584f10c2c5 | ||
|
|
c0484509b8 | ||
|
|
8eb100fa52 | ||
|
|
5e4b1619f8 | ||
|
|
d8cf6d4ebb | ||
|
|
2b92e87ac6 | ||
|
|
a05468ffcf | ||
|
|
38ad323d39 | ||
|
|
359c63bfc7 | ||
|
|
70a7f318c8 | ||
|
|
a5f12bd5cc | ||
|
|
610eb93c6c | ||
|
|
c7b6f06b57 | ||
|
|
6c8268adb2 | ||
|
|
8e776d2b66 | ||
|
|
37da925746 | ||
|
|
49a00dea09 | ||
|
|
ce44815e8e | ||
|
|
a4c37283fc | ||
|
|
d191b6c0a1 | ||
|
|
0cfd5936c5 | ||
|
|
573ba81c1e | ||
|
|
de0616bd90 | ||
|
|
8f7fa76f87 | ||
|
|
7f9886044a | ||
|
|
becf8db5df | ||
|
|
c78784afef | ||
|
|
ac2c41b83f | ||
|
|
c38c69059e | ||
|
|
f83890077a | ||
|
|
a5922d9323 | ||
|
|
866dbe4be9 | ||
|
|
bd3c618303 | ||
|
|
ffbe648b7b | ||
|
|
e080d9e1ff | ||
|
|
af88617937 | ||
|
|
7bff4573e3 | ||
|
|
098370174a | ||
|
|
14facfdbb7 | ||
|
|
ddc9c7c6c3 | ||
|
|
63f4dcbf7c | ||
|
|
7f35c17f3e | ||
|
|
4f98f13971 | ||
|
|
651918f3e8 | ||
|
|
59c652d537 | ||
|
|
bc3687c4a5 | ||
|
|
05bb1daa42 | ||
|
|
6b6d55cd22 | ||
|
|
1361b9bda0 | ||
|
|
6644f846e8 | ||
|
|
98b5096eda | ||
|
|
0cc951e1a4 | ||
|
|
24910610c4 | ||
|
|
12fd210a41 | ||
|
|
7484249328 | ||
|
|
d034244735 | ||
|
|
3b837f7593 | ||
|
|
36419de6e7 | ||
|
|
b488c1b082 | ||
|
|
869cfcc024 | ||
|
|
81a7572e0a | ||
|
|
2ff73cd864 | ||
|
|
3ebdecc2d2 | ||
|
|
f0bfd6898f | ||
|
|
1b6528c9f1 | ||
|
|
f8f2e235f6 | ||
|
|
9086aa10e9 | ||
|
|
cfa8c389b2 | ||
|
|
c670809704 | ||
|
|
1f78793830 | ||
|
|
4e342e8481 | ||
|
|
b61455bb53 | ||
|
|
18c743403e | ||
|
|
b1fd062915 | ||
|
|
15b446ab04 | ||
|
|
0f61a79a6b | ||
|
|
9ebbf353f6 | ||
|
|
be25ff92a1 | ||
|
|
11ef81fdee | ||
|
|
ce08daf26f | ||
|
|
e2b5e9dd9d | ||
|
|
49bbcf7467 | ||
|
|
757cca20db | ||
|
|
b196d10ad3 | ||
|
|
3e92c2d2ad | ||
|
|
f1d495dfde | ||
|
|
a14b9c1b3b | ||
|
|
6cffef70f5 | ||
|
|
a5a8fa1f1e | ||
|
|
6c7102aa37 | ||
|
|
809e5b6724 | ||
|
|
5f75951717 | ||
|
|
8ff78faef0 | ||
|
|
2bd68fc928 | ||
|
|
288f07c709 | ||
|
|
1f199c1dfb | ||
|
|
51efac3d49 | ||
|
|
a9ccb8def6 | ||
|
|
b2dacc7a92 | ||
|
|
d769903e13 | ||
|
|
167af6a806 | ||
|
|
e2ad85b627 | ||
|
|
c1c1926751 | ||
|
|
40dbb2dd5e | ||
|
|
f206ac8d9b | ||
|
|
c93f52c810 | ||
|
|
fac1eda9fa | ||
|
|
5baa9d4709 | ||
|
|
78272586f8 | ||
|
|
86926f69d3 | ||
|
|
0ee7e2fa48 | ||
|
|
c3f7c0e3f8 | ||
|
|
1774564beb | ||
|
|
557ce7c3c6 | ||
|
|
a60fa0c2cb | ||
|
|
39e4f68e31 | ||
|
|
072c06e2bd | ||
|
|
e781690132 | ||
|
|
68b19966fc | ||
|
|
8b5db401e8 | ||
|
|
946fc08c43 | ||
|
|
f5cbb8cc48 | ||
|
|
44d6075e8c | ||
|
|
fa2ae8e884 | ||
|
|
36e3b6ec3a | ||
|
|
ba9a841466 | ||
|
|
5ebc738f29 | ||
|
|
045e9a224f | ||
|
|
a6e22daac1 | ||
|
|
6bdd48ffe3 | ||
|
|
5bb0f1f560 | ||
|
|
2d263d19e3 | ||
|
|
5e98f44b80 | ||
|
|
da6555f5cf | ||
|
|
937f3f7b03 | ||
|
|
71d065f0a9 | ||
|
|
b30071a69a | ||
|
|
407fc2a5ab | ||
|
|
de113f72ce | ||
|
|
6199e0056f | ||
|
|
fd6b176446 | ||
|
|
42d1202c18 | ||
|
|
d1a4656cb8 | ||
|
|
cc934ea766 | ||
|
|
8bf117b047 | ||
|
|
caea0b31cf | ||
|
|
1d6dd0d16d | ||
|
|
dca47362f8 | ||
|
|
2f8b666797 | ||
|
|
f3aef4f4ad | ||
|
|
5ac6249eb3 | ||
|
|
8acbee8000 | ||
|
|
bfd2ad8f0f | ||
|
|
a982db6bf5 | ||
|
|
3e6d266a33 | ||
|
|
e197cf97af | ||
|
|
97d89cc855 | ||
|
|
955cc0b4be | ||
|
|
1b4424393d | ||
|
|
3337769411 | ||
|
|
0b07779984 | ||
|
|
2e54d68fb2 | ||
|
|
e67c3e625d | ||
|
|
325fe07ab1 | ||
|
|
dc53fec17e | ||
|
|
17c3ed6635 | ||
|
|
f661f0c21e | ||
|
|
89d1394fb7 | ||
|
|
0cac006357 | ||
|
|
c9a5ea4dd1 | ||
|
|
79f7d83c4d | ||
|
|
56a1ac2862 | ||
|
|
8d0acd1436 | ||
|
|
b6628af72f | ||
|
|
0169970a8b | ||
|
|
9cc27dfa59 | ||
|
|
9221d59b7e | ||
|
|
22047bb890 | ||
|
|
772216505c | ||
|
|
3d7541f3c7 | ||
|
|
5a59eef925 | ||
|
|
dc05f59514 | ||
|
|
f64ab4ebab | ||
|
|
19497a7470 | ||
|
|
823690bc76 | ||
|
|
82930d7f02 | ||
|
|
fa9b90beb1 | ||
|
|
d4d0c64400 | ||
|
|
cb70117c05 | ||
|
|
3d9c45e126 | ||
|
|
516df0c70e | ||
|
|
4cd2aa7c3e | ||
|
|
e6943a7389 | ||
|
|
a43c90f357 | ||
|
|
91a42b4fac | ||
|
|
34689d9a05 | ||
|
|
528b264f87 | ||
|
|
0580e2dc34 | ||
|
|
91c985eca9 | ||
|
|
3bb34fa1c9 | ||
|
|
f539b334dd | ||
|
|
9cfe59b1a2 | ||
|
|
fb59bf9325 | ||
|
|
2b2ff2dae5 | ||
|
|
0dd7d78c07 | ||
|
|
33daf1d822 | ||
|
|
9faad636cb | ||
|
|
b201dca6cf | ||
|
|
f1982e56e3 | ||
|
|
439c3f3be5 | ||
|
|
37449925fa |
@@ -1,7 +1,7 @@
|
||||
;; -*- comment-column: 100; -*-
|
||||
{:config-paths ["macros" "src" "config"]
|
||||
:linters
|
||||
{ ;;
|
||||
{;;
|
||||
;; enabled optional linters (these are disabled by default)
|
||||
;;
|
||||
|
||||
@@ -178,6 +178,11 @@
|
||||
metabase.notification.models/create-notification!
|
||||
metabase.pulse.send/send-pulse!
|
||||
metabase-enterprise.transforms-python.python-runner-test/execute!
|
||||
;; Workspace isolation functions are safe in parallel tests because they operate on
|
||||
;; isolated, test-specific workspaces and databases that don't affect other tests
|
||||
metabase-enterprise.workspaces.isolation/destroy-workspace-isolation!
|
||||
metabase-enterprise.workspaces.isolation/ensure-database-isolation!
|
||||
metabase-enterprise.workspaces.test-util/create-ready-ws!
|
||||
;; Safe in tests, where we typically use a private, temporary index per thread.
|
||||
metabase.search.core/reindex!
|
||||
metabase.util/run-count!
|
||||
@@ -191,7 +196,7 @@
|
||||
:metabase/disallow-hardcoded-driver-names-in-tests
|
||||
;; don't include `:h2` since that will probably lead to a mountain of tests where we are specifically testing
|
||||
;; middleware or compilation behavior with just H2 since it's the default.
|
||||
{:drivers #{ ;; core drivers
|
||||
{:drivers #{;; core drivers
|
||||
#_:h2
|
||||
:postgres
|
||||
:mysql
|
||||
@@ -422,6 +427,7 @@
|
||||
metabase-enterprise.transforms.api-test/with-worker-runs clojure.core/let
|
||||
metabase-enterprise.transforms.test-util/with-transform-cleanup! clojure.core/let
|
||||
metabase-enterprise.transforms.execute/with-transform-lifecycle clojure.core/let
|
||||
metabase-enterprise.workspaces.test-util/with-workspaces! clojure.core/let
|
||||
metabase.actions.test-util/with-actions clojure.core/let
|
||||
metabase.api.common/let-404 clojure.core/let
|
||||
metabase.app-db.custom-migrations.util/with-temp-scheduler! clojure.core/fn
|
||||
|
||||
@@ -534,6 +534,11 @@
|
||||
metabase.driver.sql-jdbc.execute
|
||||
metabase.driver.sql-jdbc.execute.diagnostic
|
||||
metabase.driver.sql-jdbc.sync
|
||||
;; TODO (lbrdnk 2025/12/17): We should revisit addition of the following line. I've added it to make
|
||||
;; tests pass as we are using it directly with ee/workspaces module. I believe later, when workspaces
|
||||
;; are in better state quality wise, it should become part of the driver api and that should be used
|
||||
;; instead of calling directly into the normalize module.
|
||||
metabase.driver.sql.normalize
|
||||
metabase.driver.sql.query-processor
|
||||
metabase.driver.sql.util
|
||||
metabase.driver.sync
|
||||
@@ -2279,6 +2284,7 @@
|
||||
{:team "Querying"
|
||||
:api #{metabase-enterprise.transforms.api
|
||||
metabase-enterprise.transforms.core
|
||||
metabase-enterprise.transforms.execute
|
||||
metabase-enterprise.transforms.init
|
||||
metabase-enterprise.transforms.instrumentation
|
||||
metabase-enterprise.transforms.interface
|
||||
@@ -2312,6 +2318,7 @@
|
||||
app-db
|
||||
config
|
||||
driver
|
||||
events
|
||||
lib
|
||||
lib-be
|
||||
query-processor
|
||||
@@ -2326,7 +2333,24 @@
|
||||
models
|
||||
premium-features
|
||||
upload
|
||||
util}}}
|
||||
util}}
|
||||
|
||||
enterprise/workspaces
|
||||
{:team "Metabot"
|
||||
:api #{metabase-enterprise.workspaces.api
|
||||
metabase-enterprise.workspaces.init}
|
||||
:uses #{api
|
||||
api-keys
|
||||
app-db
|
||||
driver
|
||||
lib
|
||||
lib-be
|
||||
models
|
||||
queries
|
||||
request
|
||||
system
|
||||
util
|
||||
enterprise/transforms}}}
|
||||
|
||||
;; namespaces matching these patterns (with `re-find`) are excluded from the module linter. Since regex literals
|
||||
;; aren't allowed in EDN just used the `(str <regex>)` version i.e. two slashes instead of one.
|
||||
|
||||
@@ -90,6 +90,7 @@
|
||||
metabase-enterprise.test/with-gtaps! [[:inner 0]]
|
||||
metabase-enterprise.test/with-gtaps-for-user! [[:inner 0]]
|
||||
metabase-enterprise.test/with-user-attributes [[:block 2]]
|
||||
metabase-enterprise.workspaces.models.workspace-log/track! [[:block 2]]
|
||||
metabase.actions.actions-test/with-actions-test-data-and-actions-permissively-enabled [[:block 0]]
|
||||
metabase.actions.test-util/with-actions [[:block 1]]
|
||||
metabase.actions.test-util/with-actions-disabled [[:block 0]]
|
||||
|
||||
@@ -61,6 +61,7 @@
|
||||
(eval . (put-clojure-indent 'u/prog1 1))
|
||||
(eval . (put-clojure-indent 'u/select-keys-when '(:form)))
|
||||
(eval . (put-clojure-indent 'with-meta '(:form)))
|
||||
(eval . (put-clojure-indent 'ws.log/track! 2))
|
||||
;; these ones have to be done with `define-clojure-indent' for now because of upstream bug
|
||||
;; https://github.com/clojure-emacs/clojure-mode/issues/600 once that's resolved we should use `put-clojure-indent'
|
||||
;; instead. Please don't add new entries unless they don't work with `put-clojure-indent'
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
# execute this in your shell to enable custom merge driver:
|
||||
# echo '\n[include]\npath=../.gitconfig' >> .git/config
|
||||
#
|
||||
# TODO (Bryan 12/12/25) -- reenable this when we can completely verify correctness
|
||||
# [merge "yaml-migrations"]
|
||||
# name = YAML migration changeset-aware merge
|
||||
# driver = mage -merge-yaml-migrations %O %A %B %L %P
|
||||
[merge "yaml-migrations"]
|
||||
name = YAML migration changeset-aware merge
|
||||
driver = bin/mage -merge-yaml-migrations %O %A %B %L %P
|
||||
|
||||
@@ -24,6 +24,9 @@
|
||||
"dev.failsafe" {"failsafe" {:resource "LICENSE"}}
|
||||
"colorize" {"colorize" {:resource "EPL.txt"}}
|
||||
"com.github.jnr" {"jffi$native" {:resource "apache2_0.txt"}}
|
||||
"com.google.api" {"gax" {:resource "BSD.txt"}
|
||||
"gax-grpc" {:resource "BSD.txt"}
|
||||
"gax-httpjson" {:resource "BSD.txt"}}
|
||||
"com.google.api-client" {"google-api-client" {:resource "apache2_0.txt"}}
|
||||
"com.google.auto.value" {"auto-value-annotations" {:resource "apache2_0.txt"}}
|
||||
"com.google.http-client" {"google-http-client" {:resource "apache2_0.txt"}
|
||||
@@ -60,6 +63,7 @@
|
||||
"net.thisptr" {"jackson-jq" {:resource "apache2_0.txt"}}
|
||||
"org.antlr" {"antlr-runtime" {:resource "BSD.txt"}
|
||||
"antlr4-runtime" {:resource "BSD.txt"}}
|
||||
"org.codehaus.mojo" {"animal-sniffer-annotations" {:resource "MIT.txt"}}
|
||||
"org.eclipse.jgit" {"org.eclipse.jgit" {:resource "BSD.txt"}}
|
||||
"org.eclipse.jetty" {"jetty-webapp" {:resource "EPL.txt"}}
|
||||
"org.eclipse.jetty.toolchain" {"jetty-jakarta-servlet-api" {:resource "EPL.txt"}}
|
||||
|
||||
@@ -144,6 +144,18 @@
|
||||
change-log
|
||||
#(pos? (compare % "v49.00-000"))))
|
||||
|
||||
(defn require-primary-key-exists-has-table-name
|
||||
"Ensures that all primaryKeyExists preconditions specify a tableName."
|
||||
[change-log]
|
||||
(doseq [{:keys [changeSet]} change-log
|
||||
:when changeSet
|
||||
:let [s (pr-str (:preConditions changeSet))]
|
||||
:when (and (str/includes? s ":primaryKeyExists")
|
||||
(not (str/includes? s ":tableName")))]
|
||||
(throw (validation-error
|
||||
(format "Migration '%s' has primaryKeyExists precondition without tableName" (:id changeSet))
|
||||
{:id (:id changeSet)}))))
|
||||
|
||||
(s/def ::changeSet
|
||||
(s/spec :change-set.strict/change-set))
|
||||
|
||||
@@ -163,6 +175,7 @@
|
||||
(require-no-bare-blob-or-text-types change-log)
|
||||
(require-no-bare-boolean-types change-log)
|
||||
(require-no-datetime-type change-log)
|
||||
(require-primary-key-exists-has-table-name change-log)
|
||||
(let [{:keys [changeSet]
|
||||
:as all} (group-by only-key change-log)]
|
||||
(when-not (set/subset? (set (keys all)) #{:property :objectQuotingStrategy :changeSet})
|
||||
|
||||
13
deps.edn
13
deps.edn
@@ -36,7 +36,7 @@
|
||||
com.clearspring.analytics/stream {:mvn/version "2.9.8" ; Various sketching algorithms
|
||||
:exclusions [it.unimi.dsi/fastutil
|
||||
org.slf4j/slf4j-api]}
|
||||
com.clojure-goes-fast/jvm-alloc-rate-meter{:mvn/version "0.1.4"} ; Monitor heap allocation rate.
|
||||
com.clojure-goes-fast/jvm-alloc-rate-meter {:mvn/version "0.1.4"} ; Monitor heap allocation rate.
|
||||
com.clojure-goes-fast/jvm-hiccup-meter {:mvn/version "0.1.1"} ; Monitor GC pauses and other system-induced pauses.
|
||||
com.draines/postal {:mvn/version "2.0.5" ; SMTP library
|
||||
:exclusions [com.sun.mail/jakarta.mail]}
|
||||
@@ -45,7 +45,7 @@
|
||||
com.github.albfernandez/juniversalchardet {:mvn/version "2.5.0"}
|
||||
com.github.blagerweij/liquibase-sessionlock {:mvn/version "1.6.9"} ; session lock for liquibase migrations
|
||||
com.github.jknack/handlebars ^:antq/exclude ; 4.4.0 requires java 17+
|
||||
{:mvn/version "4.3.1"} ; templating engine
|
||||
{:mvn/version "4.3.1"} ; templating engine
|
||||
com.github.oliyh/martian-clj-http {:mvn/version "0.2.1"} ; openapi client
|
||||
com.github.seancorfield/honeysql {:mvn/version "2.7.1350"} ; Honey SQL 2. SQL generation from Clojure data maps
|
||||
com.github.seancorfield/next.jdbc {:mvn/version "1.3.1070"} ; Talk to JDBC DBs
|
||||
@@ -53,12 +53,12 @@
|
||||
com.github.vertical-blank/sql-formatter {:mvn/version "2.0.5"} ; Java SQL formatting library https://github.com/vertical-blank/sql-formatter
|
||||
com.google.guava/guava {:mvn/version "33.5.0-jre"} ; dep for BigQuery, Spark, and GA. Require here rather than letting different dep versions stomp on each other — see comments on #9697
|
||||
com.h2database/h2 ^:antq/exclude ; https://metaboat.slack.com/archives/CKZEMT1MJ/p1727191010259979
|
||||
{:mvn/version "2.1.214"} ; embedded SQL database
|
||||
{:mvn/version "2.1.214"} ; embedded SQL database
|
||||
com.ibm.icu/icu4j {:mvn/version "78.1"} ; Used to transliterate Unicode characters into Latin characters
|
||||
com.knuddels/jtokkit {:mvn/version "1.1.0"} ; Java tokenizer library for OpenAI models, used for token counting in semantic search
|
||||
com.mchange/c3p0 {:mvn/version "0.11.2"} ; DB connection pool
|
||||
com.snowplowanalytics/snowplow-java-tracker ^:antq/exclude ; 2+ depend on apache httpclient 5.x, but saml and clj-http depend on 4.x
|
||||
{:mvn/version "1.0.1"} ; Snowplow analytics
|
||||
{:mvn/version "1.0.1"} ; Snowplow analytics
|
||||
com.sun.mail/jakarta.mail ^:antq/exclude ; 2+ changes imports from javax.mail to jakarta.mail
|
||||
{:mvn/version "1.6.8"} ; SMTP lib, required by postal
|
||||
com.taoensso/encore {:mvn/version "3.158.0"} ; Utilities library (used by Nippy)
|
||||
@@ -88,7 +88,7 @@
|
||||
:git/sha "ffc4edc482c057cd9412c62f018a41e9140c618b"
|
||||
:git/url "https://github.com/eerohele/pp"}
|
||||
;; TODO: replace back to normal artifact
|
||||
io.github.metabase/macaw {:mvn/version "0.2.29"} ; Parse native SQL queries
|
||||
io.github.metabase/macaw {:mvn/version "0.2.31"} ; Parse native SQL queries
|
||||
io.prometheus/simpleclient_hotspot {:mvn/version "0.16.0"} ; prometheus jvm collector
|
||||
junegunn/grouper {:mvn/version "0.1.1"} ; Batch processing helper
|
||||
kixi/stats {:mvn/version "0.5.7" ; Various statistic measures implemented as transducers
|
||||
@@ -891,8 +891,7 @@
|
||||
{:exec-args {:only ["enterprise/backend/test/metabase_enterprise/transforms"
|
||||
"enterprise/backend/test/metabase_enterprise/transforms_python"]}}
|
||||
|
||||
|
||||
;; test all MBQL related stuff: MLv2, the legacy shared `metabase.legacy-mbql` code, and the QP
|
||||
;; test all MBQL related stuff: MLv2, the legacy shared `metabase.legacy-mbql` code, and the QP
|
||||
;;
|
||||
;; clj -X:dev:ee:ee-dev:test:test/mbql
|
||||
:test/mbql
|
||||
|
||||
@@ -8,6 +8,11 @@ title: Driver interface changelog
|
||||
|
||||
- Added `sql-jdbc.execute/db-type-name` multimethod. Override this if something more than the default is needed in your sql-jdbc-based driver. See the `:mysql` implementation as an example.
|
||||
|
||||
- Added workspace isolation multimethods for enterprise workspaces feature:
|
||||
- `init-workspace-isolation!` - Initialize database isolation for a workspace (create isolated schema/database and user credentials)
|
||||
- `destroy-workspace-isolation!` - Destroy all database resources created for workspace isolation
|
||||
- `grant-workspace-read-access!` - Grant read access on specified tables to a workspace's isolated user
|
||||
|
||||
## Metabase 0.58.0
|
||||
|
||||
- Added a `:collate` feature for drivers that support collation settings on text fields
|
||||
|
||||
@@ -193,6 +193,9 @@ export function codeMirrorHelpers<T extends object>(testId: string, extra: T) {
|
||||
el[0].dispatchEvent(pasteEvent);
|
||||
});
|
||||
},
|
||||
runButton() {
|
||||
return cy.findByTestId("run-button");
|
||||
},
|
||||
...extra,
|
||||
};
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ export const DataStudio = {
|
||||
editDefinition: () => cy.findByRole("link", { name: "Edit definition" }),
|
||||
queryEditor: () => cy.findByTestId("transform-query-editor"),
|
||||
runTab: () => DataStudio.Transforms.header().findByText("Run"),
|
||||
targetTab: () => DataStudio.Transforms.header().findByText("Target"),
|
||||
settingsTab: () => DataStudio.Transforms.header().findByText("Settings"),
|
||||
dependenciesTab: () =>
|
||||
DataStudio.Transforms.header().findByText("Dependencies"),
|
||||
|
||||
@@ -78,7 +78,11 @@ export function entityPickerModalItem(level, name) {
|
||||
}
|
||||
|
||||
export function entityPickerModalTab(name) {
|
||||
return cy.findAllByRole("tab").filter(`:contains(${name})`);
|
||||
if (typeof name === "string") {
|
||||
return cy.findAllByRole("tab").filter(`:contains(${name})`);
|
||||
} else {
|
||||
return cy.findAllByRole("tab").first().parent().findByText(name);
|
||||
}
|
||||
}
|
||||
|
||||
// displays at least these tabs:
|
||||
|
||||
130
e2e/support/helpers/e2e-workspaces-helpers.ts
Normal file
130
e2e/support/helpers/e2e-workspaces-helpers.ts
Normal file
@@ -0,0 +1,130 @@
|
||||
import type { WorkspaceId } from "metabase-types/api";
|
||||
|
||||
export const Workspaces = {
|
||||
visit: (id: WorkspaceId) => {
|
||||
return cy.visit(`/data-studio/workspace/${id}`);
|
||||
},
|
||||
|
||||
visitDataStudio() {
|
||||
return cy.visit("/data-studio");
|
||||
},
|
||||
|
||||
visitWorkspaces() {
|
||||
return cy.visit("/data-studio/workspaces");
|
||||
},
|
||||
|
||||
visitTransformListPage() {
|
||||
return cy.visit("/data-studio/transforms");
|
||||
},
|
||||
|
||||
getWorkspacesPage() {
|
||||
return cy.findByTestId("workspaces-page");
|
||||
},
|
||||
|
||||
getWorkspacePage() {
|
||||
return cy.findByTestId("workspace-page");
|
||||
},
|
||||
|
||||
getWorkspaceContent() {
|
||||
return cy.findByTestId("workspace-content");
|
||||
},
|
||||
|
||||
getWorkspaceTabs() {
|
||||
return cy.findByTestId("workspace-tabs");
|
||||
},
|
||||
|
||||
getWorkspaceSidebar() {
|
||||
return cy.findByTestId("workspace-sidebar");
|
||||
},
|
||||
|
||||
openCodeTab() {
|
||||
return this.getWorkspaceSidebar().within(() => {
|
||||
return cy.findByRole("tab", { name: "Code" }).click();
|
||||
});
|
||||
},
|
||||
|
||||
openDataTab() {
|
||||
return this.getWorkspaceSidebar().within(() => {
|
||||
return cy.findByRole("tab", { name: "Data" }).click();
|
||||
});
|
||||
},
|
||||
|
||||
getWorkspacesSection() {
|
||||
return cy.findByTestId("workspaces-section");
|
||||
},
|
||||
|
||||
getWorkspaceItem(name: string | RegExp) {
|
||||
return this.getWorkspacesSection().find(`button[name="${name}"]`);
|
||||
},
|
||||
|
||||
getWorkspaceItemStatus(name: string | RegExp) {
|
||||
return this.getWorkspaceItem(name).findByTestId("workspace-status");
|
||||
},
|
||||
|
||||
getWorkspaceItemActions(name: string | RegExp) {
|
||||
return this.getWorkspaceItem(name).findByLabelText("More actions");
|
||||
},
|
||||
|
||||
getWorkspaceNameInput() {
|
||||
return cy.findByPlaceholderText("Workspace name");
|
||||
},
|
||||
|
||||
getMergeWorkspaceButton() {
|
||||
return cy.findByRole("button", { name: /Merge/ });
|
||||
},
|
||||
|
||||
getMergeCommitInput() {
|
||||
return cy.findByPlaceholderText(
|
||||
"Describe the changes you made in this workspace...",
|
||||
);
|
||||
},
|
||||
|
||||
getWorkspaceDatabaseSelect() {
|
||||
return cy.findByPlaceholderText("Select a database");
|
||||
},
|
||||
|
||||
getTransformTabHeader() {
|
||||
return cy.findByTestId("transform-tab-header");
|
||||
},
|
||||
|
||||
getTransformTargetButton() {
|
||||
return this.getTransformTabHeader().findByRole("button", {
|
||||
name: /Change target/,
|
||||
});
|
||||
},
|
||||
|
||||
getRunTransformButton() {
|
||||
return this.getTransformTabHeader().findByTestId("run-button");
|
||||
},
|
||||
|
||||
getSaveTransformButton() {
|
||||
return this.getTransformTabHeader().findByRole("button", { name: /Save/ });
|
||||
},
|
||||
|
||||
getWorkspaceTransforms() {
|
||||
return cy.findByTestId("workspace-transforms");
|
||||
},
|
||||
|
||||
getMainlandTransforms() {
|
||||
return cy.findByTestId("mainland-transforms");
|
||||
},
|
||||
|
||||
getNewWorkspaceButton() {
|
||||
return cy.findByRole("button", { name: /New workspace/ });
|
||||
},
|
||||
|
||||
getNewWorkspaceNameInput() {
|
||||
return cy.findByPlaceholderText("Enter workspace name");
|
||||
},
|
||||
|
||||
getNewWorkspaceDatabaseInput() {
|
||||
return cy.findByPlaceholderText("Select a database");
|
||||
},
|
||||
|
||||
getTransformStatusDot(name: string | RegExp) {
|
||||
return cy
|
||||
.findByText(name)
|
||||
.parent()
|
||||
.find('[class*="statusDot"]');
|
||||
},
|
||||
};
|
||||
@@ -13,9 +13,9 @@ export * from "./e2e-comment-helpers";
|
||||
export * from "./e2e-custom-column-helpers";
|
||||
export * from "./e2e-dashboard-helpers";
|
||||
export * from "./e2e-dashboard-visualizer-helpers";
|
||||
export * from "./e2e-data-studio-helpers";
|
||||
export * from "./e2e-database-metadata-helpers";
|
||||
export * from "./e2e-datamodel-helpers";
|
||||
export * from "./e2e-data-studio-helpers";
|
||||
export * from "./e2e-dependency-graph-helpers";
|
||||
export * from "./e2e-detail-view-helpers";
|
||||
export * from "./e2e-dimension-list-helpers";
|
||||
@@ -60,4 +60,5 @@ export * from "./e2e-upload-helpers";
|
||||
export * from "./e2e-users-helpers";
|
||||
export * from "./e2e-visual-tests-helpers";
|
||||
export * from "./e2e-viz-settings-helpers";
|
||||
export * from "./e2e-workspaces-helpers";
|
||||
export * from "./embedding-sdk-helpers/constants";
|
||||
|
||||
1006
e2e/test/scenarios/data-studio/workspaces.cy.spec.ts
Normal file
1006
e2e/test/scenarios/data-studio/workspaces.cy.spec.ts
Normal file
File diff suppressed because it is too large
Load Diff
@@ -40,6 +40,7 @@
|
||||
[metabase-enterprise.transforms-python.api]
|
||||
[metabase-enterprise.transforms.api]
|
||||
[metabase-enterprise.upload-management.api]
|
||||
[metabase-enterprise.workspaces.api]
|
||||
[metabase.api.macros :as api.macros]
|
||||
[metabase.api.util.handlers :as handlers]
|
||||
[metabase.util.i18n :refer [deferred-tru]]))
|
||||
@@ -134,7 +135,8 @@
|
||||
"/transform-job" (premium-handler metabase-enterprise.transforms.api/transform-job-routes :transforms)
|
||||
"/transform-tag" (premium-handler metabase-enterprise.transforms.api/transform-tag-routes :transforms)
|
||||
"/tenant" (premium-handler metabase-enterprise.tenants.api/routes :tenants)
|
||||
"/upload-management" (premium-handler metabase-enterprise.upload-management.api/routes :upload-management)})
|
||||
"/upload-management" (premium-handler metabase-enterprise.upload-management.api/routes :upload-management)
|
||||
"/workspace" metabase-enterprise.workspaces.api/routes})
|
||||
;;; ↑↑↑ KEEP THIS SORTED OR ELSE ↑↑↑
|
||||
|
||||
(def ^:private routes-map
|
||||
|
||||
@@ -22,4 +22,5 @@
|
||||
[metabase-enterprise.stale.init]
|
||||
[metabase-enterprise.support-access-grants.init]
|
||||
[metabase-enterprise.transforms-python.init]
|
||||
[metabase-enterprise.transforms.init]))
|
||||
[metabase-enterprise.transforms.init]
|
||||
[metabase-enterprise.workspaces.init]))
|
||||
|
||||
@@ -42,6 +42,12 @@
|
||||
query-deps
|
||||
param-card-ids)))
|
||||
|
||||
(mu/defn upstream-deps:python-transform :- ::deps.schema/upstream-deps
|
||||
"Given a Toucan `:model/Transform`, return its upstream dependencies as a map from the kind to a set of IDs."
|
||||
[{{tables :source-tables} :source :as _py-transform}
|
||||
:- [:map [:source-tables {:optional true} [:map-of :string :int]]]]
|
||||
{:table (set (vals tables))})
|
||||
|
||||
(mu/defn upstream-deps:transform :- ::deps.schema/upstream-deps
|
||||
"Given a Transform (in Toucan form), return its upstream dependencies."
|
||||
[{{:keys [query], source-type :type} :source :as transform} :-
|
||||
@@ -50,9 +56,15 @@
|
||||
[:query
|
||||
[:map [:query ::lib.schema/query]]]
|
||||
[:python
|
||||
[:map]]]]]]
|
||||
(if (= (keyword source-type) :query)
|
||||
[:map [:source-tables {:optional true} [:map-of :string :int]]]]]]]]
|
||||
(cond
|
||||
(= (keyword source-type) :query)
|
||||
(upstream-deps:query query)
|
||||
|
||||
(= (keyword source-type) :python)
|
||||
(upstream-deps:python-transform transform)
|
||||
|
||||
:else
|
||||
(do (log/warnf "Don't know how to analyze the deps of Transform %d with source type '%s'" (:id transform) source-type)
|
||||
{})))
|
||||
|
||||
|
||||
@@ -116,4 +116,16 @@
|
||||
"User"
|
||||
"UserKeyValue"
|
||||
"UserParameterValue"
|
||||
"ViewLog"])
|
||||
"ViewLog"
|
||||
"Workspace"
|
||||
;; TODO (lbrdnk 2025/12/17): I've added rest of the workspace models. I believe some of those will be available for
|
||||
;; serialization, hence we should revisit this later in the project.
|
||||
"WorkspaceDependency"
|
||||
"WorkspaceInput"
|
||||
"WorkspaceInputExternal"
|
||||
"WorkspaceLog"
|
||||
"WorkspaceMerge"
|
||||
"WorkspaceMergeTransform"
|
||||
"WorkspaceOutput"
|
||||
"WorkspaceOutputExternal"
|
||||
"WorkspaceTransform"])
|
||||
|
||||
@@ -81,9 +81,13 @@
|
||||
|
||||
(defn get-transforms
|
||||
"Get a list of transforms."
|
||||
[& {:keys [last_run_start_time last_run_statuses tag_ids]}]
|
||||
[& {:keys [last_run_start_time last_run_statuses tag_ids database_id type]}]
|
||||
(api/check-superuser)
|
||||
(let [transforms (t2/select :model/Transform {:order-by [[:id :asc]]})]
|
||||
(let [type (when type (map #(if (= % "query") "mbql" %) type))
|
||||
where (cond-> [:and]
|
||||
type (conj [:in :source_type type])
|
||||
database_id (conj [:= :target_db_id database_id]))
|
||||
transforms (t2/select :model/Transform {:where where :order-by [[:id :asc]]})]
|
||||
(into []
|
||||
(comp (transforms.util/->date-field-filter-xf [:last_run :start_time] last_run_start_time)
|
||||
(transforms.util/->status-filter-xf [:last_run :status] last_run_statuses)
|
||||
@@ -106,14 +110,35 @@
|
||||
[:map
|
||||
[:last_run_start_time {:optional true} [:maybe ms/NonBlankString]]
|
||||
[:last_run_statuses {:optional true} [:maybe (ms/QueryVectorOf [:enum "started" "succeeded" "failed" "timeout"])]]
|
||||
[:tag_ids {:optional true} [:maybe (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]]]]
|
||||
[:tag_ids {:optional true} [:maybe (ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]]
|
||||
[:database_id {:optional true} [:maybe ms/PositiveInt]]
|
||||
[:type {:optional true} [:maybe (ms/QueryVectorOf [:enum "query" "native" "python"])]]]]
|
||||
(get-transforms query-params))
|
||||
|
||||
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
|
||||
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
|
||||
;;
|
||||
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
|
||||
(api.macros/defendpoint :post "/"
|
||||
(defn create-transform!
|
||||
"Create new transform in the appdb.
|
||||
Optionally accepts a creator-id to use instead of the current user (for workspace merges)."
|
||||
([body]
|
||||
(create-transform! body nil))
|
||||
([body creator-id]
|
||||
(let [creator-id (or creator-id api/*current-user-id*)
|
||||
transform (t2/with-transaction [_]
|
||||
(let [tag-ids (:tag_ids body)
|
||||
transform (t2/insert-returning-instance!
|
||||
:model/Transform
|
||||
(assoc (select-keys body [:name :description :source :target :run_trigger])
|
||||
:creator_id creator-id))]
|
||||
;; Add tag associations if provided
|
||||
(when (seq tag-ids)
|
||||
(transform.model/update-transform-tags! (:id transform) tag-ids))
|
||||
;; Return with hydrated tag_ids
|
||||
(t2/hydrate transform :transform_tag_ids :creator)))]
|
||||
(events/publish-event! :event/transform-create {:object transform :user-id api/*current-user-id*})
|
||||
transform)))
|
||||
|
||||
;; TODO (chris 2025/12/16) fully populate the result schema
|
||||
;; TODO (lbrdnk 2025/12/16) relaxed result schema to unblock FE. This should be properly handled later.
|
||||
(api.macros/defendpoint :post "/" :- [:map [:name :string] [:description {:optional true} [:maybe :string]]]
|
||||
"Create a new transform."
|
||||
[_route-params
|
||||
_query-params
|
||||
@@ -183,6 +208,34 @@
|
||||
dependencies (map id->transform dep-ids)]
|
||||
(t2/hydrate dependencies :creator)))
|
||||
|
||||
(def ^:private MergeHistoryEntry
|
||||
[:map
|
||||
[:id ms/PositiveInt]
|
||||
[:workspace_merge_id ms/PositiveInt]
|
||||
[:commit_message :string]
|
||||
[:workspace_id [:maybe ms/PositiveInt]]
|
||||
[:workspace_name :string]
|
||||
[:merging_user_id ms/PositiveInt]
|
||||
[:created_at :any]])
|
||||
|
||||
(api.macros/defendpoint :get "/:id/merge-history"
|
||||
:- [:sequential MergeHistoryEntry]
|
||||
"Get merge history for a transform. Returns all merge events that affected this transform,
|
||||
ordered by created_at descending (newest first)."
|
||||
[{:keys [id]} :- [:map [:id ms/PositiveInt]]]
|
||||
(api/check-superuser)
|
||||
(api/check-404 (t2/select-one :model/Transform id))
|
||||
(t2/select [:model/WorkspaceMergeTransform
|
||||
:id
|
||||
:workspace_merge_id
|
||||
:commit_message
|
||||
:workspace_id
|
||||
:workspace_name
|
||||
:merging_user_id
|
||||
:created_at]
|
||||
{:where [:= :transform_id id]
|
||||
:order-by [[:created_at :desc]]}))
|
||||
|
||||
;; TODO (Cam 10/28/25) -- fix this endpoint so it uses kebab-case for query parameters for consistency with the rest
|
||||
;; of the REST API
|
||||
;;
|
||||
@@ -210,24 +263,10 @@
|
||||
:limit (request/limit)))
|
||||
(update :data #(map transforms.util/localize-run-timestamps %))))
|
||||
|
||||
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
|
||||
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
|
||||
;;
|
||||
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
|
||||
(api.macros/defendpoint :put "/:id"
|
||||
"Update a transform."
|
||||
[{:keys [id]} :- [:map
|
||||
[:id ms/PositiveInt]]
|
||||
_query-params
|
||||
body :- [:map
|
||||
[:name {:optional true} :string]
|
||||
[:description {:optional true} [:maybe :string]]
|
||||
[:source {:optional true} ::transforms.schema/transform-source]
|
||||
[:target {:optional true} ::transforms.schema/transform-target]
|
||||
[:run_trigger {:optional true} ::run-trigger]
|
||||
[:tag_ids {:optional true} [:sequential ms/PositiveInt]]
|
||||
[:collection_id {:optional true} [:maybe ms/PositiveInt]]]]
|
||||
(api/check-superuser)
|
||||
(defn update-transform!
|
||||
"Update a transform. Validates features, database support, cycles, and target conflicts.
|
||||
Returns the updated transform with hydrated associations."
|
||||
[id body]
|
||||
(let [transform (t2/with-transaction [_]
|
||||
;; Cycle detection should occur within the transaction to avoid race
|
||||
(let [old (t2/select-one :model/Transform id)
|
||||
@@ -252,6 +291,35 @@
|
||||
(events/publish-event! :event/transform-update {:object transform :user-id api/*current-user-id*})
|
||||
transform))
|
||||
|
||||
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
|
||||
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
|
||||
;;
|
||||
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
|
||||
(api.macros/defendpoint :put "/:id"
|
||||
"Update a transform."
|
||||
[{:keys [id]} :- [:map
|
||||
[:id ms/PositiveInt]]
|
||||
_query-params
|
||||
body :- [:map
|
||||
[:name {:optional true} :string]
|
||||
[:description {:optional true} [:maybe :string]]
|
||||
[:source {:optional true} ::transforms.schema/transform-source]
|
||||
[:target {:optional true} ::transforms.schema/transform-target]
|
||||
[:run_trigger {:optional true} ::run-trigger]
|
||||
[:tag_ids {:optional true} [:sequential ms/PositiveInt]]
|
||||
[:collection_id {:optional true} [:maybe ms/PositiveInt]]]]
|
||||
(api/check-superuser)
|
||||
(update-transform! id body))
|
||||
|
||||
(defn delete-transform!
|
||||
"Delete a transform and publish the delete event."
|
||||
[transform]
|
||||
(t2/delete! :model/Transform (:id transform))
|
||||
(events/publish-event! :event/transform-delete
|
||||
{:object transform
|
||||
:user-id api/*current-user-id*})
|
||||
nil)
|
||||
|
||||
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
|
||||
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
|
||||
;;
|
||||
@@ -261,12 +329,7 @@
|
||||
[{:keys [id]} :- [:map
|
||||
[:id ms/PositiveInt]]]
|
||||
(api/check-superuser)
|
||||
(let [transform (api/check-404 (t2/select-one :model/Transform id))]
|
||||
(t2/delete! :model/Transform id)
|
||||
(events/publish-event! :event/transform-delete
|
||||
{:object transform
|
||||
:user-id api/*current-user-id*}))
|
||||
nil)
|
||||
(delete-transform! (api/check-404 (t2/select-one :model/Transform id))))
|
||||
|
||||
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
|
||||
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
|
||||
@@ -296,18 +359,12 @@
|
||||
(transforms.canceling/cancel-run! (:id run))))
|
||||
nil)
|
||||
|
||||
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
|
||||
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
|
||||
;;
|
||||
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
|
||||
(api.macros/defendpoint :post "/:id/run"
|
||||
"Run a transform."
|
||||
[{:keys [id]} :- [:map
|
||||
[:id ms/PositiveInt]]]
|
||||
(api/check-superuser)
|
||||
(let [transform (api/check-404 (t2/select-one :model/Transform id))
|
||||
_ (check-feature-enabled! transform)
|
||||
start-promise (promise)]
|
||||
(defn run-transform!
|
||||
"Run a transform. Returns a 202 response with run_id.
|
||||
The transform must already be fetched and validated."
|
||||
[transform]
|
||||
(check-feature-enabled! transform)
|
||||
(let [start-promise (promise)]
|
||||
(u.jvm/in-virtual-thread*
|
||||
(transforms.execute/execute! transform {:start-promise start-promise
|
||||
:run-method :manual}))
|
||||
@@ -320,6 +377,17 @@
|
||||
:run_id run-id})
|
||||
(assoc :status 202)))))
|
||||
|
||||
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
|
||||
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
|
||||
;;
|
||||
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
|
||||
(api.macros/defendpoint :post "/:id/run"
|
||||
"Run a transform."
|
||||
[{:keys [id]} :- [:map
|
||||
[:id ms/PositiveInt]]]
|
||||
(api/check-superuser)
|
||||
(run-transform! (api/check-404 (t2/select-one :model/Transform id))))
|
||||
|
||||
(defn- extract-columns-from-query
|
||||
"Attempts to extract column names from an MBQL query without executing it.
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
(ns metabase-enterprise.transforms.core
|
||||
"API namespace for the `metabase-enterprise.transform` module."
|
||||
(:require
|
||||
[metabase-enterprise.transforms.models.transform]
|
||||
[metabase-enterprise.transforms.models.transform-run]
|
||||
[metabase-enterprise.transforms.settings]
|
||||
[metabase-enterprise.transforms.util]
|
||||
@@ -12,6 +13,9 @@
|
||||
[metabase-enterprise.transforms.util
|
||||
native-query-transform?
|
||||
python-transform?
|
||||
query-transform?]
|
||||
query-transform?
|
||||
transform-source-type]
|
||||
[metabase-enterprise.transforms.models.transform-run
|
||||
timeout-run!])
|
||||
timeout-run!]
|
||||
[metabase-enterprise.transforms.models.transform
|
||||
update-transform-tags!])
|
||||
|
||||
@@ -4,15 +4,6 @@
|
||||
[transform]
|
||||
(-> transform :source :type keyword))
|
||||
|
||||
(defmulti source-db-id
|
||||
"Return the ID of the source database for a given `transform`. The source database is where the data originates from
|
||||
before being transformed and written to the target destination.
|
||||
|
||||
NOTE: Currently returns a single database ID, but we may want to change this to return multiple database IDs in the
|
||||
future since transforms can have multiple input sources."
|
||||
{:added "0.57.0" :arglists '([transform])}
|
||||
transform->transform-type)
|
||||
|
||||
(defmulti target-db-id
|
||||
"Return the ID of the target database for a given `transform`. The target database is the destination where the
|
||||
transformed data will be written. Often this is the same database as the source database."
|
||||
|
||||
@@ -54,13 +54,13 @@
|
||||
#{:transforms})
|
||||
|
||||
(t2/define-before-insert :model/Transform
|
||||
[{:keys [source target collection_id] :as transform}]
|
||||
[{:keys [source collection_id] :as transform}]
|
||||
(collection/check-collection-namespace :model/Transform collection_id)
|
||||
(when collection_id
|
||||
(collection/check-allowed-content :model/Transform collection_id))
|
||||
(assoc transform
|
||||
:source_type (transforms.util/transform-source-type source)
|
||||
:target_db_id (or (:database target) (:target_db_id transform))))
|
||||
:target_db_id (transforms.i/target-db-id transform)))
|
||||
|
||||
(t2/define-before-update :model/Transform
|
||||
[{:keys [source target] :as transform}]
|
||||
|
||||
@@ -14,13 +14,11 @@
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(defmethod transforms.i/source-db-id :query
|
||||
[transform]
|
||||
(-> transform :source :query :database))
|
||||
|
||||
(defmethod transforms.i/target-db-id :query
|
||||
[transform]
|
||||
(or (get-in transform [:target :database])
|
||||
;; Fallback to source. https://github.com/metabase/metabase/pull/67084#discussion_r2630147855
|
||||
(-> transform :source :query :database)
|
||||
(:target_db_id transform)))
|
||||
|
||||
(mr/def ::transform-details
|
||||
|
||||
@@ -45,14 +45,14 @@
|
||||
[:map
|
||||
[:database {:optional true} :int]
|
||||
[:type [:= "table"]]
|
||||
[:schema {:optional true} [:or ms/NonBlankString :nil]]
|
||||
[:schema {:optional true} [:maybe ms/NonBlankString]]
|
||||
[:name :string]])
|
||||
|
||||
(mr/def ::table-incremental-target
|
||||
[:map
|
||||
[:database {:optional true} :int]
|
||||
[:type [:= "table-incremental"]]
|
||||
[:schema {:optional true} [:or ms/NonBlankString :nil]]
|
||||
[:schema {:optional true} [:maybe ms/NonBlankString]]
|
||||
[:name :string]
|
||||
[:target-incremental-strategy ::target-incremental-strategy]])
|
||||
|
||||
|
||||
@@ -436,33 +436,28 @@
|
||||
(defn- matching-timestamp?
|
||||
[job field-path {:keys [start end]}]
|
||||
(when-let [field-instant (->instant (get-in job field-path))]
|
||||
(let [start-instant (some-> start u.date/parse ->instant)
|
||||
end-instant (some-> end u.date/parse ->instant)]
|
||||
(and (or (nil? start)
|
||||
(not (.isBefore field-instant start-instant)))
|
||||
(or (nil? end)
|
||||
(.isAfter end-instant field-instant))))))
|
||||
(let [parse #(-> % u.date/parse ->instant)]
|
||||
;; logic here is to find when it's not matching and invert this
|
||||
(not (or (and start (.isBefore field-instant (parse start)))
|
||||
(and end (.isAfter field-instant (parse end))))))))
|
||||
|
||||
(defn ->date-field-filter-xf
|
||||
"Returns an xform for a date filter."
|
||||
[field-path filter-value]
|
||||
(let [range (some-> filter-value (params.dates/date-string->range {:inclusive-end? false}))]
|
||||
(if range
|
||||
(filter #(matching-timestamp? % field-path range))
|
||||
identity)))
|
||||
(if-let [range (some-> filter-value (params.dates/date-string->range {:inclusive-end? false}))]
|
||||
(filter #(matching-timestamp? % field-path range))
|
||||
identity))
|
||||
|
||||
(defn ->status-filter-xf
|
||||
"Returns an xform for a transform run status filter."
|
||||
[field-path statuses]
|
||||
(let [statuses (->> statuses (map keyword) set not-empty)]
|
||||
(if statuses
|
||||
(filter #(statuses (get-in % field-path)))
|
||||
identity)))
|
||||
(if-let [statuses (->> statuses (map keyword) set not-empty)]
|
||||
(filter #(statuses (get-in % field-path)))
|
||||
identity))
|
||||
|
||||
(defn ->tag-filter-xf
|
||||
"Returns an xform for a transform tag filter."
|
||||
[field-path tag-ids]
|
||||
(let [tag-ids (-> tag-ids set not-empty)]
|
||||
(if tag-ids
|
||||
(filter #(some tag-ids (get-in % field-path)))
|
||||
identity)))
|
||||
(if-let [tag-ids (-> tag-ids set not-empty)]
|
||||
(filter #(some tag-ids (get-in % field-path)))
|
||||
identity))
|
||||
|
||||
@@ -8,10 +8,12 @@
|
||||
[metabase-enterprise.transforms-python.settings :as transforms-python.settings]
|
||||
[metabase-enterprise.transforms.core :as transforms]
|
||||
[metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
|
||||
[metabase-enterprise.transforms.interface :as transforms.i]
|
||||
[metabase-enterprise.transforms.util :as transforms.util]
|
||||
[metabase.app-db.core :as app-db]
|
||||
[metabase.driver :as driver]
|
||||
[metabase.driver.util :as driver.u]
|
||||
[metabase.events.core :as events]
|
||||
[metabase.util :as u]
|
||||
[metabase.util.format :as u.format]
|
||||
[metabase.util.i18n :as i18n]
|
||||
@@ -353,13 +355,14 @@
|
||||
(try
|
||||
(let [message-log (empty-message-log)
|
||||
{:keys [target] transform-id :id} transform
|
||||
{driver :engine :as db} (t2/select-one :model/Database (:database target))
|
||||
{driver :engine :as db} (t2/select-one :model/Database (transforms.i/target-db-id transform))
|
||||
{run-id :id} (transforms.util/try-start-unless-already-running transform-id run-method)]
|
||||
(some-> start-promise (deliver [:started run-id]))
|
||||
(log! message-log (i18n/tru "Executing Python transform"))
|
||||
(log/info "Executing Python transform" transform-id "with target" (pr-str target))
|
||||
(let [start-ms (u/start-timer)
|
||||
transform-details {:db-id (:id db)
|
||||
:transform-id transform-id
|
||||
:transform-type (keyword (:type target))
|
||||
:conn-spec (driver/connection-spec driver db)
|
||||
:output-schema (:schema target)
|
||||
@@ -372,7 +375,8 @@
|
||||
result (transforms.instrumentation/with-stage-timing [run-id [:computation :python-execution]]
|
||||
(transforms.util/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn))]
|
||||
(transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]
|
||||
(transforms.util/sync-target! target db))
|
||||
(transforms.util/sync-target! target db)
|
||||
(events/publish-event! :event/transform-run-complete {:object transform-details}))
|
||||
{:run_id run-id
|
||||
:result result}))
|
||||
(catch Throwable t
|
||||
|
||||
@@ -8,10 +8,6 @@
|
||||
(or (get-in transform [:target :database])
|
||||
(:target_db_id transform)))
|
||||
|
||||
(defmethod transforms.i/source-db-id :python
|
||||
[transform]
|
||||
(-> transform :source :database))
|
||||
|
||||
#_{:clj-kondo/ignore [:discouraged-var]}
|
||||
(defmethod transforms.i/execute! :python
|
||||
[transform options]
|
||||
|
||||
877
enterprise/backend/src/metabase_enterprise/workspaces/api.clj
Normal file
877
enterprise/backend/src/metabase_enterprise/workspaces/api.clj
Normal file
@@ -0,0 +1,877 @@
|
||||
(ns metabase-enterprise.workspaces.api
|
||||
"`/api/ee/workspace/` routes"
|
||||
(:require
|
||||
[clojure.set :as set]
|
||||
[clojure.string :as str]
|
||||
[honey.sql.helpers :as sql.helpers]
|
||||
[metabase-enterprise.transforms.core :as transforms]
|
||||
[metabase-enterprise.transforms.util :as transforms.util]
|
||||
[metabase-enterprise.workspaces.common :as ws.common]
|
||||
[metabase-enterprise.workspaces.impl :as ws.impl]
|
||||
[metabase-enterprise.workspaces.merge :as ws.merge]
|
||||
[metabase-enterprise.workspaces.models.workspace :as ws.model]
|
||||
[metabase-enterprise.workspaces.models.workspace-input-external]
|
||||
[metabase-enterprise.workspaces.models.workspace-log]
|
||||
[metabase-enterprise.workspaces.models.workspace-output-external]
|
||||
[metabase-enterprise.workspaces.types :as ws.t]
|
||||
[metabase-enterprise.workspaces.validation :as ws.validation]
|
||||
[metabase.api.common :as api]
|
||||
[metabase.api.macros :as api.macros]
|
||||
[metabase.api.routes.common :refer [+auth]]
|
||||
[metabase.driver.util :as driver.u]
|
||||
[metabase.lib.core :as lib]
|
||||
[metabase.queries.schema :as queries.schema]
|
||||
[metabase.request.core :as request]
|
||||
[metabase.util :as u]
|
||||
[metabase.util.i18n :as i18n :refer [deferred-tru tru]]
|
||||
[metabase.util.malli.registry :as mr]
|
||||
[metabase.util.malli.schema :as ms]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(def ^:private log-limit "Maximum number of recent workspace log items to show" 20)
|
||||
|
||||
(mr/def ::appdb-or-ref-id [:or ::ws.t/appdb-id ::ws.t/ref-id])
|
||||
|
||||
(mr/def ::status (into [:enum] ws.model/workspace-statuses))
|
||||
|
||||
(def ^:private Workspace
|
||||
[:map
|
||||
[:id ::ws.t/appdb-id]
|
||||
[:name :string]
|
||||
[:collection_id ::ws.t/appdb-id]
|
||||
[:database_id ::ws.t/appdb-id]
|
||||
[:status ::status]
|
||||
[:created_at ms/TemporalInstant]
|
||||
[:updated_at ms/TemporalInstant]])
|
||||
|
||||
;; Transform-related schemas (adapted from transforms/api.clj)
|
||||
;; TODO we should reuse these schemas, by exposing common types from the transforms module. they *can* match exactly.
|
||||
|
||||
(mr/def ::transform-source
|
||||
[:multi {:dispatch (comp keyword :type)}
|
||||
[:query
|
||||
[:map
|
||||
[:type [:= "query"]]
|
||||
[:query ::queries.schema/query]]]
|
||||
[:python
|
||||
[:map {:closed true}
|
||||
[:source-database {:optional true} :int]
|
||||
[:source-tables [:map-of [:string {:min 1}] :int]]
|
||||
[:type [:= "python"]]
|
||||
[:body :string]]]])
|
||||
|
||||
(mr/def ::transform-target
|
||||
[:map {:closed true}
|
||||
[:database {:optional true} ::ws.t/appdb-id]
|
||||
[:type [:enum "table"]]
|
||||
[:schema {:optional true} [:maybe [:string {:min 1}]]]
|
||||
[:name [:string {:min 1}]]])
|
||||
|
||||
(mr/def ::run-trigger
|
||||
[:enum "none" "global-schedule"])
|
||||
|
||||
(defn- check-transforms-enabled!
|
||||
[db-id]
|
||||
(let [database (api/check-400 (t2/select-one :model/Database db-id)
|
||||
(deferred-tru "The target database cannot be found."))]
|
||||
(api/check (transforms.util/check-feature-enabled nil)
|
||||
[402 (deferred-tru "Premium features required for transforms are not enabled.")])
|
||||
(api/check-400 (not (:is_sample database))
|
||||
(deferred-tru "Cannot run transforms on the sample database."))
|
||||
(api/check-400 (not (:is_audit database))
|
||||
(deferred-tru "Cannot run transforms on audit databases."))
|
||||
(api/check-400 (driver.u/supports? (:engine database) :transforms/table database)
|
||||
(deferred-tru "The database does not support the requested transform target type."))
|
||||
(api/check-400 (not (transforms.util/db-routing-enabled? database))
|
||||
(deferred-tru "Transforms are not supported on databases with DB routing enabled."))))
|
||||
|
||||
(defn- ws->response [ws]
|
||||
(select-keys ws [:id :name :collection_id :database_id :status :created_at :updated_at]))
|
||||
|
||||
;;; routes
|
||||
|
||||
(def ^:private WorkspaceListing
|
||||
[:map {:closed true}
|
||||
[:id ::ws.t/appdb-id]
|
||||
[:database_id ::ws.t/appdb-id]
|
||||
[:name :string]
|
||||
[:status ::status]
|
||||
[:updated_at ms/TemporalInstant]])
|
||||
|
||||
(api.macros/defendpoint :get "/" :- [:map {:closed true}
|
||||
[:items [:sequential WorkspaceListing]]
|
||||
[:limit [:maybe :int]]
|
||||
[:offset [:maybe :int]]]
|
||||
"Get a list of all workspaces"
|
||||
[_route-params
|
||||
_query-params]
|
||||
{:items (t2/select [:model/Workspace :id :name :database_id :status :updated_at]
|
||||
(cond-> {:order-by [[:created_at :desc]]}
|
||||
(request/limit) (sql.helpers/limit (request/limit))
|
||||
(request/offset) (sql.helpers/offset (request/offset))))
|
||||
:limit (request/limit)
|
||||
:offset (request/offset)})
|
||||
|
||||
(mr/def ::input-table
|
||||
[:map
|
||||
;; Future-proof with cross-db Python transforms
|
||||
[:db_id ::ws.t/appdb-id]
|
||||
[:schema [:maybe :string]]
|
||||
[:table :string]
|
||||
[:table_id [:maybe ::ws.t/appdb-id]]])
|
||||
|
||||
(mr/def ::output-table
|
||||
[:map
|
||||
;; Future-proof with multi-db Workspaces, plus necessary to resolve table references when id is null.
|
||||
[:db_id ::ws.t/appdb-id]
|
||||
[:global [:map
|
||||
;; transform_id is nil for workspace outputs, int for external outputs
|
||||
[:transform_id [:maybe ::ws.t/appdb-id]]
|
||||
[:schema [:maybe :string]]
|
||||
[:table :string]
|
||||
[:table_id [:maybe ::ws.t/appdb-id]]]]
|
||||
[:isolated [:map
|
||||
;; transform_id is ref_id (string) for workspace outputs, int for external outputs
|
||||
[:transform_id [:or ::ws.t/ref-id ::ws.t/appdb-id]]
|
||||
[:schema :string]
|
||||
[:table :string]
|
||||
[:table_id [:maybe ::ws.t/appdb-id]]]]])
|
||||
|
||||
(defn- batch-lookup-table-ids
|
||||
"Given a bounded list of tables, all within the same database, return an association list of [db schema table] => id"
|
||||
[db-id schema-key table-key table-refs]
|
||||
(when (seq table-refs)
|
||||
(t2/select-fn-vec (juxt (juxt (constantly db-id) :schema :name) :id)
|
||||
[:model/Table :id :schema :name]
|
||||
:db_id db-id
|
||||
{:where (into [:or] (for [tr table-refs]
|
||||
[:and
|
||||
[:= :schema (get tr schema-key)]
|
||||
[:= :name (get tr table-key)]]))})))
|
||||
|
||||
(defn- table-ids-fallbacks
|
||||
"Given a list of maps holding [db_id schema table], return a mapping from those tuples => table_id"
|
||||
[schema-key table-key id-key table-refs]
|
||||
(when-let [table-refs (seq (remove id-key table-refs))]
|
||||
;; These are ordered by db, so this will partition fine.
|
||||
(u/for-map [table-refs (partition-by :db_id table-refs)
|
||||
:let [db_id (:db_id (first table-refs))]
|
||||
;; Guesstimating a number that prevents this query being too large.
|
||||
table-refs (partition-all 20 table-refs)
|
||||
map-entry (batch-lookup-table-ids db_id schema-key table-key table-refs)]
|
||||
map-entry)))
|
||||
|
||||
(api.macros/defendpoint :get "/:id/table"
|
||||
:- [:map {:closed true}
|
||||
[:inputs [:sequential ::input-table]]
|
||||
[:outputs [:sequential ::output-table]]]
|
||||
"Get workspace tables"
|
||||
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
|
||||
_query-params]
|
||||
(let [workspace (api/check-404 (t2/select-one :model/Workspace :id id))
|
||||
;; Trigger creation of the Workspace*External entries
|
||||
_ (ws.impl/get-or-calculate-graph workspace)
|
||||
order-by {:order-by [:db_id :global_schema :global_table]}
|
||||
outputs (t2/select [:model/WorkspaceOutput
|
||||
:db_id :global_schema :global_table :global_table_id
|
||||
:isolated_schema :isolated_table :isolated_table_id :ref_id]
|
||||
:workspace_id id order-by)
|
||||
external-outputs (t2/select [:model/WorkspaceOutputExternal
|
||||
:db_id :global_schema :global_table :global_table_id
|
||||
:isolated_schema :isolated_table :isolated_table_id :transform_id]
|
||||
:workspace_id id order-by)
|
||||
all-outputs (concat outputs external-outputs)
|
||||
raw-inputs (t2/select [:model/WorkspaceInput :db_id :schema :table :table_id]
|
||||
:workspace_id id {:order-by [:db_id :schema :table]})
|
||||
external-inputs (t2/select [:model/WorkspaceInputExternal :db_id :schema :table :table_id]
|
||||
:workspace_id id {:order-by [:db_id :schema :table]})
|
||||
all-raw-inputs (concat raw-inputs external-inputs)
|
||||
;; Some of our inputs may be shadowed by the outputs of other transforms. We only want external inputs.
|
||||
shadowed? (into #{} (map (juxt :db_id :global_schema :global_table)) all-outputs)
|
||||
inputs (remove (comp shadowed? (juxt :db_id :schema :table)) all-raw-inputs)
|
||||
;; Build a map of [d s t] => id for every table that has been synced since the output row was written.
|
||||
fallback-map (merge
|
||||
(table-ids-fallbacks :global_schema :global_table :global_table_id all-outputs)
|
||||
(table-ids-fallbacks :isolated_schema :isolated_table :isolated_table_id all-outputs))]
|
||||
{:inputs (sort-by (juxt :db_id :schema :table) inputs)
|
||||
:outputs (sort-by
|
||||
(juxt :db_id (comp (juxt :schema :table) :global))
|
||||
(concat
|
||||
;; Workspace transform outputs
|
||||
(for [{:keys [ref_id db_id global_schema global_table global_table_id
|
||||
isolated_schema isolated_table isolated_table_id]} outputs]
|
||||
{:db_id db_id
|
||||
:global {:transform_id nil
|
||||
:schema global_schema
|
||||
:table global_table
|
||||
:table_id (or global_table_id (get fallback-map [db_id global_schema global_table]))}
|
||||
:isolated {:transform_id ref_id
|
||||
:schema isolated_schema
|
||||
:table isolated_table
|
||||
:table_id (or isolated_table_id (get fallback-map [db_id isolated_schema isolated_table]))}})
|
||||
;; External transform outputs
|
||||
(for [{:keys [transform_id db_id global_schema global_table global_table_id
|
||||
isolated_schema isolated_table isolated_table_id]} external-outputs]
|
||||
{:db_id db_id
|
||||
:global {:transform_id transform_id
|
||||
:schema global_schema
|
||||
:table global_table
|
||||
:table_id (or global_table_id (get fallback-map [db_id global_schema global_table]))}
|
||||
:isolated {:transform_id transform_id
|
||||
:schema isolated_schema
|
||||
:table isolated_table
|
||||
:table_id (or isolated_table_id (get fallback-map [db_id isolated_schema isolated_table]))}})))}))
|
||||
|
||||
(api.macros/defendpoint :get "/:id" :- Workspace
|
||||
"Get a single workspace by ID"
|
||||
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
|
||||
_query-params]
|
||||
(-> (t2/select-one :model/Workspace :id id) api/check-404 ws->response))
|
||||
|
||||
(api.macros/defendpoint :get "/:id/log"
|
||||
:- [:map
|
||||
[:workspace_id ms/PositiveInt]
|
||||
[:status ::status]
|
||||
[:updated_at :any]
|
||||
[:last_completed_at [:maybe :any]]
|
||||
[:logs [:sequential [:map
|
||||
[:id ms/PositiveInt]
|
||||
[:task :keyword]
|
||||
[:description ms/LocalizedString]
|
||||
[:started_at :any]
|
||||
[:updated_at :any]
|
||||
[:completed_at [:maybe :any]]
|
||||
[:status [:maybe :keyword]]
|
||||
[:message [:maybe :string]]]]]]
|
||||
"Get workspace creation status and recent log entries for polling during async setup"
|
||||
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
|
||||
_query-params]
|
||||
(let [workspace (api/check-404 (t2/select-one :model/Workspace :id id))
|
||||
logs (t2/select [:model/WorkspaceLog
|
||||
:id :task :started_at :completed_at :status :message
|
||||
:updated_at]
|
||||
:workspace_id id
|
||||
{:order-by [[:started_at :desc]]
|
||||
:limit log-limit})]
|
||||
{:workspace_id id
|
||||
:status (:status workspace)
|
||||
:logs logs
|
||||
:updated_at (->> (map :updated_at logs) sort reverse first)
|
||||
:last_completed_at (->> (seq (keep :completed_at logs)) sort reverse first)}))
|
||||
|
||||
(def ^:private CreateWorkspace
|
||||
[:map
|
||||
[:name {:optional true} [:string {:min 1}]]
|
||||
[:database_id {:optional true} ::ws.t/appdb-id]])
|
||||
|
||||
(defn- first-supported-database-id
|
||||
"Return the ID of the first database that supports workspaces, or nil if none."
|
||||
[]
|
||||
(:id
|
||||
(u/seek #(driver.u/supports? (:engine %) :workspace %)
|
||||
(t2/select :model/Database :is_audit false :is_sample false {:order-by [:name]}))))
|
||||
|
||||
(api.macros/defendpoint :post "/" :- Workspace
|
||||
"Create a new workspace
|
||||
|
||||
Potential payload:
|
||||
{:name \"a\" :database_id 2}}"
|
||||
[_route-params
|
||||
_query-params
|
||||
{:keys [database_id] :as body} :- CreateWorkspace]
|
||||
|
||||
(when database_id
|
||||
(check-transforms-enabled! database_id))
|
||||
|
||||
;; If no database_id provided, use first supported DB as provisional default (uninitialized workspace)
|
||||
(let [provisional? (not database_id)
|
||||
database-id (or database_id
|
||||
(api/check-400 (first-supported-database-id)
|
||||
(tru "No supported databases configured. Please add a database that supports workspaces.")))]
|
||||
(ws->response (ws.common/create-workspace! api/*current-user-id*
|
||||
(assoc body
|
||||
:database_id database-id
|
||||
:provisional? provisional?)))))
|
||||
|
||||
(defn- db-unsupported-reason [db]
|
||||
(when (not (driver.u/supports? (:engine db) :workspace db))
|
||||
"Database type not supported."))
|
||||
|
||||
(api.macros/defendpoint :get "/enabled" :- [:map [:supported :boolean] [:reason {:optional true} :string]]
|
||||
"Test whether the current user can use Workspaces. Optionally takes a specific database.
|
||||
Factors include: driver support, database user privileges, metabase permissions."
|
||||
[_url-params
|
||||
{:keys [database-id]} :- [:map [:database-id {:optional true} ms/PositiveInt]]]
|
||||
(if-let [reason (or (when (not api/*is-superuser?*)
|
||||
(tru "Not allowed."))
|
||||
(when database-id
|
||||
(db-unsupported-reason (api/check-404 (t2/select-one :model/Database database-id)))))]
|
||||
{:supported false, :reason reason}
|
||||
{:supported true}))
|
||||
|
||||
(api.macros/defendpoint :get "/database" :- [:map
|
||||
[:databases
|
||||
[:sequential [:map
|
||||
[:id ms/PositiveInt]
|
||||
[:name :string]
|
||||
[:supported :boolean]
|
||||
[:reason {:optional true} :string]]]]]
|
||||
"Get a list of databases to show in the workspace picker, along with whether they're supported."
|
||||
[_url-params
|
||||
_query-params]
|
||||
{:databases (->> (t2/select :model/Database :is_audit false :is_sample false {:order-by [:name]})
|
||||
;; Omit those we don't even support
|
||||
(filter #(driver.u/supports? (:engine %) :workspace %))
|
||||
(mapv (fn [db]
|
||||
(merge (select-keys db [:id :name])
|
||||
(if-let [reason (db-unsupported-reason db)]
|
||||
{:supported false, :reason reason}
|
||||
{:supported true})))))})
|
||||
|
||||
(api.macros/defendpoint :put "/:id" :- Workspace
|
||||
"Update simple workspace properties.
|
||||
|
||||
Can set database_id only on uninitialized workspaces."
|
||||
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
|
||||
_query-params
|
||||
{:keys [name database_id]} :- [:map {:closed true}
|
||||
[:name {:optional true} [:string {:min 1}]]
|
||||
[:database_id {:optional true} ::ws.t/appdb-id]]]
|
||||
(let [workspace (api/check-404 (t2/select-one :model/Workspace :id id))
|
||||
_ (api/check-400 (not= :archived (:status workspace)) "Cannot update an archived workspace")
|
||||
data (cond-> {}
|
||||
database_id (-> (u/prog1
|
||||
(api/check-400 (= :uninitialized (keyword (:status workspace)))
|
||||
"Can only set database_id on uninitialized workspace")
|
||||
(check-transforms-enabled! database_id))
|
||||
(assoc :database_id database_id))
|
||||
name (assoc :name name))]
|
||||
(ws->response
|
||||
(if (seq data)
|
||||
(do
|
||||
(t2/update! :model/Workspace id data)
|
||||
(t2/select-one :model/Workspace :id id))
|
||||
workspace))))
|
||||
|
||||
(api.macros/defendpoint :post "/:id/archive" :- Workspace
|
||||
"Archive a workspace. Deletes the isolated schema and tables, but preserves mirrored entities."
|
||||
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
|
||||
_query-params
|
||||
_body-params]
|
||||
(let [ws (api/check-404 (t2/select-one :model/Workspace :id id))]
|
||||
(api/check-400 (not= :archived (:status ws)) "You cannot archive an archived workspace")
|
||||
(ws.model/archive! ws)
|
||||
(-> (t2/select-one :model/Workspace :id id)
|
||||
ws->response)))
|
||||
|
||||
(api.macros/defendpoint :post "/:id/unarchive" :- Workspace
|
||||
"Restore an archived workspace. Recreates the isolated schema and tables."
|
||||
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
|
||||
_query-params
|
||||
_body-params]
|
||||
(let [ws (api/check-404 (t2/select-one :model/Workspace :id id))]
|
||||
(api/check-400 (= :archived (:status ws)) "You cannot unarchive a workspace that is not archived")
|
||||
(ws.model/unarchive! ws)
|
||||
(-> (t2/select-one :model/Workspace :id id)
|
||||
ws->response)))
|
||||
|
||||
(api.macros/defendpoint :delete "/:ws-id" :- [:map [:ok [:= true]]]
|
||||
"Delete a workspace and all its contents, including mirrored entities."
|
||||
[{:keys [ws-id]} :- [:map [:ws-id ms/PositiveInt]]
|
||||
_query-params]
|
||||
(let [ws (api/check-404 (t2/select-one :model/Workspace :id ws-id))]
|
||||
(api/check-400 (= :archived (:status ws)) "You cannot delete a workspace without first archiving it")
|
||||
(ws.model/delete! ws)
|
||||
{:ok true}))
|
||||
|
||||
(def ^:private ExternalTransform
|
||||
;; Might be interesting to show whether they're enclosed, once we have the graph.
|
||||
;; When they're enclosed, it could also be interesting to know whether they're stale.
|
||||
[:map
|
||||
[:id ::ws.t/appdb-id]
|
||||
[:name :string]
|
||||
[:source_type :keyword]
|
||||
[:checkout_disabled [:maybe :string]]])
|
||||
|
||||
(api.macros/defendpoint :get "/:ws-id/external/transform" :- [:map [:transforms [:sequential ExternalTransform]]]
|
||||
"Get transforms that are external to the workspace, i.e. no matching workspace_transform row exists."
|
||||
[{:keys [ws-id]} :- [:map [:ws-id ::ws.t/appdb-id]]
|
||||
{:keys [database-id]} :- [:map [:database-id {:optional true} ::ws.t/appdb-id]]]
|
||||
(api/check-superuser)
|
||||
(let [db-id (or database-id
|
||||
(:database_id (api/check-404 (t2/select-one [:model/Workspace :database_id] ws-id))))
|
||||
transforms (t2/select [:model/Transform :id :name :source_type :source]
|
||||
{:left-join [[:workspace_transform :wt]
|
||||
[:and
|
||||
[:= :transform.id :wt.global_id]
|
||||
[:= :wt.workspace_id ws-id]]]
|
||||
;; NULL workspace_id means transform is not checked out into this workspace
|
||||
:where [:and
|
||||
[:= nil :wt.workspace_id]
|
||||
[:= :target_db_id db-id]]
|
||||
:order-by [[:id :asc]]})]
|
||||
{:transforms
|
||||
(into []
|
||||
(map #(-> %
|
||||
(dissoc :source)
|
||||
(assoc :checkout_disabled (case (:source_type %)
|
||||
:mbql "mbql"
|
||||
:native (when (seq (lib/template-tags-referenced-cards (:query (:source %))))
|
||||
"card-reference")
|
||||
:python nil
|
||||
"unknown-type"))))
|
||||
;; (Sanya 2025-12-24) - do we need this line?
|
||||
#_(map #(update % :last_run transforms.util/localize-run-timestamps))
|
||||
transforms
|
||||
;; Perhaps we want to expose some of this later?
|
||||
#_(t2/hydrate transforms :last_run :creator))}))
|
||||
|
||||
(api.macros/defendpoint :post "/:ws-id/run"
|
||||
:- [:map
|
||||
[:succeeded [:sequential ::ws.t/ref-id]]
|
||||
[:failed [:sequential ::ws.t/ref-id]]
|
||||
[:not_run [:sequential ::ws.t/ref-id]]]
|
||||
"Execute all transforms in the workspace in dependency order.
|
||||
Returns which transforms succeeded, failed, and were not run."
|
||||
[{:keys [ws-id]} :- [:map [:ws-id ::ws.t/appdb-id]]
|
||||
_query-params
|
||||
;; Hmmm, I wonder why this isn't a boolean? T_T
|
||||
{:keys [stale_only]} :- [:map [:stale_only {:optional true} [:or [:= 1] :boolean]]]]
|
||||
(let [workspace (t2/select-one :model/Workspace :id ws-id)]
|
||||
(api/check-404 workspace)
|
||||
(api/check-400 (not= :archived (:status workspace)) "Cannot execute archived workspace")
|
||||
(ws.impl/execute-workspace! workspace {:stale-only stale_only})))
|
||||
|
||||
(mr/def ::graph-node-type [:enum :input-table :external-transform :workspace-transform])
|
||||
|
||||
(mr/def ::graph-node
|
||||
[:map
|
||||
[:id ::appdb-or-ref-id]
|
||||
[:type [:enum :input-table :external-transform :workspace-transform]]
|
||||
[:dependents_count [:map-of ::graph-node-type ms/PositiveInt]]
|
||||
[:data :map]])
|
||||
|
||||
(mr/def ::graph-edge
|
||||
[:map
|
||||
[:from_entity_id ::appdb-or-ref-id]
|
||||
[:from_entity_type :string]
|
||||
[:to_entity_id ::appdb-or-ref-id]
|
||||
[:to_entity_type :string]])
|
||||
|
||||
(def ^:private GraphResult
|
||||
[:map
|
||||
[:nodes [:sequential ::graph-node]]
|
||||
[:edges [:sequential ::graph-edge]]])
|
||||
|
||||
(defn- node-type [node]
|
||||
(let [nt (:node-type node)]
|
||||
(case nt
|
||||
:table :input-table
|
||||
nt)))
|
||||
|
||||
(defn- node-id [{:keys [node-type id]}]
|
||||
(case node-type
|
||||
:workspace-transform id
|
||||
:external-transform id
|
||||
:table (str (:db id) "-" (:schema id) "-" (:table id))))
|
||||
|
||||
;; TODO we'll want to bulk query this of course...
|
||||
(defn- node-data [{:keys [node-type id]}]
|
||||
(case node-type
|
||||
:table id
|
||||
:external-transform (t2/select-one [:model/Transform :id :name] id)
|
||||
;; TODO we'll want to select by workspace as well here, to relax uniqueness assumption
|
||||
:workspace-transform (t2/select-one [:model/WorkspaceTransform :ref_id :name] :ref_id id)))
|
||||
|
||||
(api.macros/defendpoint :get "/:ws-id/graph" :- GraphResult
|
||||
"Display the dependency graph between the Changeset and the (potentially external) entities that they depend on."
|
||||
[{:keys [ws-id]} :- [:map [:ws-id ms/PositiveInt]]
|
||||
_query-params]
|
||||
(let [workspace (api/check-404 (t2/select-one :model/Workspace :id ws-id))
|
||||
{:keys [inputs entities dependencies]} (ws.impl/get-or-calculate-graph workspace)
|
||||
;; TODO Graph analysis doesn't return this currently, we need to invert the deps graph
|
||||
;; It could be cheaper to build it as we go.
|
||||
inverted (reduce
|
||||
(fn [inv [c parents]]
|
||||
(reduce (fn [inv p] (update inv p (fnil conj #{}) c)) inv parents))
|
||||
{}
|
||||
dependencies)
|
||||
dep-count #(frequencies (map node-type (get inverted %)))]
|
||||
|
||||
{:nodes (concat (for [i inputs]
|
||||
{:type :input-table
|
||||
;; Use an id that's independent of whether the table exists yet.
|
||||
:id (node-id {:id i, :node-type :table})
|
||||
:data i
|
||||
:dependents_count (dep-count {:node-type :table, :id i})})
|
||||
(for [e entities]
|
||||
{:type (node-type e)
|
||||
:id (:id e)
|
||||
:data (node-data e)
|
||||
:dependents_count (dep-count e)}))
|
||||
:edges (for [[child parents] dependencies, parent parents]
|
||||
;; Yeah, this graph points to dependents, not dependencies
|
||||
{:from_entity_type (name (node-type parent))
|
||||
:from_entity_id (node-id parent)
|
||||
:to_entity_type (name (node-type child))
|
||||
:to_entity_id (node-id child)})}))
|
||||
|
||||
;;; ---------------------------------------- Problems/Validation ----------------------------------------
|
||||
|
||||
(api.macros/defendpoint :get "/:id/problem" :- [:sequential ::ws.t/problem]
|
||||
"Detect problems in the workspace that would affect downstream transforms after merge.
|
||||
|
||||
Returns a list of problems, each with:
|
||||
- category: the problem category (e.g. 'unused', 'internal-downstream', 'external-downstream')
|
||||
- problem: the specific problem (e.g. 'not-run', 'stale', 'removed-field')
|
||||
- severity: :error, :warning, or :info
|
||||
- block-merge: whether this problem prevents merging
|
||||
- data: extra information, shape depends on the problem type
|
||||
|
||||
See `metabase-enterprise.workspaces.types/problem-types` for the full list."
|
||||
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
|
||||
_query-params]
|
||||
(let [workspace (api/check-404 (t2/select-one :model/Workspace :id id))
|
||||
graph (ws.impl/get-or-calculate-graph workspace)]
|
||||
(ws.validation/find-downstream-problems id graph)))
|
||||
|
||||
(def ^:private db+schema+table (juxt :database :schema :name))
|
||||
|
||||
(defn- internal-target-conflict?
|
||||
"Check whether the given table is the target of another transform within the workspace. Ignores global transforms."
|
||||
[ws-id target & [tx-id]]
|
||||
(contains?
|
||||
(t2/select-fn-set (comp db+schema+table :target)
|
||||
[:model/WorkspaceTransform :target]
|
||||
:ref_id [:not= tx-id]
|
||||
:workspace_id ws-id)
|
||||
(db+schema+table target)))
|
||||
|
||||
(api.macros/defendpoint :post "/:ws-id/transform/validate/target"
|
||||
:- [:map [:status :int] [:body [:or :string i18n/LocalizedString]]]
|
||||
"Validate the target table for a workspace transform"
|
||||
[{:keys [ws-id]} :- [:map [:ws-id ::ws.t/appdb-id]]
|
||||
{:keys [transform-id]} :- [:map [:transform-id {:optional true} ::ws.t/ref-id]]
|
||||
{:keys [db_id target]} :- [:map
|
||||
[:db_id {:optional true} ::ws.t/appdb-id]
|
||||
[:target [:map
|
||||
[:database {:optional true} ::ws.t/appdb-id]
|
||||
[:type :string]
|
||||
[:schema :string]
|
||||
[:name :string]]]]]
|
||||
(let [workspace (api/check-404 (t2/select-one [:model/Workspace :database_id :status] ws-id))
|
||||
target (update target :database #(or % db_id))
|
||||
tx-id (when transform-id (parse-long transform-id))
|
||||
;; For uninitialized workspaces we skip over checks for their db id
|
||||
ws-db-id (when (not= :uninitialized (:status workspace)) (:database_id workspace))]
|
||||
(cond
|
||||
(not= "table" (:type target))
|
||||
{:status 403 :body (deferred-tru "Unsupported target type")}
|
||||
|
||||
(and db_id ws-db-id (not= db_id ws-db-id))
|
||||
{:status 403 :body (deferred-tru "Must target the workspace database")}
|
||||
|
||||
(not (or db_id ws-db-id))
|
||||
{:status 403 :body (deferred-tru "Must target a database")}
|
||||
|
||||
(str/starts-with? (:schema target) "mb__isolation_")
|
||||
{:status 403 :body (deferred-tru "Must not target an isolated workspace schema")}
|
||||
|
||||
;; Within a workspace, we defer blocking on conflicts outside the workspace
|
||||
#_{:status 403 :body (deferred-tru "A table with that name already exists.")}
|
||||
|
||||
;; TODO consider deferring this validation until merge also.
|
||||
(internal-target-conflict? ws-id target tx-id)
|
||||
{:status 403 :body (deferred-tru "Another transform in this workspace already targets that table")}
|
||||
|
||||
:else
|
||||
{:status 200 :body "OK"})))
|
||||
|
||||
(defn- malli-map-keys [schema]
|
||||
(into [] (map first) (rest schema)))
|
||||
|
||||
(defn- select-malli-keys
|
||||
"Like select-keys, but with the arguments reversed, and taking the malli schema for the output map.
|
||||
Nothing smart - for example, it doesn't understand optional verus maybe."
|
||||
([schema m]
|
||||
(select-keys m (malli-map-keys schema)))
|
||||
([schema alias-map m]
|
||||
(merge (select-malli-keys schema m)
|
||||
(u/for-map [[to from] alias-map] [to (from m)]))))
|
||||
|
||||
(defn- select-model-malli-keys
|
||||
"Build the model-fields vector for a t2/select from a malli schema for the output row(s)"
|
||||
([model schema]
|
||||
(into [model] (malli-map-keys schema)))
|
||||
([model schema alias-map]
|
||||
(into [model]
|
||||
(map (fn [to] (if-let [from (to alias-map)] [from to] to)))
|
||||
(malli-map-keys schema))))
|
||||
|
||||
(def ^:private WorkspaceTransform
|
||||
[:map
|
||||
[:ref_id ::ws.t/ref-id]
|
||||
[:global_id [:maybe ::ws.t/appdb-id]]
|
||||
[:name :string]
|
||||
[:description [:maybe :string]]
|
||||
[:source :map]
|
||||
[:target :map]
|
||||
;; Not yet calculated, see https://linear.app/metabase/issue/BOT-684/mark-stale-transforms-workspace-only
|
||||
[:target_stale :boolean]
|
||||
[:workspace_id ::ws.t/appdb-id]
|
||||
[:creator_id [:maybe ::ws.t/appdb-id]]
|
||||
[:archived_at :any]
|
||||
[:created_at :any]
|
||||
[:updated_at :any]
|
||||
[:last_run_at :any]
|
||||
[:last_run_message [:maybe :string]]])
|
||||
|
||||
(def ^:private workspace-transform-alias {:target_stale :stale})
|
||||
|
||||
(defn- attach-isolated-target [{:keys [workspace_id ref_id] :as ws-transform}]
|
||||
(let [{:keys [db_id isolated_schema isolated_table]}
|
||||
(t2/select-one :model/WorkspaceOutput :workspace_id workspace_id :ref_id ref_id)]
|
||||
(assoc ws-transform :target_isolated {:type "table"
|
||||
:database db_id
|
||||
:schema isolated_schema
|
||||
:name isolated_table})))
|
||||
|
||||
(api.macros/defendpoint :post "/:ws-id/transform"
|
||||
:- WorkspaceTransform
|
||||
"Add another transform to the Changeset. This could be a fork of an existing global transform, or something new."
|
||||
[{:keys [ws-id]} :- [:map [:ws-id ::ws.t/appdb-id]]
|
||||
_query-params
|
||||
body :- [:map #_{:closed true}
|
||||
[:name :string]
|
||||
[:description {:optional true} [:maybe :string]]
|
||||
[:source ::transform-source]
|
||||
;; Not sure why this schema is giving trouble
|
||||
#_[:target ::transform-target]]]
|
||||
(api/check (transforms.util/check-feature-enabled body)
|
||||
[402 (deferred-tru "Premium features required for this transform type are not enabled.")])
|
||||
(t2/with-transaction [_tx]
|
||||
(let [workspace (u/prog1 (api/check-404 (t2/select-one :model/Workspace :id ws-id))
|
||||
(api/check-400 (not= :archived (:status <>)) "Cannot create transforms in an archived workspace"))
|
||||
;; TODO why 400 here and 403 in the validation route? T_T
|
||||
_ (api/check-400 (not (internal-target-conflict? ws-id (:target body)))
|
||||
(deferred-tru "Another transform in this workspace already targets that table"))
|
||||
global-id (:global_id body (:id body))
|
||||
;; For uninitialized workspaces, preserve the target database from the request body
|
||||
;; (add-to-changeset! will reinitialize the workspace with it if different from provisional)
|
||||
;; For initialized workspaces, ensure the target database matches the workspace database
|
||||
body (-> body (dissoc :global_id)
|
||||
(cond-> (not= :uninitialized (:status workspace))
|
||||
(update :target assoc :database (:database_id workspace))))
|
||||
transform (ws.common/add-to-changeset! api/*current-user-id* workspace :transform global-id body)]
|
||||
(attach-isolated-target (select-malli-keys WorkspaceTransform workspace-transform-alias transform)))))
|
||||
|
||||
;; TODO Confirm precisely which fields are needed by the FE
|
||||
(def ^:private WorkspaceTransformListing
|
||||
"Schema for a transform in a workspace"
|
||||
[:map {:closed true}
|
||||
[:ref_id ::ws.t/ref-id]
|
||||
[:global_id [:maybe ::ws.t/appdb-id]]
|
||||
[:name :string]
|
||||
[:source_type [:maybe :keyword]]
|
||||
[:creator_id [:maybe ::ws.t/appdb-id]]
|
||||
;[:last_run :map]
|
||||
; See https://metaboat.slack.com/archives/C099RKNLP6U/p1765205882655869?thread_ts=1765205222.888209&cid=C099RKNLP6U
|
||||
#_[:target_stale :boolean]])
|
||||
|
||||
;; Global transforms precalculate these in a toucan hook - maybe we should do that too? Let's review later.
|
||||
(defn- map-source-type [ws-tx]
|
||||
(-> ws-tx
|
||||
(assoc :source_type (transforms/transform-source-type (:source ws-tx)))
|
||||
(dissoc :source)))
|
||||
|
||||
(api.macros/defendpoint :get "/:id/transform" :- [:map [:transforms [:sequential WorkspaceTransformListing]]]
|
||||
"Get all transforms in a workspace."
|
||||
[{:keys [id]} :- [:map [:id ::ws.t/appdb-id]]]
|
||||
(api/check-404 (t2/select-one :model/Workspace :id id))
|
||||
{:transforms (->> (t2/select [:model/WorkspaceTransform :ref_id :global_id :name :source :creator_id]
|
||||
:workspace_id id {:order-by [:created_at]})
|
||||
(map map-source-type))})
|
||||
|
||||
(defn- fetch-ws-transform [ws-id tx-id]
|
||||
;; TODO We still need to do some hydration, e.g. of the target table (both internal and external)
|
||||
(-> (select-model-malli-keys :model/WorkspaceTransform WorkspaceTransform workspace-transform-alias)
|
||||
(t2/select-one :ref_id tx-id :workspace_id ws-id)
|
||||
api/check-404
|
||||
attach-isolated-target))
|
||||
|
||||
(api.macros/defendpoint :get "/:id/transform/:tx-id" :- WorkspaceTransform
|
||||
"Get a specific transform in a workspace."
|
||||
[{:keys [id tx-id]} :- [:map [:id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]
|
||||
(fetch-ws-transform id tx-id))
|
||||
|
||||
(api.macros/defendpoint :put "/:ws-id/transform/:tx-id" :- WorkspaceTransform
|
||||
"Update a transform in a workspace."
|
||||
[{:keys [ws-id tx-id]} :- [:map [:ws-id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]
|
||||
_query-params
|
||||
body :- [:map
|
||||
[:name {:optional true} :string]
|
||||
[:description {:optional true} [:maybe :string]]
|
||||
[:source {:optional true} ::transform-source]
|
||||
[:target {:optional true} ::transform-target]]]
|
||||
(t2/with-transaction [_tx]
|
||||
(api/check-404 (t2/select-one :model/WorkspaceTransform :ref_id tx-id :workspace_id ws-id))
|
||||
(t2/update! :model/WorkspaceTransform tx-id body)
|
||||
;; Being cheeky and using the API response value for the pre-validation, to save a query.
|
||||
(u/prog1 (fetch-ws-transform ws-id tx-id)
|
||||
;; NOTE: FE may send these fields even when unchanged, causing unnecessary re-syncs.
|
||||
;; This is acceptable for now, but using t2/changes in hooks might catch false positives?
|
||||
;; The most reliable thing would be to have a clear, tested contract with the FE to NOT send them if unchanged.
|
||||
(when (or (:source body) (:target body))
|
||||
;; Note that we do NOT want to couple ourselves to the response shape of this API.
|
||||
;; We want to be extremely mindful of the fields we depend on, in case we remove them from the response.
|
||||
(let [transform (select-keys <> [:ref_id :source :source_type :target])
|
||||
workspace (t2/select-one :model/Workspace :id ws-id)]
|
||||
;; Re-sync dependencies if source or target changed.
|
||||
(ws.impl/sync-transform-dependencies! workspace transform))))))
|
||||
|
||||
(api.macros/defendpoint :post "/:id/transform/:tx-id/archive" :- :nil
|
||||
"Mark the given transform to be archived when the workspace is merged.
|
||||
For provisional transforms we will skip even creating it in the first place."
|
||||
[{:keys [id tx-id]} :- [:map [:id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]
|
||||
(api/check-404 (pos? (t2/update! :model/WorkspaceTransform {:ref_id tx-id :workspace_id id} {:archived_at [:now]})))
|
||||
nil)
|
||||
|
||||
(api.macros/defendpoint :post "/:id/transform/:tx-id/unarchive" :- :nil
|
||||
"Unmark the given transform for archival. This will recall the last definition it had within the workspace."
|
||||
[{:keys [id tx-id]} :- [:map [:id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]
|
||||
(api/check-404 (pos? (t2/update! :model/WorkspaceTransform {:ref_id tx-id :workspace_id id} {:archived_at nil})))
|
||||
nil)
|
||||
|
||||
(api.macros/defendpoint :delete "/:id/transform/:tx-id" :- :nil
|
||||
"Discard a transform from the changeset.
|
||||
Equivalent to resetting a checked-out transform to its global definition, or deleting a provisional transform."
|
||||
[{:keys [id tx-id]} :- [:map [:id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]
|
||||
(api/check-404 (pos? (t2/delete! :model/WorkspaceTransform :ref_id tx-id :workspace_id id)))
|
||||
nil)
|
||||
|
||||
(api.macros/defendpoint :post "/:id/transform/:tx-id/run"
|
||||
:- ::ws.t/execution-result
|
||||
"Run a transform in a workspace.
|
||||
|
||||
App DB changes are rolled back. Warehouse DB changes persist."
|
||||
[{:keys [id tx-id]} :- [:map [:id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]]
|
||||
(let [workspace (api/check-404 (t2/select-one :model/Workspace id))
|
||||
transform (api/check-404 (t2/select-one :model/WorkspaceTransform :ref_id tx-id :workspace_id id))]
|
||||
(api/check-400 (not= :archived (:status workspace)) "Cannot execute archived workspace")
|
||||
(check-transforms-enabled! (:database_id workspace))
|
||||
(ws.impl/run-transform! workspace transform)))
|
||||
|
||||
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-route-uses-kebab-case]}
|
||||
(api.macros/defendpoint :get "/checkout"
|
||||
:- [:map
|
||||
[:transforms [:sequential
|
||||
[:map
|
||||
[:id ::ws.t/ref-id]
|
||||
[:name :string]
|
||||
[:workspace [:map
|
||||
[:id ms/PositiveInt]
|
||||
[:name :string]]]]]]]
|
||||
"Get all downstream transforms for a transform that is not in a workspace.
|
||||
Returns the transforms that were mirrored from this upstream transform, with workspace info."
|
||||
[_route-params
|
||||
{:keys [transform-id]} :- [:map {:closed true} [:transform-id ms/PositiveInt]]]
|
||||
(let [transforms (t2/select [:model/WorkspaceTransform :ref_id :name :workspace_id] :global_id transform-id)
|
||||
workspace-ids (map :workspace_id transforms)
|
||||
workspaces-by-id (when (seq transforms)
|
||||
(t2/select-fn->fn :id identity [:model/Workspace :id :name] :id [:in workspace-ids]))]
|
||||
{:transforms (for [{:keys [ref_id name workspace_id]} transforms]
|
||||
{:id ref_id
|
||||
:name name
|
||||
:workspace (get workspaces-by-id workspace_id)})}))
|
||||
|
||||
(api.macros/defendpoint :post "/:ws-id/merge"
|
||||
:- [:or
|
||||
[:map
|
||||
[:merged
|
||||
[:map
|
||||
[:transforms [:sequential
|
||||
[:map
|
||||
[:op [:enum :create :delete :update :noop]]
|
||||
[:global_id {:optional true} [:maybe ::ws.t/appdb-id]]
|
||||
[:ref_id ::ws.t/ref-id]]]]]]
|
||||
[:errors
|
||||
[:sequential
|
||||
[:map
|
||||
[:op [:enum :create :delete :update :noop]]
|
||||
[:global_id {:optional true} [:maybe ::ws.t/appdb-id]]
|
||||
[:ref_id ::ws.t/ref-id]]]]
|
||||
[:workspace [:map [:id ::ws.t/appdb-id] [:name :string]]]]
|
||||
;; error message from check-404 or check-400
|
||||
:string]
|
||||
"This will:
|
||||
1. Update original transforms with workspace versions
|
||||
2. Delete the workspace and clean up isolated resources
|
||||
Returns a report of merged entities, or error in errors key.
|
||||
|
||||
Request body may include:
|
||||
- commit-message: A description of what changes are being merged (optional, will be required in future)"
|
||||
[{:keys [ws-id] :as _query-params} :- [:map [:ws-id ::ws.t/appdb-id]]
|
||||
_query-params
|
||||
{:keys [commit-message]
|
||||
:or {commit-message "Placeholder for merge commit message. Should be required on FE"}} :- [:map
|
||||
[:commit-message {:optional true} [:string {:min 1}]]]]
|
||||
(let [ws (u/prog1 (t2/select-one :model/Workspace :id ws-id)
|
||||
(api/check-404 <>)
|
||||
(api/check-400 (not= :archived (:status <>)) "Cannot merge an archived workspace"))
|
||||
{:keys [merged
|
||||
errors]} (-> (ws.merge/merge-workspace! ws api/*current-user-id* commit-message)
|
||||
(update :errors
|
||||
(partial mapv #(-> %
|
||||
(update :error (fn [e] (.getMessage ^Throwable e)))
|
||||
(set/rename-keys {:error :message})))))]
|
||||
(u/prog1
|
||||
{:merged merged
|
||||
:errors errors
|
||||
:workspace {:id ws-id, :name (:name ws)}}
|
||||
(when-not (seq errors)
|
||||
(ws.model/delete! ws)))))
|
||||
|
||||
(api.macros/defendpoint :post "/:ws-id/transform/:tx-id/merge"
|
||||
:- [:map
|
||||
[:op [:enum :create :delete :update :noop]]
|
||||
[:global_id [:maybe ::ws.t/appdb-id]]
|
||||
[:ref_id ::ws.t/ref-id]
|
||||
[:message {:optional true} :string]]
|
||||
"Merge single transform from workspace back to the core. If workspace transform is archived
|
||||
the corresponding core transform is deleted.
|
||||
|
||||
Request body may include:
|
||||
- commit-message: A description of what changes are being merged (optional, will be required in future)"
|
||||
[{:keys [ws-id tx-id] :as _query-params} :- [:map
|
||||
[:ws-id ::ws.t/appdb-id]
|
||||
[:tx-id ::ws.t/ref-id]]
|
||||
_query-params
|
||||
{:keys [commit-message]
|
||||
:or {commit-message "Placeholder for merge commit message. Should be required on FE"}} :- [:map
|
||||
[:commit-message {:optional true} [:string {:min 1}]]]]
|
||||
(let [ws (api/check-404 (t2/select-one [:model/Workspace :id :name] :id ws-id))
|
||||
ws-transform (api/check-404 (t2/select-one :model/WorkspaceTransform :workspace_id ws-id :ref_id tx-id))
|
||||
ws-merge-id (t2/insert-returning-pk!
|
||||
:model/WorkspaceMerge
|
||||
{:workspace_id (:id ws)
|
||||
:workspace_name (:name ws)
|
||||
:commit_message commit-message
|
||||
:creator_id api/*current-user-id*})
|
||||
{:keys [error] :as result} (ws.merge/merge-transform! {:ws-transform ws-transform
|
||||
:workspace ws
|
||||
:workspace-merge-id ws-merge-id
|
||||
:merging-user-id api/*current-user-id*
|
||||
:commit-message commit-message})]
|
||||
(if error
|
||||
(throw (ex-info "Failed to merge transform."
|
||||
(-> result
|
||||
(dissoc :error)
|
||||
(assoc :status-code 500))
|
||||
error))
|
||||
result)))
|
||||
|
||||
(def ^{:arglists '([request respond raise])} routes
|
||||
"`/api/ee/workspace/` routes."
|
||||
(api.macros/ns-handler *ns* api/+check-superuser +auth))
|
||||
188
enterprise/backend/src/metabase_enterprise/workspaces/common.clj
Normal file
188
enterprise/backend/src/metabase_enterprise/workspaces/common.clj
Normal file
@@ -0,0 +1,188 @@
|
||||
(ns metabase-enterprise.workspaces.common
|
||||
(:require
|
||||
[clojure.string :as str]
|
||||
[metabase-enterprise.transforms.interface :as transforms.i]
|
||||
[metabase-enterprise.workspaces.dag :as ws.dag]
|
||||
[metabase-enterprise.workspaces.impl :as ws.impl]
|
||||
[metabase-enterprise.workspaces.isolation :as ws.isolation]
|
||||
[metabase-enterprise.workspaces.models.workspace-log :as ws.log]
|
||||
[metabase-enterprise.workspaces.util :as ws.u]
|
||||
[metabase.api-keys.core :as api-key]
|
||||
[metabase.api.common :as api]
|
||||
[metabase.util :as u]
|
||||
[metabase.util.log :as log]
|
||||
[metabase.util.quick-task :as quick-task]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
;; should be encapsulated in our dag namespace, or dependency module
|
||||
(defn check-no-card-dependencies!
|
||||
"Check that transforms don't depend on cards. Throws 400 if they do."
|
||||
[transform-ids]
|
||||
(when-let [card-ids (seq (ws.dag/unsupported-dependency? transform-ids))]
|
||||
(api/check-400 false
|
||||
(format "Cannot add transforms that depend on saved questions (cards). Found dependencies on card IDs: %s"
|
||||
(pr-str (vec card-ids))))))
|
||||
|
||||
(defn- extract-suffix-number
|
||||
"Extract the numeric suffix from a workspace name like 'Foo (3)', or nil if no valid suffix."
|
||||
[ws-name base-name]
|
||||
(when (= (str/replace ws-name #"\s*\(\d+\)$" "") base-name)
|
||||
(when-let [[_ n] (re-find #"\((\d+)\)$" ws-name)]
|
||||
(parse-long n))))
|
||||
|
||||
(defn- generate-unique-workspace-name
|
||||
"Generate a unique workspace name by appending (N) if the name already exists.
|
||||
|
||||
Strategy:
|
||||
1. If the base name doesn't exist, use it as-is
|
||||
2. If it exists, strip any existing (N) suffix and find the highest N among similar names
|
||||
3. Return the base name with (N+1) appended
|
||||
|
||||
Note: This function has a potential race condition between checking existence and insertion.
|
||||
Callers should handle unique constraint violations by retrying."
|
||||
[base-name]
|
||||
(when (str/blank? base-name)
|
||||
(throw (ex-info "Workspace name cannot be empty" {:status-code 400})))
|
||||
(if-not (t2/exists? :model/Workspace :name base-name)
|
||||
base-name
|
||||
(let [stripped-name (str/replace base-name #"\s*\(\d+\)$" "")
|
||||
existing (t2/select-fn-set :name :model/Workspace
|
||||
:name [:like (str stripped-name " (%)")])
|
||||
numbers (keep #(extract-suffix-number % stripped-name) existing)
|
||||
next-num (inc (apply max 0 numbers))]
|
||||
(str stripped-name " (" next-num ")"))))
|
||||
|
||||
;; TODO: Generate new metabase user for the workspace
|
||||
;; TODO: Should we move this to model as per the diagram?
|
||||
(defn- create-workspace-container!
|
||||
"Create the workspace and its related collection, user, and api key."
|
||||
[creator-id db-id workspace-name status]
|
||||
;; TODO (Chris 2025-11-19) Unsure API key name is unique, and remove this (insecure) workaround.
|
||||
(let [api-key (let [key-name (format "API key for Workspace %s" workspace-name)]
|
||||
(or (t2/select-one :model/ApiKey :name key-name)
|
||||
(api-key/create-api-key-with-new-user! {:key-name key-name})))
|
||||
;; TODO (Chris 2025-11-19) Associate the api-key user with the workspace as well.
|
||||
ws (t2/insert-returning-instance! :model/Workspace
|
||||
{:name workspace-name
|
||||
:creator_id creator-id
|
||||
:database_id db-id
|
||||
:api_key_id (:id api-key)
|
||||
:execution_user (:user_id api-key)
|
||||
:status status})
|
||||
coll (t2/insert-returning-instance! :model/Collection
|
||||
{:name (format "Collection for Workspace %s" workspace-name)
|
||||
:namespace "workspace"
|
||||
:workspace_id (:id ws)})
|
||||
;; Only set schema for initialized workspaces (not uninitialized)
|
||||
schema (when (not= status :uninitialized) (ws.u/isolation-namespace-name ws))
|
||||
ws (assoc ws
|
||||
:collection_id (:id coll)
|
||||
:schema schema)]
|
||||
;; Set the backlink from the workspace to the collection inside it and set the schema.
|
||||
(t2/update! :model/Workspace (:id ws) (cond-> {:collection_id (:id coll)}
|
||||
schema (assoc :schema schema)))
|
||||
;; TODO (Sanya 2025-11-18) - not sure how to transfer this api key to agent
|
||||
#_(log/infof "Generated API key for workspace: %s" (u.secret/expose (:unmasked_key api-key)))
|
||||
ws))
|
||||
|
||||
(defn- unique-constraint-violation?
|
||||
"Check if an exception is due to a unique constraint violation."
|
||||
[e]
|
||||
(let [msg (str (ex-message e) (some-> (ex-cause e) ex-message))]
|
||||
(or (str/includes? msg "unique")
|
||||
(str/includes? msg "duplicate")
|
||||
(str/includes? msg "UNIQUE"))))
|
||||
|
||||
(defn- run-workspace-setup!
|
||||
"Background job: runs isolation, grants. Updates status to :ready when done."
|
||||
[{ws-id :id :as workspace} database]
|
||||
(try
|
||||
(ws.log/track! ws-id :workspace-setup
|
||||
(let [isolation-details (ws.log/track! ws-id :database-isolation
|
||||
(ws.isolation/ensure-database-isolation! workspace database))]
|
||||
(t2/update! :model/Workspace ws-id (merge (select-keys isolation-details [:schema :database_details])
|
||||
{:status :ready}))))
|
||||
(catch Exception e
|
||||
(log/error e "Failed to setup workspace")
|
||||
(t2/update! :model/Workspace ws-id {:status :setup-failed})
|
||||
(throw e))))
|
||||
|
||||
(defn- initialize-workspace!
|
||||
"Initialize an uninitialized workspace with the given database_id.
|
||||
Updates database_id (if different from provisional), sets schema, creates isolation resources async,
|
||||
and transitions to :pending status. Returns the updated workspace with schema set."
|
||||
[workspace database-id async?]
|
||||
(let [database (t2/select-one :model/Database :id database-id)
|
||||
schema (ws.u/isolation-namespace-name workspace)
|
||||
res (t2/update! :model/Workspace {:id (:id workspace)
|
||||
:status :uninitialized}
|
||||
{:database_id database-id
|
||||
:schema schema
|
||||
:status :pending})
|
||||
new-ws (t2/select-one :model/Workspace :id (:id workspace))]
|
||||
(when (zero? res)
|
||||
(let [new-db-id (t2/select-one-fn :database_id :model/Workspace (:id workspace))]
|
||||
(when (not= database-id new-db-id)
|
||||
(throw (ex-info "Workspace has been initialized already with a different database"
|
||||
{:requested-db-id database-id
|
||||
:actual-db-id new-db-id})))))
|
||||
(if async?
|
||||
(quick-task/submit-task! #(run-workspace-setup! new-ws database))
|
||||
(run-workspace-setup! new-ws database))
|
||||
(t2/select-one :model/Workspace :id (:id workspace))))
|
||||
|
||||
(defn- create-uninitialized-workspace!
|
||||
"Create a workspace with a provisional database_id but no isolation resources.
|
||||
Retries on unique constraint violations for workspace name."
|
||||
[creator-id db-id ws-name max-retries]
|
||||
(loop [attempt 1]
|
||||
(let [unique-name (generate-unique-workspace-name ws-name)
|
||||
{:keys [retry
|
||||
workspace]} (try
|
||||
{:workspace (create-workspace-container! creator-id db-id unique-name :uninitialized)}
|
||||
(catch Exception e
|
||||
(if (and (< attempt max-retries) (unique-constraint-violation? e))
|
||||
{:retry true}
|
||||
(throw e))))]
|
||||
(if retry
|
||||
(recur (inc attempt))
|
||||
workspace))))
|
||||
|
||||
;; TODO internal: test!
|
||||
(defn create-workspace!
|
||||
"Create workspace. If :provisional is true, creates an uninitialized workspace with a
|
||||
provisional database_id (no isolation resources yet). Otherwise creates a pending
|
||||
workspace and kicks off async setup."
|
||||
[creator-id {ws-name-maybe :name
|
||||
db-id :database_id
|
||||
provisional? :provisional?}]
|
||||
(let [ws-name (or ws-name-maybe (ws.u/generate-name))
|
||||
ws (create-uninitialized-workspace! creator-id db-id ws-name 5)]
|
||||
(if provisional?
|
||||
ws
|
||||
(initialize-workspace! ws db-id true))))
|
||||
|
||||
(defn add-to-changeset!
|
||||
"Add the given transform to the workspace changeset.
|
||||
If workspace is uninitialized, initializes it with the transform's target database."
|
||||
[creator-id workspace entity-type global-id body]
|
||||
(ws.u/assert-transform! entity-type)
|
||||
;; Initialize workspace if uninitialized (outside transaction so async task can see committed data)
|
||||
(let [workspace (if (= :uninitialized (:status workspace))
|
||||
(let [target-db-id (transforms.i/target-db-id body)]
|
||||
(api/check-400 target-db-id "Transform must have a target database")
|
||||
(initialize-workspace! workspace target-db-id false))
|
||||
workspace)]
|
||||
(t2/with-transaction [_]
|
||||
(let [workspace-id (:id workspace)
|
||||
workspace-db-id (:database_id workspace)
|
||||
body (assoc-in body [:target :database] workspace-db-id)
|
||||
transform (t2/insert-returning-instance!
|
||||
:model/WorkspaceTransform
|
||||
(assoc (select-keys body [:name :description :source :target])
|
||||
:ref_id (ws.u/generate-ref-id)
|
||||
:creator_id creator-id
|
||||
:global_id global-id
|
||||
:workspace_id workspace-id))]
|
||||
(ws.impl/sync-transform-dependencies! workspace (select-keys transform [:ref_id :source_type :source :target]))
|
||||
transform))))
|
||||
@@ -0,0 +1,2 @@
|
||||
;; For now, we do not anticipate any non-API usage of this module.
|
||||
(ns metabase-enterprise.workspaces.core)
|
||||
212
enterprise/backend/src/metabase_enterprise/workspaces/dag.clj
Normal file
212
enterprise/backend/src/metabase_enterprise/workspaces/dag.clj
Normal file
@@ -0,0 +1,212 @@
|
||||
;; TODO consider renaming this to dataflow
|
||||
(ns metabase-enterprise.workspaces.dag
|
||||
"Used to do graph computations with respect to dataflow within a workspace.
|
||||
|
||||
This namespace should encapsulate all coupling to the [[metabase-enterprise.dependencies]] module, including through
|
||||
database artifacts.
|
||||
|
||||
It is purely concerned with calculation, any side effects wanted as a result of the analysis should happen outside."
|
||||
(:require
|
||||
[metabase-enterprise.workspaces.util :as ws.u]
|
||||
[metabase.util :as u]
|
||||
[toucan2.core :as t2]
|
||||
[toucan2.realize :as t2.realize]))
|
||||
|
||||
;; Is this working with appdb-ids or ref-ids?
|
||||
(defn unsupported-dependency?
|
||||
"Return a map of the entities which have unsupported dependencies, or nil if there are none."
|
||||
[{transform-ids :transforms :as _entity-map}]
|
||||
;; TODO filter out those that don't have a card dependency
|
||||
(when (seq transform-ids)
|
||||
{:transforms transform-ids}))
|
||||
|
||||
;; TODO (chris 2025/12/15) Should we leverage data-authority table metadata to speed things up?
|
||||
|
||||
(defn- table-id->coord [id]
|
||||
(t2/select-one [:model/Table :id :schema [:name :table] [:db_id :db]] id))
|
||||
|
||||
(defn- global-parents
|
||||
"Dependencies analyzed by global hooks - may be incomplete. For example, excluding transforms that have not been run."
|
||||
[ws-id node-type id]
|
||||
;; Note: from_entity_type is transformed to a keyword by the Dependency model
|
||||
(t2/select-fn-vec (fn [{id :to_entity_id entity-type :to_entity_type}]
|
||||
(case entity-type
|
||||
:card {:node-type :external-card, :id id}
|
||||
:table {:node-type :table, :id (table-id->coord id)}
|
||||
:transform {:node-type :external-transform, :id id}))
|
||||
[:model/Dependency :to_entity_type :to_entity_id]
|
||||
:from_entity_type node-type
|
||||
:from_entity_id id
|
||||
;; Exclude transforms which are being overridden in the workspace.
|
||||
;; Note: use string "transform" in SQL WHERE clause (db stores strings)
|
||||
{:where [:not [:and
|
||||
[:= "transform" :to_entity_type]
|
||||
[:exists {:select [1]
|
||||
:from [[:workspace_transform :wt]]
|
||||
:where [:and
|
||||
[:= :wt.workspace_id ws-id]
|
||||
[:= :wt.global_id :to_entity_id]]}]]]}))
|
||||
|
||||
(defn- ws-transform-parents [ws-id ref-id]
|
||||
;; We assume there are no card dependencies yet
|
||||
(t2/select-fn-vec (fn [table-coord]
|
||||
{:node-type :table, :id (t2.realize/realize table-coord)})
|
||||
[:model/WorkspaceInput [:db_id :db] :schema :table [:table_id :id]]
|
||||
:workspace_id ws-id
|
||||
:id [:in {:select [:to_entity_id]
|
||||
:from [:workspace_dependency]
|
||||
:where [:and
|
||||
[:= "transform" :from_entity_type]
|
||||
[:= ref-id :from_entity_id]
|
||||
[:= "input" :to_entity_type]]}]))
|
||||
|
||||
(defn- table-producers [ws-id id-or-coord]
|
||||
;; Work with either logical co-ords or an id
|
||||
(let [{:keys [db schema table id]} (if (map? id-or-coord) id-or-coord (table-id->coord id-or-coord))
|
||||
tx-ref-id (t2/select-one-fn :ref_id [:model/WorkspaceOutput :ref_id]
|
||||
{:where [:and
|
||||
[:= :workspace_id ws-id]
|
||||
[:or
|
||||
(when id
|
||||
[:= :global_table_id id])
|
||||
[:and
|
||||
[:= :db_id db]
|
||||
[:= :global_schema schema]
|
||||
[:= :global_table table]]]]})]
|
||||
(if tx-ref-id
|
||||
;; If there is a workspace transform that targets this table, ignore any global transform that also targets it.
|
||||
[{:node-type :workspace-transform :id tx-ref-id}]
|
||||
(global-parents ws-id "table" id))))
|
||||
|
||||
(defn- table? [{:keys [node-type]}] (= :table node-type))
|
||||
|
||||
(defn- node-parents [ws-id {:keys [node-type id]}]
|
||||
(case node-type
|
||||
:workspace-transform (ws-transform-parents ws-id id)
|
||||
:external-transform (global-parents ws-id "transform" id)
|
||||
:external-card (global-parents ws-id "card" id)
|
||||
:table (table-producers ws-id id)))
|
||||
|
||||
(defn- node->allowed-parents
|
||||
[pred deps node]
|
||||
(if-not (pred node)
|
||||
[node]
|
||||
;; Collapse to the things that *produce* node. If it is a root node, leave it in.
|
||||
(or (seq (mapcat (partial node->allowed-parents pred deps) (deps node)))
|
||||
[node])))
|
||||
|
||||
(defn- collapse
|
||||
"Remove nodes that satisfy pred, in-lining their transitive dependencies instead."
|
||||
[pred deps]
|
||||
(u/for-map [[child parents] deps
|
||||
:when (not (pred child))]
|
||||
[child (mapcat (partial node->allowed-parents pred deps) parents)]))
|
||||
|
||||
(defn- render-graph [entities parents deps & {:keys [table? table-sort unwrap-table]
|
||||
:or {table? table?
|
||||
table-sort (juxt :db :schema :table)
|
||||
unwrap-table :id}}]
|
||||
(let [table-nodes (filter table? entities)
|
||||
;; Any table that has a parent in the subgraph is an output
|
||||
outputs (filter deps table-nodes)
|
||||
;; Anything other parent table is an input
|
||||
inputs (->> entities (mapcat parents) (filter table?) (remove (set outputs)) distinct)
|
||||
entities (->> (ws.u/toposort-dfs deps) (remove table?))
|
||||
deps+input (reduce
|
||||
(fn [deps [c parents]]
|
||||
(if-not (deps c)
|
||||
deps
|
||||
(update deps c into parents)))
|
||||
deps
|
||||
parents)]
|
||||
{:inputs (sort-by table-sort (map unwrap-table inputs))
|
||||
:outputs (sort-by table-sort (map unwrap-table outputs))
|
||||
:entities entities
|
||||
;; collapse tables out, directly connecting transforms? smaller graphs are easier for humans to read
|
||||
:dependencies (collapse table? deps+input)}))
|
||||
|
||||
(defn- path-induced-subgraph*
|
||||
"Implementation for [[path-induced-subgraph]] that takes the lookup functions as arguments. Useful for testing."
|
||||
[init-nodes fns]
|
||||
(loop [members (set init-nodes)
|
||||
cache {}
|
||||
;; Association list sets from nodes to their direct dependencies
|
||||
deps (u/for-map [node init-nodes] [node #{}])
|
||||
;; Paths are vectors sorted from child to parent
|
||||
[path & paths] (for [node members] [node])]
|
||||
(if-not path
|
||||
;; Finished walking all paths, render the result.
|
||||
(render-graph members cache deps fns)
|
||||
;; Look-up from cache first
|
||||
(let [_ (when (some #(not= 1 (val %)) (frequencies path))
|
||||
(throw (ex-info "Cycle detected" {:path path})))
|
||||
head (peek path)
|
||||
node-parents* (:node-parents fns)
|
||||
parents (cache head (node-parents* head))
|
||||
cache (assoc cache head parents)
|
||||
continue (when parents (remove members parents))]
|
||||
(if (not= parents continue)
|
||||
;; At least one parent is in the enclosed subgraph, so this entire path is as well.
|
||||
(let [add-dep (fn [deps [child parent]]
|
||||
(update deps child (fnil conj #{}) parent))
|
||||
;; Break up the path into adjacent child-parent pairs.
|
||||
pairs (concat (partition 2 1 path)
|
||||
(map (fn [end] [(peek path) end]) (filter members parents)))
|
||||
;; Since everything along this path will now be added to deps already, we can truncate the paths.
|
||||
next-paths (into paths (for [c continue] [(peek path) c]))]
|
||||
(recur (into members path) cache (reduce add-dep deps pairs) next-paths))
|
||||
;; We have not reached another member of the enclosed subgraph yet, so keep extending.
|
||||
(recur members cache deps (into paths (for [c continue] (conj path c)))))))))
|
||||
|
||||
;;;; Public API
|
||||
|
||||
(defn path-induced-subgraph
|
||||
"Given a map of entity types to IDs, compute the path-induced subgraph.
|
||||
`entities-by-type` is a map like {:transform [1 2 3]}.
|
||||
|
||||
Returns a map with:
|
||||
- :inputs - tables that the subgraph directly depends on, that are not themselves produced by the subgraph.
|
||||
ordered lexically by db+schema+table, to be stable across metabase instances, and visibly sorted.
|
||||
- :outputs - outputs of both the changeset transforms, and the external transforms enclosed by the changeset.
|
||||
both the external names, and the internal (isolated) names. sorted by the external names.
|
||||
point back to their relevant transform.
|
||||
|
||||
- :entities - the full list of entities, both in the changeset, and those that are enclosed. topo-sorted.
|
||||
- :dependencies - association list for the subgraph, with keys and values both in topological order"
|
||||
[ws-id changeset]
|
||||
(ws.u/assert-transforms! changeset)
|
||||
(if (empty? changeset)
|
||||
{:inputs []
|
||||
:outputs []
|
||||
:entities []
|
||||
:dependencies {}}
|
||||
(let [tx-nodes (for [{:keys [entity-type id]} changeset]
|
||||
(case entity-type
|
||||
:transform {:node-type :workspace-transform, :id id}))
|
||||
outputs (when (seq tx-nodes)
|
||||
(t2/select-fn-vec (fn [row]
|
||||
{:node-type :table, :id (t2.realize/realize row)})
|
||||
[:model/WorkspaceOutput
|
||||
[:db_id :db]
|
||||
[:global_schema :schema]
|
||||
[:global_table :table]
|
||||
[:global_table_id :id]]
|
||||
:ref_id [:in (map :id tx-nodes)]))
|
||||
init-nodes (concat tx-nodes outputs)
|
||||
fns {:node-parents (partial node-parents ws-id)
|
||||
:table? table?}]
|
||||
(path-induced-subgraph* init-nodes fns))))
|
||||
|
||||
;; source-table ---------> checked-out-transform
|
||||
;; \____external-transform__/
|
||||
;;
|
||||
;; should external-transform be enclosed? answer: no, because it won't become stale from changes in the workspace
|
||||
;;
|
||||
;; source-table ----> model ---> checked-out-transform
|
||||
;;
|
||||
;; this time we *need* the model to be serialized, because we need to read it in a fresh instance.
|
||||
;; it doesn't need to be remapped or "run" independently though, so does it make sense to call it enclosed?
|
||||
;;
|
||||
;; enclosed: between changeset only? or including the model in the example above?
|
||||
;; ancestors: as far back as you can go
|
||||
;;
|
||||
@@ -0,0 +1,399 @@
|
||||
(ns metabase-enterprise.workspaces.dependencies
|
||||
"Workspace-local dependency tracking.
|
||||
|
||||
Unlike the global `dependency` table which links to `table_id` (requiring the table to exist),
|
||||
workspace dependencies use three tables that support logical references:
|
||||
|
||||
- `workspace_input` - external tables transforms consume (may not exist yet)
|
||||
- `workspace_output` - tables transforms produce
|
||||
- `workspace_dependency` - edges: transform → input (external) or transform → output (internal)
|
||||
|
||||
## Example: Internal Dependency (Transform B depends on Transform A's output)
|
||||
|
||||
Transform A produces `analytics.orders_summary`:
|
||||
```
|
||||
{:ref_id \"happy-dolphin-a1b2\"
|
||||
:target {:database 1 :schema \"analytics\" :name \"orders_summary\"}}
|
||||
```
|
||||
|
||||
Transform B reads from `analytics.orders_summary`:
|
||||
```
|
||||
{:ref_id \"brave-lion-g7h8\"
|
||||
:source {:type \"query\" :query \"SELECT * FROM analytics.orders_summary\"}
|
||||
:target {:database 1 :schema \"analytics\" :name \"orders_report\"}}
|
||||
```
|
||||
|
||||
After processing both transforms:
|
||||
|
||||
workspace_output:
|
||||
| id | ref_id | schema | table |
|
||||
|-----|--------------------|-----------|----------------|
|
||||
| 101 | happy-dolphin-a1b2 | analytics | orders_summary |
|
||||
| 104 | brave-lion-g7h8 | analytics | orders_report |
|
||||
|
||||
workspace_dependency:
|
||||
| from_entity_type | from_entity_id | to_entity_type | to_entity_id | meaning |
|
||||
|------------------|--------------------| ---------------|--------------|-------------------------|
|
||||
| transform | brave-lion-g7h8 | output | 101 | B depends on A's output |
|
||||
|
||||
Transform B's dependency points to `output:101` (Transform A's output), NOT to a new `input`.
|
||||
This is because `analytics.orders_summary` matches an existing `workspace_output` record.
|
||||
|
||||
If Transform A also depended on an external table ORDERS, we'd also have:
|
||||
|
||||
workspace_input:
|
||||
| id | schema | table | table_id |
|
||||
|-----|--------|--------|----------|
|
||||
| 201 | PUBLIC | ORDERS | 42 |
|
||||
|
||||
workspace_dependency:
|
||||
| from_entity_type | from_entity_id | to_entity_type | to_entity_id | meaning |
|
||||
|------------------|--------------------| ---------------|--------------|------------------------------|
|
||||
| transform | happy-dolphin-a1b2 | input | 201 | A depends on external ORDERS |
|
||||
| transform | brave-lion-g7h8 | output | 101 | B depends on A's output |"
|
||||
(:require
|
||||
[clojure.set :as set]
|
||||
[metabase-enterprise.workspaces.models.workspace-dependency]
|
||||
[metabase-enterprise.workspaces.models.workspace-input]
|
||||
[metabase-enterprise.workspaces.models.workspace-output]
|
||||
[metabase-enterprise.workspaces.util :as ws.u]
|
||||
[metabase.app-db.core :as app-db]
|
||||
[metabase.driver :as driver]
|
||||
[metabase.driver.sql :as driver.sql]
|
||||
;; TODO (chris 2025/12/17) I solemnly declare that we will clean up this coupling nightmare for table normalization
|
||||
^{:clj-kondo/ignore [:metabase/modules]}
|
||||
[metabase.driver.sql.normalize :as sql.normalize]
|
||||
[metabase.lib.core :as lib]
|
||||
[metabase.lib.metadata :as lib.metadata]
|
||||
[metabase.util :as u]
|
||||
[metabase.util.log :as log]
|
||||
[metabase.util.malli :as mu]
|
||||
[metabase.util.malli.registry :as mr]
|
||||
[metabase.util.performance :as perf]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
;;; ---------------------------------------- Schemas ----------------------------------------
|
||||
|
||||
(mr/def ::table-ref
|
||||
"A logical reference to a table by schema and name. Includes table_id if the table exists."
|
||||
[:map
|
||||
[:db_id :int]
|
||||
[:schema [:maybe :string]]
|
||||
[:table :string]
|
||||
[:table_id {:optional true} [:maybe :int]]])
|
||||
|
||||
(mr/def ::output-ref
|
||||
"A reference to an output table (transform target)."
|
||||
[:map
|
||||
[:db_id :int]
|
||||
[:schema [:maybe :string]]
|
||||
[:table :string]])
|
||||
|
||||
(mr/def ::analysis
|
||||
"The result of analyzing an entity's dependencies."
|
||||
[:map
|
||||
[:output ::output-ref]
|
||||
[:inputs [:sequential ::table-ref]]])
|
||||
|
||||
;;; ---------------------------------------- Batch table lookups ----------------------------------------
|
||||
|
||||
(defn- batch-table-refs-from-ids
|
||||
"Look up table metadata for multiple table IDs in a single query.
|
||||
Returns a map of table-id -> {:db_id :schema :table :table_id}."
|
||||
[table-ids]
|
||||
(when (seq table-ids)
|
||||
(t2/select-fn->fn :id
|
||||
(fn [{:keys [id db_id schema name]}]
|
||||
{:db_id db_id
|
||||
:schema schema
|
||||
:table name
|
||||
:table_id id})
|
||||
[:model/Table :id :db_id :schema :name]
|
||||
:id [:in table-ids])))
|
||||
|
||||
(defn- batch-lookup-table-ids
|
||||
"Look up table_ids for multiple logical table references in a single query.
|
||||
Uses the same matching logic as find-table-or-transform: if parsed schema is nil,
|
||||
it matches against the driver's default schema.
|
||||
Returns a map of [db_id search-schema table] -> table_id where search-schema
|
||||
is the schema used for lookup (defaulted if nil)."
|
||||
[driver db-id table-refs]
|
||||
(when (seq table-refs)
|
||||
(let [default-schema (driver.sql/default-schema driver)
|
||||
;; Include default-schema in filter so nil refs can match that too
|
||||
nil-schema? (some (comp nil? :schema) table-refs)
|
||||
schemas-to-query (into #{} (map #(or (:schema %) default-schema)) table-refs)
|
||||
db-table-lookup (t2/select-fn->fn (juxt :schema :name) :id
|
||||
[:model/Table :id :schema :name]
|
||||
:db_id db-id
|
||||
:name [:in (map :table table-refs)]
|
||||
{:where (if nil-schema?
|
||||
[:or [:= :schema nil] [:in :schema schemas-to-query]]
|
||||
[:in :schema schemas-to-query])})]
|
||||
(into {}
|
||||
(keep (fn [{:keys [schema table]}]
|
||||
(let [table-id (or (get db-table-lookup [schema table])
|
||||
(when (nil? schema)
|
||||
(get db-table-lookup [default-schema table])))]
|
||||
(when table-id
|
||||
[[db-id schema table] table-id]))))
|
||||
table-refs))))
|
||||
|
||||
(defn- output-from-target
|
||||
"Extract output table info from a transform's :target field."
|
||||
[{:keys [database schema name] :as _target}]
|
||||
{:db_id database
|
||||
:schema schema
|
||||
:table name})
|
||||
|
||||
;;; ---------------------------------------- Input extraction ----------------------------------------
|
||||
|
||||
(defn- inputs-from-mbql-query
|
||||
"Extract table refs from an MBQL query. Tables are looked up by ID in batch."
|
||||
[query]
|
||||
(let [table-ids (-> #{}
|
||||
(into (lib/all-source-table-ids query))
|
||||
(into (lib/all-implicitly-joined-table-ids query)))
|
||||
table-refs (batch-table-refs-from-ids table-ids)]
|
||||
(u/keepv table-refs table-ids)))
|
||||
|
||||
(defn- inputs-from-native-query
|
||||
"Extract table refs from a native query. Uses driver/native-query-table-refs to get
|
||||
schema+table directly, then batch looks up table_ids for tables that exist.
|
||||
Uses the same default schema logic as find-table-or-transform."
|
||||
[query db-id]
|
||||
(let [driver (:engine (lib.metadata/database query))
|
||||
parsed-refs (driver/native-query-table-refs driver query)
|
||||
;; Batch lookup existing table IDs (handles default schema logic)
|
||||
table-id-map (batch-lookup-table-ids driver db-id parsed-refs)]
|
||||
(perf/for [{:keys [schema table]} parsed-refs
|
||||
:let [table-id (get table-id-map [db-id schema table])]]
|
||||
(cond-> {:db_id db-id
|
||||
:schema schema
|
||||
:table table}
|
||||
table-id (assoc :table_id table-id)))))
|
||||
|
||||
(defn- inputs-from-query
|
||||
"Extract table refs from a query (MBQL or native)."
|
||||
[query db-id]
|
||||
(if (lib/native-only-query? query)
|
||||
(inputs-from-native-query query db-id)
|
||||
(inputs-from-mbql-query query)))
|
||||
|
||||
(defn- inputs-from-python-transform
|
||||
"Extract table refs from a python transform's source-tables.
|
||||
Python transforms require tables to exist (they map name -> table_id). Batch lookup."
|
||||
[source-tables]
|
||||
(let [table-ids (set (vals source-tables))
|
||||
table-refs (batch-table-refs-from-ids table-ids)]
|
||||
(u/keepv table-refs (vals source-tables))))
|
||||
|
||||
(mu/defn analyze-entity :- ::analysis
|
||||
"Analyze a workspace entity to find its dependencies.
|
||||
|
||||
Arguments:
|
||||
- entity-type: keyword, must be :transform (asserted)
|
||||
- entity: the workspace transform map with :source and :target
|
||||
|
||||
Returns:
|
||||
{:output {:db_id int :schema string :table string}
|
||||
:inputs [{:db_id int :schema string :table string} ...]}"
|
||||
[entity-type :- :keyword
|
||||
entity :- :map]
|
||||
(ws.u/assert-transform! entity-type)
|
||||
(let [{{:keys [query source-tables], source-type :type} :source
|
||||
target :target} entity
|
||||
db-id (:database target)
|
||||
inputs (cond
|
||||
(= (keyword source-type) :query)
|
||||
(inputs-from-query query db-id)
|
||||
|
||||
(= (keyword source-type) :python)
|
||||
(inputs-from-python-transform source-tables)
|
||||
|
||||
:else
|
||||
(do (log/warnf "Don't know how to analyze deps of transform with source type '%s'" source-type)
|
||||
[]))]
|
||||
{:output (output-from-target target)
|
||||
:inputs inputs}))
|
||||
|
||||
;;; ---------------------------------------- Write helpers ----------------------------------------
|
||||
|
||||
(defn- upsert-workspace-output!
|
||||
"Upsert a workspace_output record for the transform's target table.
|
||||
Stores both global (original) and isolated (workspace-specific) table identifiers.
|
||||
Returns the workspace_output id."
|
||||
[workspace-id ref-id isolated-schema {:keys [db_id schema table]} normalize-sql]
|
||||
(app-db/update-or-insert! :model/WorkspaceOutput
|
||||
{:workspace_id workspace-id
|
||||
:ref_id ref-id}
|
||||
(fn [existing]
|
||||
(let [isolated-table (ws.u/isolated-table-name schema table)
|
||||
qry-table-id (fn [schema table]
|
||||
(t2/select-one-fn :id [:model/Table :id]
|
||||
:db_id db_id
|
||||
:schema (normalize-sql schema)
|
||||
:name (normalize-sql table)))
|
||||
id-if-match (fn [schema-key schema table-key table id-key]
|
||||
(when (and (= schema (get existing schema-key))
|
||||
(= table (get existing table-key)))
|
||||
(get existing id-key)))
|
||||
id-or-fallback (fn [schema-key schema table-key table id-key]
|
||||
(or (id-if-match schema-key schema table-key table id-key)
|
||||
(qry-table-id schema table)))]
|
||||
{:db_id db_id
|
||||
:global_schema schema
|
||||
:global_table table
|
||||
:global_table_id (id-or-fallback
|
||||
:global_schema schema
|
||||
:global_table table
|
||||
:global_table_id)
|
||||
:isolated_schema isolated-schema
|
||||
:isolated_table isolated-table
|
||||
:isolated_table_id (id-or-fallback
|
||||
:isolated_schema isolated-schema
|
||||
:isolated_table isolated-table
|
||||
:isolated_table_id)}))))
|
||||
|
||||
(defn- build-output-lookup
|
||||
"Build a lookup map for workspace outputs: [db_id global_schema global_table] -> output_id.
|
||||
Single query to fetch all outputs for the workspace."
|
||||
[workspace-id]
|
||||
(t2/select-fn->fn (juxt :db_id :global_schema :global_table) :id
|
||||
:model/WorkspaceOutput
|
||||
:workspace_id workspace-id))
|
||||
|
||||
(defn- build-input-lookup
|
||||
"Build a lookup map for workspace inputs: [db_id schema table] -> {:id :table_id}.
|
||||
Single query to fetch all inputs for the workspace."
|
||||
[workspace-id]
|
||||
(t2/select-fn->fn (juxt :db_id :schema :table)
|
||||
#(select-keys % [:id :table_id])
|
||||
[:model/WorkspaceInput :id :db_id :schema :table :table_id]
|
||||
:workspace_id workspace-id))
|
||||
|
||||
(defn- normalize-input-schema
|
||||
"Normalize an input's schema: replace nil with the driver's default schema.
|
||||
This ensures consistent comparison with output schemas which are always explicit."
|
||||
[default-schema input]
|
||||
(update input :schema #(or % default-schema)))
|
||||
|
||||
(defn- ensure-workspace-inputs!
|
||||
"Ensure all external inputs exist in workspace_input table.
|
||||
Uses batch lookup to avoid N+1. Returns map of [db_id schema table] -> input_id.
|
||||
Inputs should already have normalized schemas (nil replaced with driver default)."
|
||||
[workspace-id inputs existing-input-lookup]
|
||||
(let [{existing true new-inputs false}
|
||||
(group-by (fn [{:keys [db_id schema table]}]
|
||||
(contains? existing-input-lookup [db_id schema table]))
|
||||
inputs)
|
||||
|
||||
;; TODO: This N+1 section could be optimized into one batch update using HoneySQL
|
||||
_ (doseq [{:keys [db_id schema table table_id]} existing]
|
||||
(let [existing-record (get existing-input-lookup [db_id schema table])]
|
||||
(when (and existing-record
|
||||
(not= (:table_id existing-record) table_id))
|
||||
(t2/update! :model/WorkspaceInput (:id existing-record) {:table_id table_id}))))
|
||||
|
||||
new-input-records (when (seq new-inputs)
|
||||
(t2/insert-returning-instances!
|
||||
:model/WorkspaceInput
|
||||
(for [{:keys [db_id schema table table_id]} new-inputs]
|
||||
{:workspace_id workspace-id
|
||||
:db_id db_id
|
||||
:schema schema
|
||||
:table table
|
||||
:table_id table_id})))
|
||||
|
||||
new-input-lookup (into {} (map (fn [{:keys [id db_id schema table]}]
|
||||
[[db_id schema table] id])
|
||||
new-input-records))]
|
||||
(merge (into {} (map (fn [[k v]] [k (:id v)]) existing-input-lookup))
|
||||
new-input-lookup)))
|
||||
|
||||
(defn- create-dependency-edges!
|
||||
"Batch create workspace_dependency edges."
|
||||
[workspace-id ref-id dep-specs]
|
||||
(when (seq dep-specs)
|
||||
(t2/insert! :model/WorkspaceDependency
|
||||
(for [{:keys [to_entity_type to_entity_id]} dep-specs]
|
||||
{:workspace_id workspace-id
|
||||
:from_entity_type :transform
|
||||
:from_entity_id ref-id
|
||||
:to_entity_type to_entity_type
|
||||
:to_entity_id to_entity_id}))))
|
||||
|
||||
(defn- delete-stale-edges!
|
||||
"Delete dependency edges that are no longer present.
|
||||
Uses single delete with OR conditions for batch deletion."
|
||||
[workspace-id ref-id stale-edges]
|
||||
(when (seq stale-edges)
|
||||
(doseq [{:keys [to_entity_type to_entity_id]} stale-edges]
|
||||
;; TODO: Could optimize further with raw SQL DELETE ... WHERE (type, id) IN (...)
|
||||
(t2/delete! :model/WorkspaceDependency
|
||||
:workspace_id workspace-id
|
||||
:from_entity_type :transform
|
||||
:from_entity_id ref-id
|
||||
:to_entity_type to_entity_type
|
||||
:to_entity_id to_entity_id))))
|
||||
|
||||
(defn- current-edge-specs
|
||||
"Get the current dependency edge specs for a transform."
|
||||
[workspace-id ref-id]
|
||||
(into #{}
|
||||
(map #(select-keys % [:to_entity_type :to_entity_id]))
|
||||
(t2/select [:model/WorkspaceDependency :to_entity_type :to_entity_id]
|
||||
:workspace_id workspace-id
|
||||
:from_entity_type :transform
|
||||
:from_entity_id ref-id)))
|
||||
|
||||
(mu/defn write-dependencies! :- :nil
|
||||
"Persist dependency analysis for a workspace entity.
|
||||
|
||||
Arguments:
|
||||
- workspace-id: int
|
||||
- isolated-schema: string, the workspace's isolated schema name
|
||||
- entity-type: keyword, must be :transform
|
||||
- ref-id: string, the workspace_transform.ref_id
|
||||
- analysis: result from analyze-entity
|
||||
|
||||
Side effects:
|
||||
- Upserts workspace_output for this transform (with both global and isolated identifiers)
|
||||
- Upserts workspace_input for each external table dependency
|
||||
- Creates workspace_dependency edges
|
||||
- Deletes stale edges"
|
||||
[workspace-id :- :int
|
||||
isolated-schema :- :string
|
||||
entity-type :- :keyword
|
||||
ref-id :- :string
|
||||
{:keys [output inputs]} :- ::analysis]
|
||||
(ws.u/assert-transform! entity-type)
|
||||
(t2/with-transaction [_conn]
|
||||
(let [output-lookup (build-output-lookup workspace-id)
|
||||
existing-input-lookup (build-input-lookup workspace-id)
|
||||
current-edges (current-edge-specs workspace-id ref-id)
|
||||
driver (t2/select-one-fn :engine [:model/Database :engine]
|
||||
:id [:in {:select [:database_id]
|
||||
:from [:workspace]
|
||||
:where [:= :id workspace-id]}])
|
||||
normalize (partial sql.normalize/normalize-name driver)]
|
||||
(upsert-workspace-output! workspace-id ref-id isolated-schema output normalize)
|
||||
(let [{internal-inputs true external-inputs false} (group-by (fn [{:keys [db_id schema table]}]
|
||||
(contains? output-lookup [db_id schema table]))
|
||||
inputs)
|
||||
default-schema (driver.sql/default-schema driver)
|
||||
;; Normalize external inputs so lookups use consistent schema keys
|
||||
normalized-external-inputs (map (partial normalize-input-schema default-schema) external-inputs)
|
||||
input-id-lookup (ensure-workspace-inputs! workspace-id normalized-external-inputs existing-input-lookup)
|
||||
new-dep-specs (concat
|
||||
(for [{:keys [db_id schema table]} internal-inputs]
|
||||
{:to_entity_type :output
|
||||
:to_entity_id (get output-lookup [db_id schema table])})
|
||||
(for [{:keys [db_id schema table]} normalized-external-inputs]
|
||||
{:to_entity_type :input
|
||||
:to_entity_id (get input-id-lookup [db_id schema table])}))
|
||||
new-dep-specs-set (set new-dep-specs)
|
||||
edges-to-create (set/difference new-dep-specs-set current-edges)
|
||||
edges-to-delete (set/difference current-edges new-dep-specs-set)]
|
||||
(create-dependency-edges! workspace-id ref-id edges-to-create)
|
||||
(delete-stale-edges! workspace-id ref-id edges-to-delete)))))
|
||||
@@ -0,0 +1,120 @@
|
||||
(ns metabase-enterprise.workspaces.execute
|
||||
"This namespace is concerned with executing non-AppDB transforms, with optional reference re-mappings.
|
||||
|
||||
It currently lives inside the workspace module, but eventually will become part of the transform module, so it should
|
||||
not make any assumptions about being run within a workspace, having the input / output semantics of a workspace, and
|
||||
especially it should not touch any of the Workspace entities inside AppDb, or any of the other workspace namespaces.
|
||||
|
||||
For now, it uses AppDB as a side-channel with the transforms module, but in future this module should be COMPLETELY
|
||||
decoupled from AppDb."
|
||||
(:require
|
||||
[macaw.core :as macaw]
|
||||
[metabase-enterprise.transforms.core :as transforms]
|
||||
[metabase-enterprise.transforms.execute :as transforms.execute]
|
||||
[metabase.api.common :as api]
|
||||
[metabase.util :as u]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(defn- execution-results
|
||||
"Extract execution metadata from transform_run and target table info.
|
||||
Must be called within the transaction before rollback."
|
||||
[{xf-id :id :keys [target]}]
|
||||
(let [run (t2/select-one :model/TransformRun :transform_id xf-id)]
|
||||
(merge
|
||||
(select-keys run [:status :start_time :end_time :message])
|
||||
{:table {:name (:name target)
|
||||
:schema (:schema target)}})))
|
||||
|
||||
(defn- remap-python-source
|
||||
"Remap source-tables in a Python transform source to point to isolated tables.
|
||||
Handles both legacy integer format and new map format (from PR #66934)."
|
||||
[table-mapping source]
|
||||
(letfn [(remap [{:keys [database_id schema table table_id] :as table-ref}]
|
||||
(if-let [{:keys [id db-id schema table]} (or (table-mapping table-ref)
|
||||
(some-> table_id table-mapping)
|
||||
(when table
|
||||
(table-mapping [database_id schema table])))]
|
||||
;; Prefer table ID when available; map format won't work until PR #66934 is merged:
|
||||
;; https://github.com/metabase/metabase/pull/66934
|
||||
(or id {:database_id db-id, :schema schema, :table table})
|
||||
;; Leave it un-mapped if we don't have an override.
|
||||
table-ref))]
|
||||
(update source :source-tables update-vals remap)))
|
||||
|
||||
(defn- remap-sql-source [table-mapping source]
|
||||
(let [remapping (reduce
|
||||
(fn [remapping [[_ source-schema source-table] {target-schema :schema, target-table :table}]]
|
||||
(assoc-in remapping [:tables {:schema source-schema
|
||||
:table source-table}]
|
||||
{:schema target-schema
|
||||
:table target-table}))
|
||||
{:schemas {}
|
||||
:tables {}}
|
||||
;; Strip out the numeric keys (table ids)
|
||||
(filter (comp vector? key) table-mapping))]
|
||||
;; We may need to set other options, like the case insensitivity (driver dependent)
|
||||
(update-in source [:query :stages 0 :native] #(macaw/replace-names % remapping {:allow-unused? true}))))
|
||||
|
||||
(defn- remap-mbql-source [_table-mapping _field-map source]
|
||||
(throw (ex-info "Remapping MBQL queries is not supported yet" {:source source})))
|
||||
|
||||
(defn- remap-source [table-map field-map source-type source]
|
||||
(case source-type
|
||||
:mbql (remap-mbql-source table-map field-map source)
|
||||
;; TODO make sure it's actually a SQL dialect though..
|
||||
:native (remap-sql-source table-map source)
|
||||
:python (remap-python-source table-map source)))
|
||||
|
||||
;; You might prefer a multi-method? I certainly would.
|
||||
|
||||
(defn- remap-target [table-map target-fallback {d :database, s :schema, t :name :as target}]
|
||||
(if-let [replacement (or (table-map [d s t]) (target-fallback [d s t]))]
|
||||
;; Always fallback to tables for re-mapped outputs, regardless of the type used in the original target.
|
||||
{:type (:type replacement "table")
|
||||
:database (:db-id replacement)
|
||||
:schema (:schema replacement)
|
||||
:name (:table replacement)}
|
||||
target))
|
||||
|
||||
(def ^:private no-mapping {})
|
||||
|
||||
(defn run-transform-with-remapping
|
||||
"Execute a given collection with the given table and field re-mappings.
|
||||
|
||||
This is used by Workspaces to re-route the output of each transform to a non-production table, and
|
||||
to re-write their queries where these outputs transitively become inputs to other transforms.
|
||||
|
||||
Returns an ::ws.t/execution-result map with status, timing, and table metadata.
|
||||
|
||||
-------
|
||||
|
||||
NOTE: currently execution is done using transaction-rollback pattern, as a short-term hack.
|
||||
|
||||
1. Creating temporary Transform/TransformRun records in a transaction
|
||||
2. Executing using existing transform infrastructure
|
||||
3. Scraping metadata (execution stats + table schema)
|
||||
4. Rolling back the transaction (no app DB records persist)
|
||||
|
||||
The warehouse DB changes (actual table data) DO persist in the isolated schema."
|
||||
[{:keys [source target] :as transform} remapping]
|
||||
(t2/with-transaction [_conn]
|
||||
(let [s-type (transforms/transform-source-type source)
|
||||
table-mapping (:tables remapping no-mapping)
|
||||
target-fallback (:target-fallback remapping no-mapping)
|
||||
field-mapping (:fields remapping no-mapping)
|
||||
new-xf (-> (select-keys transform [:name :description])
|
||||
(assoc :creator_id api/*current-user-id*
|
||||
:source (remap-source table-mapping field-mapping s-type source)
|
||||
:target (remap-target table-mapping target-fallback target)))
|
||||
_ (assert (:target new-xf) "Target mapping must not be nil")
|
||||
temp-xf (t2/insert-returning-instance! :model/Transform new-xf)]
|
||||
(try
|
||||
(transforms.execute/execute! temp-xf {:run-method :manual})
|
||||
(catch Exception _e
|
||||
;; Execution failed - the TransformRun record has been updated with failure status and message
|
||||
nil))
|
||||
;; Return execution results whether succeeded or failed
|
||||
(u/prog1 (execution-results temp-xf)
|
||||
(t2/delete! :model/Transform (:id temp-xf))))))
|
||||
364
enterprise/backend/src/metabase_enterprise/workspaces/impl.clj
Normal file
364
enterprise/backend/src/metabase_enterprise/workspaces/impl.clj
Normal file
@@ -0,0 +1,364 @@
|
||||
(ns metabase-enterprise.workspaces.impl
|
||||
"Glue code connecting workspace subsystems (dependencies, isolation)."
|
||||
(:require
|
||||
[clojure.set :as set]
|
||||
[metabase-enterprise.workspaces.dag :as ws.dag]
|
||||
[metabase-enterprise.workspaces.dependencies :as ws.deps]
|
||||
[metabase-enterprise.workspaces.execute :as ws.execute]
|
||||
[metabase-enterprise.workspaces.isolation :as ws.isolation]
|
||||
[metabase-enterprise.workspaces.util :as ws.u]
|
||||
[metabase.driver.sql :as driver.sql]
|
||||
[metabase.util :as u]
|
||||
[metabase.util.log :as log]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(defn- query-ungranted-external-inputs
|
||||
"Query for external inputs in a workspace that haven't been granted access yet.
|
||||
External inputs are tables that are not shadowed by any output.
|
||||
Returns seq of WorkspaceInput records where access_granted is false."
|
||||
[workspace-id]
|
||||
;; NOTE: Could optimize with table_id join if workspace_output gets that column,
|
||||
;; but not worth it given the small number of rows per workspace.
|
||||
(t2/select :model/WorkspaceInput
|
||||
:workspace_id workspace-id
|
||||
{:where [:and
|
||||
[:= :workspace_input.access_granted false]
|
||||
[:not [:exists {:select [1]
|
||||
:from [[:workspace_output :wo]]
|
||||
:where [:and
|
||||
[:= :wo.workspace_id :workspace_input.workspace_id]
|
||||
[:= :wo.db_id :workspace_input.db_id]
|
||||
[:or
|
||||
[:and [:= :wo.global_schema nil] [:= :workspace_input.schema nil]]
|
||||
[:= :wo.global_schema :workspace_input.schema]]
|
||||
[:= :wo.global_table :workspace_input.table]]}]]]}))
|
||||
|
||||
(defn- external-input->table
|
||||
[{:keys [schema table]}]
|
||||
{:schema schema
|
||||
:name table})
|
||||
|
||||
(defn sync-grant-accesses!
|
||||
"Grant read access to external input tables for a workspace that haven't been granted yet.
|
||||
External inputs are tables that are read by transforms but not produced by any transform in the workspace.
|
||||
This should be called after adding transforms to a workspace or when re-initializing workspace isolation
|
||||
(e.g., after unarchiving)."
|
||||
[{workspace-id :id :as workspace}]
|
||||
(let [ungranted-inputs (query-ungranted-external-inputs workspace-id)]
|
||||
(if-not (:database_details workspace)
|
||||
;; TODO (chris 2025/12/15) we will want to make this strict before merging to master
|
||||
#_(throw (ex-info "No database details, unable to grant read only access to the service account." {}))
|
||||
(log/warn "No database details, unable to grant read only access to the service account.")
|
||||
(when (seq ungranted-inputs)
|
||||
(let [database (t2/select-one :model/Database :id (:database_id workspace))
|
||||
tables (mapv external-input->table ungranted-inputs)]
|
||||
(try
|
||||
(ws.isolation/grant-read-access-to-tables! database workspace tables)
|
||||
;; Mark inputs as granted after successful grant
|
||||
(t2/update! :model/WorkspaceInput {:id [:in (map :id ungranted-inputs)]}
|
||||
{:access_granted true})
|
||||
(catch Exception e
|
||||
(log/warn e "Error granting RO table permissions"))))))))
|
||||
|
||||
(defn sync-transform-dependencies!
|
||||
"Analyze and persist dependencies for a workspace transform, then grant
|
||||
read access to external input tables."
|
||||
[{workspace-id :id, isolated-schema :schema :as workspace} transform]
|
||||
(let [analysis (ws.deps/analyze-entity :transform transform)]
|
||||
(ws.deps/write-dependencies! workspace-id isolated-schema :transform (:ref_id transform) analysis)
|
||||
(sync-grant-accesses! workspace)))
|
||||
|
||||
(defn- build-remapping [workspace]
|
||||
;; Build table remapping from stored WorkspaceOutput and WorkspaceOutputExternal data.
|
||||
;; Maps [db_id global_schema global_table] -> {:db-id :schema :table :id} for isolated tables.
|
||||
;; Also maps global_table_id -> same. This is more convenient and reliable for MBQL queries and Python transforms.
|
||||
;; Also maps [db_id nil global_table] for tables in the default schema, so unqualified SQL references work.
|
||||
;; This is used to remap queries, sources and targets to reflect the "isolated" tables used to seal the Workspace.
|
||||
(let [outputs (t2/select [:model/WorkspaceOutput
|
||||
:db_id :global_schema :global_table :global_table_id
|
||||
:isolated_schema :isolated_table :isolated_table_id]
|
||||
:workspace_id (:id workspace))
|
||||
external-outputs (t2/select [:model/WorkspaceOutputExternal
|
||||
:db_id :global_schema :global_table :global_table_id
|
||||
:isolated_schema :isolated_table :isolated_table_id]
|
||||
:workspace_id (:id workspace))
|
||||
all-outputs (concat outputs external-outputs)
|
||||
;; Get default schema for each database involved
|
||||
db-ids (into #{} (map :db_id) all-outputs)
|
||||
db-id->default (when (seq db-ids)
|
||||
(t2/select-fn->fn :id #(driver.sql/default-schema (:engine %))
|
||||
[:model/Database :id :engine]
|
||||
:id [:in db-ids]))
|
||||
table-map (reduce
|
||||
(fn [m {:keys [db_id global_schema global_table global_table_id
|
||||
isolated_schema isolated_table isolated_table_id]}]
|
||||
(let [replacement {:db-id db_id
|
||||
:schema isolated_schema
|
||||
:table isolated_table
|
||||
:id isolated_table_id}
|
||||
default-schema (get db-id->default db_id)]
|
||||
(cond-> (assoc m [db_id global_schema global_table] replacement)
|
||||
global_table_id (assoc global_table_id replacement)
|
||||
;; Add nil-schema entry for tables in the default schema
|
||||
(= global_schema default-schema) (assoc [db_id nil global_table] replacement))))
|
||||
{}
|
||||
all-outputs)]
|
||||
{:tables table-map
|
||||
;; We never want to write to any global tables, so remap on-the-fly if we hit an un-mapped target.
|
||||
:target-fallback (fn [[d s t]]
|
||||
(log/warn "Missing remapping for" {:db d :schema s, :table s})
|
||||
{:db-id d, :schema (:schema workspace), :table (ws.u/isolated-table-name s t), :id nil})
|
||||
;; We won't need the field-map until we support MBQL.
|
||||
:fields nil}))
|
||||
|
||||
(defn- backfill-isolated-table-id!
|
||||
"Backfill workspace_output.isolated_table_id FK in the case where we just created the table for the first time."
|
||||
[ref-id]
|
||||
(when-let [table-id (:id (t2/query-one
|
||||
{:from [[:workspace_output :wo]]
|
||||
:join [[:metabase_table :t] [:and
|
||||
[:= :t.db_id :wo.db_id]
|
||||
[:= :t.schema :wo.isolated_schema]
|
||||
[:= :t.name :wo.isolated_table]]]
|
||||
:select [:t.id]
|
||||
:where [:and
|
||||
[:= :wo.ref_id ref-id]
|
||||
[:or [:= :wo.isolated_table_id nil] [:not= :t.id :wo.isolated_table_id]]]}))]
|
||||
(t2/update! :model/WorkspaceOutput {:ref_id ref-id} {:isolated_table_id table-id})))
|
||||
|
||||
(defn- backfill-external-isolated-table-id!
|
||||
"Backfill workspace_output_external.isolated_table_id FK after running an enclosed external transform."
|
||||
[transform-id]
|
||||
(when-let [table-id (:id (t2/query-one
|
||||
{:from [[:workspace_output_external :woe]]
|
||||
:join [[:metabase_table :t] [:and
|
||||
[:= :t.db_id :woe.db_id]
|
||||
[:= :t.schema :woe.isolated_schema]
|
||||
[:= :t.name :woe.isolated_table]]]
|
||||
:select [:t.id]
|
||||
:where [:and
|
||||
[:= :woe.transform_id transform-id]
|
||||
[:or [:= :woe.isolated_table_id nil] [:not= :t.id :woe.isolated_table_id]]]}))]
|
||||
(t2/update! :model/WorkspaceOutputExternal {:transform_id transform-id} {:isolated_table_id table-id})))
|
||||
|
||||
(defn run-transform!
|
||||
"Execute the given workspace transform or enclosed external transform."
|
||||
([workspace transform]
|
||||
(run-transform! workspace transform (build-remapping workspace)))
|
||||
([workspace transform remapping]
|
||||
(let [ref-id (:ref_id transform)
|
||||
external-id (:id transform)
|
||||
result (ws.isolation/with-workspace-isolation workspace (ws.execute/run-transform-with-remapping transform remapping))]
|
||||
(when ref-id
|
||||
(t2/update! :model/WorkspaceTransform ref-id {:last_run_at (:end_time result)
|
||||
:last_run_message (:message result)}))
|
||||
(when (= :succeeded (:status result))
|
||||
(if ref-id
|
||||
;; Workspace transform
|
||||
(backfill-isolated-table-id! ref-id)
|
||||
;; External transform (enclosed in workspace)
|
||||
(backfill-external-isolated-table-id! external-id)))
|
||||
result)))
|
||||
|
||||
;;;; ---------------------------------------- External Transform Sync ----------------------------------------
|
||||
|
||||
(defn- extract-external-transform-ids
|
||||
"Extract IDs of external transforms from graph entities."
|
||||
[entities]
|
||||
(into [] (comp (filter #(= :external-transform (:node-type %)))
|
||||
(map :id))
|
||||
entities))
|
||||
|
||||
(defn- sync-external-outputs!
|
||||
"Sync workspace_output_external table based on enclosed external transforms in the graph.
|
||||
Deletes obsolete rows, upserts current ones, and backfills table IDs where possible."
|
||||
[workspace-id isolated-schema entities]
|
||||
(let [external-tx-ids (extract-external-transform-ids entities)
|
||||
existing-rows (when (seq external-tx-ids)
|
||||
(t2/select-fn->fn :transform_id identity
|
||||
:model/WorkspaceOutputExternal
|
||||
:workspace_id workspace-id))
|
||||
existing-tx-ids (set (keys existing-rows))]
|
||||
;; Delete rows for transforms no longer in the graph
|
||||
(let [obsolete-ids (set/difference existing-tx-ids (set external-tx-ids))]
|
||||
(when (seq obsolete-ids)
|
||||
(t2/delete! :model/WorkspaceOutputExternal
|
||||
:workspace_id workspace-id
|
||||
:transform_id [:in obsolete-ids])))
|
||||
;; Upsert rows for current external transforms
|
||||
(when (seq external-tx-ids)
|
||||
(let [transforms (t2/select [:model/Transform :id :target] :id [:in external-tx-ids])]
|
||||
(doseq [{tx-id :id, {:keys [database schema name]} :target} transforms]
|
||||
(let [isolated-table (ws.u/isolated-table-name schema name)
|
||||
existing (get existing-rows tx-id)
|
||||
;; Try to find table IDs - reuse existing if schema/table match
|
||||
global-table-id (or (when (and existing
|
||||
(= schema (:global_schema existing))
|
||||
(= name (:global_table existing)))
|
||||
(:global_table_id existing))
|
||||
(t2/select-one-fn :id [:model/Table :id]
|
||||
:db_id database :schema schema :name name))
|
||||
isolated-table-id (or (when (and existing
|
||||
(= isolated-schema (:isolated_schema existing))
|
||||
(= isolated-table (:isolated_table existing)))
|
||||
(:isolated_table_id existing))
|
||||
(t2/select-one-fn :id [:model/Table :id]
|
||||
:db_id database :schema isolated-schema :name isolated-table))]
|
||||
(if existing
|
||||
(t2/update! :model/WorkspaceOutputExternal (:id existing)
|
||||
{:db_id database
|
||||
:global_schema schema
|
||||
:global_table name
|
||||
:global_table_id global-table-id
|
||||
:isolated_schema isolated-schema
|
||||
:isolated_table isolated-table
|
||||
:isolated_table_id isolated-table-id})
|
||||
(t2/insert! :model/WorkspaceOutputExternal
|
||||
{:workspace_id workspace-id
|
||||
:transform_id tx-id
|
||||
:db_id database
|
||||
:global_schema schema
|
||||
:global_table name
|
||||
:global_table_id global-table-id
|
||||
:isolated_schema isolated-schema
|
||||
:isolated_table isolated-table
|
||||
:isolated_table_id isolated-table-id}))))))))
|
||||
|
||||
(defn- normalize-table-schema
|
||||
"Normalize a table's schema: replace nil with the driver's default schema.
|
||||
Takes a map of db_id -> default-schema for lookup."
|
||||
[db-id->default-schema {:keys [db_id schema] :as table}]
|
||||
(if (some? schema)
|
||||
table
|
||||
(assoc table :schema (get db-id->default-schema db_id))))
|
||||
|
||||
(defn- sync-external-inputs!
|
||||
"Sync workspace_input_external table based on enclosed external transforms in the graph.
|
||||
Tracks external tables consumed by enclosed transforms that aren't outputs of other transforms.
|
||||
Inputs are deduplicated per workspace. Normalizes nil schemas to driver defaults."
|
||||
[workspace-id entities]
|
||||
(let [external-tx-ids (extract-external-transform-ids entities)]
|
||||
(when (seq external-tx-ids)
|
||||
;; Query global Dependency table for all inputs of external transforms
|
||||
(let [input-tables (t2/select [:model/Dependency :to_entity_id]
|
||||
:from_entity_type :transform
|
||||
:from_entity_id [:in external-tx-ids]
|
||||
:to_entity_type :table)
|
||||
input-table-ids (into #{} (map :to_entity_id) input-tables)
|
||||
;; Get workspace output table IDs (both workspace and external outputs)
|
||||
workspace-output-ids (t2/select-fn-set :global_table_id
|
||||
[:model/WorkspaceOutput :global_table_id]
|
||||
:workspace_id workspace-id
|
||||
{:where [:not= :global_table_id nil]})
|
||||
external-output-ids (t2/select-fn-set :global_table_id
|
||||
[:model/WorkspaceOutputExternal :global_table_id]
|
||||
:workspace_id workspace-id
|
||||
{:where [:not= :global_table_id nil]})
|
||||
all-output-ids (set/union workspace-output-ids external-output-ids)
|
||||
;; Filter to only true external inputs (not outputs of transforms in graph)
|
||||
external-input-ids (set/difference input-table-ids all-output-ids)]
|
||||
(when (seq external-input-ids)
|
||||
;; Get table details for external inputs
|
||||
(let [tables (t2/select [:model/Table :id :db_id :schema :name] :id [:in external-input-ids])
|
||||
;; Build default schema lookup for each database involved
|
||||
db-ids (into #{} (map :db_id) tables)
|
||||
db-id->default-schema (when (seq db-ids)
|
||||
(into {}
|
||||
(map (fn [{:keys [id engine]}]
|
||||
[id (driver.sql/default-schema engine)]))
|
||||
(t2/select [:model/Database :id :engine] :id [:in db-ids])))
|
||||
;; Normalize schemas: nil -> default schema for the table's database
|
||||
normalized-tables (map (partial normalize-table-schema db-id->default-schema) tables)
|
||||
new-inputs (into {}
|
||||
(map (fn [{:keys [id db_id schema name]}]
|
||||
[[db_id schema name] {:table_id id}]))
|
||||
normalized-tables)
|
||||
;; Get existing external inputs
|
||||
existing-inputs (t2/select-fn->fn (juxt :db_id :schema :table)
|
||||
#(select-keys % [:id :table_id])
|
||||
[:model/WorkspaceInputExternal :id :db_id :schema :table :table_id]
|
||||
:workspace_id workspace-id)
|
||||
existing-keys (set (keys existing-inputs))
|
||||
new-keys (set (keys new-inputs))
|
||||
obsolete-keys (set/difference existing-keys new-keys)
|
||||
missing-keys (set/difference new-keys existing-keys)]
|
||||
;; Delete obsolete rows
|
||||
(doseq [[db_id schema table] obsolete-keys]
|
||||
(t2/delete! :model/WorkspaceInputExternal
|
||||
:workspace_id workspace-id
|
||||
:db_id db_id
|
||||
:schema schema
|
||||
:table table))
|
||||
;; Insert missing rows
|
||||
(when (seq missing-keys)
|
||||
(t2/insert! :model/WorkspaceInputExternal
|
||||
(for [[db_id schema table] missing-keys]
|
||||
{:workspace_id workspace-id
|
||||
:db_id db_id
|
||||
:schema schema
|
||||
:table table
|
||||
:table_id (:table_id (get new-inputs [db_id schema table]))
|
||||
:access_granted false})))))))))
|
||||
|
||||
;; TODO save graph with invalidation hooks
|
||||
(defn get-or-calculate-graph
|
||||
"Return the graph. Also syncs workspace_output_external and workspace_input_external tables."
|
||||
[{ws-id :id, isolated-schema :schema :as _workspace}]
|
||||
(let [graph (ws.dag/path-induced-subgraph ws-id (t2/select-fn-vec (fn [{:keys [ref_id]}] {:entity-type :transform :id ref_id})
|
||||
[:model/WorkspaceTransform :ref_id]
|
||||
:workspace_id ws-id))]
|
||||
(sync-external-outputs! ws-id isolated-schema (:entities graph))
|
||||
(sync-external-inputs! ws-id (:entities graph))
|
||||
graph))
|
||||
|
||||
(defn- transforms-to-execute
|
||||
"Given a workspace and an optional filter, return the global and workspace definitions to run, in the correct order."
|
||||
[{ws-id :id :as workspace} & {:keys [stale-only?]}]
|
||||
;; 1. Depending on what we end up storing in this field, we might not be considering stale ancestors.
|
||||
;; 2. For now, we never set this field to false, so we'll always run everything, even with the flag.
|
||||
;; Why is there all this weird code then? To avoid unused references.
|
||||
(let [stale-clause (if stale-only? {:where [:= :stale true]} {})
|
||||
entities (:entities (get-or-calculate-graph workspace))
|
||||
type->ids (u/group-by :node-type :id entities)
|
||||
id->tx (merge
|
||||
{}
|
||||
(when-let [ids (seq (type->ids :external-transform))]
|
||||
(t2/select-fn->fn :id identity
|
||||
[:model/Transform :id :name :source :target]
|
||||
:id [:in ids]
|
||||
stale-clause))
|
||||
(when-let [ref-ids (seq (type->ids :workspace-transform))]
|
||||
(t2/select-fn->fn :ref_id identity
|
||||
[:model/WorkspaceTransform :ref_id :name :source :target]
|
||||
:workspace_id ws-id
|
||||
:ref_id [:in ref-ids]
|
||||
stale-clause)))]
|
||||
(keep (comp id->tx :id) entities)))
|
||||
|
||||
(defn- id->str [ref-id-or-id]
|
||||
(if (string? ref-id-or-id)
|
||||
ref-id-or-id
|
||||
(str "global-id:" ref-id-or-id)))
|
||||
|
||||
(defn execute-workspace!
|
||||
"Execute all the transforms within a given workspace."
|
||||
[workspace & {:keys [stale-only?] :or {stale-only? false}}]
|
||||
(let [ws-id (:id workspace)
|
||||
remapping (build-remapping workspace)]
|
||||
(reduce
|
||||
(fn [acc {external-id :id ref-id :ref_id :as transform}]
|
||||
(let [node-type (if external-id :external-transform :workspace-transform)
|
||||
id-str (id->str (or external-id ref-id))]
|
||||
(try
|
||||
;; Perhaps we want to return some of the metadata from this as well?
|
||||
(if (= :succeeded (:status (run-transform! workspace transform remapping)))
|
||||
(update acc :succeeded conj id-str)
|
||||
;; Perhaps the status might indicate it never ran?
|
||||
(update acc :failed conj id-str))
|
||||
(catch Exception e
|
||||
(log/error e "Failed to execute transform" {:workspace-id ws-id :node-type node-type :id id-str})
|
||||
(update acc :failed conj id-str)))))
|
||||
{:succeeded []
|
||||
:failed []
|
||||
:not_run []}
|
||||
(transforms-to-execute workspace {:stale-only stale-only?}))))
|
||||
@@ -0,0 +1,6 @@
|
||||
(ns metabase-enterprise.workspaces.init
|
||||
"Initialize workspace models. Driver workspace isolation methods are now
|
||||
defined directly in the driver files."
|
||||
(:require
|
||||
[metabase-enterprise.workspaces.models.workspace]
|
||||
[metabase-enterprise.workspaces.models.workspace-transform]))
|
||||
@@ -0,0 +1,59 @@
|
||||
(ns metabase-enterprise.workspaces.isolation
|
||||
"Workspace database isolation functions.
|
||||
|
||||
The actual driver-specific implementations are now in the driver files:
|
||||
- metabase.driver.h2
|
||||
- metabase.driver.postgres
|
||||
- metabase.driver.redshift
|
||||
- metabase.driver.snowflake
|
||||
- metabase.driver.sqlserver
|
||||
- metabase.driver.clickhouse
|
||||
- metabase.driver.bigquery-cloud-sdk
|
||||
|
||||
This namespace provides the public API for workspace isolation."
|
||||
(:require
|
||||
[metabase.driver :as driver]
|
||||
[metabase.driver.util :as driver.u]))
|
||||
|
||||
;;;; Delegation to driver methods
|
||||
;; The actual multimethod implementations are now in the individual driver files.
|
||||
;; These functions dispatch through the driver multimethods.
|
||||
|
||||
(defn init-workspace-database-isolation!
|
||||
"Create database isolation for a workspace. Return the database details.
|
||||
Delegates to driver/init-workspace-isolation!"
|
||||
[database workspace]
|
||||
(driver/init-workspace-isolation! (driver.u/database->driver database) database workspace))
|
||||
|
||||
(defn grant-read-access-to-tables!
|
||||
"Grant read access to these tables.
|
||||
Delegates to driver/grant-workspace-read-access!"
|
||||
[database workspace tables]
|
||||
(driver/grant-workspace-read-access! (driver.u/database->driver database) database workspace tables))
|
||||
|
||||
(defn destroy-workspace-isolation!
|
||||
"Destroy all database resources created for workspace isolation.
|
||||
This includes dropping tables, schemas/databases, users, roles, and logins.
|
||||
Fails fast on first error. Should be called when deleting a workspace.
|
||||
Delegates to driver/destroy-workspace-isolation!"
|
||||
[database workspace]
|
||||
(driver/destroy-workspace-isolation! (driver.u/database->driver database) database workspace))
|
||||
|
||||
;;;; Public API
|
||||
|
||||
(defn ensure-database-isolation!
|
||||
"Wrapper around the driver method, to make migrations easier in future."
|
||||
[workspace database]
|
||||
(init-workspace-database-isolation! database workspace))
|
||||
|
||||
(defn do-with-workspace-isolation
|
||||
"Impl of* with-workspace-isolation*."
|
||||
[workspace thunk]
|
||||
(driver/with-swapped-connection-details (:database_id workspace)
|
||||
(:database_details workspace)
|
||||
(thunk)))
|
||||
|
||||
(defmacro with-workspace-isolation
|
||||
"Execute body with necessary isolation."
|
||||
[workspace & body]
|
||||
`(do-with-workspace-isolation ~workspace (fn [] ~@body)))
|
||||
129
enterprise/backend/src/metabase_enterprise/workspaces/merge.clj
Normal file
129
enterprise/backend/src/metabase_enterprise/workspaces/merge.clj
Normal file
@@ -0,0 +1,129 @@
|
||||
(ns metabase-enterprise.workspaces.merge
|
||||
"Update global entities based on the definitions within a workspace.
|
||||
All functions assumes that validation of the dependencies AFTER merge is done BEFORE calling any of these methods."
|
||||
(:require
|
||||
[metabase-enterprise.transforms.api :as transforms.api]
|
||||
[metabase-enterprise.workspaces.models.workspace-merge]
|
||||
[metabase-enterprise.workspaces.models.workspace-merge-transform]
|
||||
[metabase-enterprise.workspaces.types :as ws.types]
|
||||
[metabase.util.malli :as mu]
|
||||
[toucan2.core :as t2])
|
||||
(:import (java.sql Connection)))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(defn- create-merge-transform-history!
|
||||
"Create a workspace_merge_transform record to track this merge operation.
|
||||
Does nothing for :noop operations (archived transforms that don't exist globally)
|
||||
or :delete operations (history would be CASCADE deleted with the transform anyway)."
|
||||
[op merge-map]
|
||||
(when (#{:create :update} op)
|
||||
(t2/insert! :model/WorkspaceMergeTransform merge-map)))
|
||||
|
||||
;; TODO (crisptrutski 2025/12/10): When there are more entity types support, this should update those too.
|
||||
(mu/defn merge-transform! :- [:map
|
||||
[:op [:enum :create :delete :noop :update]]
|
||||
[:ref_id ::ws.types/ref-id]
|
||||
[:global_id {:optional true} [:maybe ::ws.types/appdb-id]]
|
||||
[:error {:optional true} :any]]
|
||||
"Make the given transform in the Changeset public, i.e. create or update the relevant model/Transform entities.
|
||||
This should also clear it out from the Changset, as it no longer has any changes.
|
||||
|
||||
Creates a WorkspaceMergeTransform history record.
|
||||
- :ws-transform - The workspace transform to merge
|
||||
- :workspace - The workspace map (must have :id and :name keys)
|
||||
- :workspace-merge-id - ID of parent WorkspaceMerge (caller must create this)
|
||||
- :merging-user-id - user performing the merge
|
||||
- :commit-message - description of the merge"
|
||||
[{:keys [ws-transform workspace workspace-merge-id merging-user-id commit-message]}]
|
||||
;; Problems this may run into:
|
||||
;; It will recalculate and validate the dag greedily as it inserts each items, and this might fail of temporary conflicts.
|
||||
;; Some of these conflicts could be solved by ordering things smartly, but in the general case (e.g. reversing a
|
||||
;; chain) this will not be possible - we would need to defer all validation and calculation, which is a bit scary.
|
||||
;; Let's just focus on the happy path for now though, as this is a fundamental problem and unrelated to the design.
|
||||
(let [{:keys [global_id ref_id archived_at]} ws-transform
|
||||
{:keys [error] :as result}
|
||||
(cond (and global_id archived_at)
|
||||
(merge
|
||||
{:op :delete :global_id global_id :ref_id ref_id}
|
||||
(try
|
||||
(transforms.api/delete-transform! (t2/select-one :model/Transform :id global_id))
|
||||
nil
|
||||
(catch Throwable e
|
||||
{:error e})))
|
||||
|
||||
global_id
|
||||
(merge
|
||||
{:op :update :global_id global_id :ref_id ref_id}
|
||||
(try
|
||||
(transforms.api/update-transform!
|
||||
global_id (select-keys ws-transform [:name :description :source :target]))
|
||||
nil
|
||||
(catch Throwable e
|
||||
{:error e})))
|
||||
|
||||
archived_at
|
||||
{:op :noop :global_id nil :ref_id ref_id}
|
||||
|
||||
:else
|
||||
(merge
|
||||
{:op :create :global_id nil :ref_id ref_id}
|
||||
(try
|
||||
{:global_id (:id (transforms.api/create-transform!
|
||||
(select-keys ws-transform [:name :description :source :target])
|
||||
(:creator_id ws-transform)))}
|
||||
(catch Throwable e
|
||||
{:error e}))))]
|
||||
|
||||
(when-not error
|
||||
(t2/delete! :model/WorkspaceTransform :ref_id ref_id)
|
||||
(create-merge-transform-history!
|
||||
(:op result)
|
||||
{:workspace_merge_id workspace-merge-id
|
||||
:workspace_id (:id workspace)
|
||||
:workspace_name (:name workspace)
|
||||
:transform_id (:global_id result)
|
||||
:commit_message commit-message
|
||||
:merging_user_id merging-user-id}))
|
||||
|
||||
result))
|
||||
|
||||
;; TODO (crisptrutski 2025-12-XX): When there are more entity types support, this should update those too.
|
||||
(defn merge-workspace!
|
||||
"Make all the transforms in the Changeset public, i.e. create or update the relevant model/Transform entities.
|
||||
This should also clear the entire Changeset, as it no longer has any changes.
|
||||
|
||||
- workspace: The workspace map (must have :id and :name keys)
|
||||
- merging-user-id: user performing the merge
|
||||
- commit-message: description of the merge"
|
||||
[workspace merging-user-id commit-message]
|
||||
;; Perhaps we want to solve the N+1?
|
||||
;; This will require reworking the lifecycle for validating and recalculating dependencies for the transforms.
|
||||
(t2/with-transaction [tx]
|
||||
(let [savepoint (.setSavepoint ^Connection tx)
|
||||
ws-merge-id (t2/insert-returning-pk!
|
||||
:model/WorkspaceMerge
|
||||
{:workspace_id (:id workspace)
|
||||
:workspace_name (:name workspace)
|
||||
:commit_message commit-message
|
||||
:creator_id merging-user-id})
|
||||
result (reduce
|
||||
(fn [acc ws-transform]
|
||||
(let [{:keys [error] :as result} (merge-transform! {:ws-transform ws-transform
|
||||
:workspace workspace
|
||||
:workspace-merge-id ws-merge-id
|
||||
:merging-user-id merging-user-id
|
||||
:commit-message commit-message})]
|
||||
(if error
|
||||
(reduced (-> acc
|
||||
(update :errors conj result)
|
||||
(assoc-in [:merged :transforms] [])
|
||||
#_(assoc :short_circuit true)))
|
||||
(update-in acc [:merged :transforms] conj result))))
|
||||
{:merged {:transforms []}
|
||||
:errors []
|
||||
#_#_:short_circuit false}
|
||||
(t2/select :model/WorkspaceTransform :workspace_id (:id workspace)))]
|
||||
(when (seq (:errors result))
|
||||
(.rollback ^Connection tx savepoint))
|
||||
result)))
|
||||
@@ -0,0 +1,71 @@
|
||||
(ns metabase-enterprise.workspaces.models.workspace
|
||||
(:require
|
||||
[metabase-enterprise.workspaces.impl :as ws.impl]
|
||||
[metabase-enterprise.workspaces.isolation :as ws.isolation]
|
||||
[metabase.models.interface :as mi]
|
||||
[metabase.util.log :as log]
|
||||
[methodical.core :as methodical]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(methodical/defmethod t2/table-name :model/Workspace [_model] :workspace)
|
||||
|
||||
(def workspace-statuses
|
||||
"Set of valid workspace statuses."
|
||||
#{:uninitialized :pending :ready :setup-failed :archived}
|
||||
#_#{:enum :uninitialized :database-not-read :graph-not-ready :ready})
|
||||
|
||||
(t2/deftransforms :model/Workspace
|
||||
{:graph mi/transform-json
|
||||
:database_details mi/transform-encrypted-json
|
||||
:status (mi/transform-validator mi/transform-keyword (partial mi/assert-enum workspace-statuses))})
|
||||
|
||||
(doto :model/Workspace
|
||||
(derive :metabase/model)
|
||||
(derive :hook/timestamped?))
|
||||
|
||||
(defn archive!
|
||||
"Archive a workspace. Destroys database isolation resources (best-effort), revokes access grants, sets status to :archived.
|
||||
Cleanup failures are logged but don't block archiving - the workspace can be unarchived to retry cleanup,
|
||||
or deleted once the underlying permission issues are resolved."
|
||||
[{workspace-id :id :as workspace}]
|
||||
;; Only destroy isolation if workspace was initialized (not uninitialized status)
|
||||
(when (not= :uninitialized (:status workspace))
|
||||
(let [database (t2/select-one :model/Database :id (:database_id workspace))]
|
||||
;; Best-effort cleanup - don't block archiving on cleanup failures
|
||||
(try
|
||||
(ws.isolation/destroy-workspace-isolation! database workspace)
|
||||
(catch Exception e
|
||||
(log/warnf e "Failed to cleanup isolation resources for workspace %s, proceeding with archive" workspace-id)))
|
||||
;; Mark all inputs as un-granted since the user may have been dropped
|
||||
(t2/update! :model/WorkspaceInput {:workspace_id workspace-id}
|
||||
{:access_granted false})))
|
||||
(t2/update! :model/Workspace workspace-id {:status :archived}))
|
||||
|
||||
(defn unarchive!
|
||||
"Unarchive a workspace. Re-initializes database isolation resources, re-grants read access
|
||||
to source tables, and changes status back to :ready.
|
||||
Cannot unarchive an uninitialized workspace (no isolation resources to restore)."
|
||||
[workspace]
|
||||
(when (= :uninitialized (:status workspace))
|
||||
(throw (ex-info "Cannot unarchive an uninitialized workspace" {:status-code 400})))
|
||||
(let [database (t2/select-one :model/Database (:database_id workspace))
|
||||
isolation-result (ws.isolation/ensure-database-isolation! workspace database)
|
||||
workspace-id (:id workspace)
|
||||
_ (t2/update! :model/Workspace workspace-id
|
||||
(merge {:status :ready}
|
||||
(select-keys isolation-result [:schema :database_details])))
|
||||
;; Re-fetch workspace to get updated database_details and schema for granting permissions
|
||||
updated-ws (t2/select-one :model/Workspace workspace-id)]
|
||||
;; Re-grant read access to source tables (inputs were marked un-granted during archive)
|
||||
(ws.impl/sync-grant-accesses! updated-ws)))
|
||||
|
||||
(defn delete!
|
||||
"Delete a workspace. Destroys database isolation resources and deletes the workspace."
|
||||
[workspace]
|
||||
;; Only destroy isolation if workspace was initialized (not uninitialized status)
|
||||
(when (not= :uninitialized (:status workspace))
|
||||
(when-let [database (t2/select-one :model/Database :id (:database_id workspace))]
|
||||
(ws.isolation/destroy-workspace-isolation! database workspace)
|
||||
(t2/update! :model/WorkspaceInput {:workspace_id (:id workspace)}
|
||||
{:access_granted false})))
|
||||
(t2/delete! :model/Workspace (:id workspace)))
|
||||
@@ -0,0 +1,17 @@
|
||||
(ns metabase-enterprise.workspaces.models.workspace-dependency
|
||||
"Model for WorkspaceDependency - edges between workspace transforms and their dependencies."
|
||||
(:require
|
||||
[metabase.models.interface :as mi]
|
||||
[methodical.core :as methodical]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(methodical/defmethod t2/table-name :model/WorkspaceDependency [_model] :workspace_dependency)
|
||||
|
||||
(doto :model/WorkspaceDependency
|
||||
(derive :metabase/model)
|
||||
(derive :hook/created-at-timestamped?))
|
||||
|
||||
(t2/deftransforms :model/WorkspaceDependency
|
||||
{:from_entity_type mi/transform-keyword
|
||||
:to_entity_type mi/transform-keyword
|
||||
:from_entity_id mi/transform-trim})
|
||||
@@ -0,0 +1,11 @@
|
||||
(ns metabase-enterprise.workspaces.models.workspace-input
|
||||
"Model for WorkspaceInput - external tables that workspace transforms consume."
|
||||
(:require
|
||||
[methodical.core :as methodical]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(methodical/defmethod t2/table-name :model/WorkspaceInput [_model] :workspace_input)
|
||||
|
||||
(doto :model/WorkspaceInput
|
||||
(derive :metabase/model)
|
||||
(derive :hook/timestamped?))
|
||||
@@ -0,0 +1,11 @@
|
||||
(ns metabase-enterprise.workspaces.models.workspace-input-external
|
||||
"Model for WorkspaceInputExternal - external tables that enclosed external transforms consume."
|
||||
(:require
|
||||
[methodical.core :as methodical]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(methodical/defmethod t2/table-name :model/WorkspaceInputExternal [_model] :workspace_input_external)
|
||||
|
||||
(doto :model/WorkspaceInputExternal
|
||||
(derive :metabase/model)
|
||||
(derive :hook/timestamped?))
|
||||
@@ -0,0 +1,70 @@
|
||||
(ns metabase-enterprise.workspaces.models.workspace-log
|
||||
(:require
|
||||
[java-time.api :as t]
|
||||
[metabase.models.interface :as mi]
|
||||
[metabase.util.i18n :refer [deferred-tru]]
|
||||
[methodical.core :as methodical]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(methodical/defmethod t2/table-name :model/WorkspaceLog [_model] :workspace_log)
|
||||
|
||||
(doto :model/WorkspaceLog
|
||||
(derive :metabase/model)
|
||||
(derive :hook/timestamped?))
|
||||
|
||||
(t2/deftransforms :model/WorkspaceLog
|
||||
{:task mi/transform-keyword
|
||||
:status mi/transform-keyword})
|
||||
|
||||
(defn- task->description
|
||||
"Return a human-readable description for a workspace task."
|
||||
[task]
|
||||
(case task
|
||||
:workspace-setup (deferred-tru "Setting up the workspace")
|
||||
:database-isolation (deferred-tru "Provisioning database isolation")
|
||||
:mirror-entities (deferred-tru "Mirroring entities")
|
||||
:grant-read-access (deferred-tru "Granting permissions")
|
||||
(name task)))
|
||||
|
||||
(t2/define-after-select :model/WorkspaceLog
|
||||
[{:keys [task] :as log}]
|
||||
(assoc log :description (task->description task)))
|
||||
|
||||
(defn start!
|
||||
"Create a log entry for a workspace setup task. Returns the created log."
|
||||
[workspace-id task]
|
||||
(t2/insert-returning-instance! :model/WorkspaceLog
|
||||
{:workspace_id workspace-id
|
||||
:task task
|
||||
:started_at (t/offset-date-time)
|
||||
:status :started}))
|
||||
|
||||
(defn- complete!
|
||||
[status log-id & [message]]
|
||||
(assert (#{:success :failure} status) "It can only be either success or failure")
|
||||
(let [data (cond-> {:completed_at (t/offset-date-time)
|
||||
:status status}
|
||||
message (assoc :message message))]
|
||||
(t2/update! :model/WorkspaceLog log-id data)))
|
||||
|
||||
(defn success!
|
||||
"Task was completed successfully"
|
||||
[log-id & [message]]
|
||||
(complete! :success log-id message))
|
||||
|
||||
(defn failure!
|
||||
"Task has failed"
|
||||
[log-id & [message]]
|
||||
(complete! :failure log-id message))
|
||||
|
||||
(defmacro track!
|
||||
"Wrap body with automatic logging: starts a log entry, marks success on completion, failure on exception."
|
||||
[workspace-id task & body]
|
||||
`(let [log# (start! ~workspace-id ~task)]
|
||||
(try
|
||||
(let [result# (do ~@body)]
|
||||
(success! (:id log#))
|
||||
result#)
|
||||
(catch Exception e#
|
||||
(failure! (:id log#) (ex-message e#))
|
||||
(throw e#)))))
|
||||
@@ -0,0 +1,14 @@
|
||||
(ns metabase-enterprise.workspaces.models.workspace-merge
|
||||
"Model for WorkspaceMerge - tracks batch merge events with commit messages.
|
||||
Survives workspace deletion through inlined workspace_name."
|
||||
(:require
|
||||
[methodical.core :as methodical]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
;;; -------------------------------------------------- Model Setup --------------------------------------------------
|
||||
|
||||
(methodical/defmethod t2/table-name :model/WorkspaceMerge [_model] :workspace_merge)
|
||||
|
||||
(doto :model/WorkspaceMerge
|
||||
(derive :metabase/model)
|
||||
(derive :hook/created-at-timestamped?))
|
||||
@@ -0,0 +1,15 @@
|
||||
(ns metabase-enterprise.workspaces.models.workspace-merge-transform
|
||||
"Model for WorkspaceMergeTransform - tracks individual transform merges.
|
||||
Linked to WorkspaceMerge for batch merges, standalone for single-transform merges.
|
||||
Survives workspace deletion through inlined workspace_name."
|
||||
(:require
|
||||
[methodical.core :as methodical]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
;;; -------------------------------------------------- Model Setup --------------------------------------------------
|
||||
|
||||
(methodical/defmethod t2/table-name :model/WorkspaceMergeTransform [_model] :workspace_merge_transform)
|
||||
|
||||
(doto :model/WorkspaceMergeTransform
|
||||
(derive :metabase/model)
|
||||
(derive :hook/created-at-timestamped?))
|
||||
@@ -0,0 +1,15 @@
|
||||
(ns metabase-enterprise.workspaces.models.workspace-output
|
||||
"Model for WorkspaceOutput - tables that workspace transforms produce."
|
||||
(:require
|
||||
[metabase.models.interface :as mi]
|
||||
[methodical.core :as methodical]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(methodical/defmethod t2/table-name :model/WorkspaceOutput [_model] :workspace_output)
|
||||
|
||||
(doto :model/WorkspaceOutput
|
||||
(derive :metabase/model)
|
||||
(derive :hook/timestamped?))
|
||||
|
||||
(t2/deftransforms :model/WorkspaceOutput
|
||||
{:ref_id mi/transform-trim})
|
||||
@@ -0,0 +1,11 @@
|
||||
(ns metabase-enterprise.workspaces.models.workspace-output-external
|
||||
"Model for WorkspaceOutputExternal - tables that enclosed external transforms produce."
|
||||
(:require
|
||||
[methodical.core :as methodical]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(methodical/defmethod t2/table-name :model/WorkspaceOutputExternal [_model] :workspace_output_external)
|
||||
|
||||
(doto :model/WorkspaceOutputExternal
|
||||
(derive :metabase/model)
|
||||
(derive :hook/timestamped?))
|
||||
@@ -0,0 +1,49 @@
|
||||
(ns metabase-enterprise.workspaces.models.workspace-transform
|
||||
"Model for WorkspaceTransform - holds the changeset of transforms being created
|
||||
and edited within a workspace."
|
||||
(:require
|
||||
[clojure.string :as str]
|
||||
[medley.core :as m]
|
||||
[metabase.lib-be.core :as lib-be]
|
||||
[metabase.lib.core :as lib]
|
||||
[metabase.models.interface :as mi]
|
||||
[methodical.core :as methodical]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(methodical/defmethod t2/table-name :model/WorkspaceTransform [_model] :workspace_transform)
|
||||
|
||||
(methodical/defmethod t2/primary-keys :model/WorkspaceTransform [_model] [:ref_id])
|
||||
|
||||
(doto :model/WorkspaceTransform
|
||||
(derive :metabase/model)
|
||||
(derive :hook/timestamped?)
|
||||
(derive :hook/entity-id))
|
||||
|
||||
;; TODO (chris 2025/12/11) we need to share a bunch of stuff with transforms, i think we'll need to reorganize modules
|
||||
;; suggestion: add a transforms-interfaces module which both transforms and workspaces depend on.
|
||||
|
||||
(defn- transform-source-out-DUPLICATED [m]
|
||||
(-> m
|
||||
mi/json-out-without-keywordization
|
||||
(update-keys keyword)
|
||||
(m/update-existing :query lib-be/normalize-query)
|
||||
(m/update-existing :type keyword)
|
||||
(m/update-existing :source-incremental-strategy #(update-keys % keyword))))
|
||||
|
||||
(defn- transform-source-in-DUPLICATED [m]
|
||||
(-> m
|
||||
(m/update-existing :query (comp lib/prepare-for-serialization lib-be/normalize-query))
|
||||
mi/json-in))
|
||||
|
||||
(t2/deftransforms :model/WorkspaceTransform
|
||||
{:ref_id {:in identity :out str/trim}
|
||||
:source {:out transform-source-out-DUPLICATED, :in transform-source-in-DUPLICATED}
|
||||
:target mi/transform-json})
|
||||
|
||||
(t2/define-before-insert :model/WorkspaceTransform
|
||||
[instance]
|
||||
;; Ref id can not be added here due to mysql t2 bug
|
||||
(when-not (string? (not-empty (:ref_id instance)))
|
||||
(throw (ex-info "ref_id required for WorkspaceTransform insertion."
|
||||
{:instance instance})))
|
||||
instance)
|
||||
161
enterprise/backend/src/metabase_enterprise/workspaces/types.clj
Normal file
161
enterprise/backend/src/metabase_enterprise/workspaces/types.clj
Normal file
@@ -0,0 +1,161 @@
|
||||
(ns metabase-enterprise.workspaces.types
|
||||
(:require
|
||||
[metabase.util.malli.registry :as mr]
|
||||
[metabase.util.malli.schema :as ms]))
|
||||
|
||||
(mr/def ::appdb-id ms/PositiveInt)
|
||||
|
||||
(mr/def ::ref-id [:string {:min 1 :api/regex #".+"}])
|
||||
|
||||
(def entity-types
|
||||
"The kinds of entities we can store within a Workspace."
|
||||
[{:name "Transform"
|
||||
:key :transform
|
||||
:group :transforms
|
||||
:model :model/Transform}])
|
||||
|
||||
(mr/def ::entity-type (into [:enum] (map :key) entity-types))
|
||||
|
||||
(mr/def ::entity-grouping (into [:enum] (map :group) entity-types))
|
||||
|
||||
;; Map like {:transforms [1 2 3]}
|
||||
(mr/def ::entity-map
|
||||
[:map-of ::entity-grouping [:sequential ms/PositiveInt]])
|
||||
|
||||
(mr/def ::execution-result
|
||||
[:map
|
||||
[:status [:enum :succeeded :failed]]
|
||||
[:start_time {:optional true} [:maybe some?]]
|
||||
[:end_time {:optional true} [:maybe some?]]
|
||||
[:message {:optional true} [:maybe :string]]
|
||||
[:table [:map
|
||||
[:name :string]
|
||||
[:schema {:optional true} [:maybe :string]]]]])
|
||||
|
||||
;;; ---------------------------------------- Graph/Problem Types ----------------------------------------
|
||||
|
||||
;; Node types used in workspace graphs and problem detection
|
||||
(mr/def ::node-type [:enum :input-table :external-transform :workspace-transform])
|
||||
|
||||
(mr/def ::table-coord
|
||||
"Logical reference to a table by database, schema, and name."
|
||||
[:map
|
||||
[:db_id ::appdb-id]
|
||||
[:schema [:maybe :string]]
|
||||
[:table :string]])
|
||||
|
||||
(mr/def ::node-ref
|
||||
"Reference to a node in the graph or a related entity."
|
||||
[:map
|
||||
[:type ::node-type]
|
||||
[:id [:or ::appdb-id ::ref-id]]
|
||||
[:name {:optional true} :string]])
|
||||
|
||||
;;; ---------------------------------------- Problem Types ----------------------------------------
|
||||
|
||||
;; Problem type definitions with metadata for FE display and merge-blocking behavior
|
||||
;; See validation.clj docstring for full documentation table
|
||||
|
||||
(def problem-types
|
||||
"All problem types with their metadata.
|
||||
|
||||
Each entry has:
|
||||
- :description - human-readable explanation
|
||||
- :severity - :error, :warning, or :info
|
||||
- :block-merge - whether this problem should prevent merging
|
||||
- :static? - true if determinable from metadata alone, false if requires runtime info
|
||||
|
||||
## Categories
|
||||
|
||||
Dependency-based (about output state) - prefix indicates who depends on the output:
|
||||
- unused-* : nothing depends on this output
|
||||
- internal-downstream-* : transforms within the workspace graph depend on this
|
||||
- external-downstream-* : transforms outside the workspace depend on this
|
||||
|
||||
Structural (about graph shape) - prefix indicates scope of the issue:
|
||||
- internal-* : between workspace transforms only
|
||||
- external-* : involves global transforms (upstream or downstream)"
|
||||
{;; No dependents (informational)
|
||||
:unused/not-run {:description "Output hasn't been created yet, but nothing depends on it"
|
||||
:severity :info
|
||||
:block-merge false
|
||||
:static? true}
|
||||
:unused/stale {:description "Output is stale (needs re-run), but nothing depends on it"
|
||||
:severity :info
|
||||
:block-merge false
|
||||
:static? true}
|
||||
:unused/failed {:description "Transform failed on last run, but nothing depends on it"
|
||||
:severity :error
|
||||
:block-merge false
|
||||
:static? false}
|
||||
|
||||
;; Internal dependents (blocks workspace progress)
|
||||
:internal-downstream/not-run {:description "Output hasn't been created yet, other workspace transforms need it"
|
||||
:severity :warning
|
||||
:block-merge true
|
||||
:static? true}
|
||||
:internal-downstream/stale {:description "Output is stale, other workspace transforms need fresh data"
|
||||
:severity :warning
|
||||
:block-merge true
|
||||
:static? true}
|
||||
:internal-downstream/failed {:description "Transform failed on last run, other workspace transforms need it"
|
||||
:severity :error
|
||||
:block-merge true
|
||||
:static? false}
|
||||
|
||||
;; External dependents (would affect things after merge)
|
||||
:external-downstream/not-run {:description "Output hasn't been created yet, external transforms depend on it"
|
||||
:severity :warning
|
||||
:block-merge true
|
||||
:static? true}
|
||||
:external-downstream/stale {:description "Output is stale, external transforms are using outdated data"
|
||||
:severity :warning
|
||||
:block-merge true
|
||||
:static? true}
|
||||
:external-downstream/removed-field {:description "Field was removed that external transforms reference"
|
||||
:severity :error
|
||||
:block-merge true
|
||||
:static? true}
|
||||
:external-downstream/removed-table {:description "Table was removed, external transforms will stop receiving updates"
|
||||
:severity :warning
|
||||
:block-merge false
|
||||
:static? true}
|
||||
|
||||
;; Structural issues (conflicts/cycles)
|
||||
:internal/target-conflict {:description "Multiple workspace transforms target the same table"
|
||||
:severity :error
|
||||
:block-merge true
|
||||
:static? true}
|
||||
:internal/cycle {:description "Circular dependency between workspace transforms"
|
||||
:severity :error
|
||||
:block-merge true
|
||||
:static? true}
|
||||
:external/target-conflict {:description "Workspace transform conflicts with an existing global transform's target"
|
||||
:severity :error
|
||||
:block-merge true
|
||||
:static? true}
|
||||
:external/cycle {:description "Workspace changes would create a circular dependency with global transforms"
|
||||
:severity :error
|
||||
:block-merge true
|
||||
:static? true}})
|
||||
|
||||
(mr/def ::problem-type (into [:enum] (keys problem-types)))
|
||||
|
||||
(mr/def ::severity [:enum :error :warning :info])
|
||||
|
||||
;; Categories derived from problem-type namespaces
|
||||
(mr/def ::problem-category
|
||||
(into [:enum] (distinct (map (comp keyword namespace) (keys problem-types)))))
|
||||
|
||||
;; Problem names derived from problem-type names
|
||||
(mr/def ::problem-name
|
||||
(into [:enum] (distinct (map (comp keyword name) (keys problem-types)))))
|
||||
|
||||
(mr/def ::problem
|
||||
"A problem detected during workspace validation."
|
||||
[:map
|
||||
[:category ::problem-category]
|
||||
[:problem ::problem-name]
|
||||
[:severity ::severity]
|
||||
[:block_merge :boolean]
|
||||
[:data :map]])
|
||||
167
enterprise/backend/src/metabase_enterprise/workspaces/util.clj
Normal file
167
enterprise/backend/src/metabase_enterprise/workspaces/util.clj
Normal file
@@ -0,0 +1,167 @@
|
||||
(ns metabase-enterprise.workspaces.util
|
||||
(:require
|
||||
[clojure.string :as str]
|
||||
[metabase.system.core :as system]))
|
||||
|
||||
(defn assert-transform!
|
||||
"Test whether we support the given entity type within workspaces yet.
|
||||
Named for the only case we support currently, to make call sites assumptions more obvious."
|
||||
[entity-type]
|
||||
(when (not= "transform" (name entity-type))
|
||||
(throw (ex-info "Only transform entity type is supported"
|
||||
{:status-code 400
|
||||
:entity-type entity-type}))))
|
||||
|
||||
(defn assert-transforms!
|
||||
"Test that only supported types are given in the given list.
|
||||
Named for the only case we support currently, to make call sites assumptions more obvious."
|
||||
[entities]
|
||||
(when-let [other-types (seq (remove #{:transform} (map :entity-type entities)))]
|
||||
(throw (ex-info "Only transform entities are currently supported"
|
||||
{:status-code 400
|
||||
:unsupported-types other-types}))))
|
||||
|
||||
(defn- toposort-visit [node child->parents visited result]
|
||||
(cond
|
||||
(visited node) [visited result]
|
||||
:else (let [parents (child->parents node [])
|
||||
[visited' result'] (reduce (fn [[v r] p]
|
||||
(toposort-visit p child->parents v r))
|
||||
[(conj visited node) result]
|
||||
parents)]
|
||||
[visited' (conj result' node)])))
|
||||
|
||||
(defn toposort-dfs
|
||||
"Perform a topological sort using depth-first search.
|
||||
Takes a map from child nodes to their parent nodes (dependencies).
|
||||
Returns nodes in topological order (dependencies before dependents)."
|
||||
[child->parents]
|
||||
;; TODO (Chris 2025-11-20): Detect cycles and throw an error. (In practice inputs will never be cyclic, but still.)
|
||||
(let [all-nodes (set (keys child->parents))]
|
||||
(loop [visited #{}
|
||||
result []
|
||||
remaining all-nodes]
|
||||
(if (empty? remaining)
|
||||
result
|
||||
(let [node (first remaining)
|
||||
[visited' result'] (toposort-visit node child->parents visited result)]
|
||||
(recur visited' result' (disj remaining node)))))))
|
||||
|
||||
;;; Naming
|
||||
|
||||
;; re-using https://github.com/metabase/metabase/pull/61887/commits/c92e4a9cc451c61a13fef19ed9d6107873b17f07
|
||||
;; (original ws isolation code)
|
||||
(defn- instance-uuid-slug
|
||||
"Create a slug from the site UUID, taking the first character of each section."
|
||||
[site-uuid-string]
|
||||
(->> (str/split site-uuid-string #"-")
|
||||
(map first)
|
||||
(apply str)))
|
||||
|
||||
;; WARNING: Changing this prefix requires backwards compatibility handling for existing workspaces.
|
||||
;; The prefix is used to identify isolation namespaces in the database, and existing workspaces
|
||||
;; will have namespaces created with the current prefix.
|
||||
(def ^:private isolated-prefix "mb__isolation")
|
||||
|
||||
(defn isolation-namespace-name
|
||||
"Generate namespace/database name for workspace isolation following mb__isolation_<slug>_<workspace-id> pattern.
|
||||
Uses 'namespace' as the generic term that maps to 'schema' in Postgres, 'database' in ClickHouse, etc."
|
||||
[workspace]
|
||||
(assert (some? (:id workspace)) "Workspace must have an :id")
|
||||
(let [instance-slug (instance-uuid-slug (str (system/site-uuid)))
|
||||
clean-workspace-id (str/replace (str (:id workspace)) #"[^a-zA-Z0-9]" "_")]
|
||||
(format "%s_%s_%s" isolated-prefix instance-slug clean-workspace-id)))
|
||||
|
||||
(defn isolated-table-name
|
||||
"Generate name for a table mirroring transform target table in the isolated database namespace.
|
||||
Returns schema__name when schema is present, or __name when schema is nil (to distinguish from global tables)."
|
||||
[schema name]
|
||||
;; TODO: This naming scheme is not guaranteed to give a unique name, even if it's likely
|
||||
;; to always be unique in practice. Consider adding a hash suffix or conflict detection
|
||||
;; before merging to master.
|
||||
(if schema
|
||||
(format "%s__%s" schema name)
|
||||
(format "__%s" name)))
|
||||
|
||||
(defn isolation-user-name
|
||||
"Generate username for workspace isolation."
|
||||
[workspace]
|
||||
(let [instance-slug (instance-uuid-slug (str (system/site-uuid)))]
|
||||
(format "%s_%s_%s" isolated-prefix instance-slug (:id workspace))))
|
||||
|
||||
(def ^:private password-char-sets
|
||||
"Character sets for password generation. Cycles through these to ensure representation from each."
|
||||
["ABCDEFGHJKLMNPQRSTUVWXYZ"
|
||||
"abcdefghjkmnpqrstuvwxyz"
|
||||
"123456789"
|
||||
"!#$%&*+-="])
|
||||
|
||||
(defn random-isolated-password
|
||||
"Generate a random password suitable for most database engines.
|
||||
Ensures the password contains characters from all sets (uppercase, lowercase, digits, special)
|
||||
by cycling through the character sets. Result is shuffled for randomness."
|
||||
[]
|
||||
(->> (cycle password-char-sets)
|
||||
(take (+ 32 (rand-int 32)))
|
||||
(map rand-nth)
|
||||
shuffle
|
||||
(apply str)))
|
||||
|
||||
;;; -------------------------------------------------- Cute Names --------------------------------------------------
|
||||
|
||||
(def ^:private adjectives
|
||||
"Adjectives for generating cute ref IDs, inspired by Docker's naming scheme."
|
||||
["admiring" "adoring" "affectionate" "agitated" "amazing" "angry" "awesome"
|
||||
"beautiful" "blissful" "bold" "boring" "brave" "busy" "charming" "clever"
|
||||
"cool" "compassionate" "competent" "condescending" "confident" "cranky"
|
||||
"crazy" "dazzling" "determined" "distracted" "dreamy" "eager" "ecstatic"
|
||||
"elastic" "elated" "elegant" "eloquent" "epic" "exciting" "fervent"
|
||||
"festive" "flamboyant" "focused" "friendly" "frosty" "funny" "gallant"
|
||||
"gifted" "goofy" "gracious" "great" "happy" "hardcore" "heuristic" "hopeful"
|
||||
"hungry" "infallible" "inspiring" "intelligent" "interesting" "jolly"
|
||||
"jovial" "keen" "kind" "laughing" "loving" "lucid" "magical" "modest"
|
||||
"musing" "mystifying" "naughty" "nervous" "nice" "nifty" "nostalgic"
|
||||
"objective" "optimistic" "peaceful" "pedantic" "pensive" "practical"
|
||||
"priceless" "quirky" "quizzical" "recursing" "relaxed" "reverent" "romantic"
|
||||
"sad" "serene" "sharp" "silly" "sleepy" "stoic" "strange" "stupefied"
|
||||
"suspicious" "sweet" "tender" "thirsty" "trusting" "unruffled" "upbeat"
|
||||
"vibrant" "vigilant" "vigorous" "wizardly" "wonderful" "xenodochial" "youthful"
|
||||
"zealous" "zen"])
|
||||
|
||||
(def ^:private animals
|
||||
"Animals for generating cute ref IDs."
|
||||
["albatross" "alligator" "alpaca" "ant" "anteater" "antelope" "armadillo"
|
||||
"baboon" "badger" "bat" "bear" "beaver" "bee" "bison" "boar" "buffalo"
|
||||
"butterfly" "camel" "capybara" "caribou" "cat" "caterpillar" "cheetah"
|
||||
"chicken" "chimpanzee" "chinchilla" "cobra" "condor" "cormorant" "coyote"
|
||||
"crab" "crane" "crocodile" "crow" "deer" "dingo" "dog" "dolphin" "donkey"
|
||||
"dove" "dragonfly" "duck" "eagle" "echidna" "eel" "elephant" "elk" "emu"
|
||||
"falcon" "ferret" "finch" "firefly" "flamingo" "fox" "frog" "gazelle"
|
||||
"gerbil" "giraffe" "goat" "goose" "gopher" "gorilla" "grasshopper" "grouse"
|
||||
"hamster" "hare" "hawk" "hedgehog" "heron" "hippopotamus" "hornet" "horse"
|
||||
"hummingbird" "hyena" "ibex" "iguana" "impala" "jackal" "jaguar" "jellyfish"
|
||||
"kangaroo" "kingfisher" "kiwi" "koala" "komodo" "kookaburra" "lemur"
|
||||
"leopard" "lion" "lizard" "llama" "lobster" "lynx" "magpie" "mallard"
|
||||
"manatee" "mandrill" "marmot" "meerkat" "mink" "mole" "mongoose" "monkey"
|
||||
"moose" "moth" "mouse" "mule" "narwhal" "newt" "nightingale" "ocelot"
|
||||
"octopus" "okapi" "opossum" "orangutan" "orca" "oryx" "osprey" "ostrich"
|
||||
"otter" "owl" "ox" "panda" "panther" "parrot" "partridge" "peacock"
|
||||
"pelican" "penguin" "pheasant" "pig" "pigeon" "platypus" "pony" "porcupine"
|
||||
"porpoise" "puma" "quail" "rabbit" "raccoon" "ram" "raven" "reindeer"
|
||||
"rhinoceros" "salamander" "salmon" "sardine" "scorpion" "seahorse" "seal"
|
||||
"shark" "sheep" "shrew" "skunk" "sloth" "snail" "snake" "sparrow" "spider"
|
||||
"squid" "squirrel" "starfish" "stingray" "stork" "swallow" "swan" "tapir"
|
||||
"tiger" "toad" "toucan" "turkey" "turtle" "viper" "vulture" "wallaby"
|
||||
"walrus" "warthog" "wasp" "weasel" "whale" "wolf" "wolverine" "wombat"
|
||||
"woodpecker" "yak" "zebra"])
|
||||
|
||||
(defn generate-name
|
||||
"Generate a cute adjective-animal style name. Making sure it's unique is your job."
|
||||
[]
|
||||
(str (rand-nth adjectives) "-" (rand-nth animals)))
|
||||
|
||||
(defn generate-ref-id
|
||||
"Generate a cute adjective-animal style ref ID with a random suffix.
|
||||
Format: adjective-animal-xxxx (where xxxx is a random 4-char hex suffix)"
|
||||
[]
|
||||
(str (generate-name) "-" (format "%04x" (rand-int 65536))))
|
||||
@@ -0,0 +1,279 @@
|
||||
(ns metabase-enterprise.workspaces.validation
|
||||
"Validation of workspace changes against downstream dependencies.
|
||||
|
||||
ASSUMPTION: All transforms in the workspace have been run, and their outputs are not stale.
|
||||
This validation checks the *current* isolated table schema against external transform requirements.
|
||||
|
||||
## Problem Types
|
||||
|
||||
See `metabase-enterprise.workspaces.types/problem-types` for the full list of problem types
|
||||
with their descriptions, severity levels, and merge-blocking behavior.
|
||||
|
||||
## Currently Implemented
|
||||
|
||||
| Category | Problem | Description |
|
||||
|---------------------|---------------|----------------------------------------------------------------|
|
||||
| unused | not-run | Output hasn't been created, nothing depends on it |
|
||||
| internal-downstream | not-run | Output hasn't been created, other workspace transforms need it |
|
||||
| external-downstream | not-run | Output hasn't been created, external transforms need it |
|
||||
| external-downstream | removed-field | Field was removed that external transforms reference |"
|
||||
(:require
|
||||
[metabase-enterprise.workspaces.types :as ws.t]
|
||||
[metabase.lib-be.core :as lib-be]
|
||||
[metabase.lib.core :as lib]
|
||||
[metabase.lib.metadata.protocols :as lib.metadata.protocols]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
;;;; ---------------------------------------- Metadata Provider Override ----------------------------------------
|
||||
|
||||
(defn- make-field-override-provider
|
||||
"Create a metadata provider that overrides a table's fields with a different set of fields.
|
||||
Used to simulate what external transforms would see after a merge.
|
||||
|
||||
- `base-provider`: the underlying metadata provider
|
||||
- `target-table-id`: the global table ID whose fields we're overriding
|
||||
- `replacement-fields`: the fields from the isolated table (what fields will exist after merge)"
|
||||
[base-provider target-table-id replacement-fields]
|
||||
(let [replacement-field-names (into #{} (map :name) replacement-fields)
|
||||
;; TODO we probably want to do one query for *all* the tables we care about, earlier in our validation.
|
||||
table-field-id? (t2/select-fn-set :id [:model/Field :id] :table_id target-table-id)]
|
||||
(reify
|
||||
lib.metadata.protocols/MetadataProvider
|
||||
(database [_this]
|
||||
(lib.metadata.protocols/database base-provider))
|
||||
(metadatas [_this metadata-spec]
|
||||
;; Yes, for some reason :id is actually a set of ids
|
||||
(let [{spec-type :lib/type spec-field-ids :id} metadata-spec]
|
||||
(if (and (= :metadata/column spec-type) (some table-field-id? spec-field-ids))
|
||||
;; Return replacement fields with :active based on whether they exist
|
||||
(let [original-fields (lib.metadata.protocols/metadatas base-provider metadata-spec)]
|
||||
(map (fn [field]
|
||||
(if (contains? replacement-field-names (:name field))
|
||||
field
|
||||
(assoc field :active false)))
|
||||
original-fields))
|
||||
;; Delegate everything else
|
||||
(lib.metadata.protocols/metadatas base-provider metadata-spec))))
|
||||
(setting [_this setting-key]
|
||||
(lib.metadata.protocols/setting base-provider setting-key))
|
||||
|
||||
lib.metadata.protocols/CachedMetadataProvider
|
||||
(cached-metadatas [_this metadata-type metadata-ids]
|
||||
(lib.metadata.protocols/cached-metadatas base-provider metadata-type metadata-ids))
|
||||
(store-metadata! [_this a-metadata]
|
||||
(lib.metadata.protocols/store-metadata! base-provider a-metadata))
|
||||
(cached-value [_this k not-found]
|
||||
(lib.metadata.protocols/cached-value base-provider k not-found))
|
||||
(cache-value! [_this k v]
|
||||
(lib.metadata.protocols/cache-value! base-provider k v))
|
||||
(has-cache? [_this]
|
||||
(lib.metadata.protocols/has-cache? base-provider))
|
||||
(clear-cache! [_this]
|
||||
(lib.metadata.protocols/clear-cache! base-provider)))))
|
||||
|
||||
;;;; ---------------------------------------- External Transform Discovery ----------------------------------------
|
||||
|
||||
;; TODO we could join these extra fields onto the graph before calling into this namespace, and save coupling
|
||||
(defn- workspace-outputs
|
||||
"Get all workspace outputs with their global and isolated table info.
|
||||
Includes both workspace transform outputs and enclosed external transform outputs."
|
||||
[workspace-id]
|
||||
(let [ws-outputs (t2/select [:model/WorkspaceOutput
|
||||
:id :ref_id :db_id
|
||||
:global_schema :global_table :global_table_id
|
||||
:isolated_schema :isolated_table :isolated_table_id]
|
||||
:workspace_id workspace-id)
|
||||
;; External outputs use transform_id (int) instead of ref_id (string)
|
||||
;; Map transform_id to ref_id for compatibility with the rest of the validation code
|
||||
ext-outputs (map #(assoc % :ref_id (:transform_id %) :external? true)
|
||||
(t2/select [:model/WorkspaceOutputExternal
|
||||
:id :transform_id :db_id
|
||||
:global_schema :global_table :global_table_id
|
||||
:isolated_schema :isolated_table :isolated_table_id]
|
||||
:workspace_id workspace-id))]
|
||||
(concat ws-outputs ext-outputs)))
|
||||
|
||||
(defn- ids-for-type [entities desired-node-type]
|
||||
(for [{:keys [node-type id]} entities, :when (= node-type desired-node-type)] id))
|
||||
|
||||
(defn- checked-out-transform-ids
|
||||
"Get the global IDs of transforms that are checked out into this workspace.
|
||||
Extracts workspace-transform ref_ids from the graph and looks up their global_ids."
|
||||
[{:keys [entities] :as _graph}]
|
||||
(if-let [ref-ids (seq (ids-for-type entities :workspace-transform))]
|
||||
;; TODO if we keep track of the global_id properties when building the graph, we can save this query.
|
||||
(t2/select-fn-set :global_id
|
||||
[:model/WorkspaceTransform :global_id]
|
||||
:ref_id [:in ref-ids]
|
||||
{:where [:not= :global_id nil]})
|
||||
#{}))
|
||||
|
||||
(defn- enclosed-transform-ids
|
||||
"Get the IDs of external transforms that are enclosed by the workspace graph.
|
||||
These transforms run within the workspace, so they don't need further static validation."
|
||||
[{:keys [entities] :as _graph}]
|
||||
(ids-for-type entities :external-transform))
|
||||
|
||||
(def ^:private table? #{:table})
|
||||
|
||||
(defn- internal-dependents
|
||||
"Build a map of non-table ids to the non-table ids that directly depend on it (internal to workspace).
|
||||
This is essentially an inversion of the dependencies map, with some filtering."
|
||||
[{:keys [dependencies] :as _graph}]
|
||||
;; :dependencies is {child #{parents...}}, we need to invert to {parent #{children...}}
|
||||
(reduce (fn [acc [{child-type :node-type, child-id :id} parents]]
|
||||
(if (table? child-type)
|
||||
acc
|
||||
(reduce (fn [acc {parent-type :node-type, parent-id :id}]
|
||||
(if (table? parent-type)
|
||||
acc
|
||||
(update acc parent-id (fnil conj #{}) child-id)))
|
||||
acc
|
||||
parents)))
|
||||
{}
|
||||
dependencies))
|
||||
|
||||
(defn- external-transform-dependents
|
||||
"Get external transforms that depend on a given table (by global_table_id).
|
||||
Excludes transforms that are checked out or enclosed in the workspace."
|
||||
[global-table-id checked-out-ids enclosed-ids]
|
||||
(when global-table-id
|
||||
(let [dependent-transform-ids (t2/select-fn-set :from_entity_id
|
||||
[:model/Dependency :from_entity_id]
|
||||
:to_entity_type :table
|
||||
:to_entity_id global-table-id
|
||||
:from_entity_type :transform)
|
||||
;; Remove checked out and enclosed transforms
|
||||
external-ids (-> dependent-transform-ids
|
||||
(disj nil)
|
||||
(as-> ids (apply disj ids checked-out-ids))
|
||||
(as-> ids (apply disj ids enclosed-ids)))]
|
||||
(when (seq external-ids)
|
||||
(t2/select [:model/Transform :id :name :source] :id [:in external-ids])))))
|
||||
|
||||
;;;; ---------------------------------------- Validation Logic ----------------------------------------
|
||||
|
||||
(defn- validate-transform-against-fields
|
||||
"Check if a transform would break given a modified set of fields.
|
||||
Returns bad refs if the transform's query references fields that don't exist in replacement-fields."
|
||||
[transform global-table-id replacement-fields]
|
||||
(let [db-id (get-in transform [:source :query :database])]
|
||||
(when db-id
|
||||
(let [base-provider (lib-be/application-database-metadata-provider db-id)
|
||||
override-provider (make-field-override-provider base-provider global-table-id replacement-fields)
|
||||
query (lib/query override-provider (get-in transform [:source :query]))]
|
||||
(lib/find-bad-refs query)))))
|
||||
|
||||
(defn- get-table-fields
|
||||
"Get all active fields for a table by ID."
|
||||
[table-id]
|
||||
(when table-id
|
||||
(t2/select [:model/Field :id :name :base_type :semantic_type]
|
||||
:table_id table-id
|
||||
:active true)))
|
||||
|
||||
(defn- make-problem
|
||||
"Construct a problem map with type metadata from ws.t/problem-types.
|
||||
Splits the namespaced problem-type into :category and :problem keys."
|
||||
[problem-type data]
|
||||
(let [{:keys [severity block-merge description]} (get ws.t/problem-types problem-type)]
|
||||
{:category (keyword (namespace problem-type))
|
||||
:problem (keyword (name problem-type))
|
||||
:severity severity
|
||||
:block_merge block-merge
|
||||
:description description
|
||||
:data data}))
|
||||
|
||||
(defn- problems-for-output
|
||||
"Find problems related to a single workspace output.
|
||||
Returns a sequence of problem maps."
|
||||
[output checked-out-ids enclosed-ids internal-dependents-map]
|
||||
(let [{:keys [ref_id db_id global_schema global_table global_table_id
|
||||
isolated_table_id external?]} output
|
||||
;; For external outputs, ref_id is actually the transform_id (an int)
|
||||
transform-type (if external? :external-transform :workspace-transform)
|
||||
external-transforms (external-transform-dependents global_table_id checked-out-ids enclosed-ids)
|
||||
has-external-dependents? (seq external-transforms)
|
||||
internal-dependents (get internal-dependents-map ref_id)
|
||||
has-internal-dependents? (seq internal-dependents)
|
||||
table-coord {:db_id db_id :schema global_schema :table global_table}]
|
||||
|
||||
(cond
|
||||
;; Case 1: Isolated table doesn't exist yet
|
||||
(nil? isolated_table_id)
|
||||
(cond
|
||||
;; External transforms depend on this output that doesn't exist
|
||||
has-external-dependents?
|
||||
[(make-problem :external-downstream/not-run
|
||||
{:output table-coord
|
||||
:transform {:type transform-type
|
||||
:id ref_id}
|
||||
:dependents (mapv (fn [{:keys [id name]}]
|
||||
{:type :external-transform
|
||||
:id id
|
||||
:name name})
|
||||
external-transforms)})]
|
||||
;; Internal workspace transforms depend on this output that doesn't exist
|
||||
has-internal-dependents?
|
||||
[(make-problem :internal-downstream/not-run
|
||||
{:output table-coord
|
||||
:transform {:type transform-type
|
||||
:id ref_id}
|
||||
:dependents (mapv (fn [dep-ref-id]
|
||||
{:type :workspace-transform
|
||||
:id dep-ref-id})
|
||||
internal-dependents)})]
|
||||
;; No dependents - just informational
|
||||
:else
|
||||
[(make-problem :unused/not-run
|
||||
{:output table-coord
|
||||
:transform {:type transform-type
|
||||
:id ref_id}})])
|
||||
|
||||
;; Case 2: Isolated table exists - check for field breakages
|
||||
has-external-dependents?
|
||||
(let [isolated-fields (get-table-fields isolated_table_id)]
|
||||
(into []
|
||||
(mapcat (fn [transform]
|
||||
(when-let [bad-refs (validate-transform-against-fields
|
||||
transform global_table_id isolated-fields)]
|
||||
[(make-problem :external-downstream/removed-field
|
||||
{:output table-coord
|
||||
:transform {:type :external-transform
|
||||
:id (:id transform)
|
||||
:name (:name transform)}
|
||||
:bad-refs bad-refs})])))
|
||||
external-transforms))
|
||||
|
||||
;; Case 3: No problems
|
||||
:else [])))
|
||||
|
||||
;;;; ---------------------------------------- Public API ----------------------------------------
|
||||
|
||||
(defn find-downstream-problems
|
||||
"Find problems that would affect transforms outside the workspace after merge.
|
||||
|
||||
Takes the workspace graph (from `ws.dag/path-induced-subgraph`) as an argument.
|
||||
|
||||
Returns a sequence of problem maps, each with:
|
||||
- :category - the problem category (e.g. :unused, :internal-downstream, :external-downstream)
|
||||
- :problem - the specific problem (e.g. :not-run, :removed-field)
|
||||
- :severity - :error, :warning, or :info
|
||||
- :block-merge - whether this problem should prevent merging
|
||||
- :data - further data, depending on type
|
||||
|
||||
Currently implemented problem types (category + problem):
|
||||
- :unused/not-run - output table hasn't been created (informational, no dependents)
|
||||
- :internal-downstream/not-run - output table hasn't been created, other workspace transforms need it
|
||||
- :external-downstream/not-run - output table hasn't been created, but external transforms depend on it
|
||||
- :external-downstream/removed-field - a field was removed that external transforms need"
|
||||
[workspace-id graph]
|
||||
(let [outputs (workspace-outputs workspace-id)
|
||||
checked-out-ids (checked-out-transform-ids graph)
|
||||
enclosed-ids (enclosed-transform-ids graph)
|
||||
dependents (internal-dependents graph)]
|
||||
(into []
|
||||
(mapcat #(problems-for-output % checked-out-ids enclosed-ids dependents))
|
||||
outputs)))
|
||||
@@ -181,6 +181,20 @@
|
||||
:table #{checkins-id venues-id}}
|
||||
(calculation/upstream-deps:transform transform)))))))
|
||||
|
||||
(deftest ^:parallel upstream-deps-python-transform-test
|
||||
(mt/with-premium-features #{:transforms}
|
||||
(let [products-id (mt/id :products)
|
||||
orders-id (mt/id :orders)]
|
||||
(mt/with-temp [:model/Transform transform {:name "Test Transform"
|
||||
:source {:type :python
|
||||
:source-tables {"PRODUCTS" products-id
|
||||
"ORDERS" orders-id}
|
||||
:body "..."}
|
||||
:target {:schema "PUBLIC"
|
||||
:name "test_output"}}]
|
||||
(is (= {:table #{products-id orders-id}}
|
||||
(calculation/upstream-deps:transform transform)))))))
|
||||
|
||||
(deftest ^:parallel upstream-deps-card-native-with-parameter-source-test
|
||||
(let [mp (mt/metadata-provider)
|
||||
products-id (mt/id :products)
|
||||
|
||||
@@ -107,7 +107,19 @@
|
||||
:model/ConnectionImpersonation
|
||||
:model/CloudMigration
|
||||
:model/Comment
|
||||
:model/CommentReaction})
|
||||
:model/CommentReaction
|
||||
;; TODO (lbrdnk 2025/12/17): I've added rest of the workspace models here as Workspace was present. I believe
|
||||
;; going forward all of that will be available for export. We should revisit this later in the project.
|
||||
:model/Workspace
|
||||
:model/WorkspaceDependency
|
||||
:model/WorkspaceInput
|
||||
:model/WorkspaceInputExternal
|
||||
:model/WorkspaceLog
|
||||
:model/WorkspaceMerge
|
||||
:model/WorkspaceMergeTransform
|
||||
:model/WorkspaceOutput
|
||||
:model/WorkspaceOutputExternal
|
||||
:model/WorkspaceTransform})
|
||||
|
||||
(deftest ^:parallel comprehensive-entity-id-test
|
||||
(let [entity-id-models (->> (v2.entity-ids/toucan-models)
|
||||
|
||||
@@ -65,9 +65,11 @@
|
||||
(testing "GET /api/ee/transform-job/:id/transforms"
|
||||
(mt/with-premium-features #{:transforms}
|
||||
(let [crowberto-id (mt/user->id :crowberto)]
|
||||
(mt/with-temp [:model/Transform {transform1-id :id} {:name "tr1" :creator_id crowberto-id}
|
||||
:model/Transform {transform2-id :id} {:name "tr2" :creator_id crowberto-id}
|
||||
:model/Transform _transform3 {}
|
||||
(mt/with-temp [:model/Transform {transform1-id :id} {:name "tr1" :creator_id crowberto-id
|
||||
:target {:database (mt/id)}}
|
||||
:model/Transform {transform2-id :id} {:name "tr2" :creator_id crowberto-id
|
||||
:target {:database (mt/id)}}
|
||||
:model/Transform _transform3 {:target {:database (mt/id)}}
|
||||
:model/TransformTag {tag-id :id} {}
|
||||
:model/TransformTransformTag _ {:transform_id transform1-id :tag_id tag-id :position 0}
|
||||
:model/TransformTransformTag _ {:transform_id transform2-id :tag_id tag-id :position 0}
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
with-transform-cleanup!]]
|
||||
[metabase-enterprise.transforms.util :as transforms.util]
|
||||
[metabase.driver :as driver]
|
||||
[metabase.lib-be.metadata.jvm :as lib.metadata.jvm]
|
||||
[metabase.lib.core :as lib]
|
||||
[metabase.lib.metadata :as lib.metadata]
|
||||
[metabase.query-processor :as qp]
|
||||
@@ -278,6 +279,139 @@
|
||||
(is (=? [{:id t2-id}]
|
||||
(mt/user-http-request :crowberto :get 200 "ee/transform" :tag_ids [tag2-id])))))))))
|
||||
|
||||
(deftest filter-transforms-by-type-test
|
||||
(testing "should be able to filter transforms by source type"
|
||||
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
|
||||
(mt/with-premium-features #{:transforms :transforms-python}
|
||||
(let [mp (mt/metadata-provider)
|
||||
nq (lib/native-query mp "select 1")]
|
||||
(mt/with-temp [:model/Transform {native-transform-id :id}
|
||||
{:name "Native Transform"
|
||||
:source {:type "query"
|
||||
:query nq}
|
||||
:target {:type "table"
|
||||
:name (str "test_native_" (u/generate-nano-id))}}
|
||||
:model/Transform {python-transform-id :id}
|
||||
{:name "Python Transform"
|
||||
:source {:type "python"
|
||||
:body "print('hello')"
|
||||
:source-tables {}}
|
||||
:target {:type "table"
|
||||
:name (str "test_python_" (u/generate-nano-id))
|
||||
:database (mt/id)}}]
|
||||
(testing "filter by native type"
|
||||
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :type ["native"])]
|
||||
(is (some #(= native-transform-id (:id %)) results))
|
||||
(is (not (some #(= python-transform-id (:id %)) results)))))
|
||||
(testing "filter by python type"
|
||||
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :type ["python"])]
|
||||
(is (some #(= python-transform-id (:id %)) results))
|
||||
(is (not (some #(= native-transform-id (:id %)) results)))))
|
||||
(testing "filter by both types returns all"
|
||||
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :type ["native" "python"])]
|
||||
(is (some #(= native-transform-id (:id %)) results))
|
||||
(is (some #(= python-transform-id (:id %)) results))))))))))
|
||||
|
||||
(deftest filter-transforms-by-database-id-test
|
||||
(testing "should be able to filter transforms by database_id"
|
||||
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
|
||||
(mt/with-premium-features #{:transforms}
|
||||
(mt/with-temp [:model/Database {other-db-id :id} {:engine :h2 :details {}}
|
||||
:model/Transform {transform1-id :id} {:name "Transform for main DB"
|
||||
:source {:type "query"
|
||||
:query {:database (mt/id)
|
||||
:type "native"
|
||||
:native {:query "SELECT 1"
|
||||
:template-tags {}}}}
|
||||
:target_db_id (mt/id)
|
||||
:target {:type "table"
|
||||
:name (str "test_main_" (u/generate-nano-id))}}
|
||||
:model/Transform {transform2-id :id} {:name "Transform for other DB"
|
||||
:source {:type "query"
|
||||
:query {:database other-db-id
|
||||
:type "native"
|
||||
:native {:query "SELECT 2"
|
||||
:template-tags {}}}}
|
||||
:target_db_id other-db-id
|
||||
:target {:type "table"
|
||||
:name (str "test_other_" (u/generate-nano-id))}}]
|
||||
(testing "filter by main database id"
|
||||
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :database_id (mt/id))]
|
||||
(is (some #(= transform1-id (:id %)) results))
|
||||
(is (not (some #(= transform2-id (:id %)) results)))))
|
||||
(testing "filter by other database id"
|
||||
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :database_id other-db-id)]
|
||||
(is (some #(= transform2-id (:id %)) results))
|
||||
(is (not (some #(= transform1-id (:id %)) results)))))
|
||||
(testing "filter by non-existent database returns empty for our transforms"
|
||||
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :database_id 999999)]
|
||||
(is (not (some #(= transform1-id (:id %)) results)))
|
||||
(is (not (some #(= transform2-id (:id %)) results))))))))))
|
||||
|
||||
(deftest filter-transforms-by-database-id-target-test
|
||||
(testing "should match transforms where target database matches"
|
||||
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
|
||||
(mt/with-premium-features #{:transforms :transforms-python}
|
||||
(mt/with-temp [:model/Database {target-db-id :id} {:engine :h2 :details {}}
|
||||
:model/Transform {transform-id :id} {:name "Python Transform with target DB"
|
||||
:source {:type "python"
|
||||
:body "print('hello')"
|
||||
:source-tables {}}
|
||||
:target_db_id target-db-id
|
||||
:target {:type "table"
|
||||
:name (str "test_target_" (u/generate-nano-id))
|
||||
:database target-db-id}}]
|
||||
(testing "filter by target database id"
|
||||
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform" :database_id target-db-id)]
|
||||
(is (some #(= transform-id (:id %)) results)))))))))
|
||||
|
||||
(deftest filter-transforms-combined-test
|
||||
(testing "should be able to combine database_id and type filters"
|
||||
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
|
||||
(mt/with-premium-features #{:transforms :transforms-python}
|
||||
(let [nq (lib/native-query (mt/metadata-provider) "select 1")]
|
||||
(mt/with-temp [:model/Database {other-db-id :id} {:engine :h2 :details {}}
|
||||
:model/Transform {query-main-id :id}
|
||||
{:name "Query on main DB"
|
||||
:source_type :native
|
||||
:source {:type "query"
|
||||
:query nq}
|
||||
:target {:type "table"
|
||||
:name (str "test_qm_" (u/generate-nano-id))}}
|
||||
:model/Transform {python-main-id :id}
|
||||
{:name "Python on main DB"
|
||||
:source_type :python
|
||||
:source {:type "python"
|
||||
:body "print('hello')"
|
||||
:source-tables {}}
|
||||
:target {:type "table"
|
||||
:name (str "test_pm_" (u/generate-nano-id))
|
||||
:database (mt/id)}}
|
||||
:model/Transform {query-other-id :id}
|
||||
{:name "Query on other DB"
|
||||
:source_type :native
|
||||
:source {:type "query"
|
||||
:query (lib/native-query
|
||||
(lib.metadata.jvm/application-database-metadata-provider other-db-id)
|
||||
"select 2")}
|
||||
:target {:type "table"
|
||||
:name (str "test_qo_" (u/generate-nano-id))
|
||||
:database other-db-id}}]
|
||||
(testing "filter by main database and query type"
|
||||
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform"
|
||||
:database_id (mt/id)
|
||||
:type ["native"])]
|
||||
(is (some #(= query-main-id (:id %)) results))
|
||||
(is (not (some #(= python-main-id (:id %)) results)))
|
||||
(is (not (some #(= query-other-id (:id %)) results)))))
|
||||
(testing "filter by main database and python type"
|
||||
(let [results (mt/user-http-request :crowberto :get 200 "ee/transform"
|
||||
:database_id (mt/id)
|
||||
:type ["python"])]
|
||||
(is (some #(= python-main-id (:id %)) results))
|
||||
(is (not (some #(= query-main-id (:id %)) results)))
|
||||
(is (not (some #(= query-other-id (:id %)) results)))))))))))
|
||||
|
||||
(deftest get-transforms-test
|
||||
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
|
||||
(mt/with-premium-features #{:transforms}
|
||||
|
||||
@@ -108,3 +108,44 @@
|
||||
(catch Exception _e
|
||||
;; Ignore cleanup errors
|
||||
nil)))))))))
|
||||
|
||||
;;; ------------------------------------------------------------
|
||||
;;; Filter xf tests
|
||||
;;; ------------------------------------------------------------
|
||||
|
||||
(deftest ^:parallel matching-timestamp?-test
|
||||
(testing "matching-timestamp? checks if a timestamp falls within a date range [start, end)"
|
||||
(let [matching-timestamp? #'transforms.util/matching-timestamp?
|
||||
field-path [:start_time]
|
||||
range-jan-feb {:start "2024-01-01T00:00:00Z" :end "2024-02-01T00:00:00Z"}
|
||||
range-start-only {:start "2024-01-01T00:00:00Z" :end nil}
|
||||
range-end-only {:start nil :end "2024-02-01T00:00:00Z"}]
|
||||
|
||||
(testing "with both start and end bounds"
|
||||
(are [expected timestamp]
|
||||
(= expected (matching-timestamp? {:start_time timestamp} field-path range-jan-feb))
|
||||
nil nil ; missing field returns nil
|
||||
true "2024-01-15T12:00:00Z" ; timestamp in middle of range
|
||||
false "2023-12-15T12:00:00Z" ; timestamp before range
|
||||
false "2024-02-15T12:00:00Z" ; timestamp after range
|
||||
true "2024-01-01T00:00:00Z" ; start boundary is inclusive
|
||||
true "2024-02-01T00:00:00Z")) ; end boundary is inclusive too 🤷
|
||||
|
||||
(testing "with only start bound"
|
||||
(are [expected timestamp]
|
||||
(= expected (matching-timestamp? {:start_time timestamp} field-path range-start-only))
|
||||
true "2024-01-15T12:00:00Z" ; timestamp after start
|
||||
true "2024-02-15T12:00:00Z" ; any timestamp after start
|
||||
false "2023-12-15T12:00:00Z")) ; timestamp before start
|
||||
|
||||
(testing "with only end bound"
|
||||
(are [expected timestamp]
|
||||
(= expected (matching-timestamp? {:start_time timestamp} field-path range-end-only))
|
||||
true "2024-01-15T12:00:00Z" ; timestamp before end
|
||||
true "2023-12-15T12:00:00Z" ; any timestamp before end
|
||||
false "2024-02-15T12:00:00Z")) ; timestamp after end
|
||||
|
||||
(testing "returns nil when field value is missing"
|
||||
(are [job] (nil? (matching-timestamp? job field-path range-jan-feb))
|
||||
{}
|
||||
{:other "value"})))))
|
||||
|
||||
1665
enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj
Normal file
1665
enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,113 @@
|
||||
(ns metabase-enterprise.workspaces.dag-abstract
|
||||
"Abstract graph solver using shorthand notation for testing.
|
||||
|
||||
Shorthand uses keywords like :x1, :t2, :m3 where:
|
||||
- :x<n> is a transform
|
||||
- :t<n> is a table (source/generated)
|
||||
- :m<n> is a model (card)"
|
||||
(:require
|
||||
[clojure.set :as set]
|
||||
[flatland.ordered.map :as ordered-map]
|
||||
[metabase-enterprise.workspaces.util :as ws.u]
|
||||
[metabase.util :as u]))
|
||||
|
||||
;;;; Shorthand notation helpers
|
||||
|
||||
(defn- related [t id]
|
||||
(keyword (str (name t) (subs (name id) 1))))
|
||||
|
||||
(def ^:private x->t (partial related :t))
|
||||
|
||||
(defn is?
|
||||
"Check if id is of type t (e.g., (is? :x :x1) => true)."
|
||||
[t id]
|
||||
(= (name t) (str (first (name id)))))
|
||||
|
||||
(defn- lexico [id]
|
||||
(let [n (name id)]
|
||||
[(parse-long (subs n 1))
|
||||
({\x 1 \t 2 \m 3} (first n))]))
|
||||
|
||||
(defn expand-shorthand
|
||||
"As shorthand, we can use transforms in the place of their induced tables.
|
||||
This function expands the graph to insert the induced tables, and replace the corresponding references."
|
||||
[dependencies]
|
||||
(let [xs (set (filter #(is? :x %) (concat (keys dependencies) (mapcat val dependencies))))]
|
||||
(into (sorted-map-by #(compare (lexico %1) (lexico %2)))
|
||||
(merge (zipmap (map x->t xs) (map vector xs))
|
||||
(into {} (filter (comp #(is? :t %) key)) dependencies)
|
||||
(update-vals
|
||||
(into {} (remove (comp #(is? :t %) key)) dependencies)
|
||||
(fn [deps] (->> deps
|
||||
(map (fn [id]
|
||||
(if (is? :x id)
|
||||
(x->t id)
|
||||
id)))
|
||||
(sort-by lexico)
|
||||
(vec))))))))
|
||||
|
||||
;;;; Graph traversal helpers
|
||||
|
||||
(defn- children [dependencies]
|
||||
(reduce
|
||||
(fn [acc [child parents]]
|
||||
(reduce (fn [acc p] (update acc p u/conjv child)) acc parents))
|
||||
(sorted-map-by #(compare (lexico %1) (lexico %2)))
|
||||
dependencies))
|
||||
|
||||
(defn- ->ts [ids]
|
||||
(into #{}
|
||||
(keep (fn [id]
|
||||
(cond (is? :t id) id
|
||||
(is? :x id) (x->t id))))
|
||||
ids))
|
||||
|
||||
;;;; Toposort
|
||||
|
||||
(defn- toposort-map [child->parents]
|
||||
(let [child->parents (if (map? child->parents) child->parents (into {} child->parents))]
|
||||
(into (ordered-map/ordered-map)
|
||||
(keep (fn [n] (when-let [deps (get child->parents n)] [n deps])))
|
||||
(ws.u/toposort-dfs child->parents))))
|
||||
|
||||
;;;; Abstract path-induced subgraph solver
|
||||
|
||||
(def ^:private max-iterations 100)
|
||||
|
||||
(defn path-induced-subgraph
|
||||
"Abstract solver for path-induced subgraph using shorthand notation."
|
||||
[{:keys [check-outs dependencies] :as _graph}]
|
||||
(let [->parents dependencies
|
||||
->children (children dependencies)]
|
||||
(loop [iteration 1
|
||||
below check-outs
|
||||
above check-outs
|
||||
seen-down? check-outs
|
||||
seen-up? check-outs
|
||||
to-descend check-outs
|
||||
to-ascend check-outs]
|
||||
(if (and (empty? to-descend) (empty? to-ascend))
|
||||
(let [nodes (set/intersection above below)
|
||||
entities (remove #(is? :t %) nodes)
|
||||
tables (sort-by lexico (filter #(is? :t %) nodes))]
|
||||
{:check-outs (sort-by lexico check-outs)
|
||||
:inputs (sort-by lexico (remove nodes (->ts (mapcat ->parents check-outs))))
|
||||
:tables tables
|
||||
:transforms (sort-by lexico (filter #(is? :x %) nodes))
|
||||
:entities (sort-by lexico entities)
|
||||
:dependencies (apply dissoc
|
||||
(toposort-map
|
||||
(for [n nodes]
|
||||
[n (vec (filter nodes (get dependencies n)))]))
|
||||
tables)})
|
||||
(if (> iteration max-iterations)
|
||||
{:error "max iterations exceeded"}
|
||||
(let [next-down (into #{} (comp (mapcat ->children) (remove seen-down?)) to-descend)
|
||||
next-up (into #{} (comp (mapcat ->parents) (remove seen-up?)) to-ascend)]
|
||||
(recur (inc iteration)
|
||||
(into below next-down)
|
||||
(into above next-up)
|
||||
(into seen-down? next-down)
|
||||
(into seen-up? next-up)
|
||||
next-down
|
||||
next-up)))))))
|
||||
@@ -0,0 +1,313 @@
|
||||
(ns metabase-enterprise.workspaces.dag-test
|
||||
(:require
|
||||
[clojure.string :as str]
|
||||
[clojure.test :refer :all]
|
||||
[flatland.ordered.map :as ordered-map]
|
||||
[metabase-enterprise.workspaces.dag :as ws.dag]
|
||||
[metabase-enterprise.workspaces.dag-abstract :as dag-abstract]
|
||||
[metabase-enterprise.workspaces.impl :as ws.impl]
|
||||
[metabase.app-db.core :as app-db]
|
||||
[metabase.driver.sql.normalize :as sql.normalize]
|
||||
[metabase.test :as mt]
|
||||
[metabase.util :as u]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(use-fixtures
|
||||
:once
|
||||
(fn [f]
|
||||
(mt/with-premium-features [:dependencies :transforms :workspaces]
|
||||
(mt/with-model-cleanup [:model/Dependency :model/Transform :model/Table]
|
||||
(f)))))
|
||||
|
||||
;;;; Example graphs for testing
|
||||
|
||||
(def ^:private example-graph
|
||||
{:x3 [:x1 :t2]
|
||||
:x4 [:x3]
|
||||
:m6 [:x4 :t5]
|
||||
:m10 [:t9]
|
||||
:x11 [:m10]
|
||||
:m12 [:x11]
|
||||
:m13 [:x11 :m12]})
|
||||
|
||||
;;;; Test data helpers
|
||||
|
||||
(defn- transform? [kw] (= \x (first (name kw))))
|
||||
(defn- table? [kw] (= \t (first (name kw))))
|
||||
(defn- kw->id [kw] (parse-long (subs (name kw) 1)))
|
||||
|
||||
(defn- create-test-graph!
|
||||
"Create test transforms and tables from shorthand notation, returning id mappings.
|
||||
|
||||
Shorthand uses keywords like :x1, :t2, :m3 where:
|
||||
- :x<n> is a transform
|
||||
- :t<n> is a table (source/generated)
|
||||
- :m<n> is a model (card)
|
||||
|
||||
Returns a map from shorthand id to real database id, e.g. {:x1 123, :t2 456}
|
||||
|
||||
NOTE: Caller should wrap in mt/with-model-cleanup for proper cleanup."
|
||||
[dependencies]
|
||||
(let [schema (str/replace (str (random-uuid)) "-" "_")
|
||||
all-ids (set (concat (keys dependencies)
|
||||
(mapcat val dependencies)))
|
||||
transforms (filter transform? all-ids)
|
||||
driver (t2/select-one-fn :engine [:model/Database :engine] (mt/id))
|
||||
normalize #(sql.normalize/normalize-name driver %)
|
||||
tables (filter table? all-ids)
|
||||
table-ids (u/for-map [t tables]
|
||||
[t (t2/insert-returning-pk! :model/Table
|
||||
{:db_id (mt/id)
|
||||
:schema (some-> schema normalize)
|
||||
:name (normalize (str "test_table_" (kw->id t)))
|
||||
:active true})])
|
||||
;; Create transforms that reference their input tables
|
||||
tx-ids (u/for-map [tx transforms]
|
||||
(let [parent-tables (->> (get dependencies tx []) (filter table?) (map table-ids))
|
||||
id (t2/insert-returning-pk! :model/Transform
|
||||
{:name (str "Test " (name tx))
|
||||
:source {:type :query
|
||||
:query (if (seq parent-tables)
|
||||
#_{:database (mt/id)
|
||||
:type :query
|
||||
:query {:source-table (first parent-tables)
|
||||
:joins (for [pt (rest parent-tables)]
|
||||
{:source-table pt
|
||||
:condition [:= 1 1]})}}
|
||||
{:database (mt/id)
|
||||
:type :native
|
||||
:native {:query (->> (t2/select-fn-vec #(str (:schema %) "." (:name %))
|
||||
[:model/Table :schema :name]
|
||||
:id [:in parent-tables])
|
||||
(str/join ", ")
|
||||
(str "SELECT * FROM "))}}
|
||||
{:database (mt/id)
|
||||
:type :native
|
||||
:native {:query "SELECT 1"}})}
|
||||
:target {:type "table"
|
||||
:database (mt/id)
|
||||
:schema schema
|
||||
:name (str "test_table_" (kw->id tx))}})]
|
||||
[tx id]))]
|
||||
|
||||
;; This is a workaround for the dependency between transforms and their output only being inserted on run.
|
||||
(doseq [[tx-kw tx-id] tx-ids]
|
||||
(app-db/update-or-insert! :model/Dependency
|
||||
{:to_entity_type "transform"
|
||||
:to_entity_id tx-id
|
||||
:from_entity_type "table"
|
||||
:from_entity_id (table-ids (keyword (str "t" (kw->id tx-kw))))}))
|
||||
|
||||
(merge tx-ids table-ids)))
|
||||
|
||||
(defn- translate-result
|
||||
"Translate result from real IDs back to shorthand notation for easier comparison."
|
||||
[{:keys [inputs outputs entities dependencies] :as _result} id-map]
|
||||
(let [reverse-map (u/for-map [[k v] id-map] [v k])
|
||||
table->kw (comp reverse-map :id)
|
||||
node->kw (fn [{:keys [node-type id]}]
|
||||
(reverse-map (case node-type
|
||||
:table (:id id)
|
||||
:external-transform id
|
||||
:workspace-transform (t2/select-one-fn :global_id [:model/WorkspaceTransform :global_id] :ref_id id))))]
|
||||
{:inputs (into #{} (map table->kw) inputs)
|
||||
:outputs (into #{} (map table->kw) outputs)
|
||||
:entities (into #{} (map node->kw) entities)
|
||||
:dependencies (u/for-map [[child parents] dependencies]
|
||||
[(node->kw child) (into #{} (map node->kw parents))])}))
|
||||
|
||||
;;;; Tests
|
||||
|
||||
(deftest path-induced-subgraph-shorthand-test
|
||||
(testing "graph built from shorthand matches abstract solver"
|
||||
(let [shorthand {:x2 [:t1]}
|
||||
id-map (create-test-graph! (dag-abstract/expand-shorthand shorthand))
|
||||
gtx (t2/select-one :model/Transform (id-map :x2))]
|
||||
;; TODO make this nice and declarative too
|
||||
(mt/with-temp [:model/Workspace ws {:name "Test Workspace", :database_id (mt/id)}
|
||||
:model/WorkspaceTransform wtx (merge
|
||||
(select-keys gtx [:name :source :target])
|
||||
{:workspace_id (:id ws)
|
||||
:global_id (:id gtx)})]
|
||||
(ws.impl/sync-transform-dependencies! ws wtx)
|
||||
(let [entity {:entity-type :transform, :id (:ref_id wtx)}
|
||||
result (ws.dag/path-induced-subgraph (:id ws) [entity])
|
||||
translated (translate-result result id-map)]
|
||||
(is (=? {:inputs #{:t1}
|
||||
:outputs #{:t2}
|
||||
:entities #{:x2}
|
||||
:dependencies {:x2 #{:t1}}}
|
||||
translated)))))))
|
||||
|
||||
(deftest path-induced-subgraph-larger-test
|
||||
(testing "graph built from shorthand matches abstract solver"
|
||||
;; check-outs x2, x4
|
||||
(let [shorthand {:x1 [:t0]
|
||||
:x2 [:x1, :t10]
|
||||
:x3 [:x2, :t8]
|
||||
:x4 [:x3]
|
||||
:x5 [:x2, :x4, :t9]}
|
||||
id-map (create-test-graph! (dag-abstract/expand-shorthand shorthand))
|
||||
gtx-2 (t2/select-one :model/Transform (id-map :x2))
|
||||
gtx-4 (t2/select-one :model/Transform (id-map :x4))
|
||||
fork (fn [ws gtx]
|
||||
(merge
|
||||
(select-keys gtx [:name :source :target])
|
||||
{:workspace_id (:id ws)
|
||||
:global_id (:id gtx)}))]
|
||||
;; TODO make a convenience method / method for doing these checkouts too
|
||||
(mt/with-temp [:model/Workspace ws {:name "Test Workspace", :database_id (mt/id)}
|
||||
:model/WorkspaceTransform wtx-2 (fork ws gtx-2)
|
||||
:model/WorkspaceTransform wtx-4 (fork ws gtx-4)]
|
||||
(ws.impl/sync-transform-dependencies! ws wtx-2)
|
||||
(ws.impl/sync-transform-dependencies! ws wtx-4)
|
||||
(let [entities (for [wtx [wtx-2 wtx-4]]
|
||||
{:entity-type :transform, :id (:ref_id wtx)})
|
||||
result (ws.dag/path-induced-subgraph (:id ws) entities)
|
||||
translated (translate-result result id-map)]
|
||||
(is (=? {:inputs #{:t1 :t8 :t10}
|
||||
:outputs #{:t2 :t3 :t4}
|
||||
:entities #{:x2 :x3 :x4}
|
||||
:dependencies {:x2 #{:t1 :t10}
|
||||
:x3 #{:x2 :t8}
|
||||
:x4 #{:x3}}}
|
||||
translated)))))))
|
||||
|
||||
(deftest expand-solver-test
|
||||
(testing "expand-shorthand inserts interstitial nodes for transform output tables"
|
||||
(is (= {:t1 [:x1]
|
||||
:x3 [:t1 :t2]
|
||||
:t3 [:x3]
|
||||
:x4 [:t3]
|
||||
:t4 [:x4]
|
||||
:m6 [:t4 :t5]
|
||||
:m10 [:t9]
|
||||
:x11 [:m10]
|
||||
:t11 [:x11]
|
||||
:m12 [:t11]
|
||||
:m13 [:t11 :m12]}
|
||||
(dag-abstract/expand-shorthand example-graph)))))
|
||||
|
||||
(deftest abstract-path-induced-subgraph-test
|
||||
(testing "path-induced-subgraph computes correct result for example graph"
|
||||
(is (= {:check-outs [:x3 :m6 :m10 :m13]
|
||||
:inputs [:t1 :t2 :t5 :t9]
|
||||
:tables [:t3 :t4 :t11]
|
||||
:transforms [:x3 :x4 :x11]
|
||||
:entities [:x3 :x4 :m6 :m10 :x11 :m12 :m13]
|
||||
:dependencies (ordered-map/ordered-map
|
||||
:m10 []
|
||||
:x11 [:m10]
|
||||
:m12 [:t11]
|
||||
:m13 [:t11 :m12]
|
||||
:x3 []
|
||||
:x4 [:t3]
|
||||
:m6 [:t4])}
|
||||
(dag-abstract/path-induced-subgraph
|
||||
{:check-outs #{:x3 :m6 :m10 :m13}
|
||||
:dependencies (dag-abstract/expand-shorthand example-graph)})))))
|
||||
|
||||
;;;; Card dependency detection tests
|
||||
|
||||
(deftest unsupported-dependency-no-transforms-test
|
||||
(testing "empty input returns nil"
|
||||
(is (nil? (ws.dag/unsupported-dependency? {})))
|
||||
(is (nil? (ws.dag/unsupported-dependency? {:transforms []})))))
|
||||
|
||||
(deftest unsupported-dependency-mbql-card-test
|
||||
#_(testing "transform with card dependencies are unsupported"
|
||||
(let [tx-with-no-dependencies 1
|
||||
;; These tests transforms must seem pretty redundant - but it's intentional: we may relax them one case at
|
||||
;; at time in the future.
|
||||
;; ----------------------
|
||||
;; We don't allow MBQL card dependencies, as we do not support re-mapping their references on execution.
|
||||
;; This is only a problem if the card depends on a table that is shadowed in the isolated schema, but we
|
||||
;; want to keep the semantics simple.
|
||||
tx-with-direct-mbql-card-dependency 2
|
||||
;; Even a transitive dependency would need to be remapped.
|
||||
tx-with-indirect-mbql-card-dependency 3
|
||||
;; In fact, we don't allow *any* card dependency for two reasons:
|
||||
;; 1. We don't yet support reference re-mapping across entity references.
|
||||
;; 2. It's an anti-pattern to build transforms on models, the relationship is intended to be the other way
|
||||
;; around.
|
||||
tx-with-sql-card-dependency 4]
|
||||
(is (= nil (ws.dag/unsupported-dependency? {:transforms [1]})))
|
||||
(is (= {:transforms [2 3 4]} (ws.dag/unsupported-dependency? {:transforms [1 2 3 4]}))))))
|
||||
|
||||
(deftest collapse-test
|
||||
(is (= {:x1 [:x2 :x3]
|
||||
:x2 [:t3]
|
||||
:x3 [:x5]
|
||||
:x4 []
|
||||
:x5 []}
|
||||
(#'ws.dag/collapse
|
||||
table?
|
||||
{:x1 [:t1 :t2]
|
||||
:t1 [:x2]
|
||||
:t2 [:x3]
|
||||
:x2 [:t3]
|
||||
:x3 [:t4]
|
||||
:t4 [:x5]
|
||||
:t5 [:x4]
|
||||
:x4 []
|
||||
:x5 []}))))
|
||||
|
||||
(defn tx->table [kw]
|
||||
(when (transform? kw)
|
||||
(keyword (str "t" (kw->id kw)))))
|
||||
|
||||
(defn- solve-in-memory [init-nodes graph]
|
||||
(let [tx-nodes (filter transform? init-nodes)
|
||||
tables (map tx->table tx-nodes)]
|
||||
(#'ws.dag/path-induced-subgraph*
|
||||
;; Include all changeset targets in the init-nodes
|
||||
(distinct (into init-nodes tables))
|
||||
{:node-parents (dag-abstract/expand-shorthand graph)
|
||||
:table? table?
|
||||
:table-sort kw->id
|
||||
:unwrap-table identity})))
|
||||
|
||||
(defn- chain->deps [chain]
|
||||
(reduce
|
||||
(fn [deps [from to]]
|
||||
(assoc deps from [to]))
|
||||
{}
|
||||
(partition 2 1 (reverse chain))))
|
||||
|
||||
(deftest in-memory-path-induced-subgraph-test
|
||||
(testing "singleton"
|
||||
(is (= {:inputs [:t1]
|
||||
:outputs [:t2]
|
||||
:entities [:x2]
|
||||
:dependencies {:x2 [:t1]}}
|
||||
(solve-in-memory [:x2] {:x2 [:t1]}))))
|
||||
|
||||
(testing "encloses middle of a chain"
|
||||
(is (= {:inputs [:t1]
|
||||
:outputs [:t2 :t3 :t4]
|
||||
:entities [:x2 :x3 :x4]
|
||||
:dependencies {:x2 [:t1]
|
||||
:x3 [:x2]
|
||||
:x4 [:x3]}}
|
||||
(solve-in-memory [:x2 :x4] (chain->deps [:x1 :x2 :x3 :x4 :x5])))))
|
||||
|
||||
(testing "larger graph"
|
||||
(is (= {:inputs [:t1 :t2 :t5 :t9]
|
||||
:outputs [:t3 :t4 :t11]
|
||||
:entities [:m10 :x11 :m12 :m13 :x3 :x4 :m6]
|
||||
:dependencies {:m10 [:t9]
|
||||
:m12 [:x11]
|
||||
:m13 [:m12 :x11]
|
||||
:m6 [:x4 :t5]
|
||||
:x11 [:m10]
|
||||
:x3 [:t2 :t1]
|
||||
:x4 [:x3]}}
|
||||
(solve-in-memory
|
||||
[:x3 :m6 :m10 :m13]
|
||||
{:x3 [:x1 :t2]
|
||||
:x4 [:x3]
|
||||
:m6 [:x4 :t5]
|
||||
:m10 [:t9]
|
||||
:x11 [:m10]
|
||||
:m12 [:x11]
|
||||
:m13 [:x11 :m12]})))))
|
||||
@@ -0,0 +1,318 @@
|
||||
(ns metabase-enterprise.workspaces.dependencies-test
|
||||
(:require
|
||||
[clojure.test :refer :all]
|
||||
[metabase-enterprise.workspaces.dependencies :as ws.deps]
|
||||
[metabase-enterprise.workspaces.test-util :as ws.tu]
|
||||
[metabase.lib.core :as lib]
|
||||
[metabase.lib.metadata :as lib.metadata]
|
||||
[metabase.test :as mt]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(ws.tu/ws-fixtures!)
|
||||
|
||||
;;; ---------------------------------------- analyze-entity tests ----------------------------------------
|
||||
|
||||
(deftest ^:parallel analyze-entity-query-transform-test
|
||||
(testing "analyze-entity extracts dependencies from a query transform"
|
||||
(let [mp (mt/metadata-provider)
|
||||
query (lib/query mp (lib.metadata/table mp (mt/id :orders)))
|
||||
transform {:source {:type "query"
|
||||
:query query}
|
||||
:target {:database (mt/id)
|
||||
:schema "public"
|
||||
:name "orders_with_products"}}
|
||||
result (ws.deps/analyze-entity :transform transform)]
|
||||
(testing "output is extracted from target"
|
||||
(is (= {:db_id (mt/id)
|
||||
:schema "public"
|
||||
:table "orders_with_products"}
|
||||
(:output result))))
|
||||
(testing "inputs contain the source table with table_id (since table exists)"
|
||||
(let [inputs (:inputs result)
|
||||
table-names (set (map :table inputs))]
|
||||
(is (= 1 (count inputs)))
|
||||
(is (contains? table-names (mt/format-name :orders)))
|
||||
;; MBQL queries have table_id since the tables must exist
|
||||
(is (every? #(and (:db_id %)
|
||||
(:table %)
|
||||
(:table_id %))
|
||||
inputs)))))))
|
||||
|
||||
(deftest ^:parallel analyze-entity-native-query-existing-table-test
|
||||
(testing "analyze-entity extracts dependencies from native query referencing existing table"
|
||||
(let [mp (mt/metadata-provider)
|
||||
query (lib/native-query mp "SELECT * FROM ORDERS WHERE total > 100")
|
||||
transform {:source {:type "query"
|
||||
:query query}
|
||||
:target {:database (mt/id)
|
||||
:schema "analytics"
|
||||
:name "high_value_orders"}}
|
||||
result (ws.deps/analyze-entity :transform transform)]
|
||||
(testing "output is extracted from target"
|
||||
(is (= {:db_id (mt/id)
|
||||
:schema "analytics"
|
||||
:table "high_value_orders"}
|
||||
(:output result))))
|
||||
(testing "inputs include table_id since ORDERS table exists"
|
||||
(let [inputs (:inputs result)]
|
||||
;; Native query parsing should detect ORDERS table reference and look up table_id
|
||||
(is (pos? (count inputs)))
|
||||
(is (every? #(and (:db_id %)
|
||||
(:table %)
|
||||
(:table_id %))
|
||||
inputs)))))))
|
||||
|
||||
(deftest ^:parallel analyze-entity-native-query-nonexistent-table-test
|
||||
(testing "analyze-entity handles native query referencing non-existent table"
|
||||
(let [mp (mt/metadata-provider)
|
||||
;; Reference a table that doesn't exist in the database
|
||||
query (lib/native-query mp "SELECT * FROM future_sales_data WHERE region = 'US'")
|
||||
transform {:source {:type "query"
|
||||
:query query}
|
||||
:target {:database (mt/id)
|
||||
:schema "analytics"
|
||||
:name "regional_sales"}}
|
||||
result (ws.deps/analyze-entity :transform transform)]
|
||||
(testing "output is extracted from target"
|
||||
(is (= {:db_id (mt/id)
|
||||
:schema "analytics"
|
||||
:table "regional_sales"}
|
||||
(:output result))))
|
||||
(testing "inputs have logical reference but no table_id (table doesn't exist)"
|
||||
(let [inputs (:inputs result)]
|
||||
;; Should still parse the table reference even though table doesn't exist
|
||||
(is (pos? (count inputs)))
|
||||
;; Has db_id and table name but NOT table_id
|
||||
(is (every? #(and (:db_id %)
|
||||
(:table %))
|
||||
inputs))
|
||||
;; Verify table_id is nil or missing for non-existent tables
|
||||
(is (every? #(nil? (:table_id %)) inputs)))))))
|
||||
|
||||
(deftest ^:parallel analyze-entity-python-transform-test
|
||||
(testing "analyze-entity extracts dependencies from a python transform"
|
||||
(let [transform {:source {:type "python"
|
||||
:source-tables {"orders" (mt/id :orders)
|
||||
"products" (mt/id :products)}
|
||||
:body "def transform(orders, products): return orders"}
|
||||
:target {:database (mt/id)
|
||||
:schema "public"
|
||||
:name "python_result"}}
|
||||
result (ws.deps/analyze-entity :transform transform)]
|
||||
(testing "output is extracted from target"
|
||||
(is (= {:db_id (mt/id)
|
||||
:schema "public"
|
||||
:table "python_result"}
|
||||
(:output result))))
|
||||
(testing "inputs contain the source tables with table_ids"
|
||||
(let [inputs (:inputs result)
|
||||
table-names (set (map :table inputs))]
|
||||
(is (= 2 (count inputs)))
|
||||
(is (contains? table-names (mt/format-name :orders)))
|
||||
(is (contains? table-names (mt/format-name :products)))
|
||||
;; Python transforms have table_ids since source-tables maps to existing table IDs
|
||||
(is (every? :table_id inputs)))))))
|
||||
|
||||
(deftest analyze-entity-asserts-transform-type-test
|
||||
(testing "analyze-entity asserts entity-type is :transform"
|
||||
(is (thrown-with-msg? Exception #"Only transform entity type is supported"
|
||||
(ws.deps/analyze-entity :card {})))))
|
||||
|
||||
;;; ---------------------------------------- write-dependencies! tests ----------------------------------------
|
||||
|
||||
(deftest write-dependencies-creates-output-test
|
||||
(testing "write-dependencies! creates workspace_output record"
|
||||
(mt/with-temp [:model/Workspace workspace {:name "Test Workspace"
|
||||
:database_id (mt/id)
|
||||
:schema "test_isolated_schema"}
|
||||
:model/WorkspaceTransform wt {:workspace_id (:id workspace)
|
||||
:name "Test Transform"
|
||||
:source {:type "query" :query {}}
|
||||
:target {:database (mt/id)
|
||||
:schema "public"
|
||||
:name "test_output"}}]
|
||||
(let [analysis {:output {:db_id (mt/id)
|
||||
:schema "public"
|
||||
:table "test_output"}
|
||||
:inputs []}]
|
||||
(ws.deps/write-dependencies! (:id workspace) "test_isolated_schema" :transform (:ref_id wt) analysis)
|
||||
(let [output (t2/select-one :model/WorkspaceOutput
|
||||
:workspace_id (:id workspace)
|
||||
:ref_id (:ref_id wt))]
|
||||
(is (some? output))
|
||||
(is (= (mt/id) (:db_id output)))
|
||||
(is (= "public" (:global_schema output)))
|
||||
(is (= "test_output" (:global_table output)))
|
||||
(is (= "test_isolated_schema" (:isolated_schema output)))
|
||||
(is (= "public__test_output" (:isolated_table output))))))))
|
||||
|
||||
(deftest write-dependencies-creates-inputs-test
|
||||
(testing "write-dependencies! creates workspace_input records for external dependencies"
|
||||
(let [iso-schema "test_isolated_schema"]
|
||||
(mt/with-temp [:model/Workspace workspace {:name "Test Workspace"
|
||||
:database_id (mt/id)
|
||||
:schema iso-schema}
|
||||
:model/WorkspaceTransform wt {:workspace_id (:id workspace)
|
||||
:name "Test Transform"
|
||||
:source {:type "query" :query {}}
|
||||
:target {:database (mt/id)
|
||||
:schema "public"
|
||||
:name "test_output"}}]
|
||||
(let [orders-table (t2/select-one [:model/Table :db_id :schema :name] :id (mt/id :orders))
|
||||
analysis {:output {:db_id (mt/id)
|
||||
:schema "public"
|
||||
:table "test_output"}
|
||||
:inputs [{:db_id (:db_id orders-table)
|
||||
:schema (:schema orders-table)
|
||||
:table (:name orders-table)}]}]
|
||||
(ws.deps/write-dependencies! (:id workspace) iso-schema :transform (:ref_id wt) analysis)
|
||||
(let [input (t2/select-one :model/WorkspaceInput
|
||||
:workspace_id (:id workspace)
|
||||
:table (:name orders-table))]
|
||||
(is (some? input))
|
||||
(is (= (:db_id orders-table) (:db_id input)))
|
||||
(is (= (:name orders-table) (:table input)))))))))
|
||||
|
||||
(deftest write-dependencies-creates-edges-test
|
||||
(testing "write-dependencies! creates workspace_dependency edges"
|
||||
(let [iso-schema "test_isolated_schema"]
|
||||
(mt/with-temp [:model/Workspace workspace {:name "Test Workspace"
|
||||
:database_id (mt/id)
|
||||
:schema iso-schema}
|
||||
:model/WorkspaceTransform wt {:workspace_id (:id workspace)
|
||||
:name "Test Transform"
|
||||
:source {:type "query" :query {}}
|
||||
:target {:database (mt/id)
|
||||
:schema "public"
|
||||
:name "test_output"}}]
|
||||
(let [orders-table (t2/select-one [:model/Table :db_id :schema :name] :id (mt/id :orders))
|
||||
analysis {:output {:db_id (mt/id)
|
||||
:schema "public"
|
||||
:table "test_output"}
|
||||
:inputs [{:db_id (:db_id orders-table)
|
||||
:schema (:schema orders-table)
|
||||
:table (:name orders-table)}]}]
|
||||
(ws.deps/write-dependencies! (:id workspace) iso-schema :transform (:ref_id wt) analysis)
|
||||
(let [edges (t2/select :model/WorkspaceDependency
|
||||
:workspace_id (:id workspace)
|
||||
:from_entity_type :transform
|
||||
:from_entity_id (:ref_id wt))]
|
||||
(is (= 1 (count edges)))
|
||||
(is (= :input (:to_entity_type (first edges))))))))))
|
||||
|
||||
(deftest write-dependencies-internal-dependency-test
|
||||
(testing "write-dependencies! links to workspace_output for internal dependencies"
|
||||
(mt/with-temp [:model/Workspace workspace {:name "Test Workspace"
|
||||
:database_id (mt/id)
|
||||
:schema "test_isolated_schema"}
|
||||
:model/WorkspaceTransform wt1 {:workspace_id (:id workspace)
|
||||
:name "Upstream Transform"
|
||||
:source {:type "query" :query {}}
|
||||
:target {:database (mt/id)
|
||||
:schema "public"
|
||||
:name "upstream_output"}}
|
||||
:model/WorkspaceTransform wt2 {:workspace_id (:id workspace)
|
||||
:name "Downstream Transform"
|
||||
:source {:type "query" :query {}}
|
||||
:target {:database (mt/id)
|
||||
:schema "public"
|
||||
:name "downstream_output"}}]
|
||||
;; First, write dependencies for the upstream transform (creates the output)
|
||||
(ws.deps/write-dependencies! (:id workspace) "test_isolated_schema" :transform (:ref_id wt1)
|
||||
{:output {:db_id (mt/id)
|
||||
:schema "public"
|
||||
:table "upstream_output"}
|
||||
:inputs []})
|
||||
|
||||
;; Now write dependencies for downstream that depends on upstream's output
|
||||
(ws.deps/write-dependencies! (:id workspace) "test_isolated_schema" :transform (:ref_id wt2)
|
||||
{:output {:db_id (mt/id)
|
||||
:schema "public"
|
||||
:table "downstream_output"}
|
||||
:inputs [{:db_id (mt/id)
|
||||
:schema "public"
|
||||
:table "upstream_output"}]})
|
||||
|
||||
(let [edges (t2/select :model/WorkspaceDependency
|
||||
:workspace_id (:id workspace)
|
||||
:from_entity_type :transform
|
||||
:from_entity_id (:ref_id wt2))]
|
||||
(testing "creates edge to output (internal dependency)"
|
||||
(is (= 1 (count edges)))
|
||||
(is (= :output (:to_entity_type (first edges)))))
|
||||
(testing "no workspace_input created for internal dependency"
|
||||
(is (not (t2/exists? :model/WorkspaceInput
|
||||
:workspace_id (:id workspace)
|
||||
:table "upstream_output"))))))))
|
||||
|
||||
(deftest write-dependencies-updates-on-change-test
|
||||
(testing "write-dependencies! updates records and removes stale edges"
|
||||
(mt/with-temp [:model/Workspace workspace {:name "Test Workspace"
|
||||
:database_id (mt/id)
|
||||
:schema "test_isolated_schema"}
|
||||
:model/WorkspaceTransform wt {:workspace_id (:id workspace)
|
||||
:name "Test Transform"
|
||||
:source {:type "query" :query {}}
|
||||
:target {:database (mt/id)
|
||||
:schema "public"
|
||||
:name "test_output"}}]
|
||||
(let [orders-table (t2/select-one [:model/Table :db_id :schema :name] :id (mt/id :orders))
|
||||
products-table (t2/select-one [:model/Table :db_id :schema :name] :id (mt/id :products))]
|
||||
;; First write with orders dependency
|
||||
(ws.deps/write-dependencies! (:id workspace) "test_isolated_schema" :transform (:ref_id wt)
|
||||
{:output {:db_id (mt/id)
|
||||
:schema "public"
|
||||
:table "test_output"}
|
||||
:inputs [{:db_id (:db_id orders-table)
|
||||
:schema (:schema orders-table)
|
||||
:table (:name orders-table)}]})
|
||||
(is (= 1 (t2/count :model/WorkspaceDependency :workspace_id (:id workspace))))
|
||||
|
||||
;; Update to depend on products instead
|
||||
(ws.deps/write-dependencies! (:id workspace) "test_isolated_schema" :transform (:ref_id wt)
|
||||
{:output {:db_id (mt/id)
|
||||
:schema "public"
|
||||
:table "test_output"}
|
||||
:inputs [{:db_id (:db_id products-table)
|
||||
:schema (:schema products-table)
|
||||
:table (:name products-table)}]})
|
||||
|
||||
(testing "old edge is removed"
|
||||
(is (= 1 (t2/count :model/WorkspaceDependency :workspace_id (:id workspace)))))
|
||||
(testing "new input is created"
|
||||
(is (t2/exists? :model/WorkspaceInput
|
||||
:workspace_id (:id workspace)
|
||||
:table (:name products-table))))))))
|
||||
|
||||
;;; ---------------------------------------- Integration test ----------------------------------------
|
||||
|
||||
(deftest analyze-and-write-integration-test
|
||||
(testing "Full flow: analyze a real transform and write its dependencies"
|
||||
(let [mp (mt/metadata-provider)
|
||||
query (lib/query mp (lib.metadata/table mp (mt/id :orders)))]
|
||||
(mt/with-temp [:model/Workspace workspace {:name "Integration Test Workspace"
|
||||
:database_id (mt/id)
|
||||
:schema "test_isolated_schema"}
|
||||
:model/WorkspaceTransform wt {:workspace_id (:id workspace)
|
||||
:name "Orders Analysis"
|
||||
:source {:type "query" :query {}}
|
||||
:target {:database (mt/id)
|
||||
:schema "analytics"
|
||||
:name "orders_analysis"}}]
|
||||
;; Create the transform entity with the proper query (not from DB)
|
||||
(let [transform-entity {:source {:type "query"
|
||||
:query query}
|
||||
:target {:database (mt/id)
|
||||
:schema "analytics"
|
||||
:name "orders_analysis"}}
|
||||
analysis (ws.deps/analyze-entity :transform transform-entity)]
|
||||
(ws.deps/write-dependencies! (:id workspace) "test_isolated_schema" :transform (:ref_id wt) analysis)
|
||||
|
||||
(testing "output record created"
|
||||
(is (= 1 (t2/count :model/WorkspaceOutput :workspace_id (:id workspace)))))
|
||||
(testing "input record created for ORDERS table"
|
||||
(is (= 1 (t2/count :model/WorkspaceInput :workspace_id (:id workspace)))))
|
||||
(testing "dependency edge created"
|
||||
(is (= 1 (t2/count :model/WorkspaceDependency :workspace_id (:id workspace))))))))))
|
||||
@@ -0,0 +1,20 @@
|
||||
(ns ^:mb/driver-tests metabase-enterprise.workspaces.drivers-test
|
||||
(:require
|
||||
[clojure.test :refer :all]
|
||||
[metabase.driver :as driver]
|
||||
[metabase.test :as mt]))
|
||||
|
||||
(deftest drivers-support-workspaces-has-to-support-db-swapping-test
|
||||
(mt/test-drivers (mt/normal-drivers-with-feature :workspace)
|
||||
(let [db-swapped? (fn []
|
||||
(try
|
||||
(mt/run-mbql-query venues {:limit 1})
|
||||
false
|
||||
(catch Exception _e
|
||||
true)))]
|
||||
(testing "db is swapped if being executed inside a macros"
|
||||
(driver/with-swapped-connection-details (mt/id) {:user "unicorn"}
|
||||
(is (db-swapped?))))
|
||||
|
||||
(testing "sanity check that it's not swapped outside of the macros"
|
||||
(is (not (db-swapped?)))))))
|
||||
@@ -0,0 +1,79 @@
|
||||
(ns ^:mb/driver-tests metabase-enterprise.workspaces.e2e-test
|
||||
(:require
|
||||
[clojure.test :refer :all]
|
||||
[metabase-enterprise.transforms.execute :as transforms.execute]
|
||||
[metabase-enterprise.transforms.test-util :as transforms.tu]
|
||||
[metabase-enterprise.workspaces.common :as ws.common]
|
||||
[metabase-enterprise.workspaces.isolation :as ws.isolation]
|
||||
[metabase-enterprise.workspaces.test-util :as ws.tu]
|
||||
[metabase.driver :as driver]
|
||||
[metabase.driver.sql.query-processor :as sql.qp]
|
||||
[metabase.query-processor.preprocess :as qp.preprocess]
|
||||
^{:clj-kondo/ignore [:deprecated-namespace]}
|
||||
[metabase.query-processor.store :as qp.store]
|
||||
[metabase.test :as mt]
|
||||
[metabase.util :as u]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(defn- execute-workspace-transform!
|
||||
"Execute a transform within workspace isolation context. For testing purposes."
|
||||
[workspace ws-transform opts]
|
||||
(ws.isolation/with-workspace-isolation workspace
|
||||
(let [result (atom nil)]
|
||||
(t2/with-transaction [_tx]
|
||||
(let [transform (t2/insert-returning-instance!
|
||||
:model/Transform
|
||||
(select-keys ws-transform [:name :description :source :target]))]
|
||||
(reset! result (transforms.execute/execute! transform opts))
|
||||
;; rather abort transaction
|
||||
(t2/delete! :model/Transform (:id transform))))
|
||||
@result)))
|
||||
|
||||
(defn- mbql->native [query]
|
||||
(qp.store/with-metadata-provider (mt/id)
|
||||
(sql.qp/mbql->native driver/*driver* (qp.preprocess/preprocess query))))
|
||||
|
||||
(deftest isolation-e2e-test
|
||||
(mt/test-drivers (mt/normal-drivers-with-feature :workspace)
|
||||
(transforms.tu/with-transform-cleanup! [output-table-name (str "test_table_1_" (mt/random-name))]
|
||||
(mt/with-model-cleanup [:model/Transform :model/Workspace]
|
||||
(mt/with-temp [:model/Transform {transform-id :id} {:name "Transform 1"
|
||||
:source {:type "query"
|
||||
:query (mt/native-query (mbql->native (mt/mbql-query orders {:limit 3})))}
|
||||
:target {:type "table"
|
||||
:name output-table-name
|
||||
:database (mt/id)
|
||||
:schema (t2/select-one-fn :schema :model/Table (mt/id :orders))}}]
|
||||
(ws.tu/with-workspaces! [workspace {:name (mt/random-name)
|
||||
:database_id (mt/id)
|
||||
:upstream {:transforms [transform-id]}}]
|
||||
(ws.common/add-to-changeset! (mt/user->id :crowberto) workspace
|
||||
:transform transform-id
|
||||
(t2/select-one :model/Transform transform-id))
|
||||
(let [_workspace (u/poll {:thunk #(t2/select-one :model/Workspace (:id workspace))
|
||||
:done? #(= :ready (:status %))
|
||||
:timeout-ms 5000})
|
||||
isolated-transform (t2/select-one :model/WorkspaceTransform :workspace_id (:id workspace))
|
||||
#_#_executed-transform (mt/with-current-user (mt/user->id :crowberto)
|
||||
(ws.impl/run-transform! workspace isolated-transform))
|
||||
#_#_output-table (-> executed-transform :table)]
|
||||
;; TODO: the table is synced but there are no fields even though the table and fields exist in the db
|
||||
#_(testing "execute the transform with the original query is fine and the output is in an isolated schema"
|
||||
(u/poll {:thunk #(t2/select-one :model/Table :name (:name output-table))
|
||||
:done? some?
|
||||
:timeout-ms 1000})
|
||||
(transforms.tu/wait-for-table (:name output-table) 1000)
|
||||
(is (str/starts-with? (:schema output-table) "mb__isolation"))
|
||||
(is (= 1 (count (transforms.tu/table-rows (name output-table))))))
|
||||
|
||||
(testing "changing the query without granting access will fail"
|
||||
(t2/update! :model/WorkspaceTransform
|
||||
{:ref_id (:ref_id isolated-transform)}
|
||||
{:source {:type "query"
|
||||
:query (mt/native-query (mbql->native (mt/mbql-query venues {:limit 1})))}})
|
||||
(is (thrown-with-msg?
|
||||
Exception
|
||||
#"ERROR: permission denied for table.*"
|
||||
(execute-workspace-transform! workspace
|
||||
(t2/select-one :model/WorkspaceTransform :ref_id (:ref_id isolated-transform))
|
||||
{:run-method :manual})))))))))))
|
||||
@@ -0,0 +1,196 @@
|
||||
(ns ^:mb/driver-tests metabase-enterprise.workspaces.execute-test
|
||||
(:require
|
||||
[clojure.string :as str]
|
||||
[clojure.test :refer :all]
|
||||
[metabase-enterprise.transforms.test-util :as transforms.tu]
|
||||
[metabase-enterprise.workspaces.common :as ws.common]
|
||||
[metabase-enterprise.workspaces.execute :as ws.execute]
|
||||
[metabase-enterprise.workspaces.impl :as ws.impl]
|
||||
[metabase-enterprise.workspaces.test-util :as ws.tu]
|
||||
[metabase.test :as mt]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(ws.tu/ws-fixtures!)
|
||||
|
||||
(deftest run-workspace-transform-no-input-test
|
||||
(testing "Executing a workspace transform returns results and rolls back app DB records"
|
||||
(transforms.tu/with-transform-cleanup! [output-table "ws_execute_test"]
|
||||
(let [workspace (ws.tu/create-ready-ws! "Execute Test Workspace")
|
||||
db-id (:database_id workspace)
|
||||
ws-schema (:schema workspace)
|
||||
body {:name "Test Transform"
|
||||
:source {:type "query"
|
||||
:query (mt/native-query {:query "SELECT 1 as id, 'hello' as name"})}
|
||||
:target {:type "table"
|
||||
:database db-id
|
||||
:schema nil
|
||||
:name output-table}}
|
||||
ws-transform (ws.common/add-to-changeset! (mt/user->id :crowberto) workspace :transform nil body)
|
||||
before {:xf (t2/count :model/Transform)
|
||||
:xfrun (t2/count :model/TransformRun)}]
|
||||
|
||||
(testing "execution returns expected result structure"
|
||||
(is (=? {:status :succeeded
|
||||
:start_time some?
|
||||
:end_time some?
|
||||
:table {:name #(str/includes? % output-table)
|
||||
:schema ws-schema}}
|
||||
(mt/with-current-user (mt/user->id :crowberto)
|
||||
(ws.impl/run-transform! workspace ws-transform))))
|
||||
(is (=? {:last_run_at some?}
|
||||
(t2/select-one :model/WorkspaceTransform :ref_id (:ref_id ws-transform))))
|
||||
(is (=? [{:workspace_id (:id workspace)
|
||||
:global_table output-table
|
||||
:global_schema nil
|
||||
:global_table_id nil
|
||||
:isolated_schema string?
|
||||
:isolated_table string?
|
||||
;; TODO I think this is broken because of normalization breaking case equality
|
||||
#_#_:isolated_table_id number?}]
|
||||
(t2/select :model/WorkspaceOutput :ref_id (:ref_id ws-transform)))))
|
||||
|
||||
(testing "app DB records are rolled back"
|
||||
(is (= before
|
||||
{:xf (t2/count :model/Transform)
|
||||
:xfrun (t2/count :model/TransformRun)})))))))
|
||||
|
||||
(deftest remap-python-source-test
|
||||
(let [remap-python-source #'ws.execute/remap-python-source]
|
||||
(testing "remaps to table ID when mapping has :id"
|
||||
(let [table-mapping {[1 "public" "orders"] {:db-id 1
|
||||
:schema "ws_isolated_123"
|
||||
:table "orders_isolated"
|
||||
:id 456}}
|
||||
source {:type "python"
|
||||
:body "import pandas as pd"
|
||||
:source-tables {"orders" {:database_id 1
|
||||
:schema "public"
|
||||
:table "orders"
|
||||
:table_id 123}}}]
|
||||
(is (= {:type "python"
|
||||
:body "import pandas as pd"
|
||||
:source-tables {"orders" 456}}
|
||||
(remap-python-source table-mapping source)))))
|
||||
|
||||
(testing "remaps to map format when mapping has nil :id"
|
||||
(let [table-mapping {[1 "public" "orders"] {:db-id 1
|
||||
:schema "ws_isolated_123"
|
||||
:table "orders_isolated"
|
||||
:id nil}}
|
||||
source {:type "python"
|
||||
:body "import pandas as pd"
|
||||
:source-tables {"orders" {:database_id 1
|
||||
:schema "public"
|
||||
:table "orders"
|
||||
:table_id 123}}}]
|
||||
(is (= {:type "python"
|
||||
:body "import pandas as pd"
|
||||
:source-tables {"orders" {:database_id 1
|
||||
:schema "ws_isolated_123"
|
||||
:table "orders_isolated"}}}
|
||||
(remap-python-source table-mapping source)))))
|
||||
|
||||
(testing "leaves source-tables unchanged when no mapping exists"
|
||||
(let [table-mapping {}
|
||||
source {:type "python"
|
||||
:body "import pandas as pd"
|
||||
:source-tables {"orders" {:database_id 1
|
||||
:schema "public"
|
||||
:table "orders"
|
||||
:table_id 123}}}]
|
||||
(is (= source (remap-python-source table-mapping source)))))
|
||||
|
||||
(testing "remaps legacy integer to table ID when mapping has :id"
|
||||
(let [table-mapping {123 {:db-id 1
|
||||
:schema "ws_isolated_123"
|
||||
:table "orders_isolated"
|
||||
:id 456}}
|
||||
source {:type "python"
|
||||
:body "import pandas as pd"
|
||||
:source-tables {"orders" 123}}]
|
||||
(is (= {:type "python"
|
||||
:body "import pandas as pd"
|
||||
:source-tables {"orders" 456}}
|
||||
(remap-python-source table-mapping source)))))
|
||||
|
||||
(testing "remaps legacy integer to map format when mapping has nil :id"
|
||||
(let [table-mapping {123 {:db-id 1
|
||||
:schema "ws_isolated_123"
|
||||
:table "orders_isolated"
|
||||
:id nil}}
|
||||
source {:type "python"
|
||||
:body "import pandas as pd"
|
||||
:source-tables {"orders" 123}}]
|
||||
(is (= {:type "python"
|
||||
:body "import pandas as pd"
|
||||
:source-tables {"orders" {:database_id 1
|
||||
:schema "ws_isolated_123"
|
||||
:table "orders_isolated"}}}
|
||||
(remap-python-source table-mapping source)))))
|
||||
|
||||
(testing "leaves legacy integer table-id unchanged when no mapping exists"
|
||||
(let [table-mapping {}
|
||||
source {:type "python"
|
||||
:body "import pandas as pd"
|
||||
:source-tables {"orders" 123}}]
|
||||
(is (= source (remap-python-source table-mapping source)))))
|
||||
|
||||
(testing "handles mixed formats with mappings for both"
|
||||
(let [table-mapping {[1 "public" "orders"] {:db-id 1
|
||||
:schema "ws_isolated_123"
|
||||
:table "orders_isolated"
|
||||
:id 456}
|
||||
789 {:db-id 1
|
||||
:schema "ws_isolated_123"
|
||||
:table "customers_isolated"
|
||||
:id 999}}
|
||||
source {:type "python"
|
||||
:body "import pandas as pd"
|
||||
:source-tables {"orders" {:database_id 1
|
||||
:schema "public"
|
||||
:table "orders"
|
||||
:table_id 123}
|
||||
"customers" 789}}]
|
||||
(is (= {:type "python"
|
||||
:body "import pandas as pd"
|
||||
:source-tables {"orders" 456
|
||||
"customers" 999}}
|
||||
(remap-python-source table-mapping source)))))
|
||||
|
||||
(testing "prefers table_id lookup over triple lookup for map format"
|
||||
(let [table-mapping {123 {:db-id 1
|
||||
:schema "ws_isolated_123"
|
||||
:table "orders_by_id"
|
||||
:id 456}
|
||||
[1 "public" "orders"] {:db-id 1
|
||||
:schema "ws_isolated_123"
|
||||
:table "orders_by_triple"
|
||||
:id 789}}
|
||||
source {:type "python"
|
||||
:body "import pandas as pd"
|
||||
:source-tables {"orders" {:database_id 1
|
||||
:schema "public"
|
||||
:table "orders"
|
||||
:table_id 123}}}]
|
||||
(is (= {:type "python"
|
||||
:body "import pandas as pd"
|
||||
:source-tables {"orders" 456}}
|
||||
(remap-python-source table-mapping source)))))
|
||||
|
||||
(testing "falls back to triple lookup when table_id not in mapping"
|
||||
(let [table-mapping {[1 "public" "orders"] {:db-id 1
|
||||
:schema "ws_isolated_123"
|
||||
:table "orders_isolated"
|
||||
:id 456}}
|
||||
source {:type "python"
|
||||
:body "import pandas as pd"
|
||||
:source-tables {"orders" {:database_id 1
|
||||
:schema "public"
|
||||
:table "orders"
|
||||
:table_id 999}}}]
|
||||
(is (= {:type "python"
|
||||
:body "import pandas as pd"
|
||||
:source-tables {"orders" 456}}
|
||||
(remap-python-source table-mapping source)))))))
|
||||
@@ -0,0 +1,84 @@
|
||||
(ns metabase-enterprise.workspaces.impl-test
|
||||
(:require
|
||||
[clojure.test :refer :all]
|
||||
[metabase-enterprise.workspaces.common :as ws.common]
|
||||
[metabase-enterprise.workspaces.impl :as ws.impl]
|
||||
[metabase-enterprise.workspaces.isolation :as ws.isolation]
|
||||
[metabase-enterprise.workspaces.test-util :as ws.tu]
|
||||
[metabase.lib.core :as lib]
|
||||
[metabase.lib.metadata :as lib.metadata]
|
||||
[metabase.test :as mt]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(ws.tu/ws-fixtures!)
|
||||
|
||||
(deftest sync-transform-dependencies-test
|
||||
(testing "sync-transform-dependencies! writes dependencies and grants access to external inputs"
|
||||
(let [mp (mt/metadata-provider)
|
||||
query-a (lib/query mp (lib.metadata/table mp (mt/id :orders)))
|
||||
query-b (lib/query mp (lib.metadata/table mp (mt/id :products)))
|
||||
workspace (ws.tu/create-ready-ws! "Test Workspace")
|
||||
grant-calls (atom [])]
|
||||
;; Set up mock BEFORE calling add-to-changeset! since it calls sync-transform-dependencies! internally
|
||||
(mt/with-dynamic-fn-redefs [ws.isolation/grant-read-access-to-tables!
|
||||
(fn [_database _workspace tables]
|
||||
(swap! grant-calls conj (set (map :name tables))))]
|
||||
(testing "Transform A depends on ORDERS"
|
||||
(ws.common/add-to-changeset! (mt/user->id :crowberto) workspace
|
||||
:transform nil
|
||||
{:name "Transform A"
|
||||
:source {:type "query" :query query-a}
|
||||
:target {:database (mt/id)
|
||||
:schema "analytics"
|
||||
:name "table_a"}})
|
||||
(testing "creates output and input records"
|
||||
(is (= 1 (t2/count :model/WorkspaceOutput :workspace_id (:id workspace))))
|
||||
(is (= 1 (t2/count :model/WorkspaceInput :workspace_id (:id workspace)))))
|
||||
(testing "input record has correct table name"
|
||||
(is (= (mt/format-name :orders)
|
||||
(t2/select-one-fn :table :model/WorkspaceInput :workspace_id (:id workspace)))))
|
||||
(testing "grants access to ORDERS"
|
||||
(is (= 1 (count @grant-calls)))
|
||||
(is (contains? (first @grant-calls) (mt/format-name :orders)))))
|
||||
|
||||
(testing "Transform B depends on PRODUCTS, table_a is internal output"
|
||||
(reset! grant-calls [])
|
||||
(ws.common/add-to-changeset! (mt/user->id :crowberto) workspace
|
||||
:transform nil
|
||||
{:name "Transform B"
|
||||
:source {:type "query" :query query-b}
|
||||
:target {:database (mt/id)
|
||||
:schema "analytics"
|
||||
:name "table_b"}})
|
||||
(testing "creates records for both transforms"
|
||||
(is (= 2 (t2/count :model/WorkspaceOutput :workspace_id (:id workspace))))
|
||||
(is (= 2 (t2/count :model/WorkspaceInput :workspace_id (:id workspace)))))
|
||||
(testing "only grants access to newly discovered inputs (PRODUCTS), not already-granted ones"
|
||||
(is (= 1 (count @grant-calls)))
|
||||
;; ORDERS was already granted in Transform A, so only PRODUCTS is in this grant call
|
||||
(is (not (contains? (first @grant-calls) (mt/format-name :orders))))
|
||||
(is (contains? (first @grant-calls) (mt/format-name :products)))
|
||||
(is (not (contains? (first @grant-calls) "table_a")))))))))
|
||||
|
||||
(deftest sync-transform-dependencies-idempotent-test
|
||||
(testing "sync-transform-dependencies! is idempotent"
|
||||
(let [mp (mt/metadata-provider)
|
||||
query (lib/query mp (lib.metadata/table mp (mt/id :orders)))
|
||||
workspace (ws.tu/create-ready-ws! "Test Workspace")]
|
||||
(mt/with-dynamic-fn-redefs [ws.isolation/grant-read-access-to-tables! (constantly nil)]
|
||||
(let [wt (ws.common/add-to-changeset! (mt/user->id :crowberto) workspace
|
||||
:transform nil
|
||||
{:name "Transform"
|
||||
:source {:type "query" :query query}
|
||||
:target {:database (mt/id)
|
||||
:schema "analytics"
|
||||
:name "output_table"}})]
|
||||
;; Call sync again to test idempotency
|
||||
(ws.impl/sync-transform-dependencies! workspace wt)
|
||||
(ws.impl/sync-transform-dependencies! workspace wt)
|
||||
(testing "still has exactly one of each after multiple syncs"
|
||||
(is (= 1 (t2/count :model/WorkspaceOutput :workspace_id (:id workspace))))
|
||||
(is (= 1 (t2/count :model/WorkspaceInput :workspace_id (:id workspace))))
|
||||
(is (= 1 (t2/count :model/WorkspaceDependency :workspace_id (:id workspace))))))))))
|
||||
@@ -0,0 +1,121 @@
|
||||
(ns ^:mb/driver-tests metabase-enterprise.workspaces.isolation-test
|
||||
(:require
|
||||
[clojure.java.jdbc :as jdbc]
|
||||
[clojure.test :refer :all]
|
||||
[metabase-enterprise.workspaces.common :as ws.common]
|
||||
[metabase-enterprise.workspaces.isolation :as isolation]
|
||||
[metabase-enterprise.workspaces.test-util :as ws.tu]
|
||||
[metabase-enterprise.workspaces.util :as ws.u]
|
||||
[metabase.driver :as driver]
|
||||
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
|
||||
[metabase.test :as mt]))
|
||||
|
||||
(ws.tu/ws-fixtures!)
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(def isolated-prefix @#'ws.u/isolated-prefix)
|
||||
|
||||
;;; Test helper multimethod to verify cleanup
|
||||
|
||||
(defmulti workspace-isolation-resources-exist?
|
||||
"Check if isolation resources still exist for the workspace.
|
||||
Returns a map with keys indicating which resources exist."
|
||||
{:arglists '([database workspace])}
|
||||
(fn [database _workspace]
|
||||
(driver/the-driver (:engine database)))
|
||||
:hierarchy #'driver/hierarchy)
|
||||
|
||||
(defmethod workspace-isolation-resources-exist? :postgres
|
||||
[database workspace]
|
||||
(let [schema-name (ws.u/isolation-namespace-name workspace)
|
||||
username (ws.u/isolation-user-name workspace)
|
||||
conn-spec (sql-jdbc.conn/db->pooled-connection-spec (:id database))]
|
||||
{:schema (-> (jdbc/query conn-spec
|
||||
["SELECT 1 FROM information_schema.schemata WHERE schema_name = ?" schema-name])
|
||||
seq boolean)
|
||||
:user (-> (jdbc/query conn-spec
|
||||
["SELECT 1 FROM pg_user WHERE usename = ?" username])
|
||||
seq boolean)}))
|
||||
|
||||
(defmethod workspace-isolation-resources-exist? :h2
|
||||
[database workspace]
|
||||
(let [schema-name (ws.u/isolation-namespace-name workspace)
|
||||
username (ws.u/isolation-user-name workspace)
|
||||
conn-spec (sql-jdbc.conn/db->pooled-connection-spec (:id database))]
|
||||
{:schema (-> (jdbc/query conn-spec
|
||||
["SELECT 1 FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = ?" schema-name])
|
||||
seq boolean)
|
||||
:user (-> (jdbc/query conn-spec
|
||||
["SELECT 1 FROM INFORMATION_SCHEMA.USERS WHERE NAME = ?" username])
|
||||
seq boolean)}))
|
||||
|
||||
(defmethod workspace-isolation-resources-exist? :snowflake
|
||||
[database workspace]
|
||||
(let [schema-name (ws.u/isolation-namespace-name workspace)
|
||||
db-name (-> database :details :db)
|
||||
role-name (format "MB_ISOLATION_ROLE_%s" (:id workspace))
|
||||
username (ws.u/isolation-user-name workspace)
|
||||
conn-spec (sql-jdbc.conn/db->pooled-connection-spec (:id database))]
|
||||
{:schema (-> (jdbc/query conn-spec
|
||||
[(format "SHOW SCHEMAS LIKE '%s' IN DATABASE \"%s\"" schema-name db-name)])
|
||||
seq boolean)
|
||||
:user (-> (jdbc/query conn-spec
|
||||
[(format "SHOW USERS LIKE '%s'" username)])
|
||||
seq boolean)
|
||||
:role (-> (jdbc/query conn-spec
|
||||
[(format "SHOW ROLES LIKE '%s'" role-name)])
|
||||
seq boolean)}))
|
||||
|
||||
(defmethod workspace-isolation-resources-exist? :sqlserver
|
||||
[database workspace]
|
||||
(let [schema-name (ws.u/isolation-namespace-name workspace)
|
||||
login-name (format "%s_%s_login" isolated-prefix (:id workspace))
|
||||
username (ws.u/isolation-user-name workspace)
|
||||
conn-spec (sql-jdbc.conn/db->pooled-connection-spec (:id database))]
|
||||
{:schema (-> (jdbc/query conn-spec
|
||||
["SELECT 1 FROM sys.schemas WHERE name = ?" schema-name])
|
||||
seq boolean)
|
||||
:user (-> (jdbc/query conn-spec
|
||||
["SELECT 1 FROM sys.database_principals WHERE name = ?" username])
|
||||
seq boolean)
|
||||
:login (-> (jdbc/query conn-spec
|
||||
["SELECT 1 FROM master.sys.server_principals WHERE name = ?" login-name])
|
||||
seq boolean)}))
|
||||
|
||||
(defmethod workspace-isolation-resources-exist? :clickhouse
|
||||
[database workspace]
|
||||
(let [db-name (ws.u/isolation-namespace-name workspace)
|
||||
username (ws.u/isolation-user-name workspace)
|
||||
conn-spec (sql-jdbc.conn/db->pooled-connection-spec (:id database))]
|
||||
{:database (-> (jdbc/query conn-spec
|
||||
["SELECT 1 FROM system.databases WHERE name = ?" db-name])
|
||||
seq boolean)
|
||||
:user (-> (jdbc/query conn-spec
|
||||
["SELECT 1 FROM system.users WHERE name = ?" username])
|
||||
seq boolean)}))
|
||||
|
||||
;;; Tests
|
||||
|
||||
(deftest destroy-workspace-isolation-test
|
||||
(mt/test-drivers (mt/normal-drivers-with-feature :workspace)
|
||||
(ws.tu/with-workspaces! [workspace {:name "Test destroy isolation"}]
|
||||
(ws.common/add-to-changeset! (mt/user->id :crowberto) workspace
|
||||
:transform nil
|
||||
{:name "Transform A"
|
||||
:source {:type "query" :query (mt/mbql-query orders {:limit 1})}
|
||||
:target {:database (mt/id)
|
||||
:schema "analytics"
|
||||
:name "table_a"}})
|
||||
(let [database (mt/db)]
|
||||
(testing "resources are creeated during workspace initialization"
|
||||
(let [resources (workspace-isolation-resources-exist? database workspace)]
|
||||
(is (every? true? (vals resources)))))
|
||||
|
||||
(testing "destroy removes all isolation resources"
|
||||
(isolation/destroy-workspace-isolation! database workspace)
|
||||
(let [resources (workspace-isolation-resources-exist? database workspace)]
|
||||
(is (every? false? (vals resources)))))
|
||||
|
||||
(testing "destroy is idempotent"
|
||||
(isolation/destroy-workspace-isolation! database workspace))))))
|
||||
@@ -0,0 +1,119 @@
|
||||
(ns ^:mb/driver-tests metabase-enterprise.workspaces.test-util
|
||||
(:require
|
||||
[clojure.test :refer :all]
|
||||
[metabase-enterprise.workspaces.common :as ws.common]
|
||||
[metabase-enterprise.workspaces.isolation :as ws.isolation]
|
||||
[metabase-enterprise.workspaces.models.workspace :as ws.model]
|
||||
[metabase.config.core :as config]
|
||||
[metabase.search.test-util :as search.tu]
|
||||
[metabase.test :as mt]
|
||||
[metabase.util :as u]
|
||||
[metabase.util.log :as log]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(defn ws-fixtures!
|
||||
"Sets up test fixtures for workspace tests. Must be called at the top level of test namespaces."
|
||||
[]
|
||||
(use-fixtures :once (fn [tests]
|
||||
;; E.g. app-db.yml tests perorm driver tests. Workspaces are not supported on mysql.
|
||||
;; Following disj suppresses those runs destined for failure.
|
||||
(mt/test-drivers (mt/normal-drivers-with-feature :workspace)
|
||||
(mt/with-premium-features [:workspaces :dependencies :transforms]
|
||||
(search.tu/with-index-disabled
|
||||
(tests))))))
|
||||
|
||||
(use-fixtures :each (fn [tests]
|
||||
(mt/with-model-cleanup [:model/Collection
|
||||
:model/Transform
|
||||
:model/TransformRun
|
||||
:model/Workspace
|
||||
:model/WorkspaceTransform
|
||||
:model/WorkspaceInput
|
||||
:model/WorkspaceOutput
|
||||
:model/WorkspaceDependency]
|
||||
(tests)))))
|
||||
|
||||
(derive :model/Workspace :model/WorkspaceCleanUpInTest)
|
||||
|
||||
(t2/define-before-delete :model/WorkspaceCleanUpInTest
|
||||
[workspace]
|
||||
(try
|
||||
(log/infof "Cleaningup workspace %d in tests" (:id workspace))
|
||||
(when (:database_details workspace)
|
||||
(ws.isolation/destroy-workspace-isolation! (t2/select-one :model/Database (:database_id workspace)) workspace))
|
||||
(catch Exception e
|
||||
(log/warn e "Failed to destroy isolation" {:workspace workspace})))
|
||||
workspace)
|
||||
|
||||
(defn ws-ready
|
||||
"Poll until workspace status becomes :ready or timeout.
|
||||
Note: uninitialized workspaces will never become ready without adding a transform."
|
||||
[ws-or-id]
|
||||
(let [ws-id (cond-> ws-or-id
|
||||
(map? ws-or-id) :id)]
|
||||
(or (u/poll {:thunk #(t2/select-one :model/Workspace :id ws-id)
|
||||
:done? #(not= :pending (:status %))
|
||||
;; some cloud drivers are really slow
|
||||
:timeout-ms (if config/is-dev? 5000 60000)})
|
||||
(throw (ex-info "Timeout waiting for workspace to be ready" {:workspace-id ws-id})))))
|
||||
|
||||
(defn create-workspace-for-test!
|
||||
"Create a workspace for testing using proper initialization.
|
||||
Returns the workspace after waiting for it to be ready."
|
||||
[props]
|
||||
(let [creator-id (or (:creator_id props) (mt/user->id :crowberto))
|
||||
props (-> props
|
||||
(dissoc :creator_id)
|
||||
(assoc :provisional? true)
|
||||
(update :database_id #(or % (mt/id))))]
|
||||
(ws-ready (mt/with-current-user creator-id
|
||||
(ws.common/create-workspace! creator-id props)))))
|
||||
|
||||
(defn create-ready-ws!
|
||||
"Create a workspace and wait for it to be ready."
|
||||
[name]
|
||||
(create-workspace-for-test! {:name name}))
|
||||
|
||||
(defn do-with-workspaces!
|
||||
"Function that sets up workspaces for testing and cleans up afterwards.
|
||||
Takes a sequence of props for workspace creation and a thunk that receives
|
||||
the created workspaces as a vector. Each workspace is cleaned up by its own
|
||||
stack frame, so cleanup naturally happens in reverse creation order."
|
||||
[props-list thunk]
|
||||
(if (empty? props-list)
|
||||
(thunk [])
|
||||
(let [ws (create-workspace-for-test! (first props-list))]
|
||||
(try
|
||||
(do-with-workspaces! (rest props-list)
|
||||
(fn [rest-workspaces]
|
||||
(thunk (into [ws] rest-workspaces))))
|
||||
(finally
|
||||
(when ws
|
||||
(try
|
||||
(ws.model/delete! (t2/select-one :model/Workspace :id (:id ws)))
|
||||
(catch Exception _
|
||||
;; Workspace may already be deleted by the test
|
||||
nil))))))))
|
||||
|
||||
(defmacro with-workspaces!
|
||||
"Execute body with properly initialized workspaces that are cleaned up afterward.
|
||||
|
||||
Creates each workspace using `ws.common/create-workspace!` and waits for it to
|
||||
be ready. After body execution (or on error), cleans up using `ws.model/delete!`
|
||||
which properly destroys database isolation resources.
|
||||
|
||||
Usage:
|
||||
(with-workspaces [ws1 {:name \"Test WS 1\"}
|
||||
ws2 {:name \"Test WS 2\" :database_id (mt/id)}]
|
||||
(testing \"workspace operations\"
|
||||
(is (= :ready (:status ws1)))))"
|
||||
[bindings & body]
|
||||
(assert (vector? bindings) "bindings must be a vector")
|
||||
(assert (even? (count bindings)) "bindings must have an even number of forms")
|
||||
(let [pairs (partition 2 bindings)
|
||||
syms (mapv first pairs)
|
||||
props-list (mapv second pairs)]
|
||||
`(do-with-workspaces!
|
||||
[~@props-list]
|
||||
(fn [[~@syms]]
|
||||
~@body))))
|
||||
@@ -0,0 +1,178 @@
|
||||
(ns ^:mb/driver-tests metabase-enterprise.workspaces.validation-test
|
||||
(:require
|
||||
[clojure.test :refer :all]
|
||||
[metabase-enterprise.workspaces.dag :as ws.dag]
|
||||
[metabase-enterprise.workspaces.test-util :as ws.tu]
|
||||
[metabase-enterprise.workspaces.validation :as ws.validation]
|
||||
[metabase.test :as mt]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(ws.tu/ws-fixtures!)
|
||||
|
||||
(defn- build-graph
|
||||
"Build the workspace graph for validation tests."
|
||||
[ws-id]
|
||||
(let [changeset (t2/select-fn-vec (fn [{:keys [ref_id]}] {:entity-type :transform, :id ref_id})
|
||||
[:model/WorkspaceTransform :ref_id] :workspace_id ws-id)]
|
||||
(when (seq changeset)
|
||||
(ws.dag/path-induced-subgraph ws-id changeset))))
|
||||
|
||||
;;; ---------------------------------------- Basic API Tests ----------------------------------------
|
||||
|
||||
(deftest find-downstream-problems-empty-workspace-test
|
||||
(testing "find-downstream-problems returns empty for workspace with no transforms"
|
||||
(ws.tu/with-workspaces! [workspace {:name "Empty Workspace"}]
|
||||
(is (= [] (ws.validation/find-downstream-problems (:id workspace) nil))))))
|
||||
|
||||
(deftest problems-endpoint-returns-empty-for-new-workspace-test
|
||||
(testing "GET /api/ee/workspace/:id/problems returns empty list for workspace with no transforms"
|
||||
(ws.tu/with-workspaces! [workspace {:name "Test Workspace"}]
|
||||
(let [response (mt/user-http-request :crowberto :get 200
|
||||
(str "ee/workspace/" (:id workspace) "/problem"))]
|
||||
(is (= [] response))))))
|
||||
|
||||
(deftest problems-endpoint-requires-superuser-test
|
||||
(testing "GET /api/ee/workspace/:id/problems requires superuser"
|
||||
(ws.tu/with-workspaces! [workspace {:name "Private Workspace"}]
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :get 403
|
||||
(str "ee/workspace/" (:id workspace) "/problem")))))))
|
||||
|
||||
(deftest problems-endpoint-404-for-nonexistent-workspace-test
|
||||
(testing "GET /api/ee/workspace/:id/problems returns 404 for non-existent workspace"
|
||||
(is (= "Not found."
|
||||
(mt/user-http-request :crowberto :get 404 "ee/workspace/999999/problem")))))
|
||||
|
||||
;;; ---------------------------------------- Problem Detection Tests ----------------------------------------
|
||||
|
||||
(deftest unused-output-not-yet-created-test
|
||||
(testing "detects when a workspace output table hasn't been created yet (no external dependents)"
|
||||
(ws.tu/with-workspaces! [workspace {:name "Test Workspace"}]
|
||||
(let [ws-id (:id workspace)
|
||||
;; Create a workspace transform
|
||||
{ref-id :ref_id} (mt/user-http-request :crowberto :post 200
|
||||
(str "ee/workspace/" ws-id "/transform")
|
||||
{:name "Test Transform"
|
||||
:source {:type "query"
|
||||
:query {:database (mt/id)
|
||||
:type :native
|
||||
:native {:query "SELECT 1 as col"}}}
|
||||
:target {:type "table"
|
||||
:schema "public"
|
||||
:name "test_output_table"}})
|
||||
;; The transform hasn't been run, so isolated_table_id is nil
|
||||
;; No external transforms depend on it
|
||||
graph (build-graph ws-id)
|
||||
problems (ws.validation/find-downstream-problems ws-id graph)]
|
||||
(is (= [{:category :unused
|
||||
:problem :not-run
|
||||
:severity :info
|
||||
:block_merge false
|
||||
:description "Output hasn't been created yet, but nothing depends on it"
|
||||
:data {:output {:db_id (mt/id), :schema "public", :table "test_output_table"}
|
||||
:transform {:type :workspace-transform, :id ref-id}}}]
|
||||
problems))))))
|
||||
|
||||
(deftest dependent-transform-output-not-yet-created-test
|
||||
(testing "detects when output table hasn't been created but external transforms depend on it"
|
||||
(ws.tu/with-workspaces! [workspace {:name "Test Workspace"}]
|
||||
(let [ws-id (:id workspace)]
|
||||
;; First, create the global output table that we'll have a dependency on
|
||||
(mt/with-temp [:model/Table output-table {:db_id (mt/id)
|
||||
:schema "public"
|
||||
:name "global_output_table"
|
||||
:active true}]
|
||||
;; Create an external transform that depends on this table
|
||||
;; Note: The dependency will be created automatically by the Transform model hooks
|
||||
(mt/with-temp [:model/Transform external-tx {:name "External Transform"
|
||||
:source {:type :query
|
||||
:query {:database (mt/id)
|
||||
:type :query
|
||||
:query {:source-table (:id output-table)}}}
|
||||
:target {:type "table"
|
||||
:database (mt/id)
|
||||
:schema "public"
|
||||
:name "external_output"}}]
|
||||
;; Create a workspace transform that targets the same table
|
||||
(let [{ref-id :ref_id} (mt/user-http-request :crowberto :post 200
|
||||
(str "ee/workspace/" ws-id "/transform")
|
||||
{:name "Workspace Transform"
|
||||
:source {:type "query"
|
||||
:query {:database (mt/id)
|
||||
:type :native
|
||||
:native {:query "SELECT 1 as col"}}}
|
||||
:target {:type "table"
|
||||
:schema "public"
|
||||
:name "global_output_table"}})]
|
||||
;; Manually update the WorkspaceOutput to point to the global table
|
||||
;; (simulating a checked-out transform scenario)
|
||||
(t2/update! :model/WorkspaceOutput
|
||||
{:workspace_id ws-id :ref_id ref-id}
|
||||
{:global_table_id (:id output-table)})
|
||||
|
||||
(is (= [{:category :external-downstream
|
||||
:problem :not-run
|
||||
:severity :warning
|
||||
:block_merge true
|
||||
:description "Output hasn't been created yet, external transforms depend on it"
|
||||
:data {:output {:db_id (mt/id), :schema "public", :table "global_output_table"}
|
||||
:transform {:type :workspace-transform, :id ref-id}
|
||||
:dependents [{:type :external-transform
|
||||
:id (:id external-tx)
|
||||
:name "External Transform"}]}}]
|
||||
(ws.validation/find-downstream-problems ws-id (build-graph ws-id)))))))))))
|
||||
|
||||
(deftest no-problems-when-output-exists-and-no-field-changes-test
|
||||
(testing "no problems when isolated table exists and has all required fields"
|
||||
(ws.tu/with-workspaces! [workspace {:name "Test Workspace"}]
|
||||
(let [ws-id (:id workspace)]
|
||||
;; Create both global and isolated tables with matching fields
|
||||
(mt/with-temp [:model/Table global-table {:db_id (mt/id)
|
||||
:schema "public"
|
||||
:name "output_table"
|
||||
:active true}
|
||||
:model/Field _ {:table_id (:id global-table)
|
||||
:name "col1"
|
||||
:database_type "TEXT"
|
||||
:base_type :type/Text
|
||||
:active true}
|
||||
:model/Table isolated-table {:db_id (mt/id)
|
||||
:schema "mb__isolation_test"
|
||||
:name "public__output_table"
|
||||
:active true}
|
||||
:model/Field _ {:table_id (:id isolated-table)
|
||||
:name "col1"
|
||||
:database_type "TEXT"
|
||||
:base_type :type/Text
|
||||
:active true}]
|
||||
;; External transform that uses col1 via MBQL source-table
|
||||
(mt/with-temp [:model/Transform _external-tx {:name "External Transform"
|
||||
:source {:type :query
|
||||
:query {:database (mt/id)
|
||||
:type :query
|
||||
:query {:source-table (:id global-table)}}}
|
||||
:target {:type "table"
|
||||
:database (mt/id)
|
||||
:schema "public"
|
||||
:name "external_output"}}]
|
||||
;; Create workspace transform and output
|
||||
(let [{ref-id :ref_id} (mt/user-http-request :crowberto :post 200
|
||||
(str "ee/workspace/" ws-id "/transform")
|
||||
{:name "Workspace Transform"
|
||||
:source {:type "query"
|
||||
:query {:database (mt/id)
|
||||
:type :native
|
||||
:native {:query "SELECT 'test' as col1"}}}
|
||||
:target {:type "table"
|
||||
:schema "public"
|
||||
:name "output_table"}})]
|
||||
;; Update WorkspaceOutput to have both global and isolated table IDs
|
||||
(t2/update! :model/WorkspaceOutput
|
||||
{:workspace_id ws-id :ref_id ref-id}
|
||||
{:global_table_id (:id global-table)
|
||||
:isolated_table_id (:id isolated-table)})
|
||||
|
||||
;; Should have no problems since isolated table has all required fields
|
||||
(is (empty? (ws.validation/find-downstream-problems ws-id (build-graph ws-id)))))))))))
|
||||
@@ -5,12 +5,27 @@ import { useMetabotAgent } from "metabase-enterprise/metabot/hooks";
|
||||
|
||||
import { trackQueryFixClicked } from "../../analytics";
|
||||
|
||||
export function FixSqlQueryButton() {
|
||||
type FixSqlQueryButtonProps = {
|
||||
rawSql?: string | null;
|
||||
errorMessage?: string | null;
|
||||
};
|
||||
|
||||
export function FixSqlQueryButton({
|
||||
rawSql,
|
||||
errorMessage,
|
||||
}: FixSqlQueryButtonProps) {
|
||||
const { submitInput } = useMetabotAgent("omnibot");
|
||||
|
||||
const handleClick = () => {
|
||||
trackQueryFixClicked();
|
||||
submitInput("Fix this SQL query");
|
||||
const promptParts = ["Fix this SQL query."];
|
||||
if (errorMessage) {
|
||||
promptParts.push(`The database returned this error: ${errorMessage}`);
|
||||
}
|
||||
if (rawSql) {
|
||||
promptParts.push(`SQL:\n${rawSql}`);
|
||||
}
|
||||
submitInput(promptParts.join("\n\n"));
|
||||
};
|
||||
|
||||
return <Button onClick={handleClick}>{t`Have Metabot fix it`}</Button>;
|
||||
|
||||
@@ -20,3 +20,4 @@ export * from "./transform";
|
||||
export * from "./transform-job";
|
||||
export * from "./transform-tag";
|
||||
export * from "./upload-management";
|
||||
export * from "./workspace";
|
||||
|
||||
@@ -28,6 +28,8 @@ import type {
|
||||
TransformJob,
|
||||
TransformRun,
|
||||
TransformTag,
|
||||
Workspace,
|
||||
WorkspaceItem,
|
||||
} from "metabase-types/api";
|
||||
|
||||
export const ENTERPRISE_TAG_TYPES = [
|
||||
@@ -42,6 +44,10 @@ export const ENTERPRISE_TAG_TYPES = [
|
||||
"transform-job",
|
||||
"transform-job-via-tag",
|
||||
"transform-run",
|
||||
"workspace-transforms",
|
||||
"workspace-transform",
|
||||
"workspace-tables",
|
||||
"external-transforms",
|
||||
"git-tree",
|
||||
"git-file-content",
|
||||
"collection-dirty-entities",
|
||||
@@ -49,6 +55,7 @@ export const ENTERPRISE_TAG_TYPES = [
|
||||
"remote-sync-branches",
|
||||
"remote-sync-current-task",
|
||||
"python-transform-library",
|
||||
"workspace",
|
||||
"support-access-grant",
|
||||
"support-access-grant-current",
|
||||
"library-collection",
|
||||
@@ -82,6 +89,24 @@ export function invalidateTags(
|
||||
return !error ? tags : [];
|
||||
}
|
||||
|
||||
export function provideWorkspacesTags(
|
||||
workspaces: Workspace[],
|
||||
): TagDescription<EnterpriseTagType>[] {
|
||||
return [listTag("workspace"), ...workspaces.flatMap(provideWorkspaceTags)];
|
||||
}
|
||||
|
||||
export function provideWorkspaceTags(
|
||||
workspace: Workspace | WorkspaceItem,
|
||||
): TagDescription<EnterpriseTagType>[] {
|
||||
return [idTag("workspace", workspace.id)];
|
||||
}
|
||||
|
||||
export function provideWorkspaceContentItemsTags(
|
||||
items: WorkspaceContentItem[],
|
||||
): TagDescription<EnterpriseTagType>[] {
|
||||
return items.flatMap((transform) => idTag("transform", transform.id));
|
||||
}
|
||||
|
||||
export function provideTransformTags(
|
||||
transform: Transform,
|
||||
): TagDescription<EnterpriseTagType>[] {
|
||||
|
||||
@@ -151,7 +151,10 @@ export const transformApi = EnterpriseApi.injectEndpoints({
|
||||
body,
|
||||
}),
|
||||
invalidatesTags: (_, error) =>
|
||||
invalidateTags(error, [listTag("transform")]),
|
||||
invalidateTags(error, [
|
||||
listTag("transform"),
|
||||
listTag("external-transforms"),
|
||||
]),
|
||||
}),
|
||||
updateTransform: builder.mutation<Transform, UpdateTransformRequest>({
|
||||
query: ({ id, ...body }) => ({
|
||||
|
||||
406
enterprise/frontend/src/metabase-enterprise/api/workspace.ts
Normal file
406
enterprise/frontend/src/metabase-enterprise/api/workspace.ts
Normal file
@@ -0,0 +1,406 @@
|
||||
import type {
|
||||
CreateWorkspaceRequest,
|
||||
CreateWorkspaceTransformRequest,
|
||||
CreateWorkspaceTransformResponse,
|
||||
ExternalTransformsRequest,
|
||||
ExternalTransformsResponse,
|
||||
TransformDownstreamMapping,
|
||||
TransformId,
|
||||
TransformUpstreamMapping,
|
||||
UpdateWorkspaceTransformRequest,
|
||||
ValidateTableNameRequest,
|
||||
ValidateTableNameResponse,
|
||||
Workspace,
|
||||
WorkspaceAllowedDatabasesResponse,
|
||||
WorkspaceCheckoutResponse,
|
||||
WorkspaceGraphResponse,
|
||||
WorkspaceId,
|
||||
WorkspaceListResponse,
|
||||
WorkspaceLogResponse,
|
||||
WorkspaceMergeResponse,
|
||||
WorkspaceProblem,
|
||||
WorkspaceRunRequest,
|
||||
WorkspaceRunResponse,
|
||||
WorkspaceTablesResponse,
|
||||
WorkspaceTransform,
|
||||
WorkspaceTransformItem,
|
||||
WorkspaceTransformMergeResponse,
|
||||
WorkspaceTransformRef,
|
||||
WorkspaceTransformRunResponse,
|
||||
WorkspaceTransformsResponse,
|
||||
} from "metabase-types/api";
|
||||
|
||||
import { EnterpriseApi } from "./api";
|
||||
import {
|
||||
idTag,
|
||||
invalidateTags,
|
||||
listTag,
|
||||
provideWorkspaceContentItemsTags,
|
||||
provideWorkspaceTags,
|
||||
provideWorkspacesTags,
|
||||
tag,
|
||||
} from "./tags";
|
||||
|
||||
export const workspaceApi = EnterpriseApi.injectEndpoints({
|
||||
endpoints: (builder) => ({
|
||||
getWorkspaces: builder.query<WorkspaceListResponse, void>({
|
||||
query: () => ({
|
||||
method: "GET",
|
||||
url: "/api/ee/workspace",
|
||||
}),
|
||||
providesTags: (response) =>
|
||||
response ? provideWorkspacesTags(response.items) : [],
|
||||
}),
|
||||
getWorkspace: builder.query<Workspace, WorkspaceId>({
|
||||
query: (id) => ({
|
||||
method: "GET",
|
||||
url: `/api/ee/workspace/${id}`,
|
||||
}),
|
||||
providesTags: (workspace) =>
|
||||
workspace ? provideWorkspaceTags(workspace) : [],
|
||||
}),
|
||||
createWorkspace: builder.mutation<Workspace, void>({
|
||||
query: () => ({
|
||||
method: "POST",
|
||||
url: "/api/ee/workspace",
|
||||
}),
|
||||
invalidatesTags: (_, error) =>
|
||||
invalidateTags(error, [listTag("workspace"), listTag("transform")]),
|
||||
}),
|
||||
updateWorkspace: builder.mutation<
|
||||
Workspace,
|
||||
{ id: WorkspaceId; name?: string; database_id?: number }
|
||||
>({
|
||||
query: ({ id, ...body }) => ({
|
||||
method: "PUT",
|
||||
url: `/api/ee/workspace/${id}`,
|
||||
body,
|
||||
}),
|
||||
invalidatesTags: (_, error, { id }) =>
|
||||
invalidateTags(error, [idTag("workspace", id)]),
|
||||
}),
|
||||
createWorkspaceTransform: builder.mutation<
|
||||
CreateWorkspaceTransformResponse,
|
||||
{ id: WorkspaceId } & CreateWorkspaceTransformRequest
|
||||
>({
|
||||
query: ({ id, ...body }) => ({
|
||||
method: "POST",
|
||||
url: `/api/ee/workspace/${id}/transform`,
|
||||
body,
|
||||
}),
|
||||
invalidatesTags: (_, error, { id, global_id }) =>
|
||||
invalidateTags(error, [
|
||||
idTag("workspace", id),
|
||||
idTag("workspace-transforms", id),
|
||||
idTag("external-transforms", id),
|
||||
idTag("workspace-tables", id),
|
||||
listTag("transform"),
|
||||
...(global_id != null ? [idTag("transform", global_id)] : []),
|
||||
]),
|
||||
transformResponse: (t: WorkspaceTransform) => ({ ...t, id: t.ref_id }),
|
||||
}),
|
||||
getTransformUpstreamMapping: builder.query<
|
||||
TransformUpstreamMapping,
|
||||
TransformId
|
||||
>({
|
||||
query: (id) => ({
|
||||
method: "GET",
|
||||
url: `/api/ee/workspace/mapping/transform/${id}/upstream`,
|
||||
}),
|
||||
providesTags: (mapping) =>
|
||||
mapping?.transform
|
||||
? provideWorkspaceContentItemsTags([mapping.transform])
|
||||
: [],
|
||||
}),
|
||||
getTransformDownstreamMapping: builder.query<
|
||||
TransformDownstreamMapping,
|
||||
TransformId
|
||||
>({
|
||||
query: (id) => ({
|
||||
method: "GET",
|
||||
url: `/api/ee/workspace/mapping/transform/${id}/downstream`,
|
||||
}),
|
||||
providesTags: (mapping) =>
|
||||
mapping ? provideWorkspaceContentItemsTags(mapping.transforms) : [],
|
||||
}),
|
||||
getWorkspaceCheckout: builder.query<WorkspaceCheckoutResponse, TransformId>(
|
||||
{
|
||||
query: (transformId) => ({
|
||||
method: "GET",
|
||||
url: `/api/ee/workspace/checkout`,
|
||||
params: { "transform-id": transformId },
|
||||
}),
|
||||
providesTags: (_, __, transformId) => [idTag("transform", transformId)],
|
||||
},
|
||||
),
|
||||
mergeWorkspace: builder.mutation<
|
||||
WorkspaceMergeResponse,
|
||||
{ id: WorkspaceId; commit_message: string }
|
||||
>({
|
||||
query: ({ id, commit_message }) => ({
|
||||
method: "POST",
|
||||
url: `/api/ee/workspace/${id}/merge`,
|
||||
body: { commit_message },
|
||||
}),
|
||||
invalidatesTags: (_, error) =>
|
||||
invalidateTags(error, [tag("workspace"), tag("transform")]),
|
||||
}),
|
||||
mergeWorkspaceTransform: builder.mutation<
|
||||
WorkspaceTransformMergeResponse,
|
||||
WorkspaceTransformRef
|
||||
>({
|
||||
query: ({ workspaceId, transformId }) => ({
|
||||
method: "POST",
|
||||
url: `/api/ee/workspace/${workspaceId}/transform/${transformId}/merge`,
|
||||
}),
|
||||
invalidatesTags: (_, error) =>
|
||||
invalidateTags(error, [tag("workspace"), tag("transform")]),
|
||||
}),
|
||||
archiveWorkspace: builder.mutation<Workspace, WorkspaceId>({
|
||||
query: (id) => ({
|
||||
method: "POST",
|
||||
url: `/api/ee/workspace/${id}/archive`,
|
||||
}),
|
||||
invalidatesTags: (_, error) =>
|
||||
invalidateTags(error, [tag("workspace"), tag("transform")]),
|
||||
}),
|
||||
unarchiveWorkspace: builder.mutation<Workspace, WorkspaceId>({
|
||||
query: (id) => ({
|
||||
method: "POST",
|
||||
url: `/api/ee/workspace/${id}/unarchive`,
|
||||
}),
|
||||
invalidatesTags: (_, error) =>
|
||||
invalidateTags(error, [tag("workspace"), tag("transform")]),
|
||||
}),
|
||||
deleteWorkspace: builder.mutation<void, WorkspaceId>({
|
||||
query: (id) => ({
|
||||
method: "DELETE",
|
||||
url: `/api/ee/workspace/${id}`,
|
||||
}),
|
||||
invalidatesTags: (_, error) =>
|
||||
invalidateTags(error, [tag("workspace"), tag("transform")]),
|
||||
}),
|
||||
validateTableName: builder.mutation<
|
||||
ValidateTableNameResponse,
|
||||
ValidateTableNameRequest
|
||||
>({
|
||||
query: ({ id, ...body }) => ({
|
||||
method: "POST",
|
||||
url: `/api/ee/workspace/${id}/transform/validate/target`,
|
||||
body,
|
||||
}),
|
||||
}),
|
||||
getWorkspaceTables: builder.query<WorkspaceTablesResponse, WorkspaceId>({
|
||||
query: (id) => ({
|
||||
method: "GET",
|
||||
url: `/api/ee/workspace/${id}/table`,
|
||||
}),
|
||||
providesTags: (_, __, id) => [idTag("workspace-tables", id)],
|
||||
}),
|
||||
getWorkspaceGraph: builder.query<WorkspaceGraphResponse, WorkspaceId>({
|
||||
query: (id) => ({
|
||||
method: "GET",
|
||||
url: `/api/ee/workspace/${id}/graph`,
|
||||
}),
|
||||
providesTags: (_, __, id) => [idTag("workspace", id)],
|
||||
}),
|
||||
getWorkspaceProblems: builder.query<WorkspaceProblem[], WorkspaceId>({
|
||||
query: (id) => ({
|
||||
method: "GET",
|
||||
url: `/api/ee/workspace/${id}/problem`,
|
||||
}),
|
||||
providesTags: (_, __, id) => [idTag("workspace", id)],
|
||||
}),
|
||||
getWorkspaceTransforms: builder.query<
|
||||
WorkspaceTransformItem[],
|
||||
WorkspaceId
|
||||
>({
|
||||
query: (id) => ({
|
||||
method: "GET",
|
||||
url: `/api/ee/workspace/${id}/transform`,
|
||||
}),
|
||||
providesTags: (_, __, id) => [idTag("workspace-transforms", id)],
|
||||
transformResponse: (response: WorkspaceTransformsResponse) =>
|
||||
response.transforms,
|
||||
}),
|
||||
getExternalTransforms: builder.query<
|
||||
ExternalTransformsResponse["transforms"],
|
||||
ExternalTransformsRequest
|
||||
>({
|
||||
query: ({ workspaceId, databaseId }) => ({
|
||||
method: "GET",
|
||||
url: `/api/ee/workspace/${workspaceId}/external/transform`,
|
||||
params: { database_id: databaseId },
|
||||
}),
|
||||
providesTags: (_, __, { workspaceId: id }) => [
|
||||
listTag("external-transforms"),
|
||||
idTag("external-transforms", id),
|
||||
],
|
||||
transformResponse: (response: ExternalTransformsResponse) =>
|
||||
response.transforms,
|
||||
}),
|
||||
getWorkspaceTransform: builder.query<
|
||||
WorkspaceTransform,
|
||||
{ workspaceId: WorkspaceId; transformId: string }
|
||||
>({
|
||||
query: ({ workspaceId, transformId }) => ({
|
||||
method: "GET",
|
||||
url: `/api/ee/workspace/${workspaceId}/transform/${transformId}`,
|
||||
}),
|
||||
providesTags: (_, __, { transformId }) => [
|
||||
// idTag("workspace-transforms", workspaceId),
|
||||
idTag("workspace-transform", transformId),
|
||||
],
|
||||
transformResponse: (response: WorkspaceTransform) => ({
|
||||
...response,
|
||||
id: response.ref_id,
|
||||
}),
|
||||
}),
|
||||
getWorkspaceLog: builder.query<WorkspaceLogResponse, WorkspaceId>({
|
||||
query: (id) => ({
|
||||
method: "GET",
|
||||
url: `/api/ee/workspace/${id}/log`,
|
||||
}),
|
||||
providesTags: (_, _error, id) => [idTag("workspace", id)],
|
||||
}),
|
||||
runWorkspace: builder.mutation<WorkspaceRunResponse, WorkspaceRunRequest>({
|
||||
query: ({ id, stale_only }) => ({
|
||||
method: "POST",
|
||||
url: `/api/ee/workspace/${id}/run`,
|
||||
params: stale_only ? { stale_only: 1 } : undefined,
|
||||
}),
|
||||
invalidatesTags: (_, error, { id }) =>
|
||||
invalidateTags(error, [
|
||||
idTag("workspace", id),
|
||||
idTag("workspace-transforms", id),
|
||||
idTag("workspace-tables", id),
|
||||
tag("workspace-transform"),
|
||||
tag("transform"),
|
||||
]),
|
||||
}),
|
||||
runWorkspaceTransform: builder.mutation<
|
||||
WorkspaceTransformRunResponse,
|
||||
WorkspaceTransformRef
|
||||
>({
|
||||
query: ({ workspaceId, transformId }) => ({
|
||||
method: "POST",
|
||||
url: `/api/ee/workspace/${workspaceId}/transform/${transformId}/run`,
|
||||
}),
|
||||
invalidatesTags: (_, error, { workspaceId, transformId }) =>
|
||||
invalidateTags(error, [
|
||||
idTag("workspace", workspaceId),
|
||||
idTag("workspace-transforms", workspaceId),
|
||||
idTag("workspace-transform", transformId),
|
||||
idTag("workspace-tables", workspaceId),
|
||||
]),
|
||||
}),
|
||||
updateWorkspaceTransform: builder.mutation<
|
||||
WorkspaceTransform,
|
||||
UpdateWorkspaceTransformRequest
|
||||
>({
|
||||
query: ({ workspaceId, transformId, ...body }) => ({
|
||||
method: "PUT",
|
||||
url: `/api/ee/workspace/${workspaceId}/transform/${transformId}`,
|
||||
body,
|
||||
}),
|
||||
invalidatesTags: (_, error, { workspaceId, transformId }) =>
|
||||
invalidateTags(error, [
|
||||
idTag("workspace", workspaceId),
|
||||
idTag("workspace-transforms", workspaceId),
|
||||
idTag("workspace-transform", transformId),
|
||||
idTag("workspace-tables", workspaceId),
|
||||
]),
|
||||
transformResponse: (t: WorkspaceTransform) => ({ ...t, id: t.ref_id }),
|
||||
}),
|
||||
archiveWorkspaceTransform: builder.mutation<void, WorkspaceTransformRef>({
|
||||
query: ({ workspaceId, transformId }) => ({
|
||||
method: "POST",
|
||||
url: `/api/ee/workspace/${workspaceId}/transform/${transformId}/archive`,
|
||||
}),
|
||||
invalidatesTags: (_, error, { workspaceId, transformId }) =>
|
||||
invalidateTags(error, [
|
||||
idTag("workspace", workspaceId),
|
||||
idTag("workspace-transforms", workspaceId),
|
||||
idTag("workspace-transform", transformId),
|
||||
idTag("workspace-tables", workspaceId),
|
||||
tag("transform"),
|
||||
]),
|
||||
}),
|
||||
unarchiveWorkspaceTransform: builder.mutation<void, WorkspaceTransformRef>({
|
||||
query: ({ workspaceId, transformId }) => ({
|
||||
method: "POST",
|
||||
url: `/api/ee/workspace/${workspaceId}/transform/${transformId}/unarchive`,
|
||||
}),
|
||||
invalidatesTags: (_, error, { workspaceId, transformId }) =>
|
||||
invalidateTags(error, [
|
||||
idTag("workspace", workspaceId),
|
||||
idTag("workspace-transforms", workspaceId),
|
||||
idTag("workspace-transform", transformId),
|
||||
idTag("workspace-tables", workspaceId),
|
||||
tag("transform"),
|
||||
]),
|
||||
}),
|
||||
deleteWorkspaceTransform: builder.mutation<void, WorkspaceTransformRef>({
|
||||
query: ({ workspaceId, transformId }) => ({
|
||||
method: "DELETE",
|
||||
url: `/api/ee/workspace/${workspaceId}/transform/${transformId}`,
|
||||
}),
|
||||
invalidatesTags: (_, error, { workspaceId, transformId }) =>
|
||||
invalidateTags(error, [
|
||||
idTag("workspace", workspaceId),
|
||||
idTag("workspace-transforms", workspaceId),
|
||||
idTag("external-transforms", workspaceId),
|
||||
idTag("workspace-transform", transformId),
|
||||
idTag("workspace-tables", workspaceId),
|
||||
tag("transform"),
|
||||
]),
|
||||
}),
|
||||
getWorkspaceAllowedDatabases: builder.query<
|
||||
WorkspaceAllowedDatabasesResponse,
|
||||
void
|
||||
>({
|
||||
query: () => ({
|
||||
method: "GET",
|
||||
url: `/api/ee/workspace/database`,
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
});
|
||||
|
||||
export const {
|
||||
useGetWorkspacesQuery,
|
||||
useGetWorkspaceQuery,
|
||||
useGetWorkspaceTransformsQuery,
|
||||
useGetExternalTransformsQuery,
|
||||
useGetWorkspaceTransformQuery,
|
||||
useLazyGetWorkspaceTransformQuery,
|
||||
useCreateWorkspaceMutation,
|
||||
useCreateWorkspaceTransformMutation,
|
||||
useUpdateWorkspaceTransformMutation,
|
||||
useArchiveWorkspaceTransformMutation,
|
||||
useUnarchiveWorkspaceTransformMutation,
|
||||
useDeleteWorkspaceTransformMutation,
|
||||
useGetTransformUpstreamMappingQuery,
|
||||
useGetTransformDownstreamMappingQuery,
|
||||
useGetWorkspaceCheckoutQuery,
|
||||
useMergeWorkspaceMutation,
|
||||
useMergeWorkspaceTransformMutation,
|
||||
useArchiveWorkspaceMutation,
|
||||
useUnarchiveWorkspaceMutation,
|
||||
useDeleteWorkspaceMutation,
|
||||
useValidateTableNameMutation,
|
||||
useUpdateWorkspaceMutation,
|
||||
useGetWorkspaceTablesQuery,
|
||||
useGetWorkspaceGraphQuery,
|
||||
useGetWorkspaceProblemsQuery,
|
||||
useGetWorkspaceLogQuery,
|
||||
useRunWorkspaceMutation,
|
||||
useRunWorkspaceTransformMutation,
|
||||
useGetWorkspaceAllowedDatabasesQuery,
|
||||
} = workspaceApi;
|
||||
|
||||
export const DEFAULT_WORKSPACE_TABLES_QUERY_RESPONSE: WorkspaceTablesResponse =
|
||||
{
|
||||
inputs: [],
|
||||
outputs: [],
|
||||
};
|
||||
@@ -33,6 +33,7 @@
|
||||
.toggle {
|
||||
position: relative;
|
||||
color: var(--mb-color-text-primary);
|
||||
font-weight: 600;
|
||||
|
||||
&:hover,
|
||||
&.selected {
|
||||
@@ -44,14 +45,103 @@
|
||||
}
|
||||
}
|
||||
|
||||
.hoverButton {
|
||||
&:hover {
|
||||
.showOnHover {
|
||||
opacity: 1;
|
||||
}
|
||||
.workspacesSection {
|
||||
border-top: 1px solid var(--mb-color-border);
|
||||
padding-top: 0.75rem;
|
||||
}
|
||||
|
||||
.hideOnHover {
|
||||
opacity: 0;
|
||||
.workspacesSectionHeader {
|
||||
background-color: var(--mb-color-bg-secondary);
|
||||
color: var(--mb-color-text-primary);
|
||||
font-weight: 600;
|
||||
transition: background-color 150ms ease;
|
||||
|
||||
&:hover {
|
||||
background-color: var(--mb-color-brand-light);
|
||||
color: var(--mb-color-brand);
|
||||
}
|
||||
|
||||
&.selected {
|
||||
background-color: var(--mb-color-brand-light);
|
||||
color: var(--mb-color-brand);
|
||||
}
|
||||
}
|
||||
|
||||
.workspaceItem {
|
||||
background-color: var(--mb-color-bg-white);
|
||||
color: var(--mb-color-text-primary);
|
||||
cursor: pointer;
|
||||
border: 1px solid var(--mb-color-border);
|
||||
transition:
|
||||
background-color 150ms ease,
|
||||
border-color 150ms ease,
|
||||
color 150ms ease;
|
||||
|
||||
&:hover {
|
||||
background-color: var(--mb-color-bg-light);
|
||||
border-color: var(--mb-color-border-dark);
|
||||
}
|
||||
|
||||
&:hover .workspaceMenuButton {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
&.selected {
|
||||
background-color: var(--mb-color-brand-lighter);
|
||||
border-color: var(--mb-color-brand);
|
||||
color: var(--mb-color-brand);
|
||||
}
|
||||
|
||||
&.selected .workspaceMenuButton {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
&:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
}
|
||||
|
||||
.workspaceMenuButton {
|
||||
color: var(--mb-color-text-secondary);
|
||||
opacity: 0;
|
||||
transition:
|
||||
opacity 150ms ease,
|
||||
background-color 150ms ease,
|
||||
color 150ms ease;
|
||||
flex-shrink: 0;
|
||||
|
||||
&:hover {
|
||||
background-color: var(--mb-color-bg-medium);
|
||||
color: var(--mb-color-text-primary);
|
||||
}
|
||||
}
|
||||
|
||||
.newWorkspaceButton {
|
||||
background-color: transparent;
|
||||
color: var(--mb-color-text-primary);
|
||||
cursor: pointer;
|
||||
border: none;
|
||||
transition: color 150ms ease;
|
||||
|
||||
&:hover {
|
||||
color: var(--mb-color-brand);
|
||||
}
|
||||
|
||||
&:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.hoverButton {
|
||||
&:hover {
|
||||
.showOnHover {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
.hideOnHover {
|
||||
opacity: 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ import {
|
||||
} from "metabase/ui";
|
||||
|
||||
import S from "./DataStudioLayout.module.css";
|
||||
import { WorkspacesSection } from "./WorkspaceSection";
|
||||
import { getCurrentTab } from "./utils";
|
||||
|
||||
type DataStudioLayoutProps = {
|
||||
@@ -87,7 +88,7 @@ function DataStudioNav({ isNavbarOpened, onNavbarToggle }: DataStudioNavProps) {
|
||||
justify="space-between"
|
||||
data-testid="data-studio-nav"
|
||||
>
|
||||
<Stack gap="0.75rem">
|
||||
<Stack gap="0.75rem" flex={1} mih={0}>
|
||||
<DataStudioNavbarToggle
|
||||
isNavbarOpened={isNavbarOpened}
|
||||
onNavbarToggle={onNavbarToggle}
|
||||
@@ -144,6 +145,9 @@ function DataStudioNav({ isNavbarOpened, onNavbarToggle }: DataStudioNavProps) {
|
||||
showLabel={isNavbarOpened}
|
||||
/>
|
||||
)}
|
||||
{canAccessTransforms && (
|
||||
<WorkspacesSection showLabel={isNavbarOpened} />
|
||||
)}
|
||||
</Stack>
|
||||
<Stack gap="0.75rem">
|
||||
{canAccessTransforms && (
|
||||
@@ -164,6 +168,7 @@ function DataStudioNav({ isNavbarOpened, onNavbarToggle }: DataStudioNavProps) {
|
||||
showLabel={isNavbarOpened}
|
||||
/>
|
||||
)}
|
||||
|
||||
<DataStudioTab
|
||||
label={t`Exit`}
|
||||
icon="exit"
|
||||
|
||||
@@ -0,0 +1,381 @@
|
||||
import { useDisclosure } from "@mantine/hooks";
|
||||
import cx from "classnames";
|
||||
import dayjs from "dayjs";
|
||||
import { useCallback, useMemo, useState } from "react";
|
||||
import { push } from "react-router-redux";
|
||||
import { t } from "ttag";
|
||||
|
||||
import { ConfirmModal } from "metabase/common/components/ConfirmModal/ConfirmModal";
|
||||
import { ForwardRefLink } from "metabase/common/components/Link";
|
||||
import { useDispatch, useSelector } from "metabase/lib/redux";
|
||||
import * as Urls from "metabase/lib/urls";
|
||||
import { useMetadataToasts } from "metabase/metadata/hooks";
|
||||
import { getLocation } from "metabase/selectors/routing";
|
||||
import {
|
||||
ActionIcon,
|
||||
FixedSizeIcon,
|
||||
Flex,
|
||||
Group,
|
||||
Icon,
|
||||
type IconName,
|
||||
Loader,
|
||||
Menu,
|
||||
Skeleton,
|
||||
Stack,
|
||||
Text,
|
||||
Tooltip,
|
||||
UnstyledButton,
|
||||
} from "metabase/ui";
|
||||
import {
|
||||
useArchiveWorkspaceMutation,
|
||||
useCreateWorkspaceMutation,
|
||||
useDeleteWorkspaceMutation,
|
||||
useGetWorkspacesQuery,
|
||||
useUnarchiveWorkspaceMutation,
|
||||
} from "metabase-enterprise/api/workspace";
|
||||
import { TOOLTIP_OPEN_DELAY } from "metabase-enterprise/dependencies/components/DependencyGraph/constants";
|
||||
import type { Workspace, WorkspaceId } from "metabase-types/api/workspace";
|
||||
|
||||
import S from "./DataStudioLayout.module.css";
|
||||
|
||||
type WorkspacesSectionProps = {
|
||||
showLabel: boolean;
|
||||
};
|
||||
|
||||
function WorkspacesSection({ showLabel }: WorkspacesSectionProps) {
|
||||
const dispatch = useDispatch();
|
||||
const [isWorkspacesExpanded, setIsWorkspacesExpanded] = useState(true);
|
||||
const { pathname } = useSelector(getLocation);
|
||||
const { data: workspacesData, isLoading: areWorkspacesLoading } =
|
||||
useGetWorkspacesQuery();
|
||||
const [createWorkspace, { isLoading: isCreatingWorkspace }] =
|
||||
useCreateWorkspaceMutation();
|
||||
|
||||
const workspaces = useMemo(() => {
|
||||
const isArchived = (ws: (typeof workspacesData)["items"][0]) =>
|
||||
ws.status === "archived";
|
||||
return [...(workspacesData?.items ?? [])].sort((a, b) => {
|
||||
// Archived workspaces should be placed at the end
|
||||
if (isArchived(a) !== isArchived(b)) {
|
||||
return isArchived(a) ? 1 : -1;
|
||||
}
|
||||
|
||||
// Within each group (archived/unarchived), sort by updated_at descending
|
||||
const aDate = a.updated_at ? new Date(a.updated_at).getTime() : 0;
|
||||
const bDate = b.updated_at ? new Date(b.updated_at).getTime() : 0;
|
||||
return bDate - aDate;
|
||||
});
|
||||
}, [workspacesData]);
|
||||
|
||||
const handleOpenWorkspace = useCallback(
|
||||
(workspaceId: number) => {
|
||||
dispatch(push(Urls.dataStudioWorkspace(workspaceId)));
|
||||
},
|
||||
[dispatch],
|
||||
);
|
||||
|
||||
const { sendErrorToast, sendSuccessToast } = useMetadataToasts();
|
||||
|
||||
const handleCreateWorkspace = useCallback(async () => {
|
||||
try {
|
||||
const workspace = await createWorkspace().unwrap();
|
||||
handleOpenWorkspace(workspace.id);
|
||||
} catch (error) {
|
||||
sendErrorToast(t`Failed to create workspace`);
|
||||
}
|
||||
}, [createWorkspace, handleOpenWorkspace, sendErrorToast]);
|
||||
|
||||
const [archiveWorkspace] = useArchiveWorkspaceMutation();
|
||||
const [unarchiveWorkspace] = useUnarchiveWorkspaceMutation();
|
||||
const [deleteWorkspace] = useDeleteWorkspaceMutation();
|
||||
|
||||
const handleWorkspaceArchive = async (id: WorkspaceId) => {
|
||||
try {
|
||||
await archiveWorkspace(id).unwrap();
|
||||
sendSuccessToast(t`Workspace archived successfully`);
|
||||
dispatch(push(Urls.dataStudioWorkspaceList()));
|
||||
} catch (error) {
|
||||
sendErrorToast(t`Failed to archive workspace`);
|
||||
}
|
||||
};
|
||||
|
||||
const handleWorkspaceUnarchive = async (id: WorkspaceId) => {
|
||||
try {
|
||||
await unarchiveWorkspace(id).unwrap();
|
||||
sendSuccessToast(t`Workspace restored successfully`);
|
||||
} catch (error) {
|
||||
sendErrorToast(t`Failed to restore workspace`);
|
||||
}
|
||||
};
|
||||
|
||||
const handleWorkspaceDelete = async (id: WorkspaceId) => {
|
||||
try {
|
||||
await deleteWorkspace(id).unwrap();
|
||||
sendSuccessToast(t`Workspace deleted successfully`);
|
||||
dispatch(push(Urls.dataStudioWorkspaceList()));
|
||||
} catch (error) {
|
||||
sendErrorToast(t`Failed to delete workspace`);
|
||||
}
|
||||
};
|
||||
|
||||
const isWorkspaceListPage = pathname === Urls.dataStudioWorkspaceList();
|
||||
|
||||
if (!showLabel) {
|
||||
// In collapsed mode, show only an icon button with tooltip
|
||||
return (
|
||||
<Tooltip
|
||||
label={t`Workspaces`}
|
||||
position="right"
|
||||
openDelay={TOOLTIP_OPEN_DELAY}
|
||||
>
|
||||
<UnstyledButton
|
||||
className={cx(S.tab, { [S.selected]: isWorkspaceListPage })}
|
||||
component={ForwardRefLink}
|
||||
to={Urls.dataStudioWorkspaceList()}
|
||||
p="0.75rem"
|
||||
bdrs="md"
|
||||
>
|
||||
<Flex align="center" justify="center">
|
||||
<FixedSizeIcon name="git_branch" display="block" />
|
||||
</Flex>
|
||||
</UnstyledButton>
|
||||
</Tooltip>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<Stack
|
||||
data-testid="workspaces-section"
|
||||
gap="0.5rem"
|
||||
className={S.workspacesSection}
|
||||
flex={1}
|
||||
mih={0}
|
||||
>
|
||||
<UnstyledButton
|
||||
className={cx(S.workspacesSectionHeader, {
|
||||
[S.selected]: isWorkspaceListPage,
|
||||
})}
|
||||
onClick={() => setIsWorkspacesExpanded(!isWorkspacesExpanded)}
|
||||
p="0.75rem"
|
||||
bdrs="md"
|
||||
>
|
||||
<Flex align="center" justify="space-between" w="100%">
|
||||
<Text size="sm" fw={600}>
|
||||
{t`Workspaces`}
|
||||
</Text>
|
||||
<Icon
|
||||
name={isWorkspacesExpanded ? "chevrondown" : "chevronright"}
|
||||
size={16}
|
||||
/>
|
||||
</Flex>
|
||||
</UnstyledButton>
|
||||
|
||||
{isWorkspacesExpanded && (
|
||||
<>
|
||||
<UnstyledButton
|
||||
className={S.newWorkspaceButton}
|
||||
onClick={handleCreateWorkspace}
|
||||
disabled={isCreatingWorkspace || areWorkspacesLoading}
|
||||
p="0.5rem"
|
||||
bdrs="md"
|
||||
>
|
||||
<Flex align="center" gap="xs">
|
||||
{isCreatingWorkspace ? (
|
||||
<Loader size="xs" />
|
||||
) : (
|
||||
<Icon name="add" size={16} />
|
||||
)}
|
||||
<Text size="sm" fw={500}>
|
||||
{t`New workspace`}
|
||||
</Text>
|
||||
</Flex>
|
||||
</UnstyledButton>
|
||||
<Stack
|
||||
gap="0.75rem"
|
||||
style={{ flex: 1, overflowY: "auto", minHeight: 0 }}
|
||||
>
|
||||
{areWorkspacesLoading ? (
|
||||
<>
|
||||
<Skeleton height={80} radius="md" />
|
||||
<Skeleton height={80} radius="md" />
|
||||
</>
|
||||
) : workspaces.length === 0 ? (
|
||||
<Text c="text-secondary" size="xs" px="0.5rem">
|
||||
{t`No workspaces yet`}
|
||||
</Text>
|
||||
) : (
|
||||
workspaces.map((workspace) => {
|
||||
const isSelected =
|
||||
pathname === Urls.dataStudioWorkspace(workspace.id);
|
||||
|
||||
return (
|
||||
<WorkspaceItem
|
||||
key={workspace.id}
|
||||
workspace={workspace}
|
||||
isSelected={isSelected}
|
||||
onOpen={handleOpenWorkspace}
|
||||
onArchive={handleWorkspaceArchive}
|
||||
onUnarchive={handleWorkspaceUnarchive}
|
||||
onDelete={handleWorkspaceDelete}
|
||||
/>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</Stack>
|
||||
</>
|
||||
)}
|
||||
</Stack>
|
||||
);
|
||||
}
|
||||
|
||||
export { WorkspacesSection };
|
||||
|
||||
interface WorkspaceItemProps {
|
||||
workspace: Workspace;
|
||||
isSelected: boolean;
|
||||
onOpen: (workspaceId: WorkspaceId) => void;
|
||||
onArchive: (workspaceId: WorkspaceId) => Promise<void>;
|
||||
onUnarchive: (workspaceId: WorkspaceId) => Promise<void>;
|
||||
onDelete: (workspaceId: WorkspaceId) => Promise<void>;
|
||||
}
|
||||
|
||||
function WorkspaceItem({
|
||||
workspace,
|
||||
isSelected,
|
||||
onOpen,
|
||||
onArchive,
|
||||
onUnarchive,
|
||||
onDelete,
|
||||
}: WorkspaceItemProps) {
|
||||
const [
|
||||
deleteModalOpened,
|
||||
{ open: openDeleteModal, close: closeDeleteModal },
|
||||
] = useDisclosure();
|
||||
const timeAgo = workspace.updated_at
|
||||
? dayjs(workspace.updated_at).fromNow()
|
||||
: null;
|
||||
|
||||
const handleArchive = () => {
|
||||
onArchive(workspace.id);
|
||||
};
|
||||
|
||||
const handleUnarchive = () => {
|
||||
onUnarchive(workspace.id);
|
||||
};
|
||||
|
||||
const handleDelete = () => {
|
||||
openDeleteModal();
|
||||
};
|
||||
|
||||
const handleConfirmDelete = async () => {
|
||||
await onDelete(workspace.id);
|
||||
closeDeleteModal();
|
||||
};
|
||||
|
||||
const status = getWorkspaceListStatus(workspace);
|
||||
|
||||
return (
|
||||
<UnstyledButton
|
||||
className={cx(S.workspaceItem, { [S.selected]: isSelected })}
|
||||
onClick={() => onOpen(workspace.id)}
|
||||
p="0.75rem"
|
||||
bdrs="md"
|
||||
name={workspace.name}
|
||||
>
|
||||
<Flex align="flex-start" justify="space-between" gap="xs">
|
||||
<Stack gap="xs" style={{ flex: 1, minWidth: 0 }}>
|
||||
<Text size="sm" fw={600} truncate>
|
||||
{workspace.name}
|
||||
</Text>
|
||||
{status && (
|
||||
<Group
|
||||
gap="xs"
|
||||
align="center"
|
||||
wrap="nowrap"
|
||||
data-testid="workspace-status"
|
||||
>
|
||||
<Icon name={status.icon} size={10} c={status.color} />
|
||||
<Text size="xs" fw={500} c={status.color}>
|
||||
{status.label}
|
||||
</Text>
|
||||
</Group>
|
||||
)}
|
||||
{timeAgo && (
|
||||
<Text size="xs" c="text-secondary" truncate>
|
||||
{t`Updated ${timeAgo}`}
|
||||
</Text>
|
||||
)}
|
||||
</Stack>
|
||||
<Menu position="right" withinPortal>
|
||||
<Menu.Target>
|
||||
<ActionIcon
|
||||
className={S.workspaceMenuButton}
|
||||
component="span"
|
||||
aria-label={t`More actions`}
|
||||
onClick={(e: React.MouseEvent) => e.stopPropagation()}
|
||||
size="sm"
|
||||
variant="subtle"
|
||||
>
|
||||
<Icon name="ellipsis" size={16} />
|
||||
</ActionIcon>
|
||||
</Menu.Target>
|
||||
<Menu.Dropdown onClick={(e: React.MouseEvent) => e.stopPropagation()}>
|
||||
{workspace.status === "archived" ? (
|
||||
<>
|
||||
<Menu.Item
|
||||
leftSection={<Icon name="revert" />}
|
||||
onClick={handleUnarchive}
|
||||
>
|
||||
{t`Restore`}
|
||||
</Menu.Item>
|
||||
<Menu.Divider />
|
||||
<Menu.Item
|
||||
c="error"
|
||||
leftSection={<Icon name="trash" />}
|
||||
onClick={handleDelete}
|
||||
color="danger"
|
||||
>
|
||||
{t`Delete`}
|
||||
</Menu.Item>
|
||||
</>
|
||||
) : (
|
||||
<Menu.Item
|
||||
leftSection={<Icon name="archive" />}
|
||||
onClick={handleArchive}
|
||||
>
|
||||
{t`Archive`}
|
||||
</Menu.Item>
|
||||
)}
|
||||
</Menu.Dropdown>
|
||||
</Menu>
|
||||
</Flex>
|
||||
|
||||
<ConfirmModal
|
||||
confirmButtonText={t`Delete`}
|
||||
message={t`This can't be undone.`}
|
||||
opened={deleteModalOpened}
|
||||
title={t`Delete ${workspace.name}?`}
|
||||
onClose={closeDeleteModal}
|
||||
onConfirm={handleConfirmDelete}
|
||||
/>
|
||||
</UnstyledButton>
|
||||
);
|
||||
}
|
||||
|
||||
type WorkspaceListStatus = {
|
||||
label: string;
|
||||
icon: IconName;
|
||||
color: string;
|
||||
};
|
||||
|
||||
function getWorkspaceListStatus(workspace: Workspace): WorkspaceListStatus {
|
||||
if (workspace.status === "archived") {
|
||||
return { label: t`Archived`, icon: "archive", color: "text-light" };
|
||||
}
|
||||
|
||||
if (workspace.status === "pending") {
|
||||
return { label: t`Pending setup`, icon: "clock", color: "warning" };
|
||||
}
|
||||
|
||||
return { label: t`Ready`, icon: "check", color: "success" };
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import { t } from "ttag";
|
||||
|
||||
import { usePageTitle } from "metabase/hooks/use-page-title";
|
||||
|
||||
import { SectionLayout } from "../../components/SectionLayout";
|
||||
|
||||
type WorkspacesSectionLayoutProps = {
|
||||
location: Location;
|
||||
children?: React.ReactNode;
|
||||
};
|
||||
|
||||
export function WorkspacesSectionLayout({
|
||||
children,
|
||||
}: WorkspacesSectionLayoutProps) {
|
||||
usePageTitle(t`Workspaces`);
|
||||
|
||||
return (
|
||||
<SectionLayout>
|
||||
{children}
|
||||
</SectionLayout>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export * from "./WorkspacesSectionLayout";
|
||||
@@ -95,6 +95,7 @@ type PaneHeaderInputProps = {
|
||||
"data-testid"?: string;
|
||||
onChange?: (value: string) => void;
|
||||
onContentChange?: (value: string) => void;
|
||||
disabled?: boolean;
|
||||
};
|
||||
|
||||
export function PaneHeaderInput({
|
||||
@@ -105,6 +106,7 @@ export function PaneHeaderInput({
|
||||
isOptional,
|
||||
onChange,
|
||||
onContentChange,
|
||||
disabled,
|
||||
}: PaneHeaderInputProps) {
|
||||
return (
|
||||
<EditableText
|
||||
@@ -118,6 +120,7 @@ export function PaneHeaderInput({
|
||||
px={isOptional ? "xs" : undefined}
|
||||
bd={isOptional ? "1px solid var(--mb-color-border)" : undefined}
|
||||
isOptional={isOptional}
|
||||
isDisabled={disabled}
|
||||
data-testid={dataTestId}
|
||||
onChange={onChange}
|
||||
onContentChange={onContentChange}
|
||||
|
||||
@@ -15,11 +15,13 @@ import { DataStudioLayout } from "./app/pages/DataStudioLayout";
|
||||
import { DependenciesSectionLayout } from "./app/pages/DependenciesSectionLayout";
|
||||
import { LibrarySectionLayout } from "./app/pages/LibrarySectionLayout";
|
||||
import { TransformsSectionLayout } from "./app/pages/TransformsSectionLayout";
|
||||
import { WorkspacesSectionLayout } from "./app/pages/WorkspacesSectionLayout";
|
||||
import { getDataStudioMetadataRoutes } from "./data-model/routes";
|
||||
import { getDataStudioGlossaryRoutes } from "./glossary/routes";
|
||||
import { getDataStudioMetricRoutes } from "./metrics/routes";
|
||||
import { getDataStudioSnippetRoutes } from "./snippets/routes";
|
||||
import { getDataStudioTableRoutes } from "./tables/routes";
|
||||
import { getDataStudioWorkspaceRoutes } from "./workspaces/routes";
|
||||
|
||||
export function getDataStudioRoutes(
|
||||
store: Store<State>,
|
||||
@@ -54,6 +56,9 @@ export function getDataStudioRoutes(
|
||||
{getDataStudioMetricRoutes()}
|
||||
{getDataStudioSnippetRoutes()}
|
||||
</Route>
|
||||
<Route path="workspaces" component={WorkspacesSectionLayout}>
|
||||
{getDataStudioWorkspaceRoutes()}
|
||||
</Route>
|
||||
{PLUGIN_DEPENDENCIES.isEnabled && (
|
||||
<Route path="dependencies" component={DependenciesSectionLayout}>
|
||||
{PLUGIN_DEPENDENCIES.getDataStudioDependencyRoutes()}
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
.sidebarItem {
|
||||
cursor: pointer;
|
||||
border-left: 2px solid transparent;
|
||||
|
||||
&:hover {
|
||||
background: var(--mb-color-bg-light);
|
||||
}
|
||||
}
|
||||
|
||||
.sidebarItemActive {
|
||||
background: var(--mb-color-bg-light);
|
||||
border-left-color: var(--mb-color-brand);
|
||||
}
|
||||
|
||||
.diffEditor {
|
||||
height: 100%;
|
||||
|
||||
:global {
|
||||
.cm-content {
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.cm-collapsedLines {
|
||||
padding-left: 0;
|
||||
}
|
||||
|
||||
.cm-scroller {
|
||||
font-family: monospace;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,652 @@
|
||||
import { unifiedMergeView } from "@codemirror/merge";
|
||||
import cx from "classnames";
|
||||
import { diffLines } from "diff";
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
import { msgid, ngettext, t } from "ttag";
|
||||
import _ from "underscore";
|
||||
import * as Yup from "yup";
|
||||
|
||||
import { CodeMirror } from "metabase/common/components/CodeMirror";
|
||||
import {
|
||||
Form,
|
||||
FormProvider,
|
||||
FormSubmitButton,
|
||||
FormTextarea,
|
||||
} from "metabase/forms";
|
||||
import { useSelector } from "metabase/lib/redux";
|
||||
import { useMetadataToasts } from "metabase/metadata/hooks";
|
||||
import EditorS from "metabase/query_builder/components/NativeQueryEditor/CodeMirrorEditor/CodeMirrorEditor.module.css";
|
||||
import { getMetadata } from "metabase/selectors/metadata";
|
||||
import {
|
||||
Box,
|
||||
Collapse,
|
||||
Flex,
|
||||
Group,
|
||||
Icon,
|
||||
Loader,
|
||||
Modal,
|
||||
Stack,
|
||||
Text,
|
||||
Title,
|
||||
} from "metabase/ui";
|
||||
import {
|
||||
useGetWorkspaceProblemsQuery,
|
||||
useLazyGetTransformQuery,
|
||||
useLazyGetWorkspaceTransformQuery,
|
||||
} from "metabase-enterprise/api";
|
||||
import * as Lib from "metabase-lib";
|
||||
import type Metadata from "metabase-lib/v1/metadata/Metadata";
|
||||
import type {
|
||||
TransformSource,
|
||||
WorkspaceId,
|
||||
WorkspaceTransformItem,
|
||||
} from "metabase-types/api";
|
||||
|
||||
import S from "./MergeWorkspaceModal.module.css";
|
||||
import type { ProblemCheckCategory } from "./problemUtils";
|
||||
import {
|
||||
formatProblemDetails,
|
||||
getCheckName,
|
||||
getCheckStatus,
|
||||
getCheckStatusWithUnused,
|
||||
groupProblemsByCategory,
|
||||
} from "./problemUtils";
|
||||
|
||||
const MAX_COMMIT_MESSAGE_LENGTH = 255;
|
||||
|
||||
type ReviewChangesModalProps = {
|
||||
onClose: VoidFunction;
|
||||
onSubmit: (commitMessage: string) => Promise<void>;
|
||||
isLoading?: boolean;
|
||||
workspaceId: WorkspaceId;
|
||||
workspaceName: string;
|
||||
workspaceTransforms: WorkspaceTransformItem[];
|
||||
};
|
||||
|
||||
type MergeWorkspaceFormValues = {
|
||||
commit_message: string;
|
||||
};
|
||||
|
||||
const initialValues: MergeWorkspaceFormValues = {
|
||||
commit_message: "",
|
||||
};
|
||||
|
||||
const getMergeWorkspaceSchema = () =>
|
||||
Yup.object().shape({
|
||||
commit_message: Yup.string()
|
||||
.trim()
|
||||
.max(MAX_COMMIT_MESSAGE_LENGTH, t`Commit message is too long.`)
|
||||
.required(t`Please provide a commit message.`),
|
||||
});
|
||||
|
||||
function getSourceCode(
|
||||
transform: { source: TransformSource },
|
||||
metadata: Metadata,
|
||||
): string {
|
||||
if (transform.source.type === "python") {
|
||||
return transform.source.body;
|
||||
}
|
||||
if (transform.source.type === "query") {
|
||||
const metadataProvider = Lib.metadataProvider(
|
||||
transform.source.query.database,
|
||||
metadata,
|
||||
);
|
||||
const query = Lib.fromJsQuery(metadataProvider, transform.source.query);
|
||||
if (Lib.queryDisplayInfo(query).isNative) {
|
||||
return Lib.rawNativeQuery(query);
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
function computeDiffStats(
|
||||
oldSource: string,
|
||||
newSource: string,
|
||||
): { additions: number; deletions: number } {
|
||||
const changes = diffLines(oldSource, newSource);
|
||||
let additions = 0;
|
||||
let deletions = 0;
|
||||
|
||||
for (const change of changes) {
|
||||
const lineCount = change.count ?? 0;
|
||||
if (change.added) {
|
||||
additions += lineCount;
|
||||
} else if (change.removed) {
|
||||
deletions += lineCount;
|
||||
}
|
||||
}
|
||||
|
||||
return { additions, deletions };
|
||||
}
|
||||
|
||||
const QualityChecksSection = ({
|
||||
workspaceId,
|
||||
}: {
|
||||
workspaceId: WorkspaceId;
|
||||
}) => {
|
||||
const { data: problems = [], isLoading } =
|
||||
useGetWorkspaceProblemsQuery(workspaceId);
|
||||
const grouped = useMemo(() => groupProblemsByCategory(problems), [problems]);
|
||||
const [expandedChecks, setExpandedChecks] = useState<
|
||||
Set<ProblemCheckCategory>
|
||||
>(new Set());
|
||||
|
||||
const checkCategories: ProblemCheckCategory[] = [
|
||||
"external-dependencies",
|
||||
"internal-dependencies",
|
||||
"structural-issues",
|
||||
"unused-outputs",
|
||||
];
|
||||
|
||||
const toggleCheck = (category: ProblemCheckCategory) => {
|
||||
setExpandedChecks((prev) => {
|
||||
const next = new Set(prev);
|
||||
if (next.has(category)) {
|
||||
next.delete(category);
|
||||
} else {
|
||||
next.add(category);
|
||||
}
|
||||
return next;
|
||||
});
|
||||
};
|
||||
|
||||
return (
|
||||
<Stack gap="sm">
|
||||
<Text fw="bold">{t`Quality Checks`}</Text>
|
||||
<Stack gap="sm">
|
||||
{checkCategories.map((category) => {
|
||||
const categoryProblems = grouped[category];
|
||||
const checkStatus =
|
||||
category === "unused-outputs"
|
||||
? getCheckStatusWithUnused(categoryProblems)
|
||||
: getCheckStatus(categoryProblems);
|
||||
const isExpanded = expandedChecks.has(category);
|
||||
const hasProblems = checkStatus.count > 0;
|
||||
|
||||
return (
|
||||
<Box key={category}>
|
||||
<Group
|
||||
justify="space-between"
|
||||
style={{ cursor: hasProblems ? "pointer" : "default" }}
|
||||
onClick={() => hasProblems && toggleCheck(category)}
|
||||
>
|
||||
<Text>{getCheckName(category)}</Text>
|
||||
{isLoading ? (
|
||||
<Loader size="xs" />
|
||||
) : (
|
||||
<Group gap="xs">
|
||||
{checkStatus.status === "passed" && (
|
||||
<Icon name="check" size={14} c="success" />
|
||||
)}
|
||||
<Text
|
||||
c={
|
||||
checkStatus.status === "passed"
|
||||
? "success"
|
||||
: checkStatus.problems.some(
|
||||
(p) => p.severity === "error",
|
||||
)
|
||||
? "error"
|
||||
: "warning"
|
||||
}
|
||||
>
|
||||
{checkStatus.status === "passed"
|
||||
? category === "unused-outputs" && checkStatus.count > 0
|
||||
? t`${checkStatus.count} info`
|
||||
: t`Passed`
|
||||
: checkStatus.count === 1
|
||||
? t`Failed`
|
||||
: t`${checkStatus.count} issues`}
|
||||
</Text>
|
||||
</Group>
|
||||
)}
|
||||
</Group>
|
||||
{hasProblems && (
|
||||
<Collapse in={isExpanded}>
|
||||
<Stack gap="xs" mt="xs" pl="md">
|
||||
{checkStatus.problems.map((problem, idx) => (
|
||||
<Box key={idx}>
|
||||
<Text size="sm" c="text-medium">
|
||||
{formatProblemDetails(problem)}
|
||||
</Text>
|
||||
</Box>
|
||||
))}
|
||||
</Stack>
|
||||
</Collapse>
|
||||
)}
|
||||
</Box>
|
||||
);
|
||||
})}
|
||||
</Stack>
|
||||
</Stack>
|
||||
);
|
||||
};
|
||||
|
||||
const OverviewPanel = ({
|
||||
commitMessageLength = 0,
|
||||
hasCommitMessageError,
|
||||
workspaceId,
|
||||
transformCount,
|
||||
}: {
|
||||
commitMessageLength?: number;
|
||||
hasCommitMessageError: boolean;
|
||||
workspaceId: WorkspaceId;
|
||||
transformCount: number;
|
||||
}) => {
|
||||
return (
|
||||
<Box p="xl" h="100%" style={{ overflowY: "auto" }}>
|
||||
<Stack>
|
||||
<Box>
|
||||
<Text>{t`This will merge all changes from this workspace back to the source transforms.`}</Text>
|
||||
<Text mt="xs">
|
||||
{t`The commit message will be used to display the history of transform changes.`}
|
||||
</Text>
|
||||
</Box>
|
||||
<Stack mt="md" gap="xs">
|
||||
<FormTextarea
|
||||
data-autofocus
|
||||
label={t`Commit message`}
|
||||
name="commit_message"
|
||||
placeholder={t`Describe the changes you made in this workspace...`}
|
||||
minRows={4}
|
||||
required
|
||||
/>
|
||||
{!hasCommitMessageError && (
|
||||
<Text size="xs" c="text-tertiary">
|
||||
{commitMessageLength}/{MAX_COMMIT_MESSAGE_LENGTH}
|
||||
</Text>
|
||||
)}
|
||||
</Stack>
|
||||
<QualityChecksSection workspaceId={workspaceId} />
|
||||
<Box
|
||||
p="md"
|
||||
style={{
|
||||
backgroundColor: "var(--mb-color-bg-light)",
|
||||
borderRadius: "var(--mb-radius-sm)",
|
||||
}}
|
||||
>
|
||||
<Text>
|
||||
{ngettext(
|
||||
msgid`${transformCount} transform will be merged`,
|
||||
`${transformCount} transforms will be merged`,
|
||||
transformCount,
|
||||
)}
|
||||
</Text>
|
||||
</Box>
|
||||
</Stack>
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
const DiffView = ({
|
||||
transform,
|
||||
workspaceId,
|
||||
}: {
|
||||
transform: WorkspaceTransformItem;
|
||||
workspaceId: WorkspaceId;
|
||||
}) => {
|
||||
const metadata = useSelector(getMetadata);
|
||||
const isNewTransform = transform.global_id == null;
|
||||
|
||||
const [fetchWorkspaceTransform, workspaceTransformResult] =
|
||||
useLazyGetWorkspaceTransformQuery();
|
||||
const [fetchGlobalTransform, globalTransformResult] =
|
||||
useLazyGetTransformQuery();
|
||||
|
||||
useEffect(() => {
|
||||
fetchWorkspaceTransform({
|
||||
workspaceId,
|
||||
transformId: transform.ref_id,
|
||||
});
|
||||
if (transform.global_id) {
|
||||
fetchGlobalTransform(transform.global_id);
|
||||
}
|
||||
}, [
|
||||
transform.global_id,
|
||||
transform.ref_id,
|
||||
workspaceId,
|
||||
fetchWorkspaceTransform,
|
||||
fetchGlobalTransform,
|
||||
]);
|
||||
|
||||
const isLoading =
|
||||
workspaceTransformResult.isLoading ||
|
||||
(!isNewTransform && globalTransformResult.isLoading);
|
||||
const hasError =
|
||||
workspaceTransformResult.error ||
|
||||
(!isNewTransform && globalTransformResult.error);
|
||||
|
||||
const oldSource = globalTransformResult.data
|
||||
? getSourceCode(globalTransformResult.data, metadata)
|
||||
: "";
|
||||
const newSource = workspaceTransformResult.data
|
||||
? getSourceCode(workspaceTransformResult.data, metadata)
|
||||
: "";
|
||||
|
||||
const extensions = useMemo(
|
||||
() =>
|
||||
_.compact([
|
||||
oldSource &&
|
||||
unifiedMergeView({
|
||||
original: oldSource,
|
||||
mergeControls: false,
|
||||
}),
|
||||
]),
|
||||
[oldSource],
|
||||
);
|
||||
|
||||
if (isLoading) {
|
||||
return (
|
||||
<Flex
|
||||
align="center"
|
||||
justify="center"
|
||||
h="100%"
|
||||
direction="column"
|
||||
gap="sm"
|
||||
>
|
||||
<Loader size="sm" />
|
||||
<Text c="text-medium">{t`Loading diff...`}</Text>
|
||||
</Flex>
|
||||
);
|
||||
}
|
||||
|
||||
if (hasError) {
|
||||
return (
|
||||
<Flex align="center" justify="center" h="100%">
|
||||
<Text c="danger">{t`Failed to load diff`}</Text>
|
||||
</Flex>
|
||||
);
|
||||
}
|
||||
|
||||
if (!newSource) {
|
||||
return (
|
||||
<Flex align="center" justify="center" h="100%">
|
||||
<Text c="text-medium">{t`No source code to display`}</Text>
|
||||
</Flex>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<CodeMirror
|
||||
className={cx(EditorS.editor, S.diffEditor)}
|
||||
extensions={extensions}
|
||||
value={newSource}
|
||||
readOnly
|
||||
autoCorrect="off"
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
const TransformListItem = ({
|
||||
transform,
|
||||
isSelected,
|
||||
onClick,
|
||||
workspaceId,
|
||||
}: {
|
||||
transform: WorkspaceTransformItem;
|
||||
isSelected: boolean;
|
||||
onClick: () => void;
|
||||
workspaceId: WorkspaceId;
|
||||
}) => {
|
||||
const metadata = useSelector(getMetadata);
|
||||
const isNewTransform = transform.global_id == null;
|
||||
const [fetchWorkspaceTransform, workspaceTransformResult] =
|
||||
useLazyGetWorkspaceTransformQuery();
|
||||
const [fetchGlobalTransform, globalTransformResult] =
|
||||
useLazyGetTransformQuery();
|
||||
|
||||
useEffect(() => {
|
||||
fetchWorkspaceTransform({
|
||||
workspaceId,
|
||||
transformId: transform.ref_id,
|
||||
});
|
||||
if (transform.global_id) {
|
||||
fetchGlobalTransform(transform.global_id);
|
||||
}
|
||||
}, [
|
||||
transform.global_id,
|
||||
transform.ref_id,
|
||||
workspaceId,
|
||||
fetchWorkspaceTransform,
|
||||
fetchGlobalTransform,
|
||||
]);
|
||||
|
||||
const oldSource = globalTransformResult.data
|
||||
? getSourceCode(globalTransformResult.data, metadata)
|
||||
: "";
|
||||
const newSource = workspaceTransformResult.data
|
||||
? getSourceCode(workspaceTransformResult.data, metadata)
|
||||
: "";
|
||||
|
||||
const stats = useMemo(() => {
|
||||
if (isNewTransform && newSource) {
|
||||
// For new transforms, count all lines as additions
|
||||
const lineCount = newSource.split("\n").length;
|
||||
return { additions: lineCount, deletions: 0 };
|
||||
}
|
||||
if (oldSource && newSource) {
|
||||
return computeDiffStats(oldSource, newSource);
|
||||
}
|
||||
return null;
|
||||
}, [isNewTransform, oldSource, newSource]);
|
||||
|
||||
return (
|
||||
<Flex
|
||||
align="center"
|
||||
justify="space-between"
|
||||
px="md"
|
||||
py="sm"
|
||||
className={cx(S.sidebarItem, isSelected && S.sidebarItemActive)}
|
||||
onClick={onClick}
|
||||
>
|
||||
<Flex align="center" gap="sm" style={{ overflow: "hidden" }}>
|
||||
<Icon name="code_block" size={14} c="text-medium" />
|
||||
<Text truncate>{transform.name}</Text>
|
||||
</Flex>
|
||||
{stats && (stats.additions > 0 || stats.deletions > 0) && (
|
||||
<Flex gap="xs" fz="xs" style={{ flexShrink: 0 }}>
|
||||
{stats.additions > 0 && <Text c="success">+{stats.additions}</Text>}
|
||||
{stats.deletions > 0 && <Text c="danger">-{stats.deletions}</Text>}
|
||||
</Flex>
|
||||
)}
|
||||
</Flex>
|
||||
);
|
||||
};
|
||||
|
||||
export const ReviewChangesModal = ({
|
||||
onClose,
|
||||
onSubmit,
|
||||
isLoading = false,
|
||||
workspaceId,
|
||||
workspaceName,
|
||||
workspaceTransforms,
|
||||
}: ReviewChangesModalProps) => {
|
||||
const { sendErrorToast } = useMetadataToasts();
|
||||
const validationSchema = useMemo(() => getMergeWorkspaceSchema(), []);
|
||||
|
||||
const { data: workspaceProblems = [] } =
|
||||
useGetWorkspaceProblemsQuery(workspaceId);
|
||||
const hasBlockingProblems = useMemo(
|
||||
() => workspaceProblems.some((p) => p.block_merge === true),
|
||||
[workspaceProblems],
|
||||
);
|
||||
|
||||
const updatedTransforms = useMemo(
|
||||
() => workspaceTransforms.filter((t) => t.global_id != null),
|
||||
[workspaceTransforms],
|
||||
);
|
||||
|
||||
const newTransforms = useMemo(
|
||||
() => workspaceTransforms.filter((t) => t.global_id == null),
|
||||
[workspaceTransforms],
|
||||
);
|
||||
|
||||
const [selectedTransformId, setSelectedTransformId] = useState<
|
||||
string | "overview"
|
||||
>("overview");
|
||||
|
||||
const selectedTransform =
|
||||
selectedTransformId !== "overview"
|
||||
? workspaceTransforms.find((t) => t.ref_id === selectedTransformId)
|
||||
: null;
|
||||
|
||||
const handleSubmit = async (values: MergeWorkspaceFormValues) => {
|
||||
try {
|
||||
await onSubmit(values.commit_message.trim());
|
||||
onClose();
|
||||
} catch (error) {
|
||||
sendErrorToast(t`Failed to merge workspace`);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<Modal
|
||||
data-testid="review-changes-modal"
|
||||
onClose={onClose}
|
||||
opened
|
||||
title={
|
||||
<Flex align="center" gap="sm">
|
||||
<Icon name="git_branch" c="brand" />
|
||||
<Flex direction="column" gap="xs">
|
||||
<Title order={3}>{t`Review Changes`}</Title>
|
||||
<Text size="sm" c="text-medium">
|
||||
{workspaceName}
|
||||
</Text>
|
||||
</Flex>
|
||||
</Flex>
|
||||
}
|
||||
size="xl"
|
||||
padding={0}
|
||||
styles={{
|
||||
header: {
|
||||
padding: "1.25rem 1.5rem",
|
||||
borderBottom: "1px solid var(--mb-color-border)",
|
||||
},
|
||||
}}
|
||||
>
|
||||
<FormProvider
|
||||
initialValues={initialValues}
|
||||
validationSchema={validationSchema}
|
||||
onSubmit={handleSubmit}
|
||||
>
|
||||
{({ values, touched, errors }) => (
|
||||
<Form style={{ display: "flex", flexDirection: "column" }}>
|
||||
<Flex h={500}>
|
||||
<Box
|
||||
pt="md"
|
||||
w={220}
|
||||
miw={220}
|
||||
style={{
|
||||
borderRight: "1px solid var(--mb-color-border)",
|
||||
overflowY: "auto",
|
||||
}}
|
||||
>
|
||||
<Flex
|
||||
align="center"
|
||||
gap="sm"
|
||||
px="md"
|
||||
py="sm"
|
||||
className={cx(
|
||||
S.sidebarItem,
|
||||
selectedTransformId === "overview" && S.sidebarItemActive,
|
||||
)}
|
||||
onClick={() => setSelectedTransformId("overview")}
|
||||
>
|
||||
<Icon name="document" size={14} c="text-medium" />
|
||||
<Text>{t`Overview`}</Text>
|
||||
</Flex>
|
||||
{newTransforms.length > 0 && (
|
||||
<>
|
||||
<Text
|
||||
px="md"
|
||||
py="sm"
|
||||
fz="xs"
|
||||
fw={700}
|
||||
tt="uppercase"
|
||||
lts="0.5px"
|
||||
c="text-medium"
|
||||
>
|
||||
{t`New transforms`}
|
||||
</Text>
|
||||
{newTransforms.map((transform) => (
|
||||
<TransformListItem
|
||||
key={transform.ref_id}
|
||||
transform={transform}
|
||||
isSelected={selectedTransformId === transform.ref_id}
|
||||
onClick={() => setSelectedTransformId(transform.ref_id)}
|
||||
workspaceId={workspaceId}
|
||||
/>
|
||||
))}
|
||||
</>
|
||||
)}
|
||||
{updatedTransforms.length > 0 && (
|
||||
<>
|
||||
<Text
|
||||
px="md"
|
||||
py="sm"
|
||||
fz="xs"
|
||||
fw={700}
|
||||
tt="uppercase"
|
||||
lts="0.5px"
|
||||
c="text-medium"
|
||||
>
|
||||
{t`Modified transforms`}
|
||||
</Text>
|
||||
{updatedTransforms.map((transform) => (
|
||||
<TransformListItem
|
||||
key={transform.ref_id}
|
||||
transform={transform}
|
||||
isSelected={selectedTransformId === transform.ref_id}
|
||||
onClick={() => setSelectedTransformId(transform.ref_id)}
|
||||
workspaceId={workspaceId}
|
||||
/>
|
||||
))}
|
||||
</>
|
||||
)}
|
||||
</Box>
|
||||
<Box flex={1} miw={0} style={{ overflow: "auto" }}>
|
||||
{selectedTransformId === "overview" ? (
|
||||
<OverviewPanel
|
||||
commitMessageLength={values.commit_message?.length}
|
||||
hasCommitMessageError={!!errors.commit_message}
|
||||
workspaceId={workspaceId}
|
||||
transformCount={workspaceTransforms.length}
|
||||
/>
|
||||
) : selectedTransform ? (
|
||||
<DiffView
|
||||
key={selectedTransform.ref_id}
|
||||
transform={selectedTransform}
|
||||
workspaceId={workspaceId}
|
||||
/>
|
||||
) : (
|
||||
<Flex align="center" justify="center" h="100%">
|
||||
<Text c="text-medium">{t`Transform not found`}</Text>
|
||||
</Flex>
|
||||
)}
|
||||
</Box>
|
||||
</Flex>
|
||||
<Flex
|
||||
justify="flex-end"
|
||||
align="center"
|
||||
p="md"
|
||||
style={{ borderTop: "1px solid var(--mb-color-border)" }}
|
||||
>
|
||||
<FormSubmitButton
|
||||
disabled={
|
||||
!values.commit_message?.trim() ||
|
||||
(touched.commit_message && !!errors.commit_message) ||
|
||||
hasBlockingProblems
|
||||
}
|
||||
label={t`Merge`}
|
||||
loading={isLoading}
|
||||
variant="filled"
|
||||
/>
|
||||
</Flex>
|
||||
</Form>
|
||||
)}
|
||||
</FormProvider>
|
||||
</Modal>
|
||||
);
|
||||
};
|
||||
|
||||
export { ReviewChangesModal as MergeWorkspaceModal };
|
||||
@@ -0,0 +1,163 @@
|
||||
import type { WorkspaceProblem } from "metabase-types/api";
|
||||
|
||||
export type ProblemCheckCategory =
|
||||
| "external-dependencies"
|
||||
| "internal-dependencies"
|
||||
| "structural-issues"
|
||||
| "unused-outputs";
|
||||
|
||||
export type GroupedProblems = {
|
||||
"external-dependencies": WorkspaceProblem[];
|
||||
"internal-dependencies": WorkspaceProblem[];
|
||||
"structural-issues": WorkspaceProblem[];
|
||||
"unused-outputs": WorkspaceProblem[];
|
||||
};
|
||||
|
||||
export type CheckStatus = {
|
||||
status: "passed" | "failed";
|
||||
count: number;
|
||||
problems: WorkspaceProblem[];
|
||||
};
|
||||
|
||||
/**
|
||||
* Groups problems by category into logical Quality Check categories
|
||||
*/
|
||||
export function groupProblemsByCategory(
|
||||
problems: WorkspaceProblem[],
|
||||
): GroupedProblems {
|
||||
const grouped: GroupedProblems = {
|
||||
"external-dependencies": [],
|
||||
"internal-dependencies": [],
|
||||
"structural-issues": [],
|
||||
"unused-outputs": [],
|
||||
};
|
||||
|
||||
for (const problem of problems) {
|
||||
if (problem.category === "external-downstream") {
|
||||
grouped["external-dependencies"].push(problem);
|
||||
} else if (problem.category === "internal-downstream") {
|
||||
grouped["internal-dependencies"].push(problem);
|
||||
} else if (
|
||||
problem.category === "internal" ||
|
||||
problem.category === "external"
|
||||
) {
|
||||
grouped["structural-issues"].push(problem);
|
||||
} else if (problem.category === "unused") {
|
||||
grouped["unused-outputs"].push(problem);
|
||||
}
|
||||
}
|
||||
|
||||
return grouped;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the status for a group of problems
|
||||
* Unused problems are informational and don't affect status (always "passed")
|
||||
*/
|
||||
export function getCheckStatus(problems: WorkspaceProblem[]): CheckStatus {
|
||||
// Filter out unused problems for status calculation - they're informational only
|
||||
const relevantProblems = problems.filter((p) => p.category !== "unused");
|
||||
|
||||
// If all problems are unused, return passed with empty array
|
||||
// If there are any non-unused problems, return failed with those problems
|
||||
return {
|
||||
status: relevantProblems.length === 0 ? "passed" : "failed",
|
||||
count: relevantProblems.length,
|
||||
problems: relevantProblems,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates status including unused problems (for unused-outputs check)
|
||||
*/
|
||||
export function getCheckStatusWithUnused(
|
||||
problems: WorkspaceProblem[],
|
||||
): CheckStatus {
|
||||
return {
|
||||
status: problems.length === 0 ? "passed" : "passed", // Unused is always passed (informational)
|
||||
count: problems.length,
|
||||
problems: problems, // Include all problems for display
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats problem details for display
|
||||
*/
|
||||
export function formatProblemDetails(problem: WorkspaceProblem): string {
|
||||
const parts: string[] = [];
|
||||
|
||||
// Add transform reference if available
|
||||
if ("transform" in problem.data && problem.data.transform) {
|
||||
const transform = problem.data.transform;
|
||||
const transformName =
|
||||
"name" in transform && transform.name
|
||||
? transform.name
|
||||
: transform.id.toString();
|
||||
const transformType =
|
||||
transform.type === "workspace-transform" ? "workspace" : "external";
|
||||
parts.push(`${transformType} transform "${transformName}"`);
|
||||
}
|
||||
|
||||
// Add table reference if available
|
||||
if ("output" in problem.data && problem.data.output) {
|
||||
const output = problem.data.output;
|
||||
if ("global" in output && output.global) {
|
||||
const schema = output.global.schema || "";
|
||||
const table = output.global.table || "";
|
||||
if (schema && table) {
|
||||
parts.push(`table "${schema}.${table}"`);
|
||||
} else if (table) {
|
||||
parts.push(`table "${table}"`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add bad-refs for removed-field problems
|
||||
if ("bad-refs" in problem.data && problem.data["bad-refs"]) {
|
||||
const badRefs = problem.data["bad-refs"];
|
||||
if (Array.isArray(badRefs) && badRefs.length > 0) {
|
||||
const fieldNames = badRefs
|
||||
.map((ref) => (ref.data?.name ? ref.data.name : null))
|
||||
.filter(Boolean);
|
||||
if (fieldNames.length > 0) {
|
||||
parts.push(`removed fields: ${fieldNames.join(", ")}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add dependents if available
|
||||
if ("dependents" in problem.data && problem.data.dependents) {
|
||||
const dependents = problem.data.dependents;
|
||||
if (Array.isArray(dependents) && dependents.length > 0) {
|
||||
const dependentNames = dependents
|
||||
.map((dep: { name?: string; id?: string | number }) =>
|
||||
dep.name ? dep.name : dep.id?.toString(),
|
||||
)
|
||||
.filter(Boolean);
|
||||
if (dependentNames.length > 0) {
|
||||
parts.push(`affects: ${dependentNames.join(", ")}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If we have the description and no specific details, use description
|
||||
if (parts.length === 0 && problem.description) {
|
||||
return problem.description;
|
||||
}
|
||||
|
||||
return parts.length > 0
|
||||
? `${problem.description}. ${parts.join(". ")}`
|
||||
: problem.description;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets user-friendly check name for a category
|
||||
*/
|
||||
export function getCheckName(category: ProblemCheckCategory): string {
|
||||
return category
|
||||
.split("-")
|
||||
.map((word, index) =>
|
||||
index === 0 ? word.charAt(0).toUpperCase() + word.slice(1) : word,
|
||||
)
|
||||
.join(" ");
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import { RunButton } from "metabase-enterprise/transforms/components/RunButton";
|
||||
import type {
|
||||
TransformId,
|
||||
TransformJobId,
|
||||
TransformRun,
|
||||
} from "metabase-types/api";
|
||||
|
||||
type WorkspaceRunButtonProps = {
|
||||
id: TransformId | TransformJobId | undefined;
|
||||
run: TransformRun | null | undefined;
|
||||
isDisabled?: boolean;
|
||||
allowCancellation?: boolean;
|
||||
onRun: () => void;
|
||||
onCancel?: () => void;
|
||||
};
|
||||
|
||||
// A workspace-friendly wrapper that shows visual feedback during and after runs
|
||||
export function WorkspaceRunButton({ run, ...rest }: WorkspaceRunButtonProps) {
|
||||
// Pass through active/completed run states for visual feedback
|
||||
// - started/canceling: show in-progress state
|
||||
// - succeeded/failed: show completion state (controlled by parent)
|
||||
const activeRun =
|
||||
run?.status === "started" ||
|
||||
run?.status === "canceling" ||
|
||||
run?.status === "succeeded" ||
|
||||
run?.status === "failed"
|
||||
? run
|
||||
: null;
|
||||
|
||||
return <RunButton run={activeRun} {...rest} />;
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { useWorkspaceTransformRun } from "./use-workspace-transform-run";
|
||||
@@ -0,0 +1,185 @@
|
||||
import { useMemo, useState } from "react";
|
||||
import { t } from "ttag";
|
||||
|
||||
import { useDispatch } from "metabase/lib/redux";
|
||||
import { useMetadataToasts } from "metabase/metadata/hooks";
|
||||
import {
|
||||
DEFAULT_WORKSPACE_TABLES_QUERY_RESPONSE,
|
||||
useGetWorkspaceTablesQuery,
|
||||
useGetWorkspaceTransformQuery,
|
||||
useRunWorkspaceTransformMutation,
|
||||
workspaceApi,
|
||||
} from "metabase-enterprise/api";
|
||||
import type {
|
||||
TransformRun,
|
||||
WorkspaceId,
|
||||
WorkspaceTransform,
|
||||
} from "metabase-types/api";
|
||||
|
||||
type UseWorkspaceTransformRunOptions = {
|
||||
workspaceId: WorkspaceId;
|
||||
transform: WorkspaceTransform;
|
||||
};
|
||||
|
||||
type UseWorkspaceTransformRunResult = {
|
||||
/** Run object for RunStatus component - always shows last run info */
|
||||
statusRun: TransformRun | null;
|
||||
/** Run object for button - only shows feedback during/after user-triggered runs */
|
||||
buttonRun: TransformRun | null;
|
||||
/** Whether the run status is still loading */
|
||||
isRunStatusLoading: boolean;
|
||||
/** Whether a run is currently in progress */
|
||||
isRunning: boolean;
|
||||
/** Trigger a transform run */
|
||||
handleRun: () => Promise<void>;
|
||||
/** Output table info */
|
||||
output: {
|
||||
db_id: number;
|
||||
table_id: number;
|
||||
table_name: string;
|
||||
schema: string;
|
||||
} | null;
|
||||
};
|
||||
|
||||
export function useWorkspaceTransformRun({
|
||||
workspaceId,
|
||||
transform,
|
||||
}: UseWorkspaceTransformRunOptions): UseWorkspaceTransformRunResult {
|
||||
const dispatch = useDispatch();
|
||||
const { sendErrorToast } = useMetadataToasts();
|
||||
const [runTransform] = useRunWorkspaceTransformMutation();
|
||||
const { data: workspaceTables = DEFAULT_WORKSPACE_TABLES_QUERY_RESPONSE } =
|
||||
useGetWorkspaceTablesQuery(workspaceId);
|
||||
|
||||
const [isRunTriggered, setIsRunTriggered] = useState(false);
|
||||
const [lastRunResult, setLastRunResult] = useState<{
|
||||
status: "succeeded" | "failed";
|
||||
end_time: string;
|
||||
message: string | null;
|
||||
} | null>(null);
|
||||
|
||||
// Fetch workspace transform to get updated last_run_at
|
||||
const { data: fetchedWorkspaceTransform, isFetching } =
|
||||
useGetWorkspaceTransformQuery(
|
||||
{
|
||||
workspaceId,
|
||||
transformId: transform.ref_id,
|
||||
},
|
||||
{ skip: !transform.ref_id },
|
||||
);
|
||||
|
||||
const lastRunAt =
|
||||
fetchedWorkspaceTransform?.last_run_at ?? transform.last_run_at;
|
||||
const lastRunMessage =
|
||||
fetchedWorkspaceTransform?.last_run_message ?? transform.last_run_message;
|
||||
|
||||
const lastRunStatus = lastRunMessage ? "failed" : "succeeded";
|
||||
|
||||
const statusRun: TransformRun | null = useMemo(
|
||||
() =>
|
||||
lastRunResult
|
||||
? {
|
||||
id: -1,
|
||||
status: lastRunResult.status,
|
||||
start_time: lastRunResult.end_time,
|
||||
end_time: lastRunResult.end_time,
|
||||
message: lastRunResult.message,
|
||||
run_method: "manual",
|
||||
}
|
||||
: lastRunAt
|
||||
? {
|
||||
id: -1,
|
||||
status: lastRunStatus,
|
||||
start_time: lastRunAt,
|
||||
end_time: lastRunAt,
|
||||
message: lastRunMessage,
|
||||
run_method: "manual",
|
||||
}
|
||||
: null,
|
||||
[lastRunResult, lastRunAt, lastRunMessage, lastRunStatus],
|
||||
);
|
||||
|
||||
const output = useMemo(() => {
|
||||
const table = workspaceTables.outputs.find(
|
||||
(table) => table.isolated.transform_id === transform.ref_id,
|
||||
);
|
||||
if (!table || statusRun?.status !== "succeeded") {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
db_id: table?.db_id,
|
||||
table_id: table?.isolated.table_id,
|
||||
table_name: table?.isolated.table,
|
||||
schema: table?.isolated.schema,
|
||||
};
|
||||
}, [workspaceTables, transform.ref_id, statusRun]);
|
||||
|
||||
// Run object for button - only shows feedback during/after user-triggered runs
|
||||
const buttonRun: TransformRun | null = isRunTriggered
|
||||
? {
|
||||
id: -1,
|
||||
status: "started",
|
||||
start_time: new Date().toISOString(),
|
||||
end_time: null,
|
||||
message: null,
|
||||
run_method: "manual",
|
||||
}
|
||||
: lastRunResult
|
||||
? {
|
||||
id: -1,
|
||||
status: lastRunResult.status,
|
||||
start_time: lastRunResult.end_time,
|
||||
end_time: lastRunResult.end_time,
|
||||
message: lastRunResult.message,
|
||||
run_method: "manual",
|
||||
}
|
||||
: null;
|
||||
|
||||
const handleRun = async () => {
|
||||
try {
|
||||
setIsRunTriggered(true);
|
||||
setLastRunResult(null);
|
||||
|
||||
const result = await runTransform({
|
||||
workspaceId,
|
||||
transformId: transform.ref_id,
|
||||
}).unwrap();
|
||||
|
||||
// Transform completed - store the result
|
||||
const endTime = result.end_time ?? new Date().toISOString();
|
||||
setLastRunResult({
|
||||
status: result.status,
|
||||
end_time: endTime,
|
||||
message: result.message ?? null,
|
||||
});
|
||||
setIsRunTriggered(false);
|
||||
|
||||
// Invalidate caches to refetch updated data
|
||||
dispatch(
|
||||
workspaceApi.util.invalidateTags([
|
||||
{ type: "workspace", id: workspaceId },
|
||||
{ type: "workspace-transform", id: transform.ref_id },
|
||||
{ type: "workspace-tables", id: workspaceId },
|
||||
]),
|
||||
);
|
||||
|
||||
if (result.status === "failed") {
|
||||
sendErrorToast(t`Transform run failed`);
|
||||
}
|
||||
} catch (error) {
|
||||
setIsRunTriggered(false);
|
||||
setLastRunResult(null);
|
||||
sendErrorToast(t`Failed to run transform`);
|
||||
console.error("Failed to run transform", error);
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
statusRun,
|
||||
buttonRun,
|
||||
isRunStatusLoading: statusRun == null && isFetching,
|
||||
isRunning: isRunTriggered,
|
||||
handleRun,
|
||||
output,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import { useEffect, useMemo } from "react";
|
||||
import { replace } from "react-router-redux";
|
||||
import { t } from "ttag";
|
||||
|
||||
import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
|
||||
import { useDispatch } from "metabase/lib/redux";
|
||||
import * as Urls from "metabase/lib/urls";
|
||||
import { Box, Text, Title } from "metabase/ui";
|
||||
import { useGetWorkspacesQuery } from "metabase-enterprise/api";
|
||||
|
||||
export function WorkspaceListPage() {
|
||||
const dispatch = useDispatch();
|
||||
const {
|
||||
data: workspacesData,
|
||||
error,
|
||||
isLoading,
|
||||
isFetching,
|
||||
} = useGetWorkspacesQuery();
|
||||
|
||||
const workspaces = useMemo(
|
||||
() => workspacesData?.items ?? [],
|
||||
[workspacesData],
|
||||
);
|
||||
|
||||
const firstWorkspaceId = workspaces[0]?.id;
|
||||
|
||||
useEffect(() => {
|
||||
if (error || firstWorkspaceId == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
dispatch(replace(Urls.dataStudioWorkspace(firstWorkspaceId)));
|
||||
}, [dispatch, error, firstWorkspaceId]);
|
||||
|
||||
if (error) {
|
||||
return <LoadingAndErrorWrapper error={error} loading={false} />;
|
||||
}
|
||||
|
||||
if (firstWorkspaceId != null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (isLoading || isFetching) {
|
||||
return (
|
||||
<LoadingAndErrorWrapper error={error} loading={isLoading || isFetching} />
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<Box data-testid="workspaces-page" p="lg">
|
||||
<Title size="lg">{t`No active workspaces`}</Title>
|
||||
<Text c="text-secondary">
|
||||
{t`Create a new one from the left panel or by editing an existing transform.`}
|
||||
</Text>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
import { t } from "ttag";
|
||||
|
||||
import { ActionIcon, Icon, Menu } from "metabase/ui";
|
||||
import {
|
||||
getInitialNativeSource,
|
||||
getInitialPythonSource,
|
||||
} from "metabase-enterprise/transforms/pages/NewTransformPage/utils";
|
||||
import type {
|
||||
DatabaseId,
|
||||
TransformSource,
|
||||
WorkspaceId,
|
||||
} from "metabase-types/api";
|
||||
import { createMockTransform } from "metabase-types/api/mocks/transform";
|
||||
|
||||
import { useWorkspace } from "./WorkspaceProvider";
|
||||
|
||||
type TransformType = "sql" | "python";
|
||||
|
||||
interface Props {
|
||||
databaseId: DatabaseId;
|
||||
workspaceId: WorkspaceId;
|
||||
disabled?: boolean;
|
||||
}
|
||||
|
||||
export const AddTransformMenu = ({
|
||||
databaseId,
|
||||
workspaceId,
|
||||
disabled,
|
||||
}: Props) => {
|
||||
const { addUnsavedTransform } = useWorkspace();
|
||||
|
||||
const getSource = (type: TransformType): TransformSource => {
|
||||
if (type === "sql") {
|
||||
const source = getInitialNativeSource();
|
||||
return {
|
||||
...source,
|
||||
query: { ...source.query, database: databaseId },
|
||||
};
|
||||
}
|
||||
return {
|
||||
...getInitialPythonSource(),
|
||||
"source-database": databaseId,
|
||||
};
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<Menu position="bottom-end">
|
||||
<Menu.Target>
|
||||
<ActionIcon
|
||||
size="2rem"
|
||||
p="0"
|
||||
ml="auto"
|
||||
aria-label={t`Add transform`}
|
||||
disabled={disabled}
|
||||
>
|
||||
<Icon name="add" size={16} aria-hidden tooltip={t`Add transform`} />
|
||||
</ActionIcon>
|
||||
</Menu.Target>
|
||||
<Menu.Dropdown>
|
||||
<Menu.Item
|
||||
leftSection={<Icon name="sql" />}
|
||||
onClick={() => {
|
||||
const mockTransform = createMockTransform({
|
||||
source: getSource("sql"),
|
||||
workspace_id: workspaceId,
|
||||
});
|
||||
addUnsavedTransform(mockTransform);
|
||||
}}
|
||||
>
|
||||
{t`SQL Transform`}
|
||||
</Menu.Item>
|
||||
<Menu.Item
|
||||
leftSection={<Icon name="code_block" />}
|
||||
onClick={() => {
|
||||
const mockTransform = createMockTransform({
|
||||
source: getSource("python"),
|
||||
workspace_id: workspaceId,
|
||||
});
|
||||
addUnsavedTransform(mockTransform);
|
||||
}}
|
||||
>
|
||||
{t`Python Script`}
|
||||
</Menu.Item>
|
||||
</Menu.Dropdown>
|
||||
</Menu>
|
||||
</>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,183 @@
|
||||
import { useCallback } from "react";
|
||||
import { t } from "ttag";
|
||||
|
||||
import EmptyState from "metabase/common/components/EmptyState";
|
||||
import { Stack, Text } from "metabase/ui";
|
||||
import {
|
||||
useLazyGetTransformQuery,
|
||||
useLazyGetWorkspaceTransformQuery,
|
||||
} from "metabase-enterprise/api";
|
||||
import type {
|
||||
ExternalTransform,
|
||||
Transform,
|
||||
WorkspaceId,
|
||||
WorkspaceTransform,
|
||||
} from "metabase-types/api";
|
||||
|
||||
import { useWorkspace } from "../WorkspaceProvider";
|
||||
|
||||
import { TransformListItem } from "./TransformListItem";
|
||||
import { TransformListItemMenu } from "./TransformListItemMenu";
|
||||
|
||||
type CodeTabProps = {
|
||||
activeTransformId?: number | string;
|
||||
availableTransforms: ExternalTransform[];
|
||||
workspaceId: WorkspaceId;
|
||||
workspaceTransforms: WorkspaceTransform[];
|
||||
readOnly: boolean;
|
||||
onTransformClick: (transform: Transform | WorkspaceTransform) => void;
|
||||
};
|
||||
|
||||
export const CodeTab = ({
|
||||
activeTransformId,
|
||||
availableTransforms,
|
||||
workspaceId,
|
||||
workspaceTransforms,
|
||||
readOnly,
|
||||
onTransformClick,
|
||||
}: CodeTabProps) => {
|
||||
const { editedTransforms, hasTransformEdits } = useWorkspace();
|
||||
|
||||
const [fetchWorkspaceTransform] = useLazyGetWorkspaceTransformQuery();
|
||||
const [fetchTransform] = useLazyGetTransformQuery();
|
||||
|
||||
const normalizeTransformId = useCallback(
|
||||
(transform: Transform | WorkspaceTransform) =>
|
||||
"ref_id" in transform ? transform.ref_id : transform.id,
|
||||
[],
|
||||
);
|
||||
|
||||
const handleTransformClick = useCallback(
|
||||
(transform: Transform | WorkspaceTransform) => {
|
||||
const transformId = normalizeTransformId(transform);
|
||||
const edited = editedTransforms.get(transformId);
|
||||
const transformToOpen =
|
||||
edited != null ? { ...transform, ...edited } : transform;
|
||||
|
||||
onTransformClick(transformToOpen);
|
||||
},
|
||||
[editedTransforms, normalizeTransformId, onTransformClick],
|
||||
);
|
||||
|
||||
const handleExternalTransformClick = useCallback(
|
||||
async (externalTransform: ExternalTransform) => {
|
||||
if (externalTransform.checkout_disabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const { data: transform } = await fetchTransform(
|
||||
externalTransform.id,
|
||||
true,
|
||||
);
|
||||
|
||||
if (transform) {
|
||||
handleTransformClick(transform);
|
||||
}
|
||||
},
|
||||
[fetchTransform, handleTransformClick],
|
||||
);
|
||||
|
||||
const handleWorkspaceTransformClick = useCallback(
|
||||
async (workspaceTransform: WorkspaceTransform) => {
|
||||
if (
|
||||
typeof workspaceTransform.id === "number" &&
|
||||
workspaceTransform.id < 0
|
||||
) {
|
||||
return handleTransformClick(workspaceTransform);
|
||||
}
|
||||
|
||||
const { data: transform } = await fetchWorkspaceTransform(
|
||||
{
|
||||
workspaceId,
|
||||
transformId: workspaceTransform.ref_id,
|
||||
},
|
||||
true,
|
||||
);
|
||||
|
||||
if (transform) {
|
||||
handleTransformClick(transform);
|
||||
}
|
||||
},
|
||||
[fetchWorkspaceTransform, workspaceId, handleTransformClick],
|
||||
);
|
||||
|
||||
return (
|
||||
<Stack h="100%" gap={0}>
|
||||
<Stack
|
||||
data-testid="workspace-transforms"
|
||||
gap="xs"
|
||||
pb="sm"
|
||||
style={{
|
||||
borderBottom: "1px solid var(--mb-color-border)",
|
||||
}}
|
||||
>
|
||||
<Stack gap={0}>
|
||||
<Text fw={600}>{t`Workspace transforms`}</Text>
|
||||
{workspaceTransforms.map((transform) => {
|
||||
const isEdited = hasTransformEdits(transform);
|
||||
const isActive =
|
||||
typeof activeTransformId === "string" &&
|
||||
activeTransformId === transform.ref_id;
|
||||
|
||||
return (
|
||||
<TransformListItem
|
||||
key={transform.ref_id ?? transform.id}
|
||||
name={transform.name}
|
||||
icon="pivot_table"
|
||||
fw={600}
|
||||
isActive={isActive}
|
||||
isEdited={isEdited}
|
||||
menu={
|
||||
readOnly ? null : (
|
||||
<TransformListItemMenu
|
||||
transform={transform}
|
||||
workspaceId={workspaceId}
|
||||
/>
|
||||
)
|
||||
}
|
||||
onClick={() => {
|
||||
handleWorkspaceTransformClick(transform);
|
||||
}}
|
||||
/>
|
||||
);
|
||||
})}
|
||||
</Stack>
|
||||
|
||||
{workspaceTransforms.length === 0 && (
|
||||
<EmptyState message={t`Workspace is empty`} />
|
||||
)}
|
||||
</Stack>
|
||||
|
||||
<Stack data-testid="mainland-transforms" py="sm" gap="xs">
|
||||
<Text fw={600} mt="sm">{t`Available transforms`}</Text>
|
||||
{availableTransforms.map((transform) => {
|
||||
const tooltipLabel =
|
||||
transform.checkout_disabled === "mbql"
|
||||
? t`This transform cannot be edited in a workspace because it uses MBQL.`
|
||||
: transform.checkout_disabled === "card-reference"
|
||||
? t`This transform cannot be edited in a workspace because it references other questions.`
|
||||
: transform.checkout_disabled
|
||||
? t`This transform cannot be edited in a workspace.`
|
||||
: undefined;
|
||||
|
||||
return (
|
||||
<TransformListItem
|
||||
key={transform.id}
|
||||
name={transform.name}
|
||||
isActive={activeTransformId === transform.id}
|
||||
isEdited={editedTransforms.has(transform.id)}
|
||||
onClick={() => {
|
||||
handleExternalTransformClick(transform);
|
||||
}}
|
||||
opacity={transform.checkout_disabled ? 0.5 : 1}
|
||||
tooltipLabel={tooltipLabel}
|
||||
/>
|
||||
);
|
||||
})}
|
||||
{availableTransforms.length === 0 && (
|
||||
<EmptyState message={t`No available transforms`} />
|
||||
)}
|
||||
</Stack>
|
||||
</Stack>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,33 @@
|
||||
.root {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--mantine-spacing-sm);
|
||||
padding: var(--mantine-spacing-xs) var(--mantine-spacing-sm);
|
||||
border-radius: var(--mantine-radius-sm);
|
||||
cursor: default;
|
||||
flex-wrap: nowrap;
|
||||
|
||||
&:hover .menu {
|
||||
visibility: visible;
|
||||
}
|
||||
|
||||
&:hover {
|
||||
background-color: var(--mb-color-bg-light);
|
||||
}
|
||||
}
|
||||
|
||||
.label {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--mantine-spacing-sm);
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.cursorPointer {
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.menu {
|
||||
visibility: hidden;
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
import cx from "classnames";
|
||||
import type { ReactNode } from "react";
|
||||
|
||||
import {
|
||||
Box,
|
||||
type BoxProps,
|
||||
Icon,
|
||||
type IconName,
|
||||
Text,
|
||||
Tooltip,
|
||||
} from "metabase/ui";
|
||||
|
||||
import { StatusDot } from "../components/StatusDot/StatusDot";
|
||||
|
||||
import S from "./TransformListItem.module.css";
|
||||
|
||||
type TransformListItemProps = {
|
||||
name: string;
|
||||
icon?: IconName;
|
||||
isActive?: boolean;
|
||||
isEdited?: boolean;
|
||||
tooltipLabel?: string;
|
||||
menu?: ReactNode;
|
||||
onClick?: () => void;
|
||||
} & BoxProps;
|
||||
|
||||
export const TransformListItem = ({
|
||||
name,
|
||||
icon = "database",
|
||||
isActive,
|
||||
isEdited,
|
||||
tooltipLabel,
|
||||
menu,
|
||||
onClick,
|
||||
...props
|
||||
}: TransformListItemProps) => {
|
||||
const label = (
|
||||
<Box className={cx(S.label, { [S.cursorPointer]: !tooltipLabel })}>
|
||||
<Icon name={icon} size={14} c="brand" />
|
||||
<Text c={isActive ? "brand" : "text-dark"} truncate>
|
||||
{name}
|
||||
</Text>
|
||||
{isEdited && <StatusDot status="changed" />}
|
||||
</Box>
|
||||
);
|
||||
|
||||
return (
|
||||
<Box className={S.root} onClick={onClick} pl="sm" {...props}>
|
||||
{tooltipLabel ? (
|
||||
<Tooltip label={tooltipLabel} position="top">
|
||||
{label}
|
||||
</Tooltip>
|
||||
) : (
|
||||
label
|
||||
)}
|
||||
|
||||
<Box className={S.menu} flex="0 0 auto">
|
||||
{menu}
|
||||
</Box>
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,75 @@
|
||||
import { t } from "ttag";
|
||||
|
||||
import { useMetadataToasts } from "metabase/metadata/hooks";
|
||||
import { ActionIcon, Icon, Menu } from "metabase/ui";
|
||||
import { useDeleteWorkspaceTransformMutation } from "metabase-enterprise/api";
|
||||
import type {
|
||||
Transform,
|
||||
WorkspaceId,
|
||||
WorkspaceTransformItem,
|
||||
} from "metabase-types/api";
|
||||
|
||||
import { useWorkspace } from "../WorkspaceProvider";
|
||||
|
||||
interface Props {
|
||||
transform: WorkspaceTransformItem | Transform;
|
||||
workspaceId: WorkspaceId;
|
||||
}
|
||||
|
||||
export function TransformListItemMenu({ transform, workspaceId }: Props) {
|
||||
const [deleteWorkspaceTransform] = useDeleteWorkspaceTransformMutation();
|
||||
|
||||
const {
|
||||
removeEditedTransform,
|
||||
removeOpenedTransform,
|
||||
removeUnsavedTransform,
|
||||
} = useWorkspace();
|
||||
const { sendErrorToast, sendSuccessToast } = useMetadataToasts();
|
||||
|
||||
const handleRemove = async () => {
|
||||
// Handle unsaved transforms (negative IDs) locally
|
||||
if ("id" in transform && transform.id < 0) {
|
||||
removeUnsavedTransform(transform.id);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!("ref_id" in transform)) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await deleteWorkspaceTransform({
|
||||
workspaceId,
|
||||
transformId: transform.ref_id,
|
||||
}).unwrap();
|
||||
|
||||
sendSuccessToast(t`Transform removed from the workspace`);
|
||||
removeEditedTransform(transform.ref_id);
|
||||
removeOpenedTransform(transform.ref_id);
|
||||
} catch (error) {
|
||||
console.error("Failed to remove transform:", error);
|
||||
sendErrorToast(t`Failed to remove transform from the workspace`);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<Menu position="bottom-end" withinPortal>
|
||||
<Menu.Target>
|
||||
<ActionIcon
|
||||
onClick={(event) => event.stopPropagation()}
|
||||
size="sm"
|
||||
variant="subtle"
|
||||
aria-label={t`More actions`}
|
||||
>
|
||||
<Icon name="ellipsis" />
|
||||
</ActionIcon>
|
||||
</Menu.Target>
|
||||
|
||||
<Menu.Dropdown onClick={(event) => event.stopPropagation()}>
|
||||
<Menu.Item leftSection={<Icon name="trash" />} onClick={handleRemove}>
|
||||
{t`Remove`}
|
||||
</Menu.Item>
|
||||
</Menu.Dropdown>
|
||||
</Menu>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
import { useMemo } from "react";
|
||||
import { t } from "ttag";
|
||||
|
||||
import { createMockMetadata } from "__support__/metadata";
|
||||
import { skipToken, useGetTableQuery } from "metabase/api";
|
||||
import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
|
||||
import CS from "metabase/css/core/index.css";
|
||||
import { Box, Stack, Text } from "metabase/ui";
|
||||
import Visualization from "metabase/visualizations/components/Visualization";
|
||||
import { useGetWorkspaceTransformQuery } from "metabase-enterprise/api";
|
||||
import type { DatabaseId, TableId } from "metabase-types/api";
|
||||
|
||||
import { useTablePreview } from "./useTablePreview";
|
||||
|
||||
interface DataTabProps {
|
||||
workspaceId: number;
|
||||
databaseId: DatabaseId | null;
|
||||
tableId: TableId | null;
|
||||
transformId?: string;
|
||||
}
|
||||
|
||||
export function DataTab({
|
||||
workspaceId,
|
||||
databaseId,
|
||||
tableId,
|
||||
transformId,
|
||||
}: DataTabProps) {
|
||||
const { data: table } = useGetTableQuery(
|
||||
tableId ? { id: tableId } : skipToken,
|
||||
);
|
||||
const { data: transform } = useGetWorkspaceTransformQuery(
|
||||
transformId ? { workspaceId, transformId } : skipToken,
|
||||
);
|
||||
const metadata = useMemo(
|
||||
() =>
|
||||
createMockMetadata({
|
||||
tables: table
|
||||
? [
|
||||
{
|
||||
...table,
|
||||
visibility_type: null,
|
||||
},
|
||||
]
|
||||
: [],
|
||||
}),
|
||||
[table],
|
||||
);
|
||||
|
||||
const { rawSeries, isFetching, error } = useTablePreview({
|
||||
databaseId,
|
||||
tableId,
|
||||
metadata,
|
||||
last_transform_run_time: transform?.last_run_at,
|
||||
});
|
||||
|
||||
if (!databaseId || !tableId) {
|
||||
return (
|
||||
<Stack h="100%" align="center" justify="center">
|
||||
<Text c="text-medium">{t`Select a table to view its data`}</Text>
|
||||
</Stack>
|
||||
);
|
||||
}
|
||||
|
||||
if (isFetching) {
|
||||
return <LoadingAndErrorWrapper loading />;
|
||||
}
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<Stack h="100%" align="center" justify="center">
|
||||
<Text c="error">{t`Error loading data`}</Text>
|
||||
</Stack>
|
||||
);
|
||||
}
|
||||
|
||||
if (!rawSeries) {
|
||||
return (
|
||||
<Stack h="100%" align="center" justify="center">
|
||||
<Text c="text-medium">{t`No data available`}</Text>
|
||||
</Stack>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<Box h="100%" className={CS.relative}>
|
||||
<Visualization queryBuilderMode="dataset" rawSeries={rawSeries} />
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
import { t } from "ttag";
|
||||
|
||||
import { Stack, Text } from "metabase/ui";
|
||||
import type {
|
||||
TableId,
|
||||
Transform,
|
||||
WorkspaceTablesResponse,
|
||||
WorkspaceTransformItem,
|
||||
} from "metabase-types/api";
|
||||
|
||||
import { type OpenTable, useWorkspace } from "../WorkspaceProvider";
|
||||
|
||||
import { TableListItem } from "./TableListItem";
|
||||
|
||||
type DataTabSidebarProps = {
|
||||
tables: WorkspaceTablesResponse;
|
||||
workspaceTransforms: WorkspaceTransformItem[];
|
||||
dbTransforms: Transform[];
|
||||
selectedTableId?: TableId | null;
|
||||
runningTransforms?: Set<string>;
|
||||
onTransformClick?: (transform: WorkspaceTransformItem) => void;
|
||||
onTableSelect?: (table: OpenTable) => void;
|
||||
onRunTransform?: (transform: WorkspaceTransformItem) => void;
|
||||
};
|
||||
|
||||
export const DataTabSidebar = ({
|
||||
tables,
|
||||
workspaceTransforms,
|
||||
dbTransforms,
|
||||
selectedTableId,
|
||||
runningTransforms,
|
||||
onTransformClick,
|
||||
onTableSelect,
|
||||
onRunTransform,
|
||||
}: DataTabSidebarProps) => {
|
||||
const { hasTransformEdits } = useWorkspace();
|
||||
return (
|
||||
<Stack h="100%" gap="sm">
|
||||
<Stack
|
||||
gap="xs"
|
||||
pb="sm"
|
||||
style={{ borderBottom: "1px solid var(--mb-color-border)" }}
|
||||
>
|
||||
<Text fw={600}>{t`Data active in this workspace`}</Text>
|
||||
{tables.outputs.length > 0 && (
|
||||
<Text size="sm" fw={600} c="text-tertiary">{t`Output tables`}</Text>
|
||||
)}
|
||||
<Stack gap={0}>
|
||||
{tables.outputs.map((table, index: number) => {
|
||||
const workspaceTransform = workspaceTransforms.find(
|
||||
(t) => t["ref_id"] === table.isolated.transform_id,
|
||||
);
|
||||
const originalTransform = workspaceTransform?.global_id
|
||||
? dbTransforms.find((t) => t.id === workspaceTransform.global_id)
|
||||
: undefined;
|
||||
const hasChanges = originalTransform
|
||||
? hasTransformEdits(originalTransform)
|
||||
: false;
|
||||
const tableId = table.isolated.table_id;
|
||||
|
||||
return (
|
||||
<TableListItem
|
||||
key={`output-${index}`}
|
||||
name={table.global.table}
|
||||
schema={table.global.schema}
|
||||
icon="pivot_table"
|
||||
type="output"
|
||||
hasChanges={hasChanges}
|
||||
transform={workspaceTransform}
|
||||
tableId={tableId ?? undefined}
|
||||
isSelected={tableId === selectedTableId}
|
||||
isRunning={
|
||||
workspaceTransform
|
||||
? runningTransforms?.has(workspaceTransform.ref_id)
|
||||
: false
|
||||
}
|
||||
onTransformClick={onTransformClick}
|
||||
onTableClick={onTableSelect}
|
||||
onRunTransform={onRunTransform}
|
||||
/>
|
||||
);
|
||||
})}
|
||||
</Stack>
|
||||
{tables.inputs.length > 0 && (
|
||||
<Text size="sm" fw={600} c="text-tertiary">{t`Input tables`}</Text>
|
||||
)}
|
||||
<Stack gap={0}>
|
||||
{tables.inputs.map((table, index) => (
|
||||
<TableListItem
|
||||
key={`input-${index}`}
|
||||
name={table.table}
|
||||
schema={table.schema}
|
||||
icon="table"
|
||||
type="input"
|
||||
isSelected={table.table_id === selectedTableId}
|
||||
tableId={table.table_id ?? undefined}
|
||||
onTransformClick={onTransformClick}
|
||||
onTableClick={onTableSelect}
|
||||
/>
|
||||
))}
|
||||
</Stack>
|
||||
</Stack>
|
||||
</Stack>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,34 @@
|
||||
.root {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--mantine-spacing-sm);
|
||||
padding: var(--mantine-spacing-xs) var(--mantine-spacing-sm);
|
||||
border-radius: var(--mantine-radius-sm);
|
||||
flex-wrap: nowrap;
|
||||
|
||||
&:hover .menu {
|
||||
visibility: visible;
|
||||
}
|
||||
|
||||
&:hover {
|
||||
background-color: var(--mb-color-bg-light);
|
||||
}
|
||||
}
|
||||
|
||||
.clickable {
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.selected {
|
||||
background-color: var(--mb-color-bg-medium);
|
||||
}
|
||||
.name {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.actionIcon {
|
||||
&:hover {
|
||||
background-color: var(--mb-color-bg-medium) !important;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
import cx from "classnames";
|
||||
import { t } from "ttag";
|
||||
|
||||
import {
|
||||
ActionIcon,
|
||||
Box,
|
||||
Icon,
|
||||
type IconName,
|
||||
Loader,
|
||||
Text,
|
||||
Tooltip,
|
||||
} from "metabase/ui";
|
||||
import type { TableId, WorkspaceTransformItem } from "metabase-types/api";
|
||||
|
||||
import type { OpenTable } from "../WorkspaceProvider";
|
||||
import { StatusDot } from "../components/StatusDot/StatusDot";
|
||||
|
||||
import S from "./TableListItem.module.css";
|
||||
|
||||
type TableListItemProps = {
|
||||
name: string;
|
||||
schema?: string | null;
|
||||
icon?: IconName;
|
||||
type: "input" | "output";
|
||||
hasChanges?: boolean;
|
||||
transform?: WorkspaceTransformItem;
|
||||
tableId?: TableId;
|
||||
isSelected?: boolean;
|
||||
isRunning?: boolean;
|
||||
onTransformClick?: (transform: WorkspaceTransformItem) => void;
|
||||
onTableClick?: (table: OpenTable) => void;
|
||||
onRunTransform?: (transform: WorkspaceTransformItem) => void;
|
||||
};
|
||||
|
||||
export const TableListItem = ({
|
||||
name,
|
||||
schema,
|
||||
icon = "table",
|
||||
type,
|
||||
hasChanges = false,
|
||||
transform,
|
||||
tableId,
|
||||
isSelected = false,
|
||||
isRunning = false,
|
||||
onTransformClick,
|
||||
onTableClick,
|
||||
onRunTransform,
|
||||
}: TableListItemProps) => {
|
||||
const displayName = schema ? `${schema}.${name}` : name;
|
||||
|
||||
const handleTransformClick = (e: React.MouseEvent) => {
|
||||
e.stopPropagation();
|
||||
if (transform && onTransformClick) {
|
||||
onTransformClick(transform);
|
||||
}
|
||||
};
|
||||
|
||||
const handleTableClick = () => {
|
||||
if (isRunning) {
|
||||
return;
|
||||
}
|
||||
if (tableId && onTableClick) {
|
||||
onTableClick({ tableId, name, schema, transformId: transform?.ref_id });
|
||||
} else if (type === "output" && transform && onRunTransform) {
|
||||
onRunTransform(transform);
|
||||
}
|
||||
};
|
||||
|
||||
const hasResults = type === "output" && !!tableId;
|
||||
const canRunTransform =
|
||||
type === "output" && !hasResults && transform && onRunTransform;
|
||||
const isClickable = (tableId && onTableClick) || canRunTransform;
|
||||
const displayTooltip = type !== "input" && !hasResults && !isRunning;
|
||||
|
||||
return (
|
||||
<Box
|
||||
className={cx(S.root, {
|
||||
[S.selected]: isSelected,
|
||||
[S.clickable]: isClickable,
|
||||
})}
|
||||
aria-label={displayName}
|
||||
onClick={isClickable ? handleTableClick : undefined}
|
||||
>
|
||||
{isRunning ? (
|
||||
<Loader size="sm" />
|
||||
) : (
|
||||
<Icon
|
||||
name={icon}
|
||||
size={14}
|
||||
c={type === "input" ? "saturated-green" : "saturated-yellow"}
|
||||
/>
|
||||
)}
|
||||
<Tooltip
|
||||
label={
|
||||
isRunning
|
||||
? t`Running transform...`
|
||||
: t`Run transform to see the results`
|
||||
}
|
||||
position="top"
|
||||
disabled={!displayTooltip && !isRunning}
|
||||
>
|
||||
<Text
|
||||
className={S.name}
|
||||
c={displayTooltip ? "text-light" : "text-dark"}
|
||||
truncate
|
||||
>
|
||||
{displayName}
|
||||
</Text>
|
||||
</Tooltip>
|
||||
{hasChanges && <StatusDot status="changed" />}
|
||||
{type === "output" && transform && (
|
||||
<Tooltip label={t`Open transform`} position="top">
|
||||
<ActionIcon
|
||||
className={S.actionIcon}
|
||||
ml="auto"
|
||||
size="sm"
|
||||
variant="subtle"
|
||||
onClick={handleTransformClick}
|
||||
>
|
||||
<Icon name="code_block" size={14} c="text-medium" />
|
||||
</ActionIcon>
|
||||
</Tooltip>
|
||||
)}
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,4 @@
|
||||
export { DataTab } from "./DataTab";
|
||||
export { DataTabSidebar } from "./DataTabSidebar";
|
||||
export { useTablePreview as useTableQuestion } from "./useTablePreview";
|
||||
export type { OpenTable } from "../WorkspaceProvider";
|
||||
@@ -0,0 +1,91 @@
|
||||
import { useMemo } from "react";
|
||||
import _ from "underscore";
|
||||
|
||||
import { skipToken, useGetAdhocQueryQuery } from "metabase/api";
|
||||
import { getErrorMessage } from "metabase/api/utils";
|
||||
import * as Lib from "metabase-lib";
|
||||
import type Metadata from "metabase-lib/v1/metadata/Metadata";
|
||||
import type { DatabaseId, RawSeries, TableId } from "metabase-types/api";
|
||||
import { createMockCard } from "metabase-types/api/mocks";
|
||||
|
||||
interface UseTableQuestionProps {
|
||||
databaseId: DatabaseId | null;
|
||||
tableId: TableId | null;
|
||||
metadata: Metadata;
|
||||
last_transform_run_time?: string | null;
|
||||
}
|
||||
|
||||
interface UseTablePreviewResult {
|
||||
rawSeries?: RawSeries;
|
||||
isFetching: boolean;
|
||||
isLoading: boolean;
|
||||
isError: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export function useTablePreview({
|
||||
databaseId,
|
||||
tableId,
|
||||
metadata,
|
||||
last_transform_run_time,
|
||||
}: UseTableQuestionProps): UseTablePreviewResult {
|
||||
const metadataProvider =
|
||||
Object.keys(metadata.tables).length > 0
|
||||
? Lib.metadataProvider(databaseId, metadata)
|
||||
: null;
|
||||
|
||||
const table =
|
||||
metadataProvider && tableId
|
||||
? Lib.tableOrCardMetadata(metadataProvider, tableId)
|
||||
: null;
|
||||
|
||||
const query =
|
||||
table && metadataProvider
|
||||
? Lib.queryFromTableOrCardMetadata(metadataProvider, table)
|
||||
: null;
|
||||
|
||||
const { data, ...rest } = useGetAdhocQueryQuery(
|
||||
query == null
|
||||
? skipToken
|
||||
: {
|
||||
...Lib.toJsQuery(query),
|
||||
ignore_error: true,
|
||||
_refetchDeps: last_transform_run_time,
|
||||
},
|
||||
);
|
||||
const base = {
|
||||
...rest,
|
||||
error: rest.error ? getErrorMessage(rest.error) : undefined,
|
||||
rawSeries: undefined,
|
||||
};
|
||||
|
||||
const rawSeries = useMemo(
|
||||
() =>
|
||||
query
|
||||
? [
|
||||
{
|
||||
card: createMockCard({
|
||||
dataset_query: Lib.toJsQuery(query),
|
||||
display: "table",
|
||||
visualization_settings: {},
|
||||
}),
|
||||
data: {
|
||||
...data?.data,
|
||||
rows: _.uniq(data?.data?.rows ?? []),
|
||||
},
|
||||
},
|
||||
]
|
||||
: ([] as RawSeries),
|
||||
[data?.data, query],
|
||||
);
|
||||
|
||||
if (data?.status === "failed") {
|
||||
return { ...base, isError: true, error: getErrorMessage(data) };
|
||||
}
|
||||
|
||||
if (query == null || !data?.data || data.data.cols.length === 0) {
|
||||
return base;
|
||||
}
|
||||
|
||||
return { ...base, rawSeries };
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import { PLUGIN_METABOT } from "metabase/plugins";
|
||||
import { useRegisterMetabotTransformContext } from "metabase-enterprise/transforms/hooks/use-register-transform-metabot-context";
|
||||
import type { DraftTransformSource, Transform } from "metabase-types/api";
|
||||
|
||||
type MetabotTabProps = {
|
||||
transform?: Transform;
|
||||
source?: DraftTransformSource;
|
||||
};
|
||||
|
||||
const MetabotContextRegistration = ({ transform, source }: MetabotTabProps) => {
|
||||
useRegisterMetabotTransformContext(transform, source);
|
||||
return null;
|
||||
};
|
||||
|
||||
export const MetabotTab = ({ transform, source }: MetabotTabProps) => {
|
||||
const MetabotProvider = PLUGIN_METABOT.getMetabotProvider();
|
||||
const MetabotChat = PLUGIN_METABOT.MetabotChat;
|
||||
const metabotConfig = {
|
||||
// Show top-level mention menu for transforms and databases.
|
||||
suggestionModels: ["transform", "database"],
|
||||
};
|
||||
|
||||
return (
|
||||
<MetabotProvider>
|
||||
<MetabotContextRegistration transform={transform} source={source} />
|
||||
<MetabotChat config={metabotConfig} />
|
||||
</MetabotProvider>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,50 @@
|
||||
import { useMemo } from "react";
|
||||
import { t } from "ttag";
|
||||
|
||||
import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
|
||||
import CS from "metabase/css/core/index.css";
|
||||
import { Box, Stack, Text } from "metabase/ui";
|
||||
import Visualization from "metabase/visualizations/components/Visualization";
|
||||
import type { Dataset, RawSeries } from "metabase-types/api";
|
||||
import { createMockCard } from "metabase-types/api/mocks";
|
||||
|
||||
interface PreviewTabProps {
|
||||
dataset: Dataset | null;
|
||||
isLoading?: boolean;
|
||||
}
|
||||
|
||||
export function PreviewTab({ dataset, isLoading }: PreviewTabProps) {
|
||||
const rawSeries: RawSeries = useMemo(() => {
|
||||
if (!dataset?.data || dataset.data.cols.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return [
|
||||
{
|
||||
card: createMockCard({
|
||||
display: "table",
|
||||
visualization_settings: {},
|
||||
}),
|
||||
data: dataset.data,
|
||||
},
|
||||
];
|
||||
}, [dataset?.data]);
|
||||
|
||||
if (isLoading) {
|
||||
return <LoadingAndErrorWrapper loading />;
|
||||
}
|
||||
|
||||
if (!dataset?.data || dataset.data.cols.length === 0) {
|
||||
return (
|
||||
<Stack h="100%" align="center" justify="center">
|
||||
<Text c="text-medium">{t`No data available`}</Text>
|
||||
</Stack>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<Box h="100%" className={CS.relative}>
|
||||
<Visualization queryBuilderMode="dataset" rawSeries={rawSeries} />
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { PreviewTab } from "./PreviewTab";
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user