Compare commits
99 Commits
python-sec
...
workspaces
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
640365f194 | ||
|
|
557ce7c3c6 | ||
|
|
a60fa0c2cb | ||
|
|
39e4f68e31 | ||
|
|
072c06e2bd | ||
|
|
e781690132 | ||
|
|
68b19966fc | ||
|
|
8b5db401e8 | ||
|
|
946fc08c43 | ||
|
|
f5cbb8cc48 | ||
|
|
44d6075e8c | ||
|
|
fa2ae8e884 | ||
|
|
36e3b6ec3a | ||
|
|
ba9a841466 | ||
|
|
5ebc738f29 | ||
|
|
045e9a224f | ||
|
|
a6e22daac1 | ||
|
|
6bdd48ffe3 | ||
|
|
5bb0f1f560 | ||
|
|
2d263d19e3 | ||
|
|
5e98f44b80 | ||
|
|
da6555f5cf | ||
|
|
937f3f7b03 | ||
|
|
71d065f0a9 | ||
|
|
b30071a69a | ||
|
|
407fc2a5ab | ||
|
|
de113f72ce | ||
|
|
6199e0056f | ||
|
|
fd6b176446 | ||
|
|
42d1202c18 | ||
|
|
d1a4656cb8 | ||
|
|
cc934ea766 | ||
|
|
8bf117b047 | ||
|
|
caea0b31cf | ||
|
|
1d6dd0d16d | ||
|
|
dca47362f8 | ||
|
|
2f8b666797 | ||
|
|
f3aef4f4ad | ||
|
|
5ac6249eb3 | ||
|
|
8acbee8000 | ||
|
|
bfd2ad8f0f | ||
|
|
a982db6bf5 | ||
|
|
3e6d266a33 | ||
|
|
e197cf97af | ||
|
|
97d89cc855 | ||
|
|
955cc0b4be | ||
|
|
1b4424393d | ||
|
|
3337769411 | ||
|
|
0b07779984 | ||
|
|
2e54d68fb2 | ||
|
|
e67c3e625d | ||
|
|
325fe07ab1 | ||
|
|
dc53fec17e | ||
|
|
17c3ed6635 | ||
|
|
f661f0c21e | ||
|
|
89d1394fb7 | ||
|
|
0cac006357 | ||
|
|
c9a5ea4dd1 | ||
|
|
79f7d83c4d | ||
|
|
56a1ac2862 | ||
|
|
8d0acd1436 | ||
|
|
b6628af72f | ||
|
|
0169970a8b | ||
|
|
9cc27dfa59 | ||
|
|
9221d59b7e | ||
|
|
22047bb890 | ||
|
|
772216505c | ||
|
|
3d7541f3c7 | ||
|
|
5a59eef925 | ||
|
|
dc05f59514 | ||
|
|
f64ab4ebab | ||
|
|
19497a7470 | ||
|
|
823690bc76 | ||
|
|
82930d7f02 | ||
|
|
fa9b90beb1 | ||
|
|
d4d0c64400 | ||
|
|
cb70117c05 | ||
|
|
3d9c45e126 | ||
|
|
516df0c70e | ||
|
|
4cd2aa7c3e | ||
|
|
e6943a7389 | ||
|
|
a43c90f357 | ||
|
|
91a42b4fac | ||
|
|
34689d9a05 | ||
|
|
528b264f87 | ||
|
|
0580e2dc34 | ||
|
|
91c985eca9 | ||
|
|
3bb34fa1c9 | ||
|
|
f539b334dd | ||
|
|
9cfe59b1a2 | ||
|
|
fb59bf9325 | ||
|
|
2b2ff2dae5 | ||
|
|
0dd7d78c07 | ||
|
|
33daf1d822 | ||
|
|
9faad636cb | ||
|
|
b201dca6cf | ||
|
|
f1982e56e3 | ||
|
|
439c3f3be5 | ||
|
|
37449925fa |
@@ -1325,6 +1325,7 @@
|
||||
metabase.sync.core
|
||||
metabase.sync.init
|
||||
metabase.sync.schedules
|
||||
metabase.sync.sync-metadata.tables
|
||||
metabase.sync.task.sync-databases
|
||||
metabase.sync.util}
|
||||
:uses #{analyze
|
||||
@@ -2277,6 +2278,7 @@
|
||||
app-db
|
||||
config
|
||||
driver
|
||||
events
|
||||
lib
|
||||
lib-be
|
||||
query-processor
|
||||
@@ -2291,6 +2293,22 @@
|
||||
models
|
||||
premium-features
|
||||
upload
|
||||
util}}
|
||||
|
||||
enterprise/workspaces
|
||||
{:team "Metabot"
|
||||
:api #{metabase-enterprise.workspaces.api
|
||||
metabase-enterprise.workspaces.init}
|
||||
:uses #{api
|
||||
api-keys
|
||||
driver
|
||||
enterprise/transforms
|
||||
events
|
||||
models
|
||||
queries
|
||||
request
|
||||
sync
|
||||
system
|
||||
util}}}
|
||||
|
||||
;; namespaces matching these patterns (with `re-find`) are excluded from the module linter. Since regex literals
|
||||
|
||||
@@ -144,6 +144,18 @@
|
||||
change-log
|
||||
#(pos? (compare % "v49.00-000"))))
|
||||
|
||||
(defn require-primary-key-exists-has-table-name
|
||||
"Ensures that all primaryKeyExists preconditions specify a tableName."
|
||||
[change-log]
|
||||
(doseq [{:keys [changeSet]} change-log
|
||||
:when changeSet
|
||||
:let [s (pr-str (:preConditions changeSet))]
|
||||
:when (and (str/includes? s ":primaryKeyExists")
|
||||
(not (str/includes? s ":tableName")))]
|
||||
(throw (validation-error
|
||||
(format "Migration '%s' has primaryKeyExists precondition without tableName" (:id changeSet))
|
||||
{:id (:id changeSet)}))))
|
||||
|
||||
(s/def ::changeSet
|
||||
(s/spec :change-set.strict/change-set))
|
||||
|
||||
@@ -163,6 +175,7 @@
|
||||
(require-no-bare-blob-or-text-types change-log)
|
||||
(require-no-bare-boolean-types change-log)
|
||||
(require-no-datetime-type change-log)
|
||||
(require-primary-key-exists-has-table-name change-log)
|
||||
(let [{:keys [changeSet]
|
||||
:as all} (group-by only-key change-log)]
|
||||
(when-not (set/subset? (set (keys all)) #{:property :objectQuotingStrategy :changeSet})
|
||||
|
||||
@@ -42,6 +42,7 @@
|
||||
[metabase-enterprise.transforms-python.api]
|
||||
[metabase-enterprise.transforms.api]
|
||||
[metabase-enterprise.upload-management.api]
|
||||
[metabase-enterprise.workspaces.api]
|
||||
[metabase.api.macros :as api.macros]
|
||||
[metabase.api.routes.common :as api.routes.common]
|
||||
[metabase.api.util.handlers :as handlers]
|
||||
@@ -139,7 +140,8 @@
|
||||
"/transform" (premium-handler metabase-enterprise.transforms.api/routes :transforms)
|
||||
"/transform-job" (premium-handler metabase-enterprise.transforms.api/transform-job-routes :transforms)
|
||||
"/transform-tag" (premium-handler metabase-enterprise.transforms.api/transform-tag-routes :transforms)
|
||||
"/upload-management" (premium-handler metabase-enterprise.upload-management.api/routes :upload-management)})
|
||||
"/upload-management" (premium-handler metabase-enterprise.upload-management.api/routes :upload-management)
|
||||
"/workspace" metabase-enterprise.workspaces.api/routes})
|
||||
;;; ↑↑↑ KEEP THIS SORTED OR ELSE ↑↑↑
|
||||
|
||||
(def ^:private routes-map
|
||||
|
||||
@@ -24,4 +24,5 @@
|
||||
[metabase-enterprise.stale.init]
|
||||
[metabase-enterprise.support-access-grants.init]
|
||||
[metabase-enterprise.transforms-python.init]
|
||||
[metabase-enterprise.transforms.init]))
|
||||
[metabase-enterprise.transforms.init]
|
||||
[metabase-enterprise.workspaces.init]))
|
||||
|
||||
@@ -41,6 +41,12 @@
|
||||
query-deps
|
||||
param-card-ids)))
|
||||
|
||||
(mu/defn upstream-deps:python-transform :- ::deps.schema/upstream-deps
|
||||
"Given a Toucan `:model/Transform`, return its upstream dependencies as a map from the kind to a set of IDs."
|
||||
[{{tables :source-tables} :source :as _py-transform}
|
||||
:- [:map [:source-tables {:optional true} [:map-of :string :int]]]]
|
||||
{:table (set (vals tables))})
|
||||
|
||||
(mu/defn upstream-deps:transform :- ::deps.schema/upstream-deps
|
||||
"Given a Transform (in Toucan form), return its upstream dependencies."
|
||||
[{{:keys [query], source-type :type} :source :as transform} :-
|
||||
@@ -49,9 +55,15 @@
|
||||
[:query
|
||||
[:map [:query ::lib.schema/query]]]
|
||||
[:python
|
||||
[:map]]]]]]
|
||||
(if (= (keyword source-type) :query)
|
||||
[:map [:source-tables {:optional true} [:map-of :string :int]]]]]]]]
|
||||
(cond
|
||||
(= (keyword source-type) :query)
|
||||
(upstream-deps:query query)
|
||||
|
||||
(= (keyword source-type) :python)
|
||||
(upstream-deps:python-transform transform)
|
||||
|
||||
:else
|
||||
(do (log/warnf "Don't know how to analyze the deps of Transform %d with source type '%s'" (:id transform) source-type)
|
||||
{})))
|
||||
|
||||
|
||||
@@ -115,4 +115,5 @@
|
||||
"User"
|
||||
"UserKeyValue"
|
||||
"UserParameterValue"
|
||||
"ViewLog"])
|
||||
"ViewLog"
|
||||
"Workspace"])
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
(ns metabase-enterprise.transforms.core
|
||||
"API namespace for the `metabase-enterprise.transform` module."
|
||||
(:require
|
||||
[metabase-enterprise.transforms.models.transform]
|
||||
[metabase-enterprise.transforms.models.transform-run]
|
||||
[metabase-enterprise.transforms.settings]
|
||||
[metabase-enterprise.transforms.util]
|
||||
@@ -14,4 +15,6 @@
|
||||
python-transform?
|
||||
query-transform?]
|
||||
[metabase-enterprise.transforms.models.transform-run
|
||||
timeout-run!])
|
||||
timeout-run!]
|
||||
[metabase-enterprise.transforms.models.transform
|
||||
update-transform-tags!])
|
||||
|
||||
@@ -223,12 +223,14 @@
|
||||
[_model-name opts]
|
||||
{:copy [:name :description :entity_id]
|
||||
:skip [:dependency_analysis_version :source_type]
|
||||
:transform {:created_at (serdes/date)
|
||||
:creator_id (serdes/fk :model/User)
|
||||
:source {:export #(update % :query serdes/export-mbql)
|
||||
:import #(update % :query serdes/import-mbql)}
|
||||
:target {:export serdes/export-mbql :import serdes/import-mbql}
|
||||
:tags (serdes/nested :model/TransformTransformTag :transform_id opts)}})
|
||||
:transform {:created_at (serdes/date)
|
||||
:creator_id (serdes/fk :model/User)
|
||||
:workspace_id (serdes/fk :model/Workspace)
|
||||
:collection_id (serdes/fk :model/Collection)
|
||||
:source {:export #(update % :query serdes/export-mbql)
|
||||
:import #(update % :query serdes/import-mbql)}
|
||||
:target {:export serdes/export-mbql :import serdes/import-mbql}
|
||||
:tags (serdes/nested :model/TransformTransformTag :transform_id opts)}})
|
||||
|
||||
(defmethod serdes/dependencies "Transform"
|
||||
[{:keys [source tags]}]
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
[metabase.app-db.core :as app-db]
|
||||
[metabase.driver :as driver]
|
||||
[metabase.driver.util :as driver.u]
|
||||
[metabase.events.core :as events]
|
||||
[metabase.util :as u]
|
||||
[metabase.util.format :as u.format]
|
||||
[metabase.util.i18n :as i18n]
|
||||
@@ -360,6 +361,7 @@
|
||||
(log/info "Executing Python transform" transform-id "with target" (pr-str target))
|
||||
(let [start-ms (u/start-timer)
|
||||
transform-details {:db-id (:id db)
|
||||
:transform-id transform-id
|
||||
:transform-type (keyword (:type target))
|
||||
:conn-spec (driver/connection-spec driver db)
|
||||
:output-schema (:schema target)
|
||||
@@ -372,7 +374,8 @@
|
||||
result (transforms.instrumentation/with-stage-timing [run-id [:computation :python-execution]]
|
||||
(transforms.util/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn))]
|
||||
(transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]
|
||||
(transforms.util/sync-target! target db))
|
||||
(transforms.util/sync-target! target db)
|
||||
(events/publish-event! :event/transform-run-complete {:object transform-details}))
|
||||
{:run_id run-id
|
||||
:result result}))
|
||||
(catch Throwable t
|
||||
|
||||
402
enterprise/backend/src/metabase_enterprise/workspaces/api.clj
Normal file
402
enterprise/backend/src/metabase_enterprise/workspaces/api.clj
Normal file
@@ -0,0 +1,402 @@
|
||||
(ns metabase-enterprise.workspaces.api
|
||||
"`/api/ee/workspace/` routes"
|
||||
(:require
|
||||
[honey.sql.helpers :as sql.helpers]
|
||||
[metabase-enterprise.transforms.util :as transforms.util]
|
||||
[metabase-enterprise.workspaces.common :as ws.common]
|
||||
[metabase-enterprise.workspaces.dag :as ws.dag]
|
||||
[metabase-enterprise.workspaces.promotion :as ws.promotion]
|
||||
[metabase.api.common :as api]
|
||||
[metabase.api.macros :as api.macros]
|
||||
[metabase.api.routes.common :refer [+auth]]
|
||||
[metabase.driver.util :as driver.u]
|
||||
[metabase.queries.schema :as queries.schema]
|
||||
[metabase.request.core :as request]
|
||||
[metabase.util :as u]
|
||||
[metabase.util.i18n :refer [deferred-tru]]
|
||||
[metabase.util.malli.registry :as mr]
|
||||
[metabase.util.malli.schema :as ms]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
;;; schemas
|
||||
|
||||
(mr/def ::entity-type [:enum :transform])
|
||||
|
||||
(mr/def ::entity-grouping [:enum :transforms])
|
||||
|
||||
;; Map like {:transforms [1 2 3]}
|
||||
(mr/def ::entity-map
|
||||
[:map-of ::entity-grouping [:sequential {:min 1} ms/PositiveInt]])
|
||||
|
||||
;; Entities that live within the Workspace
|
||||
(mr/def ::downstream-entity
|
||||
[:map
|
||||
[:id ms/PositiveInt]
|
||||
[:upstream_id ms/PositiveInt]
|
||||
[:type ::entity-type]
|
||||
[:name :string]])
|
||||
|
||||
;; Entity reference used in requests
|
||||
(mr/def ::entity-reference
|
||||
[:map
|
||||
[:type ::entity-type]
|
||||
[:id ms/PositiveInt]])
|
||||
|
||||
;; Graph node for view-graph endpoint
|
||||
(mr/def ::graph-node
|
||||
[:map
|
||||
[:node_id :string]
|
||||
[:id ms/PositiveInt]
|
||||
[:type [:enum :table :transform]]
|
||||
[:title :string]
|
||||
[:dependents [:map-of [:enum :table :transform] ms/PositiveInt]]])
|
||||
|
||||
;; Graph edge for view-graph endpoint
|
||||
(mr/def ::graph-edge
|
||||
[:map
|
||||
[:from :string]
|
||||
[:to :string]])
|
||||
|
||||
;; Error: entity that needs to be checked out
|
||||
(mr/def ::unchecked-out-entity
|
||||
[:map
|
||||
[:type ::entity-type]
|
||||
[:id ms/PositiveInt]
|
||||
[:name :string]])
|
||||
|
||||
;; Error: entity that cannot be cloned
|
||||
(mr/def ::uncloneable-entity
|
||||
[:map
|
||||
[:type ::entity-type]
|
||||
[:id ms/PositiveInt]
|
||||
[:name :string]
|
||||
[:error :string]])
|
||||
|
||||
;; Error response for graph-not-closed
|
||||
(mr/def ::graph-not-closed-error
|
||||
[:map
|
||||
[:error [:= :graph-not-closed]]
|
||||
[:message :string]
|
||||
[:entities [:sequential ::unchecked-out-entity]]])
|
||||
|
||||
;; Error response for uncloneable entities
|
||||
(mr/def ::uncloneable-error
|
||||
[:map
|
||||
[:error [:= :contains-uncloneable-entities]]
|
||||
[:message :string]
|
||||
[:entities [:sequential ::uncloneable-entity]]])
|
||||
|
||||
(def ^:private CreateWorkspace
|
||||
[:map
|
||||
[:name {:optional true} [:string {:min 1}]]
|
||||
[:database_id {:optional true} :int]
|
||||
[:upstream {:optional true} ::entity-map]])
|
||||
|
||||
(def ^:private ModifyEntities
|
||||
[:map {:closed true}
|
||||
[:add {:optional true} ::entity-map]
|
||||
[:remove {:optional true} ::entity-map]])
|
||||
|
||||
(def ^:private Workspace
|
||||
[:map
|
||||
[:id ms/PositiveInt]
|
||||
[:name :string]
|
||||
[:collection_id :int]
|
||||
[:database_id :int]
|
||||
[:created_at :any]
|
||||
[:updated_at :any]
|
||||
[:archived_at [:maybe :any]]])
|
||||
|
||||
(def ^:private FullWorkspace
|
||||
[:and Workspace
|
||||
[:map
|
||||
[:contents [:map-of ::entity-grouping [:sequential ::downstream-entity]]]]])
|
||||
|
||||
(def ^:private ExecuteResult
|
||||
"Schema for workspace execution result"
|
||||
[:map
|
||||
[:succeeded ::entity-map]
|
||||
[:failed ::entity-map]
|
||||
[:not_run ::entity-map]])
|
||||
|
||||
(def ^:private GraphResult
|
||||
"Schema for workspace graph visualization"
|
||||
[:map
|
||||
[:nodes [:sequential ::graph-node]]
|
||||
[:edges [:sequential ::graph-edge]]])
|
||||
|
||||
;; Transform-related schemas (adapted from transforms/api.clj)
|
||||
|
||||
(mr/def ::transform-source
|
||||
[:multi {:dispatch (comp keyword :type)}
|
||||
[:query
|
||||
[:map
|
||||
[:type [:= "query"]]
|
||||
[:query ::queries.schema/query]]]
|
||||
[:python
|
||||
[:map {:closed true}
|
||||
[:source-database {:optional true} :int]
|
||||
[:source-tables [:map-of :string :int]]
|
||||
[:type [:= "python"]]
|
||||
[:body :string]]]])
|
||||
|
||||
(mr/def ::ws-transform-target
|
||||
[:map
|
||||
[:type [:enum "table"]]
|
||||
[:name :string]])
|
||||
|
||||
(mr/def ::run-trigger
|
||||
[:enum "none" "global-schedule"])
|
||||
|
||||
(defn- check-transforms-enabled!
|
||||
[db-id]
|
||||
(let [database (api/check-400 (t2/select-one :model/Database db-id)
|
||||
(deferred-tru "The target database cannot be found."))]
|
||||
(api/check (transforms.util/check-feature-enabled nil)
|
||||
[402 (deferred-tru "Premium features required for transforms are not enabled.")])
|
||||
(api/check-400 (not (:is_sample database))
|
||||
(deferred-tru "Cannot run transforms on the sample database."))
|
||||
(api/check-400 (not (:is_audit database))
|
||||
(deferred-tru "Cannot run transforms on audit databases."))
|
||||
(api/check-400 (driver.u/supports? (:engine database) :transforms/table database)
|
||||
(deferred-tru "The database does not support the requested transform target type."))
|
||||
(api/check-400 (not (transforms.util/db-routing-enabled? database))
|
||||
(deferred-tru "Transforms are not supported on databases with DB routing enabled."))))
|
||||
|
||||
(defn- ws->response [ws]
|
||||
(select-keys ws
|
||||
[:id :name :collection_id :database_id :created_at :updated_at :archived_at :contents]))
|
||||
|
||||
;;; routes
|
||||
|
||||
(api.macros/defendpoint :get "/" :- [:map [:items [:sequential Workspace]]]
|
||||
"Get a list of all workspaces"
|
||||
[_route-params
|
||||
_query-params]
|
||||
{:items (->> (t2/select :model/Workspace :archived_at [:is nil]
|
||||
(cond-> {:order-by [[:created_at :desc]]}
|
||||
(request/limit) (sql.helpers/limit (request/limit))
|
||||
(request/offset) (sql.helpers/offset (request/offset))))
|
||||
(mapv ws->response))
|
||||
:limit (request/limit)
|
||||
:offset (request/offset)})
|
||||
|
||||
(api.macros/defendpoint :get "/:id" :- FullWorkspace
|
||||
"Get a single workspace by ID"
|
||||
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
|
||||
_query-params]
|
||||
(-> (api/check-404 (t2/select-one :model/Workspace :id id))
|
||||
(t2/hydrate :contents)
|
||||
ws->response))
|
||||
|
||||
(api.macros/defendpoint :post "/" :- Workspace
|
||||
"Create a new workspace
|
||||
|
||||
Potential payload:
|
||||
{:name \"a\" :database_id 2 :upstream {:transforms [1 2 3]}}"
|
||||
[_route-params
|
||||
_query-params
|
||||
{:keys [database_id upstream] :as body} :- CreateWorkspace]
|
||||
|
||||
(check-transforms-enabled! database_id)
|
||||
|
||||
(when-let [transform-ids (seq (get upstream :transforms []))]
|
||||
(let [workspace-transforms (t2/select-fn-set :id :model/Transform
|
||||
:id [:in transform-ids]
|
||||
:workspace_id [:not= nil])]
|
||||
(when (seq workspace-transforms)
|
||||
(api/check-400 false "Cannot add transforms that belong to another workspace")))
|
||||
(ws.common/check-no-card-dependencies! transform-ids))
|
||||
|
||||
(-> (ws.common/create-workspace! api/*current-user-id* body)
|
||||
ws->response))
|
||||
|
||||
(api.macros/defendpoint :post "/:id/archive" :- Workspace
|
||||
"Archive a workspace. Deletes the isolated schema and tables, but preserves mirrored entities."
|
||||
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
|
||||
_query-params
|
||||
_body-params]
|
||||
(let [ws (api/check-404 (t2/select-one :model/Workspace :id id))]
|
||||
(api/check-400 (nil? (:archived_at ws)) "You cannot archive an archived workspace")
|
||||
(t2/update! :model/Workspace id {:archived_at [:now]})
|
||||
(-> (t2/select-one :model/Workspace :id id)
|
||||
ws->response)))
|
||||
|
||||
(api.macros/defendpoint :post "/:id/unarchive" :- Workspace
|
||||
"Restore an archived workspace. Recreates the isolated schema and tables."
|
||||
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
|
||||
_query-params
|
||||
_body-params]
|
||||
(let [ws (api/check-404 (t2/select-one :model/Workspace :id id))]
|
||||
(api/check-400 (some? (:archived_at ws)) "You cannot unarchive a workspace that is not archived")
|
||||
(t2/update! :model/Workspace id {:archived_at nil})
|
||||
(-> (t2/select-one :model/Workspace :id id)
|
||||
ws->response)))
|
||||
|
||||
(api.macros/defendpoint :delete "/:id" :- [:map [:ok [:= true]]]
|
||||
"Delete a workspace and all its contents, including mirrored entities."
|
||||
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
|
||||
_query-params]
|
||||
(api/check-404 (t2/select-one :model/Workspace :id id))
|
||||
;; TODO (Chris 11/21/25) -- implement actual deletion logic
|
||||
{:ok true})
|
||||
|
||||
(api.macros/defendpoint :post "/:id/execute" :- ExecuteResult
|
||||
"Execute all transforms in the workspace in dependency order.
|
||||
Returns which transforms succeeded, failed, and were not run."
|
||||
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
|
||||
_query-params
|
||||
_body-params]
|
||||
(u/prog1 (t2/select-one :model/Workspace :id id)
|
||||
(api/check-404 <>)
|
||||
(api/check-400 (nil? (:archived_at <>)) "Cannot execute archived workspace"))
|
||||
;; TODO (Chris 11/21/25) -- implement execution logic
|
||||
{:succeeded []
|
||||
:failed []
|
||||
:not_run []})
|
||||
|
||||
(api.macros/defendpoint :get "/:id/graph" :- GraphResult
|
||||
"Get the dependency graph for a workspace, for visualization.
|
||||
Shows tables and transforms the workspace depends on, with edges representing dependencies.
|
||||
Tables produced by transforms in the workspace are not shown; instead, dependencies appear
|
||||
directly between transforms."
|
||||
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
|
||||
_query-params]
|
||||
(api/check-404 (t2/select-one :model/Workspace :id id))
|
||||
;; TODO (Chris 11/21/25) -- implement graph generation logic
|
||||
{:nodes []
|
||||
:edges []})
|
||||
|
||||
(api.macros/defendpoint :post "/:id/contents"
|
||||
:- [:map [:contents [:map-of ::entity-grouping [:sequential ::downstream-entity]]]]
|
||||
"Add upstream entities to workspace by mirroring them into the workspace's isolated environment.
|
||||
|
||||
The entities and their dependencies will be mirrored into the workspace.
|
||||
Returns the workspace's updated contents.
|
||||
|
||||
Removal is not implemented yet."
|
||||
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
|
||||
_query-params
|
||||
body :- ModifyEntities]
|
||||
|
||||
(api/check-400 (nil? (:remove body)) "Not implemented yet")
|
||||
|
||||
(let [workspace (api/check-404 (t2/select-one :model/Workspace :id id))
|
||||
upstream (:add body)]
|
||||
(api/check-400 (nil? (:archived_at workspace)) "Cannot add entities to an archived workspace")
|
||||
|
||||
(when-let [transform-ids (seq (get upstream :transforms []))]
|
||||
(let [workspace-transforms (t2/select-fn-set :id :model/Transform
|
||||
:id [:in transform-ids]
|
||||
:workspace_id [:not= nil])]
|
||||
(when (seq workspace-transforms)
|
||||
(api/check-400 false "Cannot add transforms that belong to another workspace")))
|
||||
(ws.common/check-no-card-dependencies! transform-ids))
|
||||
|
||||
(let [existing-upstream-ids (t2/select-fn-set :upstream_id :model/WorkspaceMappingTransform
|
||||
:workspace_id id)
|
||||
combined-upstream (update upstream :transforms #(into (vec existing-upstream-ids) %))
|
||||
graph (ws.dag/path-induced-subgraph combined-upstream)
|
||||
table-ids (seq (keep :id (concat (:inputs graph) (:outputs graph))))
|
||||
db-ids (when table-ids (t2/select-fn-set :db_id :model/Table :id [:in table-ids]))
|
||||
db-id (first db-ids)]
|
||||
(when db-id
|
||||
(api/check-400 (= db-id (:database_id workspace)) "All entities must belong to the workspace's database")))
|
||||
|
||||
(ws.common/add-entities! workspace upstream)
|
||||
|
||||
{:contents (:contents (t2/hydrate (t2/select-one :model/Workspace :id id) :contents))}))
|
||||
|
||||
(api.macros/defendpoint :post "/:id/transform"
|
||||
"Create a new transform directly within a workspace.
|
||||
|
||||
This creates a transform that exists only in the workspace's isolated schema.
|
||||
The transform is not mirrored from an existing transform, but created from scratch."
|
||||
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
|
||||
_query-params
|
||||
body :- [:map
|
||||
[:name :string]
|
||||
[:description {:optional true} [:maybe :string]]
|
||||
[:source ::transform-source]
|
||||
[:target ::ws-transform-target]
|
||||
[:run_trigger {:optional true} ::run-trigger]
|
||||
[:tag_ids {:optional true} [:sequential ms/PositiveInt]]]]
|
||||
(api/check (transforms.util/check-feature-enabled body)
|
||||
[402 (deferred-tru "Premium features required for this transform type are not enabled.")])
|
||||
|
||||
(let [workspace (u/prog1 (api/check-404 (t2/select-one :model/Workspace :id id))
|
||||
(api/check-400 (nil? (:archived_at <>)) "Cannot create transforms in an archived workspace"))
|
||||
target-name (get-in body [:target :name])
|
||||
existing-target (some #(= (:name (:target %)) target-name) (t2/select :model/Transform :workspace_id id))
|
||||
body (update body :target
|
||||
assoc :database_id (:database_id workspace) :schema (:schema workspace))]
|
||||
(api/check-400 (not existing-target)
|
||||
(deferred-tru "Another transform in this workspace already targets that table."))
|
||||
|
||||
(ws.common/create-transform! workspace body api/*current-user-id*)))
|
||||
|
||||
(api.macros/defendpoint :get "/mapping/transform/:id/downstream"
|
||||
:- [:map
|
||||
[:transforms [:sequential
|
||||
[:map
|
||||
[:id ms/PositiveInt]
|
||||
[:name :string]
|
||||
[:workspace [:map
|
||||
[:id ms/PositiveInt]
|
||||
[:name :string]]]]]]]
|
||||
"Get all downstream transforms for a transform that is not in a workspace.
|
||||
Returns the transforms that were mirrored from this upstream transform, with workspace info."
|
||||
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
|
||||
_query-params]
|
||||
(let [mappings (t2/select :model/WorkspaceMappingTransform :upstream_id id)
|
||||
_ (when (empty? mappings) (api/check-404 (t2/exists? :model/Workspace id)))
|
||||
tid->wid (u/for-map [m mappings]
|
||||
[(:downstream_id m) (:workspace_id m)])
|
||||
transform-ids (map :downstream_id mappings)
|
||||
workspace-ids (map :workspace_id mappings)
|
||||
transforms (when (seq transform-ids)
|
||||
(t2/select [:model/Transform :id :name] :id [:in transform-ids] {:order-by [:created_at]}))
|
||||
workspaces-by-id (when (seq workspace-ids)
|
||||
(u/index-by :id (t2/select [:model/Workspace :id :name] :id [:in workspace-ids])))]
|
||||
{:transforms (for [transform transforms]
|
||||
(assoc transform :workspace (get workspaces-by-id (tid->wid (:id transform)))))}))
|
||||
|
||||
(api.macros/defendpoint :post "/:id/merge"
|
||||
#_#_:- [:or
|
||||
[:map
|
||||
[:promoted [:sequential [:map [:id ms/PositiveInt] [:name :string]]]]
|
||||
[:errors {:optional true} [:sequential [:map [:id ms/PositiveInt] [:name :string] [:error :string]]]]
|
||||
[:workspace [:map [:id ms/PositiveInt] [:name :string]]]
|
||||
[:archived_at [:maybe :any]]]
|
||||
;; error message from check-404 or check-400
|
||||
:string]
|
||||
"Promote workspace transforms back to main Metabase and archive the workspace.
|
||||
|
||||
This will:
|
||||
1. Update original transforms with workspace versions
|
||||
2. Re-execute transforms in the original schema
|
||||
3. Archive the workspace and clean up isolated resources
|
||||
|
||||
Returns a report of promoted transforms and any errors."
|
||||
[{:keys [id]} :- [:map [:id ms/PositiveInt]]
|
||||
_query-params
|
||||
_body-params]
|
||||
(let [ws (u/prog1 (t2/select-one :model/Workspace :id id)
|
||||
(api/check-404 <>)
|
||||
(api/check-400 (nil? (:archived_at <>)) "Cannot promote an already archived workspace"))
|
||||
{:keys [promoted
|
||||
errors]} (ws.promotion/promote-transforms! ws)]
|
||||
(u/prog1
|
||||
{:promoted (vec promoted)
|
||||
:errors errors
|
||||
:workspace {:id id :name (:name ws)}
|
||||
:archived_at (when-not (seq errors)
|
||||
(t2/update! :model/Workspace :id id {:archived_at [:now]})
|
||||
(t2/select-one-fn :archived_at [:model/Workspace :archived_at] :id id))}
|
||||
(when-not (seq errors)
|
||||
;; Most of the APIs and the FE are not respecting when a Workspace is archived yet.
|
||||
(t2/delete! :model/Workspace id)))))
|
||||
|
||||
(def ^{:arglists '([request respond raise])} routes
|
||||
"`/api/ee/workspace/` routes."
|
||||
(api.macros/ns-handler *ns* api/+check-superuser +auth))
|
||||
241
enterprise/backend/src/metabase_enterprise/workspaces/common.clj
Normal file
241
enterprise/backend/src/metabase_enterprise/workspaces/common.clj
Normal file
@@ -0,0 +1,241 @@
|
||||
(ns metabase-enterprise.workspaces.common
|
||||
(:require
|
||||
[clojure.string :as str]
|
||||
[metabase-enterprise.transforms.core :as transforms]
|
||||
[metabase-enterprise.workspaces.dag :as ws.dag]
|
||||
[metabase-enterprise.workspaces.isolation :as ws.isolation]
|
||||
[metabase-enterprise.workspaces.mirroring :as ws.mirroring]
|
||||
[metabase.api-keys.core :as api-key]
|
||||
[metabase.api.common :as api]
|
||||
[metabase.events.core :as events]
|
||||
[metabase.util :as u]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(defn check-no-card-dependencies!
|
||||
"Check that transforms don't depend on cards. Throws 400 if they do."
|
||||
[transform-ids]
|
||||
(when-let [card-ids (seq (ws.dag/card-dependencies transform-ids))]
|
||||
(api/check-400 false
|
||||
(format "Cannot add transforms that depend on saved questions (cards). Found dependencies on card IDs: %s"
|
||||
(pr-str (vec card-ids))))))
|
||||
|
||||
(defn- extract-suffix-number
|
||||
"Extract the numeric suffix from a workspace name like 'Foo (3)', or nil if no valid suffix."
|
||||
[ws-name base-name]
|
||||
(when (= (str/replace ws-name #"\s*\(\d+\)$" "") base-name)
|
||||
(when-let [[_ n] (re-find #"\((\d+)\)$" ws-name)]
|
||||
(parse-long n))))
|
||||
|
||||
(defn- generate-unique-workspace-name
|
||||
"Generate a unique workspace name by appending (N) if the name already exists.
|
||||
|
||||
Strategy:
|
||||
1. If the base name doesn't exist, use it as-is
|
||||
2. If it exists, strip any existing (N) suffix and find the highest N among similar names
|
||||
3. Return the base name with (N+1) appended
|
||||
|
||||
Note: This function has a potential race condition between checking existence and insertion.
|
||||
Callers should handle unique constraint violations by retrying."
|
||||
[base-name]
|
||||
(when (str/blank? base-name)
|
||||
(throw (ex-info "Workspace name cannot be empty" {:status-code 400})))
|
||||
(if-not (t2/exists? :model/Workspace :name base-name)
|
||||
base-name
|
||||
(let [stripped-name (str/replace base-name #"\s*\(\d+\)$" "")
|
||||
existing (t2/select-fn-set :name :model/Workspace
|
||||
:name [:like (str stripped-name " (%)")])
|
||||
numbers (keep #(extract-suffix-number % stripped-name) existing)
|
||||
next-num (inc (apply max 0 numbers))]
|
||||
(str stripped-name " (" next-num ")"))))
|
||||
|
||||
;; TODO (Chris 2025-11-20) Just subsume the rest of this up into ws.dag/path-induced-subgraph
|
||||
(defn- build-graph
|
||||
"Thin wrapper around the dag module, that should probably be absorbed by it."
|
||||
[upstream]
|
||||
(if (not-any? seq (vals upstream))
|
||||
{:db_id nil
|
||||
:transforms []
|
||||
:inputs []
|
||||
:outputs []}
|
||||
(let [graph (ws.dag/path-induced-subgraph upstream)
|
||||
db-ids (when-let [table-ids (seq (keep :id (concat (:inputs graph) (:outputs graph))))]
|
||||
(t2/select-fn-set :db_id :model/Table :id [:in table-ids]))
|
||||
_ (assert (<= (count db-ids) 1) "All inputs and outputs must belong to the same database.")]
|
||||
;; One reason this is here, is that I don't want the DAG module to have the single-DWH assumption.
|
||||
(assoc graph :db_id (first db-ids)))))
|
||||
|
||||
;; TODO: Generate new metabase user for the workspace
|
||||
;; TODO: Should we move this to model as per the diagram?
|
||||
(defn- create-workspace-container!
|
||||
"Create the workspace and its related collection, user, and api key."
|
||||
[creator-id database-id workspace-name]
|
||||
;; TODO (Chris 2025-11-19) Unsure API key name is unique, and remove this (insecure) workaround.
|
||||
(let [api-key (let [key-name (format "API key for Workspace %s" workspace-name)]
|
||||
(or (t2/select-one :model/ApiKey :name key-name)
|
||||
(api-key/create-api-key-with-new-user! {:key-name key-name})))
|
||||
;; TODO (Chris 2025-11-19) Associate the api-key user with the workspace as well.
|
||||
ws (t2/insert-returning-instance! :model/Workspace
|
||||
{:name workspace-name
|
||||
:creator_id creator-id
|
||||
:database_id database-id
|
||||
:api_key_id (:id api-key)
|
||||
:execution_user (:user_id api-key)})
|
||||
coll (t2/insert-returning-instance! :model/Collection
|
||||
{:name (format "Collection for Workspace %s" workspace-name)
|
||||
:namespace "workspace"
|
||||
:workspace_id (:id ws)})
|
||||
schema (ws.isolation/isolation-schema-name (:id ws))
|
||||
ws (assoc ws
|
||||
:collection_id (:id coll)
|
||||
:schema schema)]
|
||||
;; Set the backlink from the workspace to the collection inside it and set the schema.
|
||||
(t2/update! :model/Workspace (:id ws) {:collection_id (:id coll)
|
||||
:schema schema})
|
||||
;; TODO (Sanya 2025-11-18) - not sure how to transfer this api key to agent
|
||||
#_(log/infof "Generated API key for workspace: %s" (u.secret/expose (:unmasked_key api-key)))
|
||||
ws))
|
||||
|
||||
(defn- unique-constraint-violation?
|
||||
"Check if an exception is due to a unique constraint violation."
|
||||
[e]
|
||||
(let [msg (str (ex-message e) (some-> (ex-cause e) ex-message))]
|
||||
(or (str/includes? msg "unique")
|
||||
(str/includes? msg "duplicate")
|
||||
(str/includes? msg "UNIQUE"))))
|
||||
|
||||
;; TODO (Chris 2025-11-20) We have not added a uniqueness constraint to the db, we should either do that, or remove
|
||||
;; the whole retry song and dance.
|
||||
(defn- create-workspace-with-unique-name!
|
||||
"Create a workspace, retrying with a new unique name if there's a constraint violation."
|
||||
[creator-id db-id database ws-name graph max-retries]
|
||||
(loop [attempt 1]
|
||||
(let [unique-name (generate-unique-workspace-name ws-name)
|
||||
result (try
|
||||
{:workspace (create-workspace-container! creator-id db-id unique-name)}
|
||||
(catch Exception e
|
||||
(if (and (< attempt max-retries) (unique-constraint-violation? e))
|
||||
{:retry true}
|
||||
(throw e))))]
|
||||
(if (:retry result)
|
||||
(recur (inc attempt))
|
||||
(let [workspace (:workspace result)]
|
||||
(ws.isolation/ensure-database-isolation! workspace database)
|
||||
;; temp hack to avoid running mirror and failing if there are no checkouts
|
||||
(if (not-empty (:check-outs graph))
|
||||
(let [graph (ws.mirroring/mirror-entities! workspace database graph)]
|
||||
(t2/update! :model/Workspace {:id (:id workspace)} {:graph graph})
|
||||
(assoc workspace :graph graph))
|
||||
(assoc workspace :graph {})))))))
|
||||
|
||||
;; TODO internal: test!
|
||||
(defn create-workspace!
|
||||
"Create workspace"
|
||||
[creator-id {ws-name-maybe :name
|
||||
maybe-db-id :database_id
|
||||
upstream :upstream}]
|
||||
;; TODO put this in the malli schema for a request
|
||||
(assert (or maybe-db-id (some seq (vals upstream))) "Must provide a database_id unless initial entities are given.")
|
||||
(let [ws-name (or ws-name-maybe (str (random-uuid)))
|
||||
graph (build-graph upstream)
|
||||
db-id (or (:db_id graph) maybe-db-id)
|
||||
_ (when (and maybe-db-id (:db_id graph))
|
||||
(assert (= maybe-db-id (:db_id graph))
|
||||
"The database_id provided must match that of the upstream entities."))
|
||||
_ (assert db-id "Was not given and could not infer a database_id for the workspace.")
|
||||
database (api/check-500 (t2/select-one :model/Database :id db-id))]
|
||||
(create-workspace-with-unique-name! creator-id db-id database ws-name graph 5)))
|
||||
|
||||
(defn- rebuild-workspace-graph!
|
||||
"Rebuild the workspace graph from scratch based on all upstream transforms.
|
||||
Returns the new graph."
|
||||
[workspace-id]
|
||||
(let [all-upstream-ids (t2/select-fn-vec :upstream_id :model/WorkspaceMappingTransform
|
||||
:workspace_id workspace-id)
|
||||
graph (build-graph {:transforms all-upstream-ids})
|
||||
;; Add from-scratch transforms (those created directly in workspace, not mirrored)
|
||||
;; These have workspace_id set but no entry in WorkspaceMappingTransform
|
||||
mirrored-ids (t2/select-fn-set :downstream_id :model/WorkspaceMappingTransform
|
||||
:workspace_id workspace-id)
|
||||
from-scratch-txs (->> (t2/select [:model/Transform :id] :workspace_id workspace-id
|
||||
{:where (if (seq mirrored-ids)
|
||||
[:not [:in :id mirrored-ids]]
|
||||
true)})
|
||||
(mapv #(assoc % :type :transform)))
|
||||
graph (update graph :transforms into from-scratch-txs)]
|
||||
(t2/update! :model/Workspace workspace-id {:graph graph})
|
||||
graph))
|
||||
|
||||
(defn add-entities!
|
||||
"Add upstream entities to an existing workspace by mirroring them.
|
||||
Returns the workspace with updated graph."
|
||||
[workspace upstream]
|
||||
(let [new-graph (ws.dag/path-induced-subgraph upstream)
|
||||
database (t2/select-one :model/Database (:database_id workspace))]
|
||||
(t2/with-transaction [_]
|
||||
(ws.mirroring/mirror-entities! workspace database new-graph)
|
||||
(let [graph (rebuild-workspace-graph! (:id workspace))]
|
||||
(assoc workspace :graph graph)))))
|
||||
|
||||
(defn create-transform!
|
||||
"Create a new transform directly within a workspace.
|
||||
Returns the created transform with hydrated tag_ids and creator."
|
||||
[workspace body creator-id]
|
||||
(u/prog1
|
||||
(t2/with-transaction [_]
|
||||
(let [workspace-id (:id workspace)
|
||||
workspace-db-id (:database_id workspace)
|
||||
database (t2/select-one :model/Database workspace-db-id)
|
||||
tag-ids (:tag_ids body)
|
||||
body (assoc-in body [:target :database] workspace-db-id)
|
||||
transform (t2/insert-returning-instance!
|
||||
:model/Transform
|
||||
(assoc (select-keys body [:name :description :source :target :run_trigger])
|
||||
:creator_id creator-id
|
||||
:workspace_id workspace-id))]
|
||||
(when (seq tag-ids)
|
||||
(transforms/update-transform-tags! (:id transform) tag-ids))
|
||||
|
||||
;; Create isolated output table for the new transform
|
||||
(let [target (:target transform)
|
||||
new-graph {:db_id workspace-db-id
|
||||
:transforms [transform]
|
||||
:inputs []
|
||||
:outputs [{:id nil
|
||||
:schema (:schema target)
|
||||
:name (:name target)}]
|
||||
:check-outs []}]
|
||||
(ws.isolation/create-isolated-output-tables! workspace database new-graph))
|
||||
|
||||
;; Rebuild the full graph from scratch
|
||||
(rebuild-workspace-graph! workspace-id)
|
||||
|
||||
(t2/hydrate transform :transform_tag_ids :creator)))
|
||||
(events/publish-event! :event/transform-create {:object <> :user-id creator-id})))
|
||||
|
||||
#_:clj-kondo/ignore
|
||||
(comment
|
||||
(defn- clean-up-ws!* [& [ws-id]]
|
||||
;; Pass nil to clean up all workspaces
|
||||
(let [ws-clause (or ws-id [:not= nil])]
|
||||
(t2/delete! :model/Collection :workspace_id ws-clause)
|
||||
(t2/delete! :model/Transform :workspace_id ws-clause)
|
||||
(doseq [[db_id schema] (t2/select-fn-set (juxt :db_id :schema) :model/Table :workspace_id [:not= nil])]
|
||||
(t2/delete! :model/Table :schema schema)
|
||||
(let [db (t2/select-one :model/Database db_id)
|
||||
driver (metabase.driver.util/database->driver db)
|
||||
make-spec (requiring-resolve 'metabase.driver.sql-jdbc.connection/connection-details->spec)
|
||||
jdbc-spec (make-spec driver (:details db))]
|
||||
(clojure.java.jdbc/execute! jdbc-spec [(str "DROP SCHEMA \"" schema "\" CASCADE")])))
|
||||
(t2/delete! :model/Workspace :id ws-clause)))
|
||||
|
||||
(def upstream-id (t2/select-one-pk :model/Transform :workspace_id nil {:order-by [:id]}))
|
||||
(def upstream-ids (t2/select-pks-vec :model/Transform :workspace_id nil {:order-by [:id]}))
|
||||
|
||||
;; Ensure output table exists
|
||||
(#'metabase-enterprise.transforms.execute/run-mbql-transform! (t2/select-one :model/Transform upstream-id))
|
||||
|
||||
(let [admin-id (t2/select-one-pk :model/User :is_superuser true {:order-by [:id]})]
|
||||
(binding [api/*current-user-id* admin-id]
|
||||
(create-workspace! admin-id {:name "Workplace Workspace", :upstream {:transforms upstream-ids #_[upstream-id]}})))
|
||||
|
||||
(clean-up-ws!*))
|
||||
@@ -0,0 +1,93 @@
|
||||
(ns metabase-enterprise.workspaces.context
|
||||
(:require
|
||||
[metabase.util.i18n :refer [tru]]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
;; TODO (lbrdnk 2025-11-26): Following should have its own module
|
||||
;;;; Fetching the graph data
|
||||
|
||||
(defn- fetch-graph-transforms
|
||||
[transform-ids]
|
||||
(if (empty? transform-ids)
|
||||
{}
|
||||
(let [id->transform (t2/select-fn->fn :id identity :model/Transform :id [:in transform-ids])]
|
||||
(when (not= (count transform-ids) (count id->transform))
|
||||
(throw (ex-info (tru "Unable to fetch all transforms within the graph")
|
||||
{:status-code 500
|
||||
:transform-ids transform-ids
|
||||
:missing-transform-ids (remove id->transform transform-ids)})))
|
||||
id->transform)))
|
||||
|
||||
(defn- fetch-graph-tables
|
||||
[table-ids]
|
||||
(when (seq table-ids)
|
||||
(let [id->table (into {}
|
||||
(t2/select-fn->fn :id identity [:model/Table :id :schema :name]
|
||||
:id [:in table-ids]))]
|
||||
(assert (= (count table-ids) (count id->table)))
|
||||
id->table)))
|
||||
|
||||
(defn- fetch-graph-resources
|
||||
[graph]
|
||||
;; TODO: keep -> map when we have guards ensuring outputs have id?
|
||||
;; figure out the details
|
||||
{:id->output-table (fetch-graph-tables (keep :id (:outputs graph)))
|
||||
:id->input-table (fetch-graph-tables (keep :id (:inputs graph)))
|
||||
:id->transform (fetch-graph-transforms (map :id (:transforms graph)))})
|
||||
|
||||
;;;; Public
|
||||
|
||||
(defn ->context-with-resources
|
||||
"TODO (lbrdnk): Add docstring."
|
||||
[graph]
|
||||
{:data (fetch-graph-resources graph)
|
||||
:graph graph})
|
||||
|
||||
;;;; Misc
|
||||
|
||||
(defn transform-node->data
|
||||
"TODO (lbrdnk): Add docstring."
|
||||
[ctx transform-node]
|
||||
;; TODO: :var
|
||||
(get-in ctx [:data :id->transform (:id transform-node)]))
|
||||
|
||||
(defn direct-dependencies
|
||||
"Get direct dependencies of a node"
|
||||
[ctx node]
|
||||
(get-in ctx [:graph :dependencies node]))
|
||||
|
||||
(defn has-dependencies?
|
||||
"TODO (lbrdnk): Add docstring."
|
||||
[ctx node]
|
||||
(boolean (seq (direct-dependencies ctx node))))
|
||||
|
||||
(defn- transform-node->remapped-target-rf
|
||||
[ctx acc transform-dep-node]
|
||||
(let [transform-data (transform-node->data ctx transform-dep-node)
|
||||
{:keys [schema name]} (:target transform-data)
|
||||
k [schema name]
|
||||
v (get-in ctx [:src-schema+table->dst->schema+table k])]
|
||||
(assoc acc k v)))
|
||||
|
||||
(defn transform-node->src->dst-remapping
|
||||
"TODO (lbrdnk): Add docstring."
|
||||
[ctx transform-node]
|
||||
(let [transform-deps-nodes (direct-dependencies ctx transform-node)]
|
||||
(assert (every? (comp #{:transform} :type) transform-deps-nodes))
|
||||
(reduce
|
||||
(partial transform-node->remapped-target-rf ctx)
|
||||
{}
|
||||
transform-deps-nodes)))
|
||||
|
||||
;; for when :dependencies hold table-nodes, now we go with transform nodes
|
||||
#_(defn schema+table->schema+table-outputs-remapping
|
||||
[ctx transform-node]
|
||||
(not-empty
|
||||
(let [deps (direct-dependencies ctx transform-node)
|
||||
deps-ids (map :id deps)
|
||||
src-output-id->dst-output (:src-output-id->dst-output ctx)
|
||||
src-id->schema+table (-> (get-in ctx [:data :id->output-table])
|
||||
(update-vals (juxt :schema :name)))]
|
||||
(-> (select-keys src-output-id->dst-output deps-ids)
|
||||
(update-keys src-id->schema+table)
|
||||
(update-vals (juxt :schema :name))))))
|
||||
262
enterprise/backend/src/metabase_enterprise/workspaces/dag.clj
Normal file
262
enterprise/backend/src/metabase_enterprise/workspaces/dag.clj
Normal file
@@ -0,0 +1,262 @@
|
||||
(ns metabase-enterprise.workspaces.dag
|
||||
(:require
|
||||
[clojure.set :as set]
|
||||
[flatland.ordered.map :as ordered-map]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
;;;; Internal helpers
|
||||
|
||||
(def ^:private group->db-type
|
||||
"Mapping from our entity type grouping keywords to the dependency table's type strings."
|
||||
{:transforms "transform"
|
||||
#_#_:models "card"})
|
||||
|
||||
(defn- rows->entity-set
|
||||
"Convert query result rows to a set of {:id :type} maps."
|
||||
[rows]
|
||||
(into #{}
|
||||
(map (fn [{:keys [entity_type entity_id]}]
|
||||
{:id entity_id :type (keyword entity_type)}))
|
||||
rows))
|
||||
|
||||
(defn- entities-map->tuples
|
||||
"Convert a map of {type ids} to a sequence of [db-type id] tuples.
|
||||
`entities-by-type` is a map like {:transform [1 2 3]}."
|
||||
[entities-by-type]
|
||||
(assert (every? group->db-type (keys entities-by-type)) "Not all entity types are supported")
|
||||
(into []
|
||||
(mapcat (fn [[entity-type ids]]
|
||||
(let [dep-type (group->db-type entity-type)]
|
||||
(map (fn [id] [dep-type id]) ids))))
|
||||
entities-by-type))
|
||||
|
||||
(defn- tuples->entity-set
|
||||
"Convert a sequence of [type id] tuples to a set of {:id :type} maps."
|
||||
[tuples]
|
||||
(into #{}
|
||||
(map (fn [[entity-type entity-id]]
|
||||
{:id entity-id :type (keyword entity-type)}))
|
||||
tuples))
|
||||
|
||||
(defn- tuples->starting-cte
|
||||
"Build a UNION ALL of SELECT statements for the starting tuples."
|
||||
[tuples]
|
||||
{:union-all (mapv (fn [[t id]]
|
||||
{:select [[[:inline t] :entity_type]
|
||||
[[:inline id] :entity_id]]})
|
||||
tuples)})
|
||||
|
||||
;;;; Toposort
|
||||
|
||||
(defn- toposort-visit [node child->parents visited result]
|
||||
(cond
|
||||
(visited node) [visited result]
|
||||
:else (let [parents (child->parents node [])
|
||||
[visited' result'] (reduce (fn [[v r] p]
|
||||
(toposort-visit p child->parents v r))
|
||||
[(conj visited node) result]
|
||||
parents)]
|
||||
[visited' (conj result' node)])))
|
||||
|
||||
(defn- toposort-dfs [child->parents]
|
||||
;; TODO (Chris 2025-11-20): Detect cycles and throw an error. (In practice inputs will never be cyclic, but still.)
|
||||
(let [all-nodes (set (keys child->parents))]
|
||||
(loop [visited #{}
|
||||
result []
|
||||
remaining all-nodes]
|
||||
(if (empty? remaining)
|
||||
result
|
||||
(let [node (first remaining)
|
||||
[visited' result'] (toposort-visit node child->parents visited result)]
|
||||
(recur visited' result' (disj remaining node)))))))
|
||||
|
||||
;;;; Transitive closure queries
|
||||
|
||||
(defn- transitive-closure
|
||||
"Compute transitive closure in the given direction.
|
||||
`direction` is :upstream (follow dependencies) or :downstream (follow dependents)."
|
||||
[entities-by-type direction]
|
||||
(let [starting-tuples (entities-map->tuples entities-by-type)
|
||||
[cte-name match-type match-id result-type result-id]
|
||||
(case direction
|
||||
:upstream [:upstream, :d.from_entity_type, :d.from_entity_id, :d.to_entity_type, :d.to_entity_id]
|
||||
:downstream [:downstream, :d.to_entity_type, :d.to_entity_id, :d.from_entity_type, :d.from_entity_id])]
|
||||
(when (seq starting-tuples)
|
||||
(rows->entity-set
|
||||
(t2/query {:with-recursive
|
||||
[[:starting (tuples->starting-cte starting-tuples)]
|
||||
[cte-name {:union-all
|
||||
[{:select [:entity_type :entity_id]
|
||||
:from [:starting]}
|
||||
{:select [[result-type :entity_type]
|
||||
[result-id :entity_id]]
|
||||
:from [[:dependency :d]]
|
||||
:join [[cte-name :r]
|
||||
[:and
|
||||
[:= match-type :r.entity_type]
|
||||
[:= match-id :r.entity_id]]]}]}]]
|
||||
:select [:entity_type :entity_id]
|
||||
:from [cte-name]})))))
|
||||
|
||||
(defn- upstream-entities
|
||||
"Given a map of entity types to IDs, find all upstream entities (dependencies).
|
||||
Returns a set of {:id :type} maps."
|
||||
[entities-by-type]
|
||||
(transitive-closure entities-by-type :upstream))
|
||||
|
||||
(defn- downstream-entities
|
||||
"Given a map of entity types to IDs, find all downstream entities (dependents).
|
||||
Returns a set of {:id :type} maps."
|
||||
[entities-by-type]
|
||||
(transitive-closure entities-by-type :downstream))
|
||||
|
||||
;;;; Dependency fetching
|
||||
|
||||
(defn- entity-dependencies
|
||||
"Fetch all dependencies between the given entities within the dependency table.
|
||||
Returns a map from {:id :type} to a vector of {:id :type} dependencies."
|
||||
[entities]
|
||||
(if (empty? entities)
|
||||
{}
|
||||
(let [conditions (mapv (fn [{:keys [id type]}]
|
||||
[:and
|
||||
[:= :from_entity_type (name type)]
|
||||
[:= :from_entity_id id]])
|
||||
entities)
|
||||
rows (t2/query {:select [:from_entity_type :from_entity_id
|
||||
:to_entity_type :to_entity_id]
|
||||
:from [:dependency]
|
||||
:where (into [:or] conditions)})]
|
||||
(reduce (fn [acc {:keys [from_entity_type from_entity_id to_entity_type to_entity_id]}]
|
||||
(let [from-entity {:id from_entity_id, :type (keyword from_entity_type)}
|
||||
to-entity {:id to_entity_id, :type (keyword to_entity_type)}]
|
||||
(if (contains? entities to-entity)
|
||||
(update acc from-entity (fnil conj []) to-entity)
|
||||
acc)))
|
||||
{}
|
||||
rows))))
|
||||
|
||||
(defn- toposort-entities
|
||||
"Toposort entities based on their dependencies.
|
||||
Returns entities in topological order (dependencies before dependents)."
|
||||
[entity-set dependencies-map]
|
||||
(let [child->parents (into {}
|
||||
(map (fn [entity]
|
||||
[entity (vec (get dependencies-map entity []))]))
|
||||
entity-set)]
|
||||
(toposort-dfs child->parents)))
|
||||
|
||||
(defn- table? [entity] (= :table (:type entity)))
|
||||
|
||||
(defn- transform? [entity] (= :transform (:type entity)))
|
||||
|
||||
(defn- fetch-dependent-tables
|
||||
"Fetch tables that are the output targets of the given transforms.
|
||||
Returns tables with their IDs if they exist in the database, or with :id nil if they don't exist yet.
|
||||
This supports workspace checkout for transforms that haven't been executed yet."
|
||||
[entities]
|
||||
(when (seq entities)
|
||||
(let [transform-ids (map :id (filter transform? entities))
|
||||
transforms (when (seq transform-ids)
|
||||
(t2/select [:model/Transform :source :target] :id [:in transform-ids]))]
|
||||
(vec
|
||||
(for [{:keys [target]} transforms, :when target]
|
||||
(case (:type target)
|
||||
"table"
|
||||
;; Note: id will be nil if the table has not been created yet
|
||||
{:id (t2/select-one-pk :model/Table
|
||||
:db_id (:database target)
|
||||
:schema (:schema target)
|
||||
:name (:name target))
|
||||
:schema (:schema target)
|
||||
:name (:name target)
|
||||
:type :table}
|
||||
(throw (ex-info "Unsupported target type" {:target target}))))))))
|
||||
|
||||
(defn- toposort-key-fn [ordering]
|
||||
(let [index-map (zipmap ordering (range))
|
||||
index-of #(get index-map % Integer/MAX_VALUE)]
|
||||
(fn [e]
|
||||
[(index-of e) (:type e) (:id e)])))
|
||||
|
||||
;;;; Card dependency detection
|
||||
|
||||
;; TODO this may be overkill:
|
||||
;; 1. We currently care only that there is at-least-one card being depended on.
|
||||
;; 2. We know for transforms that we can only have any card dependencies if there is a first-order card dependency.
|
||||
;;
|
||||
;; ie. we could just do an existence check on direct ancestors.
|
||||
;;
|
||||
;; If we only disallow MBQL cards however, we would still need to walk transitive dependencies in
|
||||
;; case we hit one further up. We could still short-circuit however.
|
||||
;;
|
||||
(defn card-dependencies
|
||||
"Find all card IDs that the given transforms transitively depend on.
|
||||
Queries the dependency table for upstream cards.
|
||||
Returns a set of card IDs, or empty set if no card dependencies."
|
||||
[transform-ids]
|
||||
(when (seq transform-ids)
|
||||
(let [starting-tuples (mapv (fn [id] ["transform" id]) transform-ids)
|
||||
rows (t2/query {:with-recursive
|
||||
[[:starting (tuples->starting-cte starting-tuples)]
|
||||
[:upstream {:union-all
|
||||
[{:select [:entity_type :entity_id]
|
||||
:from [:starting]}
|
||||
{:select [[:d.to_entity_type :entity_type]
|
||||
[:d.to_entity_id :entity_id]]
|
||||
:from [[:dependency :d]]
|
||||
:join [[:upstream :r]
|
||||
[:and
|
||||
[:= :d.from_entity_type :r.entity_type]
|
||||
[:= :d.from_entity_id :r.entity_id]]]}]}]]
|
||||
:select-distinct [:entity_id]
|
||||
:from [:upstream]
|
||||
:where [:= :entity_type "card"]})]
|
||||
(into #{} (map :entity_id) rows))))
|
||||
|
||||
;;;; Public API
|
||||
|
||||
(defn path-induced-subgraph
|
||||
"Given a map of entity types to IDs, compute the path-induced subgraph.
|
||||
`entities-by-type` is a map like {:transform [1 2 3]}.
|
||||
|
||||
Returns a map with:
|
||||
- :check-outs - the entities we want to make editable within the workspace, in topological order
|
||||
- :inputs - external tables that entities in the subgraph depend on, ordered by id
|
||||
- :outputs - output tables generated by entities in the subgraph, in topological order
|
||||
- :transforms - transforms in the subgraph, in topological order
|
||||
- :dependencies - map from each subgraph entity to its dependencies within subgraph, in topological order"
|
||||
[entities-by-type]
|
||||
(let [starting-set (tuples->entity-set (entities-map->tuples entities-by-type))
|
||||
upstream (or (upstream-entities entities-by-type) #{})
|
||||
downstream (or (downstream-entities entities-by-type) #{})
|
||||
subgraph (set/intersection upstream downstream)
|
||||
transforms (filterv transform? subgraph)
|
||||
output-tables (or (fetch-dependent-tables transforms) [])
|
||||
inputs (->> upstream
|
||||
(filter table?)
|
||||
(remove subgraph)
|
||||
(sort-by :id)
|
||||
vec)
|
||||
deps-map (entity-dependencies subgraph)
|
||||
sorted (toposort-entities subgraph deps-map)
|
||||
;; Right now, this doesn't quite yield the comparator we want, since we have a partial ordering, and the
|
||||
;; toposort is rather arbitrary in this regard. It also excludes the upstream and downstream dependencies.
|
||||
;; What we'd prefer is to have the following:
|
||||
;; 1. Dependencies come before their dependents.
|
||||
;; 2. For independent nodes, order by type (tables before transforms), then by ID.
|
||||
sort-key-fn (toposort-key-fn sorted)]
|
||||
{:check-outs (vec (sort-by sort-key-fn starting-set))
|
||||
;; By definition, these are all tables, and all external to the subgraph, so ID is the only thing to sort by.
|
||||
;; Ideally, they'd be ordered by their own dependency relationships too, but for now this is good enough.
|
||||
:inputs (vec (sort-by :id inputs))
|
||||
:outputs (vec (sort-by sort-key-fn output-tables))
|
||||
:transforms (vec (sort-by sort-key-fn transforms))
|
||||
:dependencies (into (ordered-map/ordered-map)
|
||||
(keep (fn [entity]
|
||||
(when-not (table? entity)
|
||||
[entity (vec (map #(if (table? %)
|
||||
(first (get deps-map % []))
|
||||
%)
|
||||
(get deps-map entity [])))])))
|
||||
sorted)}))
|
||||
@@ -0,0 +1,4 @@
|
||||
(ns metabase-enterprise.workspaces.init
|
||||
(:require
|
||||
[metabase-enterprise.workspaces.models.workspace]
|
||||
[metabase-enterprise.workspaces.models.workspace-mapping]))
|
||||
@@ -0,0 +1,143 @@
|
||||
(ns metabase-enterprise.workspaces.isolation
|
||||
(:require
|
||||
;; TODO: should we go with next.jdbc instead?
|
||||
[clojure.java.jdbc :as jdbc]
|
||||
[clojure.string :as str]
|
||||
[metabase-enterprise.workspaces.sync :as ws.sync]
|
||||
[metabase.driver :as driver]
|
||||
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
|
||||
[metabase.driver.util :as driver.u]
|
||||
[metabase.system.core :as system]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
;;;; Naming
|
||||
|
||||
;; re-using https://github.com/metabase/metabase/pull/61887/commits/c92e4a9cc451c61a13fef19ed9d6107873b17f07
|
||||
;; (original ws isolation code)
|
||||
(defn- instance-uuid-slug
|
||||
"Create a slug from the site UUID, taking the first character of each section."
|
||||
[site-uuid-string]
|
||||
(apply str (map first (str/split site-uuid-string #"-"))))
|
||||
|
||||
(defn isolation-schema-name
|
||||
"Generate schema/database name for workspace isolation following mb__isolation_<slug>_<workspace-id> pattern."
|
||||
[workspace-id]
|
||||
(let [instance-slug (instance-uuid-slug (str (system/site-uuid)))
|
||||
clean-workspace-id (str/replace (str workspace-id) #"[^a-zA-Z0-9]" "_")]
|
||||
(format "mb__isolation_%s_%s" instance-slug clean-workspace-id)))
|
||||
|
||||
(defn isolated-table-name
|
||||
"Generate name for a table mirroring transform target table in the isolated database namespace."
|
||||
[{:keys [schema name] :as _source-table}]
|
||||
;; the schema that original transform target lives in
|
||||
(format "%s__%s" schema name))
|
||||
|
||||
;;;; Dispatch for database/driver/... multimethods
|
||||
|
||||
;; I'm making this to have a var so var can be passed as dispatch to `duplicate-transform-table!` during
|
||||
;; development as this is subject to change.
|
||||
(defn dispatch-on-engine
|
||||
"Take engine from database `db` and dispatch on that."
|
||||
[database & _args]
|
||||
(driver.u/database->driver database))
|
||||
|
||||
;;;; Isolation init
|
||||
|
||||
(defmulti init-workspace-database-isolation!
|
||||
"For now create only the new schema"
|
||||
{:added "0.59.0" :arglists '([database workspace])}
|
||||
#'dispatch-on-engine
|
||||
:hierarchy #'driver/hierarchy)
|
||||
|
||||
(defmethod init-workspace-database-isolation! :postgres [database workspace]
|
||||
(let [driver (driver.u/database->driver database)
|
||||
schema-name (isolation-schema-name (:id workspace))
|
||||
jdbc-spec (sql-jdbc.conn/connection-details->spec driver (:details database))]
|
||||
(jdbc/execute! jdbc-spec [(format "CREATE SCHEMA %s" schema-name)])))
|
||||
|
||||
;;;; Transform table duplication
|
||||
|
||||
(defmulti duplicate-output-table!
|
||||
"Create an isolated copy of the given output tables, for a workspace transform to write to."
|
||||
{:added "0.59.0" :arglists '([database transform])}
|
||||
#'dispatch-on-engine
|
||||
:hierarchy #'driver/hierarchy)
|
||||
|
||||
(defmethod duplicate-output-table! :postgres [database workspace output]
|
||||
(let [details (:details database)
|
||||
driver* (driver.u/database->driver database)
|
||||
jdbc-spec (sql-jdbc.conn/connection-details->spec driver* details)
|
||||
source-schema (:schema output)
|
||||
source-table (:name output)
|
||||
isolated-schema (isolation-schema-name (:id workspace))
|
||||
isolated-table (isolated-table-name output)]
|
||||
(assert (every? some? [source-schema source-table isolated-schema isolated-table]) "Figured out table")
|
||||
;; TODO: execute the following only if the transform was previously executed and its table exists.
|
||||
(jdbc/execute! jdbc-spec [(format (str "CREATE TABLE \"%s\".\"%s\""
|
||||
" AS SELECT * FROM \"%s\".\"%s\""
|
||||
"WITH NO DATA")
|
||||
isolated-schema
|
||||
isolated-table
|
||||
source-schema
|
||||
source-table)])
|
||||
(let [table-metadata (ws.sync/sync-transform-mirror! database isolated-schema isolated-table)]
|
||||
(select-keys table-metadata [:id :schema :name]))))
|
||||
|
||||
;;;; To be public when things are settled
|
||||
|
||||
;; TODO: Should be part of mirroring!
|
||||
(defn create-isolated-output-tables!
|
||||
"Create new _isolated tables_ to correspond to the outputs of the upstream graph.
|
||||
Decorate the graph outputs with the mapping to the new tables.
|
||||
For outputs where the table doesn't exist yet (id=nil), includes them in the graph with the intended
|
||||
isolated table location, but skips actual table duplication."
|
||||
[workspace database ctx]
|
||||
(let [graph (:graph ctx)
|
||||
existing-output-ids (keep :id (:outputs graph))
|
||||
;; TODO (Chris 2025-11-20) Avoid querying again here, let's have this data passed down as part of the graph
|
||||
table-by-id (when (seq existing-output-ids)
|
||||
(into {}
|
||||
(map (juxt :id identity))
|
||||
(t2/select [:model/Table :id :name :schema] :id [:in existing-output-ids])))
|
||||
outputs (for [upstream-output (:outputs graph)]
|
||||
(if (:id upstream-output)
|
||||
;; Table exists, duplicate it
|
||||
(let [hydrated-output (merge upstream-output (get table-by-id (:id upstream-output)))
|
||||
isolated-table (duplicate-output-table! database workspace hydrated-output)]
|
||||
(t2/insert! :model/WorkspaceMappingTable
|
||||
{:upstream_id (:id upstream-output)
|
||||
:downstream_id (:id isolated-table)
|
||||
:workspace_id (:id workspace)})
|
||||
(assoc hydrated-output :mapping isolated-table))
|
||||
;; Table doesn't exist yet, provide the intended isolated location
|
||||
(let [isolated-schema (isolation-schema-name (:id workspace))
|
||||
isolated-name (isolated-table-name upstream-output)]
|
||||
(assoc upstream-output :mapping {:id nil
|
||||
:schema isolated-schema
|
||||
:name isolated-name}))))
|
||||
src-output-id->dst-output
|
||||
(into {}
|
||||
(comp (filter :id)
|
||||
(map (fn [{:keys [id mapping]}]
|
||||
[id mapping])))
|
||||
outputs)
|
||||
src-schema+table->dst->schema+table
|
||||
(into {}
|
||||
(map (fn [{:keys [mapping] :as output}]
|
||||
(let [src-schema (:schema output)
|
||||
src-table (:name output)
|
||||
dst-schema (:schema mapping)
|
||||
dst-table (:name mapping)]
|
||||
[[src-schema src-table]
|
||||
[dst-schema dst-table]])))
|
||||
outputs)]
|
||||
(-> ctx
|
||||
(update :graph assoc :outputs (vec outputs))
|
||||
(assoc :src-output-id->dst-output src-output-id->dst-output)
|
||||
(assoc :src-schema+table->dst->schema+table src-schema+table->dst->schema+table))))
|
||||
|
||||
(defn ensure-database-isolation!
|
||||
"Wrapper around the driver method, to make migrations easier in future."
|
||||
[workspace database]
|
||||
;; TODO: Make this check the ws existence aka fail closed ~atm
|
||||
(init-workspace-database-isolation! database workspace))
|
||||
@@ -0,0 +1,48 @@
|
||||
(ns metabase-enterprise.workspaces.mirroring
|
||||
(:require
|
||||
[metabase-enterprise.workspaces.context :as ws.ctx]
|
||||
[metabase-enterprise.workspaces.isolation :as ws.isolation]
|
||||
[metabase-enterprise.workspaces.remap :as ws.remap]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(def ^:private transform-mirrored-keys
|
||||
#{:description :name :source :target :source_type})
|
||||
|
||||
(defn- mirror-data
|
||||
[transform-node ctx workspace-id]
|
||||
(merge {:workspace_id workspace-id}
|
||||
(select-keys (ws.ctx/transform-node->data ctx transform-node)
|
||||
transform-mirrored-keys)
|
||||
(ws.remap/remap-for-transform transform-node ctx)))
|
||||
|
||||
;; TODO (lbrdnk 2025-11-26): Make this logic pure, move inserts upstream into single insert.
|
||||
(defn- mirror-transform!
|
||||
"Create the mirroring transform and transform mapping rows for a single transform."
|
||||
[workspace ctx transform-node]
|
||||
(let [mirror-data (mirror-data transform-node ctx (:id workspace))
|
||||
mirror (t2/insert-returning-instance! :model/Transform mirror-data)
|
||||
;; TODO (lbrdnk 2025-11-26): Strong gut feeling we want "mapping" represented differently.
|
||||
;; Motivation: Transforms _created_ in the workspace have no "original" map in graph
|
||||
;; where they could live. Having some transforms in "mapping" and some
|
||||
;; elsewhere feels dirty.
|
||||
graph-node (assoc transform-node :mapping (select-keys mirror [:id :name]))]
|
||||
(t2/insert! :model/WorkspaceMappingTransform
|
||||
{:upstream_id (:id transform-node)
|
||||
:downstream_id (:id mirror)
|
||||
:workspace_id (:id workspace)})
|
||||
(update-in ctx [:graph :transforms] #(conj (or % []) graph-node))))
|
||||
|
||||
(defn- mirror-transforms!
|
||||
[workspace ctx]
|
||||
(reduce #(mirror-transform! workspace %1 %2)
|
||||
(update ctx :graph dissoc :transforms)
|
||||
(-> ctx :graph :transforms)))
|
||||
|
||||
(defn mirror-entities!
|
||||
"TODO (lbrdnk): Add docstring."
|
||||
[workspace database graph]
|
||||
(let [ctx (ws.ctx/->context-with-resources graph)]
|
||||
(->> ctx
|
||||
(ws.isolation/create-isolated-output-tables! workspace database)
|
||||
(mirror-transforms! workspace)
|
||||
:graph)))
|
||||
@@ -0,0 +1,43 @@
|
||||
(ns metabase-enterprise.workspaces.models.workspace
|
||||
(:require
|
||||
[metabase.models.interface :as mi]
|
||||
[methodical.core :as methodical]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(def type->t2-model "Models supported by workspaces" {:transform :model/Transform})
|
||||
(def type->mapping "Mapping models for supported models" {:transform :model/WorkspaceMappingTransform})
|
||||
(def type->grouping "How models are grouped in lists" {:transform :transforms})
|
||||
|
||||
(methodical/defmethod t2/table-name :model/Workspace [_model] :workspace)
|
||||
|
||||
(t2/deftransforms :model/Workspace
|
||||
{:graph mi/transform-json})
|
||||
|
||||
(doto :model/Workspace
|
||||
(derive :metabase/model)
|
||||
(derive :hook/timestamped?))
|
||||
|
||||
(defn- field [table field-name]
|
||||
(keyword (name table) (name field-name)))
|
||||
|
||||
(defn- entities [entity-type ids]
|
||||
(let [t2-model (type->t2-model entity-type)
|
||||
t2-table (t2/table-name t2-model)
|
||||
mapping (t2/table-name (type->mapping entity-type))]
|
||||
(->> (t2/select [t2-model :id :name (field t2-table :workspace_id) (field mapping :upstream_id)]
|
||||
(field t2-table :workspace_id) [:in ids]
|
||||
{:left-join [mapping [:= (field mapping :downstream_id) (field t2-table :id)]]})
|
||||
(mapv (fn [e] (assoc e :type entity-type))))))
|
||||
|
||||
(defn- ws-contents [ids]
|
||||
(fn []
|
||||
(->> (mapcat #(entities % ids) (keys type->grouping))
|
||||
(reduce (fn [acc e]
|
||||
(update-in acc [(:workspace_id e) (type->grouping (:type e))] (fnil conj []) e))
|
||||
{}))))
|
||||
|
||||
(methodical/defmethod t2/batched-hydrate [:model/Workspace :contents]
|
||||
"Batch hydrate `Workspace` contents"
|
||||
[_model k wses]
|
||||
(let [ids (mapv :id wses)]
|
||||
(mi/instances-with-hydrated-data wses k (ws-contents ids) :id {:default {}})))
|
||||
@@ -0,0 +1,18 @@
|
||||
(ns metabase-enterprise.workspaces.models.workspace-mapping
|
||||
(:require
|
||||
[methodical.core :as methodical]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(methodical/defmethod t2/table-name :model/WorkspaceMappingTable [_model] :workspace_mapping_table)
|
||||
|
||||
(methodical/defmethod t2/primary-keys :model/WorkspaceMappingTable [_model] [:upstream_id :downstream_id])
|
||||
|
||||
(doto :model/WorkspaceMappingTable
|
||||
(derive :metabase/model))
|
||||
|
||||
(methodical/defmethod t2/table-name :model/WorkspaceMappingTransform [_model] :workspace_mapping_transform)
|
||||
|
||||
(methodical/defmethod t2/primary-keys :model/WorkspaceMappingTransform [_model] [:upstream_id :downstream_id])
|
||||
|
||||
(doto :model/WorkspaceMappingTransform
|
||||
(derive :metabase/model))
|
||||
@@ -0,0 +1,100 @@
|
||||
(ns metabase-enterprise.workspaces.promotion
|
||||
"Functionality for promoting workspace transforms back to main Metabase."
|
||||
(:require
|
||||
[metabase.util :as u]
|
||||
[metabase.util.log :as log]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(defn- find-upstream-xf
|
||||
"Find the original transform that was mirrored into this workspace.
|
||||
Uses the workspace_mapping_transform table to find the upstream transform."
|
||||
[xf workspace-id]
|
||||
(or (when-let [mapping (t2/select-one :model/WorkspaceMappingTransform
|
||||
:downstream_id (:id xf)
|
||||
:workspace_id workspace-id)]
|
||||
(t2/select-one :model/Transform :id (:upstream_id mapping)))
|
||||
(throw (ex-info "Original transform not found" {:transform-id (:id xf)
|
||||
:workspace-id workspace-id}))))
|
||||
|
||||
(defn- promote-single-transform!
|
||||
"Promote a single workspace transform to replace its original.
|
||||
Returns a map with :id, :name, and :status."
|
||||
[downstream-xf upstream-xf new->old-s+n]
|
||||
(try
|
||||
(let [upstream-id (:id upstream-xf)
|
||||
remapped-xf (-> downstream-xf
|
||||
;; TODO: do query remapping here, if the transform is dependent on an isolated table.
|
||||
;; ... with the current FE this is not possible! :-D
|
||||
(update :target (fn [target]
|
||||
;; TODO assert database is what we expect
|
||||
(assert (= "table" (:type target)) "Only table targets are supported for promotion.")
|
||||
(let [[old-schema old-name] (new->old-s+n [(:schema target) (:name target)])]
|
||||
(assert old-name "Unable to find the upstream table to sub for the isolated one.")
|
||||
(assoc target
|
||||
:schema old-schema
|
||||
:name old-name)))))
|
||||
;; TODO revisit whether there are any other fields we want
|
||||
updates (select-keys remapped-xf [:name :description :source :target])]
|
||||
(log/infof "Promoting transform %d -> %d: %s" (:id downstream-xf) upstream-id (:name upstream-xf))
|
||||
(t2/update! :model/Transform upstream-id updates)
|
||||
(assoc updates :id upstream-id))
|
||||
(catch Throwable e
|
||||
(log/errorf e "Failed to promote transform %s" (:name downstream-xf))
|
||||
{:id (:id downstream-xf)
|
||||
:name (:name downstream-xf)
|
||||
:error (ex-message e)})))
|
||||
|
||||
(defn- transform-ids [graph]
|
||||
(->> (:transforms graph) (keep :mapping) (mapv :id)))
|
||||
|
||||
(defn promote-transforms!
|
||||
"Promote all transforms in a workspace back to main Metabase.
|
||||
|
||||
This will:
|
||||
1. Get all workspace transforms
|
||||
2. Order them by dependencies (using the workspace graph)
|
||||
3. For each transform:
|
||||
- Find the original transform
|
||||
- Update it with the workspace version's definition
|
||||
- Re-execute it in the original schema
|
||||
|
||||
Returns a map with:
|
||||
- :promoted - sequence of {:id, :name} maps
|
||||
- :errors - any transforms that failed to promote"
|
||||
[ws]
|
||||
(log/infof "Starting promotion of transforms from workspace %s" (:id ws))
|
||||
|
||||
(let [ids (transform-ids (:graph ws))
|
||||
xs (if (seq ids)
|
||||
(t2/select :model/Transform :id [:in ids])
|
||||
;; fallback for when there is no graph
|
||||
(t2/select :model/Transform :workspace_id (:id ws)))
|
||||
_ (log/infof "Found %d workspace transforms to promote" (count xs))
|
||||
new->old-s+n (u/for-map [{:keys [old_schema old_name new_schema new_name]}
|
||||
(t2/query {:select [[:t1.schema :old_schema]
|
||||
[:t1.name :old_name]
|
||||
[:t2.schema :new_schema]
|
||||
[:t2.name :new_name]]
|
||||
:from [[(t2/table-name :model/WorkspaceMappingTable) :m]
|
||||
[(t2/table-name :model/Table) :t1]
|
||||
[(t2/table-name :model/Table) :t2]]
|
||||
:where [:and
|
||||
[:= :m.workspace_id (:id ws)]
|
||||
[:= :t1.id :m.upstream_id]
|
||||
[:= :t2.id :m.downstream_id]]})]
|
||||
[[new_schema new_name]
|
||||
[old_schema old_name]])
|
||||
results (for [xf xs
|
||||
:let [upstream-xf (find-upstream-xf xf (:id ws))]]
|
||||
(promote-single-transform! xf upstream-xf new->old-s+n))
|
||||
{promoted false
|
||||
errors true} (group-by #(contains? % :error) results)]
|
||||
|
||||
(log/infof "Promotion complete. %d succeeded, %d failed"
|
||||
(count promoted)
|
||||
(count errors))
|
||||
|
||||
{:promoted promoted
|
||||
:errors errors}))
|
||||
@@ -0,0 +1,86 @@
|
||||
(ns metabase-enterprise.workspaces.remap
|
||||
(:require
|
||||
[macaw.core :as macaw]
|
||||
[metabase-enterprise.workspaces.context :as ws.ctx]))
|
||||
|
||||
;;;; Transform source remapping
|
||||
|
||||
(defn- remapped-transform-source-dispatch
|
||||
[transform-node graph]
|
||||
(let [{source-type :source_type} (ws.ctx/transform-node->data graph transform-node)]
|
||||
source-type))
|
||||
|
||||
(defmulti remapped-transform-source
|
||||
"todo"
|
||||
{:added "0.59.0" :arglists '([transform-node graph])}
|
||||
#'remapped-transform-source-dispatch)
|
||||
|
||||
;;;; Transform native source remaping impl.
|
||||
|
||||
(defn- mappings-for-macaw-rf
|
||||
[acc [[src-schema src-table]
|
||||
[dst-schema dst-table]]]
|
||||
(assert (or (not (contains? (:schemas acc) src-schema))
|
||||
(= dst-schema (get-in acc [:schemas src-schema])))
|
||||
"Mapping mismatch")
|
||||
(-> acc
|
||||
(assoc-in [:schemas src-schema] dst-schema)
|
||||
(assoc-in [:tables {:schema src-schema :table src-table}] dst-table)))
|
||||
|
||||
(defn- macaw-renames
|
||||
[src-schema+table->dst-schema+table]
|
||||
(reduce mappings-for-macaw-rf
|
||||
{}
|
||||
src-schema+table->dst-schema+table))
|
||||
|
||||
(defn- native-mappings
|
||||
[transform-node ctx]
|
||||
(ws.ctx/transform-node->src->dst-remapping ctx transform-node)
|
||||
;; for when :dependencies hold table-nodes, now we go with transform nodes
|
||||
#_(ws.ctx/schema+table->schema+table-outputs-remapping ctx transform-node))
|
||||
|
||||
(defn- remap-sql-tables
|
||||
[sql-str macaw-mappings]
|
||||
(macaw/replace-names sql-str macaw-mappings))
|
||||
|
||||
(def ^:private query-sql-path
|
||||
[:query :stages 0 :native])
|
||||
|
||||
(defmethod remapped-transform-source :native
|
||||
[transform-node ctx]
|
||||
(let [mappings (native-mappings transform-node ctx)
|
||||
renames (macaw-renames mappings)
|
||||
transform-data (ws.ctx/transform-node->data ctx transform-node)
|
||||
{:keys [source]} transform-data
|
||||
native-str (get-in source query-sql-path)
|
||||
remapped-str (remap-sql-tables native-str renames)]
|
||||
(assoc-in source query-sql-path remapped-str)))
|
||||
|
||||
;;;; Public
|
||||
|
||||
(defn- source-for-mirrored-transform
|
||||
[transform-node ctx]
|
||||
(let [transform-data (ws.ctx/transform-node->data ctx transform-node)
|
||||
{source :source} transform-data]
|
||||
(if-not (ws.ctx/has-dependencies? ctx transform-node)
|
||||
source
|
||||
(remapped-transform-source transform-node ctx))))
|
||||
|
||||
(defn target-for-mirrored-transform
|
||||
"This should happen to every transform"
|
||||
[transform-node ctx]
|
||||
(let [data (ws.ctx/transform-node->data ctx transform-node)
|
||||
target (:target data)
|
||||
[dst-schema dst-table] (get (:src-schema+table->dst->schema+table ctx)
|
||||
[(:schema target) (:name target)])]
|
||||
(-> target
|
||||
(assoc :schema dst-schema)
|
||||
(assoc :name dst-table))))
|
||||
|
||||
(defn remap-for-transform
|
||||
"Return _transform data_ to be used to create transform duplicate."
|
||||
[transform-node ctx]
|
||||
(let [dst-target (target-for-mirrored-transform transform-node ctx)
|
||||
dst-source (source-for-mirrored-transform transform-node ctx)]
|
||||
{:target dst-target
|
||||
:source dst-source}))
|
||||
@@ -0,0 +1,26 @@
|
||||
(ns metabase-enterprise.workspaces.sync
|
||||
;; TODO (Chris 2025-11-20) -- Decide whether to mirror metadata manually, and drop sync dependency
|
||||
(:require
|
||||
[metabase.driver :as driver]
|
||||
[metabase.driver.util :as driver.u]
|
||||
^{:clj-kondo/ignore [:metabase/modules]}
|
||||
[metabase.sync.core :as sync]
|
||||
^{:clj-kondo/ignore [:metabase/modules]}
|
||||
[metabase.sync.sync-metadata.tables :as sync-tables]))
|
||||
|
||||
;; this ns will be probably moved
|
||||
|
||||
;; will figure out whether that is a good idea soonish
|
||||
;; this may be tricky!!!
|
||||
(defn sync-transform-mirror!
|
||||
"TODO"
|
||||
[database schema-name table-name]
|
||||
(let [driver (driver.u/database->driver database)
|
||||
table {:name table-name
|
||||
:schema schema-name}]
|
||||
(if (driver/table-exists? driver database table)
|
||||
(let [created (sync-tables/create-or-reactivate-table! database table)]
|
||||
(doto created sync/sync-table!))
|
||||
(throw (ex-info "Can not find transform mirror table"
|
||||
{:schema schema-name
|
||||
:table table-name})))))
|
||||
@@ -169,6 +169,20 @@
|
||||
:table #{checkins-id venues-id}}
|
||||
(calculation/upstream-deps:transform transform)))))))
|
||||
|
||||
(deftest ^:parallel upstream-deps-python-transform-test
|
||||
(mt/with-premium-features #{:transforms}
|
||||
(let [products-id (mt/id :products)
|
||||
orders-id (mt/id :orders)]
|
||||
(mt/with-temp [:model/Transform transform {:name "Test Transform"
|
||||
:source {:type :python
|
||||
:source-tables {"PRODUCTS" products-id
|
||||
"ORDERS" orders-id}
|
||||
:body "..."}
|
||||
:target {:schema "PUBLIC"
|
||||
:name "test_output"}}]
|
||||
(is (= {:table #{products-id orders-id}}
|
||||
(calculation/upstream-deps:transform transform)))))))
|
||||
|
||||
(deftest ^:parallel upstream-deps-card-native-with-parameter-source-test
|
||||
(let [mp (mt/metadata-provider)
|
||||
products-id (mt/id :products)
|
||||
|
||||
@@ -96,6 +96,8 @@
|
||||
:model/TransformJobRun
|
||||
:model/TransformJobTransformTag
|
||||
:model/TransformTransformTag
|
||||
:model/WorkspaceMappingTable
|
||||
:model/WorkspaceMappingTransform
|
||||
:model/PythonLibrary
|
||||
:model/Undo
|
||||
:model/User
|
||||
@@ -106,7 +108,8 @@
|
||||
:model/ConnectionImpersonation
|
||||
:model/CloudMigration
|
||||
:model/Comment
|
||||
:model/CommentReaction})
|
||||
:model/CommentReaction
|
||||
:model/Workspace})
|
||||
|
||||
(deftest ^:parallel comprehensive-entity-id-test
|
||||
(let [entity-id-models (->> (v2.entity-ids/toucan-models)
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
(ns ^:mb/driver-tests metabase-enterprise.workspaces.api-test
|
||||
(:require
|
||||
[clojure.test :refer :all]
|
||||
[metabase-enterprise.transforms.test-dataset :as transforms-dataset]
|
||||
[metabase-enterprise.transforms.test-util :refer [with-transform-cleanup!]]
|
||||
[metabase.test :as mt]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(deftest workspace-endpoints-require-superuser-test
|
||||
(mt/with-temp [:model/Workspace workspace {:name "Private Workspace"}]
|
||||
(testing "GET /api/ee/workspace requires superuser"
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :get 403 "ee/workspace"))))
|
||||
|
||||
(testing "GET /api/ee/workspace/:id requires superuser"
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :get 403 (str "ee/workspace/" (:id workspace))))))
|
||||
|
||||
(testing "POST /api/ee/workspace requires superuser"
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :post 403 "ee/workspace"
|
||||
{:name "Unauthorized Workspace"})))))
|
||||
|
||||
(mt/with-temp [:model/Workspace workspace {:name "Put Workspace"}]
|
||||
(testing "PUT /api/ee/workspace/:id requires superuser"
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :put 403 (str "ee/workspace/" (:id workspace))
|
||||
{:name "Updated"})))))
|
||||
|
||||
(mt/with-temp [:model/Workspace workspace {:name "Delete Workspace"}]
|
||||
(testing "DELETE /api/ee/workspace/:id requires superuser"
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :delete 403 (str "ee/workspace/" (:id workspace)))))))
|
||||
|
||||
(mt/with-temp [:model/Workspace workspace {:name "Promote Workspace"}]
|
||||
(testing "POST /api/ee/workspace/:id/promote requires superuser"
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :post 403 (str "ee/workspace/" (:id workspace) "/merge")))))))
|
||||
|
||||
(deftest workspace-crud-flow-test
|
||||
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
|
||||
(mt/with-premium-features [:workspaces :dependencies :transforms]
|
||||
(mt/with-model-cleanup [:model/Collection :model/Workspace :model/Transform]
|
||||
(let [tx-id (->> (t2/select :model/Transform :workspace_id nil)
|
||||
(filter #(= (mt/id) (get-in % [:target :database])))
|
||||
(sort-by :id >)
|
||||
first
|
||||
:id)
|
||||
workspace-name (str "Workspace " (random-uuid))
|
||||
;; Create workspace with or without upstream transforms depending on availability
|
||||
created (mt/user-http-request :crowberto :post 200 "ee/workspace"
|
||||
(cond-> {:name workspace-name
|
||||
:database_id (mt/id)}
|
||||
tx-id (assoc :upstream {:transforms [tx-id]})))
|
||||
workspace-id (:id created)
|
||||
collection-id (:collection_id created)]
|
||||
(is (=? {:id int?
|
||||
:collection_id int?
|
||||
:name workspace-name}
|
||||
created))
|
||||
(is (t2/exists? :model/Workspace :id workspace-id :collection_id collection-id))
|
||||
(is (t2/exists? :model/Collection :id collection-id :workspace_id workspace-id))
|
||||
|
||||
(testing "workspace appears in list response"
|
||||
(let [{:keys [items]} (mt/user-http-request :crowberto :get 200 "ee/workspace")]
|
||||
(is (some #(= workspace-id (:id %)) items))))
|
||||
|
||||
(testing "workspace can be fetched individually"
|
||||
(let [response (mt/user-http-request :crowberto :get 200 (str "ee/workspace/" workspace-id))]
|
||||
(is (= workspace-id (:id response)))))
|
||||
|
||||
#_(testing "workspace can be archived"
|
||||
(let [updated (mt/user-http-request :crowberto :post 200 (str "ee/workspace/" workspace-id "/archive"))]
|
||||
(is (some? (:archived_at updated))))))))))
|
||||
|
||||
(deftest ^:parallel promote-workspace-test
|
||||
(testing "POST /api/ee/workspace/:id/promote requires superuser"
|
||||
(mt/with-temp [:model/Workspace workspace {:name "Promote Test"}]
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :post 403 (str "ee/workspace/" (:id workspace) "/merge"))))))
|
||||
|
||||
(testing "Cannot promote an already archived workspace"
|
||||
(mt/with-temp [:model/Workspace workspace {:name "Archived Workspace"
|
||||
:archived_at (java.time.OffsetDateTime/now)}]
|
||||
(is (= "Cannot promote an already archived workspace"
|
||||
(mt/user-http-request :crowberto :post 400 (str "ee/workspace/" (:id workspace) "/merge")))))))
|
||||
|
||||
(deftest create-workspace-transform-permissions-test
|
||||
(testing "POST /api/ee/workspace/:id/transform requires superuser"
|
||||
(mt/with-temp [:model/Workspace workspace {:name "Transform Test"}]
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :post 403
|
||||
(str "ee/workspace/" (:id workspace) "/transform")
|
||||
{:name "Should Fail"
|
||||
:source {:type "query"
|
||||
:query {}}
|
||||
:target {:type "table"
|
||||
:name "should_fail"}}))))))
|
||||
|
||||
(deftest create-workspace-transform-archived-test
|
||||
(testing "Cannot create transform in archived workspace"
|
||||
(mt/with-temp [:model/Workspace workspace {:name "Archived"
|
||||
:archived_at (java.time.OffsetDateTime/now)}]
|
||||
(is (= "Cannot create transforms in an archived workspace"
|
||||
(mt/user-http-request :crowberto :post 400
|
||||
(str "ee/workspace/" (:id workspace) "/transform")
|
||||
{:name "Should Fail"
|
||||
:source {:type "query"
|
||||
:query (mt/mbql-query venues)}
|
||||
:target {:type "table"
|
||||
:name "should_fail"}}))))))
|
||||
|
||||
(deftest add-entities-to-workspace-test
|
||||
(testing "Add entities to workspace"
|
||||
(mt/with-premium-features [:workspaces :dependencies :transforms]
|
||||
(mt/with-model-cleanup [:model/Collection :model/Workspace :model/Transform :model/WorkspaceMappingTransform]
|
||||
(let [transform-ids (t2/select-pks-vec :model/Transform :workspace_id nil {:limit 2 :order-by [[:id :asc]]})
|
||||
first-tx-id (first transform-ids)
|
||||
second-tx-id (second transform-ids)]
|
||||
(when (and first-tx-id second-tx-id)
|
||||
(let [workspace-id (:id (mt/user-http-request :crowberto :post 200 "ee/workspace"
|
||||
{:name "Add Entities Test"
|
||||
:upstream {:transforms [first-tx-id]}}))]
|
||||
|
||||
(testing "Can add new entities to workspace"
|
||||
(is (=? {:contents {:transforms #(>= (count %) 2)}}
|
||||
(mt/user-http-request :crowberto :post 200
|
||||
(str "ee/workspace/" workspace-id "/contents")
|
||||
{:add {:transforms [second-tx-id]}}))))
|
||||
|
||||
(testing "Cannot add duplicate entities"
|
||||
(is (= "Transforms 1 are already in workspace"
|
||||
(mt/user-http-request :crowberto :post 400
|
||||
(str "ee/workspace/" workspace-id "/contents")
|
||||
{:add {:transforms [first-tx-id]}}))))
|
||||
|
||||
(testing "Cannot add entities to archived workspace"
|
||||
(t2/update! :model/Workspace workspace-id {:archived_at (java.time.OffsetDateTime/now)})
|
||||
(is (= "Cannot add entities to an archived workspace"
|
||||
(mt/user-http-request :crowberto :post 400
|
||||
(str "ee/workspace/" workspace-id "/contents")
|
||||
{:add {:transforms [second-tx-id]}})))))))))))
|
||||
|
||||
(deftest add-entities-requires-superuser-test
|
||||
(testing "POST /api/ee/workspace/:id/add requires superuser"
|
||||
(mt/with-temp [:model/Workspace workspace {:name "Permission Test"}]
|
||||
(is (= "You don't have permissions to do that."
|
||||
(mt/user-http-request :rasta :post 403
|
||||
(str "ee/workspace/" (:id workspace) "/contents")
|
||||
{:add {:transforms [1]}}))))))
|
||||
|
||||
(deftest add-entities-no-nested-branching-test
|
||||
(testing "Cannot add transforms that belong to another workspace (no nested branching)"
|
||||
(mt/with-temp [:model/Workspace workspace-1 {:name "Workspace 1"}
|
||||
:model/Workspace workspace-2 {:name "Workspace 2"}
|
||||
:model/Transform transform {:name "Downstream Transform"
|
||||
:workspace_id (:id workspace-1)}]
|
||||
(is (= "Cannot add transforms that belong to another workspace"
|
||||
(mt/user-http-request :crowberto :post 400
|
||||
(str "ee/workspace/" (:id workspace-2) "/contents")
|
||||
{:add {:transforms [(:id transform)]}}))))))
|
||||
|
||||
(deftest create-workspace-transform-test
|
||||
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
|
||||
(mt/with-premium-features #{:workspaces :transforms}
|
||||
(mt/dataset transforms-dataset/transforms-test
|
||||
(mt/with-model-cleanup [:model/Workspace :model/Transform :model/Collection]
|
||||
(let [workspace-id (:id (mt/user-http-request :crowberto :post 200 "ee/workspace"
|
||||
{:name "Test Workspace"
|
||||
:database_id (mt/id)}))]
|
||||
(with-transform-cleanup! [table-name "workspace_transform_test"]
|
||||
(is (=? {:id pos-int?
|
||||
:workspace_id workspace-id
|
||||
:creator_id (mt/user->id :crowberto)
|
||||
:target {:database (mt/id)}}
|
||||
(mt/user-http-request :crowberto :post 200
|
||||
(str "ee/workspace/" workspace-id "/transform")
|
||||
{:name "Workspace Transform"
|
||||
:source {:type "query"
|
||||
:query (mt/mbql-query transforms_products)}
|
||||
:target {:type "table"
|
||||
:name table-name}}))))))))))
|
||||
|
||||
;;;; Card dependency rejection tests
|
||||
|
||||
(defn- query-with-source-card
|
||||
"Create a pMBQL query that uses a card as its source."
|
||||
[card-id]
|
||||
{:lib/type :mbql/query
|
||||
:database (mt/id)
|
||||
:stages [{:lib/type :mbql.stage/mbql
|
||||
:source-card card-id}]})
|
||||
|
||||
(defn- create-transform-with-card-source!
|
||||
"Create a transform whose source query depends on a card.
|
||||
The after-insert hook triggers dependency calculation automatically."
|
||||
[card]
|
||||
(t2/insert-returning-instance! :model/Transform
|
||||
{:name "Transform depending on card"
|
||||
:source {:type :query
|
||||
:query (query-with-source-card (:id card))}
|
||||
:target {:type "table"
|
||||
:database (mt/id)
|
||||
:schema "public"
|
||||
:name "card_dep_output"}}))
|
||||
|
||||
(deftest create-workspace-rejects-card-dependencies-test
|
||||
(testing "Cannot create workspace with transforms that depend on cards"
|
||||
(mt/with-premium-features #{:workspaces :dependencies :transforms}
|
||||
(mt/with-model-cleanup [:model/Dependency :model/Transform]
|
||||
(mt/with-temp [:model/Card card {:name "Test Card"
|
||||
:database_id (mt/id)
|
||||
:dataset_query (mt/mbql-query venues)}]
|
||||
(let [tx (create-transform-with-card-source! card)
|
||||
response (mt/user-http-request :crowberto :post 400 "ee/workspace"
|
||||
{:name "Card Dep Workspace"
|
||||
:database_id (mt/id)
|
||||
:upstream {:transforms [(:id tx)]}})]
|
||||
(is (re-find #"Cannot add transforms that depend on saved questions" response))))))))
|
||||
|
||||
(deftest create-workspace-rejects-transitive-card-dependencies-test
|
||||
(testing "Cannot create workspace with transforms that transitively depend on cards"
|
||||
(mt/with-premium-features #{:workspaces :dependencies :transforms}
|
||||
(mt/with-model-cleanup [:model/Dependency :model/Transform]
|
||||
(mt/with-temp [:model/Card card {:name "Base Card"
|
||||
:database_id (mt/id)
|
||||
:dataset_query (mt/mbql-query venues)}]
|
||||
;; tx1 depends on card
|
||||
(let [tx1 (create-transform-with-card-source! card)
|
||||
;; tx2 depends on tx1 (via a manually created dependency - simulating transform chain)
|
||||
tx2 (t2/insert-returning-instance! :model/Transform
|
||||
{:name "Transform 2 - depends on tx1"
|
||||
:source {:type :query
|
||||
:query {:database (mt/id)
|
||||
:type :native
|
||||
:native {:query "SELECT 1"}}}
|
||||
:target {:type "table"
|
||||
:database (mt/id)
|
||||
:schema "public"
|
||||
:name "tx2_output"}})]
|
||||
;; Create dependency: tx2 depends on tx1
|
||||
(t2/insert! :model/Dependency
|
||||
{:from_entity_type "transform"
|
||||
:from_entity_id (:id tx2)
|
||||
:to_entity_type "transform"
|
||||
:to_entity_id (:id tx1)})
|
||||
;; Try to create workspace with tx2 (which transitively depends on card via tx1)
|
||||
(let [response (mt/user-http-request :crowberto :post 400 "ee/workspace"
|
||||
{:name "Transitive Card Dep Workspace"
|
||||
:database_id (mt/id)
|
||||
:upstream {:transforms [(:id tx2)]}})]
|
||||
(is (re-find #"Cannot add transforms that depend on saved questions" response)))))))))
|
||||
|
||||
(deftest add-entities-rejects-card-dependencies-test
|
||||
(testing "Cannot add transforms with card dependencies to existing workspace"
|
||||
(mt/with-premium-features #{:workspaces :dependencies :transforms}
|
||||
(mt/with-model-cleanup [:model/Workspace :model/Dependency :model/Collection :model/Transform]
|
||||
(mt/with-temp [:model/Card card {:name "Test Card"
|
||||
:database_id (mt/id)
|
||||
:dataset_query (mt/mbql-query venues)}]
|
||||
(let [tx (create-transform-with-card-source! card)
|
||||
;; Create a workspace without the card-dependent transform
|
||||
workspace-id (:id (mt/user-http-request :crowberto :post 200 "ee/workspace"
|
||||
{:name "Empty Workspace"
|
||||
:database_id (mt/id)}))
|
||||
response (mt/user-http-request :crowberto :post 400
|
||||
(str "ee/workspace/" workspace-id "/contents")
|
||||
{:add {:transforms [(:id tx)]}})]
|
||||
(is (re-find #"Cannot add transforms that depend on saved questions" response))))))))
|
||||
@@ -0,0 +1,222 @@
|
||||
(ns ^:mb/driver-tests metabase-enterprise.workspaces.common-test
|
||||
(:require
|
||||
[clojure.java.jdbc :as jdbc]
|
||||
[clojure.test :refer [deftest testing is]]
|
||||
[medley.core :as m]
|
||||
[metabase-enterprise.transforms.interface :as transforms.i]
|
||||
[metabase-enterprise.workspaces.common :as ws.common]
|
||||
[metabase.driver.sql-jdbc.connection :as driver.conn]
|
||||
[metabase.lib.core :as lib]
|
||||
[metabase.test :as mt]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(defn- setup!
|
||||
[schema]
|
||||
(let [spec (driver.conn/connection-details->spec :postgres (:details (mt/db)))]
|
||||
(jdbc/execute! spec (format "DROP SCHEMA IF EXISTS %s CASCADE" schema))
|
||||
(jdbc/execute! spec (format "CREATE SCHEMA %s" schema))))
|
||||
|
||||
(defn teardown!
|
||||
[ws-id]
|
||||
(let [[ws-graph ws-schema] (t2/select-one-fn (juxt :graph :schema) :model/Workspace :id ws-id)
|
||||
spec (driver.conn/connection-details->spec :postgres (:details (mt/db)))
|
||||
output-ids (-> ws-graph :outputs (->> (map :id)))
|
||||
dup-tables-ids (into #{}
|
||||
(map #(-> % :mapping :id))
|
||||
(-> ws-graph :outputs))]
|
||||
(t2/delete! :model/Table :id [:in dup-tables-ids])
|
||||
(t2/delete! :model/Table :id [:in output-ids])
|
||||
(t2/delete! :model/Workspace :id ws-id)
|
||||
(jdbc/execute! spec (format "DROP SCHEMA IF EXISTS %s CASCADE" ws-schema))))
|
||||
|
||||
(deftest create-workspace-independent-native-transforms-test
|
||||
(mt/test-driver
|
||||
:postgres
|
||||
(let [mp (mt/metadata-provider)
|
||||
native-base (lib/native-query mp "select * from ORDERS LIMIT 10")
|
||||
test-schema "test_schema"
|
||||
new-workspace (atom nil)]
|
||||
(try
|
||||
(setup! test-schema)
|
||||
(mt/with-temp
|
||||
[:model/Transform x1 {:name "x1"
|
||||
:source_type :native
|
||||
:source {:type :query
|
||||
:query native-base}
|
||||
:target {:type :table
|
||||
:database (mt/id)
|
||||
:schema test-schema
|
||||
:name "t1"}}
|
||||
:model/Transform x2 {:name "x2"
|
||||
:source_type :native
|
||||
:source {:type :query
|
||||
:query (assoc-in native-base [:stages 0 :native]
|
||||
(str "SELECT *\n"
|
||||
"FROM PRODUCTS\n"
|
||||
"LIMIT 10\n"))}
|
||||
:target {:type :table
|
||||
:database (mt/id)
|
||||
:schema test-schema
|
||||
:name "t2"}}
|
||||
:model/Transform x3 {:name "x3"
|
||||
:source_type :native
|
||||
:source {:type :query
|
||||
:query (assoc-in native-base [:stages 0 :native]
|
||||
(str "SELECT *\n"
|
||||
"FROM PEOPLE\n"
|
||||
"LIMIT 10\n"))}
|
||||
:target {:type :table
|
||||
:database (mt/id)
|
||||
:schema test-schema
|
||||
:name "t3"}}]
|
||||
(transforms.i/execute! x1 {:run-method :manual})
|
||||
(transforms.i/execute! x2 {:run-method :manual})
|
||||
(transforms.i/execute! x3 {:run-method :manual})
|
||||
(let [result (mt/with-current-user (mt/user->id :crowberto)
|
||||
(ws.common/create-workspace! (mt/user->id :crowberto)
|
||||
{:name "test_ws"
|
||||
:database_id (mt/id)
|
||||
:upstream {:transforms (mapv :id [x1 x2 x3])}}))
|
||||
checkouts (-> result :graph :check-outs)
|
||||
transforms (-> result :graph :transforms)]
|
||||
(reset! new-workspace result)
|
||||
(testing "Name is correct"
|
||||
(is (= "test_ws" (:name result))))
|
||||
(testing "Checkouts are as expected"
|
||||
(is (= [{:type :transform :id (:id x1)}
|
||||
{:type :transform :id (:id x2)}
|
||||
{:type :transform :id (:id x3)}]
|
||||
(sort-by :id checkouts))))
|
||||
(testing "Transforms look as expected"
|
||||
(is (= #{{:id (:id x1), :type :transform, :mapping {#_#_:id pos-int?, :name "x1"}}
|
||||
{:id (:id x2), :type :transform, :mapping {#_#_:id pos-int?, :name "x2"}}
|
||||
{:id (:id x3), :type :transform, :mapping {#_#_:id pos-int?, :name "x3"}}}
|
||||
(into #{} (map #(m/update-existing % :mapping dissoc :id)) transforms))))))
|
||||
;; For development, subject to change
|
||||
(catch Throwable t
|
||||
(def ttt t)
|
||||
(throw t))
|
||||
(finally
|
||||
(teardown! (:id @new-workspace)))))))
|
||||
|
||||
(deftest mirroring-multiple-interdependent-native-transforms-test
|
||||
(mt/test-driver
|
||||
:postgres
|
||||
(let [mp (mt/metadata-provider)
|
||||
native-base (lib/native-query mp "select * from ORDERS LIMIT 10")
|
||||
test-schema "test_schema"
|
||||
new-workspace (atom nil)]
|
||||
(try
|
||||
(setup! test-schema)
|
||||
(testing "Interdependent native transforms"
|
||||
(mt/with-temp
|
||||
[:model/Transform x1 {:name "x1"
|
||||
:source_type :native
|
||||
:source {:type :query
|
||||
:query native-base}
|
||||
:target {:type :table
|
||||
:database (mt/id)
|
||||
:schema test-schema
|
||||
:name "t1"}}
|
||||
:model/Transform x2 {:name "x2"
|
||||
:source_type :native
|
||||
:source {:type :query
|
||||
:query (assoc-in native-base [:stages 0 :native]
|
||||
(str "SELECT *\n"
|
||||
(format "FROM \"%s\".\"%s\"\n" test-schema "t1")
|
||||
"LIMIT 10\n"))}
|
||||
:target {:type :table
|
||||
:database (mt/id)
|
||||
:schema test-schema
|
||||
:name "t2"}}
|
||||
:model/Transform x3 {:name "x3"
|
||||
:source_type :native
|
||||
:source {:type :query
|
||||
:query (assoc-in native-base [:stages 0 :native]
|
||||
(str "SELECT *\n"
|
||||
(format "FROM \"%s\".\"%s\"\n" test-schema "t2")
|
||||
"LIMIT 10\n"))}
|
||||
:target {:type :table
|
||||
:database (mt/id)
|
||||
:schema test-schema
|
||||
:name "t3"}}]
|
||||
(transforms.i/execute! x1 {:run-method :manual})
|
||||
(transforms.i/execute! x2 {:run-method :manual})
|
||||
(transforms.i/execute! x3 {:run-method :manual})
|
||||
(let [result-ws (mt/with-current-user (mt/user->id :crowberto)
|
||||
(ws.common/create-workspace! (mt/user->id :crowberto)
|
||||
{:name "test_ws"
|
||||
:database_id (mt/id)
|
||||
:upstream {:transforms (mapv :id [x1 x3])}}))
|
||||
checkouts (-> result-ws :graph :check-outs)
|
||||
transforms (-> result-ws :graph :transforms)]
|
||||
(reset! new-workspace result-ws)
|
||||
(testing "Workspace has expected name"
|
||||
(is (= "test_ws" (:name result-ws))))
|
||||
(testing "Checkouts contain only the explicitly checked out transforms"
|
||||
(is (= #{{:type :transform :id (:id x1)}
|
||||
{:type :transform :id (:id x3)}}
|
||||
(set checkouts))))
|
||||
(testing "Workspace has correct mappings in graph"
|
||||
(is (= #{{:id (:id x1), :type :transform, :mapping {#_#_:id pos-int?, :name "x1"}}
|
||||
{:id (:id x2), :type :transform, :mapping {#_#_:id pos-int?, :name "x2"}}
|
||||
{:id (:id x3), :type :transform, :mapping {#_#_:id pos-int?, :name "x3"}}}
|
||||
(into #{} (map #(m/update-existing % :mapping dissoc :id)) transforms))))
|
||||
(let [orig-id->mirror-id (into {}
|
||||
(map (fn [{:keys [id mapping]}]
|
||||
[id (:id mapping)]))
|
||||
transforms)
|
||||
mirror-id->mirror (t2/select-fn->fn :id identity :model/Transform :id
|
||||
[:in (vals orig-id->mirror-id)])
|
||||
x1-mirror (-> (:id x1) orig-id->mirror-id mirror-id->mirror)
|
||||
x2-mirror (-> (:id x2) orig-id->mirror-id mirror-id->mirror)
|
||||
x3-mirror (-> (:id x3) orig-id->mirror-id mirror-id->mirror)]
|
||||
(testing "X1 was properly mirrored"
|
||||
(testing "name"
|
||||
(is (= "x1" (:name x1-mirror))))
|
||||
(testing "target adjusted"
|
||||
(is (=? {:type "table"
|
||||
:database (mt/id)
|
||||
:schema (:schema result-ws)
|
||||
;; table name = orig schema + __ + orig target name atm
|
||||
:name (str test-schema "__" (-> x1 :target :name))}
|
||||
(:target x1-mirror))))
|
||||
(testing "source not modifed"
|
||||
(is (=? (:source x1)
|
||||
(:source x1-mirror)))))
|
||||
(testing "X2 was properly mirrored"
|
||||
(testing "name"
|
||||
(is (= (:name x2) (:name x2-mirror))))
|
||||
(testing "target"
|
||||
(is (=? {:type "table"
|
||||
:database (mt/id)
|
||||
:schema (:schema result-ws)
|
||||
:name (str test-schema "__" (-> x2 :target :name))}
|
||||
(:target x2-mirror))))
|
||||
(testing "source query referencing target of x1 mirror"
|
||||
;; TODO (lbrdnk 2025-11-27): Quotes are removed, figure out whether what has
|
||||
;; some unwanted consequences.
|
||||
(is (=? {:query {:stages [{:native (format "SELECT *\nFROM %s.%s\nLIMIT 10\n"
|
||||
(-> x1-mirror :target :schema)
|
||||
(-> x1-mirror :target :name))}]}}
|
||||
(:source x2-mirror)))))
|
||||
(testing "X3 was properly mirrored"
|
||||
(testing "name"
|
||||
(is (= (:name x3) (:name x3-mirror))))
|
||||
(testing "target"
|
||||
(is (=? {:type "table"
|
||||
:database (mt/id)
|
||||
:schema (:schema result-ws)
|
||||
:name (str test-schema "__" (-> x3 :target :name))}
|
||||
(:target x3-mirror))))
|
||||
(testing "source query referencing target of x2 mirror"
|
||||
(is (=? {:query {:stages [{:native (format "SELECT *\nFROM %s.%s\nLIMIT 10\n"
|
||||
(-> x2-mirror :target :schema)
|
||||
(-> x2-mirror :target :name))}]}}
|
||||
(:source x3-mirror)))))))))
|
||||
;; For development, subject to change
|
||||
(catch Throwable t
|
||||
(def ttt t)
|
||||
(throw t))
|
||||
(finally
|
||||
(teardown! (:id @new-workspace)))))))
|
||||
@@ -0,0 +1,114 @@
|
||||
(ns metabase-enterprise.workspaces.dag-abstract
|
||||
"Abstract graph solver using shorthand notation for testing.
|
||||
|
||||
Shorthand uses keywords like :x1, :t2, :m3 where:
|
||||
- :x<n> is a transform
|
||||
- :t<n> is a table (source/generated)
|
||||
- :m<n> is a model (card)"
|
||||
(:require
|
||||
[clojure.set :as set]
|
||||
[flatland.ordered.map :as ordered-map]
|
||||
[metabase-enterprise.workspaces.dag :as ws.dag]
|
||||
[metabase.util :as u]))
|
||||
|
||||
;;;; Shorthand notation helpers
|
||||
|
||||
(defn- related [t id]
|
||||
(keyword (str (name t) (subs (name id) 1))))
|
||||
|
||||
(def ^:private x->t (partial related :t))
|
||||
|
||||
(defn is?
|
||||
"Check if id is of type t (e.g., (is? :x :x1) => true)."
|
||||
[t id]
|
||||
(= (name t) (str (first (name id)))))
|
||||
|
||||
(defn- lexico [id]
|
||||
(let [n (name id)]
|
||||
[(parse-long (subs n 1))
|
||||
({\x 1 \t 2 \m 3} (first n))]))
|
||||
|
||||
(defn expand-shorthand
|
||||
"As shorthand, we can use transforms in the place of their induced tables.
|
||||
This function expands the graph to insert the induced tables, and replace the corresponding references."
|
||||
[{:keys [dependencies] :as graph}]
|
||||
(assoc graph :dependencies
|
||||
(let [xs (set (filter #(is? :x %) (concat (keys dependencies) (mapcat val dependencies))))]
|
||||
(into (sorted-map-by #(compare (lexico %1) (lexico %2)))
|
||||
(merge (zipmap (map x->t xs) (map vector xs))
|
||||
(into {} (filter (comp #(is? :t %) key)) dependencies)
|
||||
(update-vals
|
||||
(into {} (remove (comp #(is? :t %) key)) dependencies)
|
||||
(fn [deps] (->> deps
|
||||
(map (fn [id]
|
||||
(if (is? :x id)
|
||||
(x->t id)
|
||||
id)))
|
||||
(sort-by lexico)
|
||||
(vec)))))))))
|
||||
|
||||
;;;; Graph traversal helpers
|
||||
|
||||
(defn- children [dependencies]
|
||||
(reduce
|
||||
(fn [acc [child parents]]
|
||||
(reduce (fn [acc p] (update acc p u/conjv child)) acc parents))
|
||||
(sorted-map-by #(compare (lexico %1) (lexico %2)))
|
||||
dependencies))
|
||||
|
||||
(defn- ->ts [ids]
|
||||
(into #{}
|
||||
(keep (fn [id]
|
||||
(cond (is? :t id) id
|
||||
(is? :x id) (x->t id))))
|
||||
ids))
|
||||
|
||||
;;;; Toposort
|
||||
|
||||
(defn- toposort-map [child->parents]
|
||||
(let [child->parents (if (map? child->parents) child->parents (into {} child->parents))]
|
||||
(into (ordered-map/ordered-map)
|
||||
(keep (fn [n] (when-let [deps (get child->parents n)] [n deps])))
|
||||
(#'ws.dag/toposort-dfs child->parents))))
|
||||
|
||||
;;;; Abstract path-induced subgraph solver
|
||||
|
||||
(def ^:private max-iterations 100)
|
||||
|
||||
(defn path-induced-subgraph
|
||||
"Abstract solver for path-induced subgraph using shorthand notation."
|
||||
[{:keys [check-outs dependencies] :as _graph}]
|
||||
(let [->parents dependencies
|
||||
->children (children dependencies)]
|
||||
(loop [iteration 1
|
||||
below check-outs
|
||||
above check-outs
|
||||
seen-down? check-outs
|
||||
seen-up? check-outs
|
||||
to-descend check-outs
|
||||
to-ascend check-outs]
|
||||
(if (and (empty? to-descend) (empty? to-ascend))
|
||||
(let [nodes (set/intersection above below)
|
||||
entities (remove #(is? :t %) nodes)
|
||||
tables (sort-by lexico (filter #(is? :t %) nodes))]
|
||||
{:check-outs (sort-by lexico check-outs)
|
||||
:inputs (sort-by lexico (remove nodes (->ts (mapcat ->parents check-outs))))
|
||||
:tables tables
|
||||
:transforms (sort-by lexico (filter #(is? :x %) nodes))
|
||||
:entities (sort-by lexico entities)
|
||||
:dependencies (apply dissoc
|
||||
(toposort-map
|
||||
(for [n nodes]
|
||||
[n (vec (filter nodes (get dependencies n)))]))
|
||||
tables)})
|
||||
(if (> iteration max-iterations)
|
||||
{:error "max iterations exceeded"}
|
||||
(let [next-down (into #{} (comp (mapcat ->children) (remove seen-down?)) to-descend)
|
||||
next-up (into #{} (comp (mapcat ->parents) (remove seen-up?)) to-ascend)]
|
||||
(recur (inc iteration)
|
||||
(into below next-down)
|
||||
(into above next-up)
|
||||
(into seen-down? next-down)
|
||||
(into seen-up? next-up)
|
||||
next-down
|
||||
next-up)))))))
|
||||
@@ -0,0 +1,337 @@
|
||||
(ns metabase-enterprise.workspaces.dag-test
|
||||
(:require
|
||||
[clojure.string :as str]
|
||||
[clojure.test :refer :all]
|
||||
[flatland.ordered.map :as ordered-map]
|
||||
[metabase-enterprise.dependencies.models.dependency :as deps]
|
||||
[metabase-enterprise.workspaces.dag :as ws.dag]
|
||||
[metabase-enterprise.workspaces.dag-abstract :as dag-abstract]
|
||||
[metabase.test :as mt]
|
||||
[metabase.util :as u]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
;;;; Example graphs for testing
|
||||
|
||||
(def ^:private example
|
||||
{:check-outs #{:x3 :m6 :m10 :m13}
|
||||
:dependencies {:x3 [:x1 :t2]
|
||||
:x4 [:x3]
|
||||
:m6 [:x4 :t5]
|
||||
:m10 [:t9]
|
||||
:x11 [:m10]
|
||||
:m12 [:x11]
|
||||
:m13 [:x11 :m12]}})
|
||||
|
||||
;;;; Test data helpers
|
||||
|
||||
(defn- transform? [kw] (= \x (first (name kw))))
|
||||
(defn- table? [kw] (= \t (first (name kw))))
|
||||
(defn- kw->id [kw] (parse-long (subs (name kw) 1)))
|
||||
|
||||
(defn- create-test-graph!
|
||||
"Create test transforms and tables from shorthand notation, returning id mappings.
|
||||
|
||||
Shorthand uses keywords like :x1, :t2, :m3 where:
|
||||
- :x<n> is a transform
|
||||
- :t<n> is a table (source/generated)
|
||||
- :m<n> is a model (card)
|
||||
|
||||
Returns a map from shorthand id to real database id, e.g. {:x1 123, :t2 456}
|
||||
|
||||
NOTE: Caller should wrap in mt/with-model-cleanup for proper cleanup."
|
||||
[{:keys [check-outs dependencies]}]
|
||||
(let [schema (str/replace (str (random-uuid)) "-" "_")
|
||||
all-ids (set (concat (keys dependencies)
|
||||
(mapcat val dependencies)
|
||||
check-outs))
|
||||
transforms (filter transform? all-ids)
|
||||
tables (filter table? all-ids)
|
||||
table-ids (u/for-map [t tables]
|
||||
[t (t2/insert-returning-pk! :model/Table
|
||||
{:db_id (mt/id)
|
||||
:schema schema
|
||||
:name (str "test_table_" (kw->id t))
|
||||
:active true})])
|
||||
;; Create transforms that reference their input tables
|
||||
tx-ids (u/for-map [tx transforms]
|
||||
(let [parent-tables (->> (get dependencies tx []) (filter table?) (map table-ids))
|
||||
id (t2/insert-returning-pk! :model/Transform
|
||||
{:name (str "Test " (name tx))
|
||||
:source {:type :query
|
||||
:query (if (seq parent-tables)
|
||||
{:database (mt/id)
|
||||
:type :query
|
||||
:query {:source-table (first parent-tables)
|
||||
:joins (for [pt (rest parent-tables)]
|
||||
{:source-table pt
|
||||
:condition [:= 1 1]})}}
|
||||
{:database (mt/id)
|
||||
:type :native
|
||||
:native {:query "SELECT 1"}})}
|
||||
:target {:type "table"
|
||||
:database (mt/id)
|
||||
:schema schema
|
||||
:name (str "test_table_" (kw->id tx))}})]
|
||||
[tx id]))]
|
||||
|
||||
;; TODO This is a workaround for the dependency between transforms and their output only being inserted on run.
|
||||
;; We will need to do something about this when we mirror as well - ideally the deps module would "just work"
|
||||
(doseq [[tx-kw tx-id] tx-ids]
|
||||
(t2/insert! :model/Dependency
|
||||
{:from_entity_type "table"
|
||||
:from_entity_id (table-ids (keyword (str "t" (kw->id tx-kw))))
|
||||
:to_entity_type "transform"
|
||||
:to_entity_id tx-id}))
|
||||
|
||||
(merge tx-ids table-ids)))
|
||||
|
||||
(defn- translate-result
|
||||
"Translate result from real IDs back to shorthand notation for easier comparison."
|
||||
[result id-map]
|
||||
(let [reverse-map (u/for-map [[k v] id-map] [v k])
|
||||
translate (fn [{:keys [id]}] (get reverse-map id))]
|
||||
(-> (reduce (fn [m k] (update m k #(set (map translate %))))
|
||||
result
|
||||
[:check-outs :inputs :outputs :transforms])
|
||||
(update :dependencies
|
||||
(fn [deps]
|
||||
(into {}
|
||||
(map (fn [[k v]]
|
||||
[(translate k) (set (map translate v))]))
|
||||
deps))))))
|
||||
|
||||
;;;; Tests
|
||||
|
||||
(deftest path-induced-subgraph-basic-test
|
||||
(mt/with-premium-features #{:workspaces :dependencies}
|
||||
(mt/with-temp [:model/Transform tx {:name "Test Transform"}]
|
||||
(let [table-id (mt/id :orders)]
|
||||
;; Set up: transform depends on orders table
|
||||
(deps/replace-dependencies! "transform" (:id tx) {"table" #{table-id}})
|
||||
|
||||
(testing "single transform returns correct subgraph"
|
||||
(let [result (ws.dag/path-induced-subgraph {:transforms [(:id tx)]})]
|
||||
(is (= [{:id (:id tx) :type :transform}] (:check-outs result)))
|
||||
(is (= [{:id (:id tx) :type :transform}] (:transforms result)))
|
||||
;; Input should be the orders table
|
||||
(is (some #(= table-id (:id %)) (:inputs result)))))))))
|
||||
|
||||
(deftest path-induced-subgraph-chain-test
|
||||
(mt/with-premium-features #{:workspaces :dependencies}
|
||||
(mt/with-model-cleanup [:model/Transform :model/Dependency]
|
||||
(mt/with-temp [:model/Transform tx1 {:name "Transform 1"}
|
||||
:model/Transform tx2 {:name "Transform 2"}
|
||||
:model/Transform tx3 {:name "Transform 3"}]
|
||||
(let [table-id (mt/id :orders)]
|
||||
;; Chain: orders -> tx1 -> tx2 -> tx3
|
||||
(deps/replace-dependencies! "transform" (:id tx1) {"table" #{table-id}})
|
||||
(deps/replace-dependencies! "transform" (:id tx2) {"transform" #{(:id tx1)}})
|
||||
(deps/replace-dependencies! "transform" (:id tx3) {"transform" #{(:id tx2)}})
|
||||
|
||||
(testing "middle transform includes upstream and downstream"
|
||||
(let [result (ws.dag/path-induced-subgraph {:transforms [(:id tx2)]})]
|
||||
;; tx2 is checked out
|
||||
(is (= [{:id (:id tx2) :type :transform}] (:check-outs result)))
|
||||
;; Should include tx1 (upstream) and tx3 (downstream) since tx2 is on path between them
|
||||
;; Actually no - tx2 is the only thing checked out, so we only get what's on paths through tx2
|
||||
;; tx1 -> tx2 -> tx3 means tx2 has tx1 upstream and tx3 downstream
|
||||
;; The intersection is just tx2 itself
|
||||
(is (= #{{:id (:id tx2) :type :transform}} (set (:transforms result)))))))))))
|
||||
|
||||
(deftest path-induced-subgraph-diamond-test
|
||||
(mt/with-premium-features #{:workspaces :dependencies}
|
||||
(mt/with-model-cleanup [:model/Transform :model/Dependency]
|
||||
(testing "diamond dependency pattern"
|
||||
;; Create: t1 -> x1 -> t2
|
||||
;; \-> x2 -> t3
|
||||
;; both x1 and x2 depend on t1
|
||||
;; if we check out x1 and x2, we should get both
|
||||
(mt/with-temp [:model/Transform tx1 {:name "Transform 1"}
|
||||
:model/Transform tx2 {:name "Transform 2"}]
|
||||
(let [source-table (mt/id :orders)]
|
||||
(deps/replace-dependencies! "transform" (:id tx1) {"table" #{source-table}})
|
||||
(deps/replace-dependencies! "transform" (:id tx2) {"table" #{source-table}})
|
||||
|
||||
(let [result (ws.dag/path-induced-subgraph {:transforms [(:id tx1) (:id tx2)]})]
|
||||
(is (= 2 (count (:check-outs result))))
|
||||
(is (= 2 (count (:transforms result)))))))))))
|
||||
|
||||
(deftest upstream-entities-test
|
||||
(mt/with-premium-features #{:workspaces :dependencies}
|
||||
(mt/with-model-cleanup [:model/Transform :model/Dependency]
|
||||
(mt/with-temp [:model/Transform tx1 {:name "Transform 1"}
|
||||
:model/Transform tx2 {:name "Transform 2"}]
|
||||
(let [table-id (mt/id :orders)]
|
||||
(deps/replace-dependencies! "transform" (:id tx1) {"table" #{table-id}})
|
||||
(deps/replace-dependencies! "transform" (:id tx2) {"transform" #{(:id tx1)}})
|
||||
|
||||
(testing "upstream includes transitive dependencies"
|
||||
(let [upstream (#'ws.dag/upstream-entities {:transforms [(:id tx2)]})]
|
||||
(is (contains? upstream {:id (:id tx2) :type :transform}))
|
||||
(is (contains? upstream {:id (:id tx1) :type :transform}))
|
||||
(is (contains? upstream {:id table-id :type :table})))))))))
|
||||
|
||||
(deftest downstream-entities-test
|
||||
(mt/with-premium-features #{:workspaces :dependencies}
|
||||
(mt/with-model-cleanup [:model/Transform :model/Dependency]
|
||||
(mt/with-temp [:model/Transform tx1 {:name "Transform 1"}
|
||||
:model/Transform tx2 {:name "Transform 2"}]
|
||||
(deps/replace-dependencies! "transform" (:id tx2) {"transform" #{(:id tx1)}})
|
||||
|
||||
(testing "downstream includes transitive dependents"
|
||||
(let [downstream (#'ws.dag/downstream-entities {:transforms [(:id tx1)]})]
|
||||
(is (contains? downstream {:id (:id tx1) :type :transform}))
|
||||
(is (contains? downstream {:id (:id tx2) :type :transform}))))))))
|
||||
|
||||
(deftest path-induced-subgraph-shorthand-test
|
||||
(mt/with-premium-features #{:workspaces :dependencies}
|
||||
(mt/with-model-cleanup [:model/Table :model/Transform :model/Dependency]
|
||||
(testing "graph built from shorthand matches abstract solver"
|
||||
(let [shorthand {:check-outs #{:x1}
|
||||
:dependencies {:x1 [:t1]}}
|
||||
id-map (create-test-graph! shorthand)
|
||||
result (ws.dag/path-induced-subgraph {:transforms [(id-map :x1)]})
|
||||
translated (translate-result result id-map)]
|
||||
;; Check-outs should be :x1
|
||||
(is (= #{:x1} (:check-outs translated)))
|
||||
;; Transforms should be :x1
|
||||
(is (= #{:x1} (:transforms translated)))
|
||||
;; Inputs should be :t1
|
||||
(is (contains? (:inputs translated) :t1)))))))
|
||||
|
||||
(deftest path-induced-subgraph-larger-test
|
||||
(mt/with-premium-features #{:workspaces :dependencies}
|
||||
(mt/with-model-cleanup [:model/Table :model/Transform :model/Dependency]
|
||||
(testing "graph built from shorthand matches abstract solver"
|
||||
(let [shorthand {:check-outs #{:x2, :x4}
|
||||
:dependencies {:x1 [:t0]
|
||||
:x2 [:x1, :t10]
|
||||
:x3 [:x2, :t8]
|
||||
:x4 [:x3]
|
||||
:x5 [:x2, :x4, :t9]}}
|
||||
id-map (create-test-graph! (dag-abstract/expand-shorthand shorthand))
|
||||
result (ws.dag/path-induced-subgraph {:transforms (mapv id-map (:check-outs shorthand))})
|
||||
translated (translate-result result id-map)]
|
||||
(is (=? {:check-outs #{:x2, :x4}
|
||||
:transforms #{:x2, :x3, :x4}
|
||||
:inputs #{:t0, :t1, :t8, :t10}
|
||||
:outputs #{:t2, :t3, :t4}
|
||||
;; Dependencies outside the subgraph are not listed
|
||||
:dependencies {:x2 #{}
|
||||
:x3 #{:x2}
|
||||
:x4 #{:x3}}}
|
||||
translated)))))))
|
||||
|
||||
(deftest expand-solver-test
|
||||
(testing "expand-shorthand inserts interstitial nodes for transform output tables"
|
||||
(is (= {:check-outs #{:x3 :m6 :m10 :m13}
|
||||
:dependencies {:t1 [:x1]
|
||||
:x3 [:t1 :t2]
|
||||
:t3 [:x3]
|
||||
:x4 [:t3]
|
||||
:t4 [:x4]
|
||||
:m6 [:t4 :t5]
|
||||
:m10 [:t9]
|
||||
:x11 [:m10]
|
||||
:t11 [:x11]
|
||||
:m12 [:t11]
|
||||
:m13 [:t11 :m12]}}
|
||||
(dag-abstract/expand-shorthand example)))))
|
||||
|
||||
(deftest abstract-path-induced-subgraph-test
|
||||
(testing "path-induced-subgraph computes correct result for example graph"
|
||||
(is (= {:check-outs [:x3 :m6 :m10 :m13]
|
||||
:inputs [:t1 :t2 :t5 :t9]
|
||||
:tables [:t3 :t4 :t11]
|
||||
:transforms [:x3 :x4 :x11]
|
||||
:entities [:x3 :x4 :m6 :m10 :x11 :m12 :m13]
|
||||
:dependencies (ordered-map/ordered-map
|
||||
:m10 []
|
||||
:x11 [:m10]
|
||||
:m12 [:t11]
|
||||
:m13 [:t11 :m12]
|
||||
:x3 []
|
||||
:x4 [:t3]
|
||||
:m6 [:t4])}
|
||||
(dag-abstract/path-induced-subgraph (dag-abstract/expand-shorthand example))))))
|
||||
|
||||
;;;; Card dependency detection tests
|
||||
|
||||
(defn- query-with-source-card
|
||||
"Create a pMBQL query that uses a card as its source."
|
||||
[card-id]
|
||||
{:lib/type :mbql/query
|
||||
:database (mt/id)
|
||||
:stages [{:lib/type :mbql.stage/mbql
|
||||
:source-card card-id}]})
|
||||
|
||||
(defn- create-transform-with-card-source!
|
||||
"Create a transform whose source query depends on a card.
|
||||
The after-insert hook triggers dependency calculation automatically."
|
||||
[card]
|
||||
(t2/insert-returning-instance! :model/Transform
|
||||
{:name "Transform depending on card"
|
||||
:source {:type :query
|
||||
:query (query-with-source-card (:id card))}
|
||||
:target {:type "table"
|
||||
:database (mt/id)
|
||||
:schema "public"
|
||||
:name "card_dep_output"}}))
|
||||
|
||||
(deftest transforms-depend-on-cards-no-deps-test
|
||||
(mt/with-premium-features #{:workspaces :dependencies}
|
||||
(testing "transform with no card dependencies returns empty set"
|
||||
(mt/with-temp [:model/Transform tx {:name "Transform without card deps"
|
||||
:source {:type :query
|
||||
:query {:database (mt/id)
|
||||
:type :query
|
||||
:query {:source-table (mt/id :orders)}}}
|
||||
:target {:type "table"
|
||||
:database (mt/id)
|
||||
:schema "public"
|
||||
:name "no_card_output"}}]
|
||||
;; after-insert hook triggers dependency calculation automatically
|
||||
(is (empty? (ws.dag/card-dependencies [(:id tx)])))))))
|
||||
|
||||
(deftest transforms-depend-on-cards-direct-dep-test
|
||||
(mt/with-premium-features #{:workspaces :dependencies}
|
||||
(testing "transform with direct card dependency returns card IDs"
|
||||
(mt/with-model-cleanup [:model/Dependency :model/Transform]
|
||||
(mt/with-temp [:model/Card card {:name "Test Card"
|
||||
:database_id (mt/id)
|
||||
:dataset_query (mt/mbql-query venues)}]
|
||||
(let [tx (create-transform-with-card-source! card)
|
||||
result (ws.dag/card-dependencies [(:id tx)])]
|
||||
(is (seq result))
|
||||
(is (contains? result (:id card)))))))))
|
||||
|
||||
(deftest transforms-depend-on-cards-transitive-dep-test
|
||||
(mt/with-premium-features #{:workspaces :dependencies}
|
||||
(testing "transform with transitive card dependency (via another transform) returns card IDs"
|
||||
(mt/with-model-cleanup [:model/Dependency :model/Transform]
|
||||
(mt/with-temp [:model/Card card {:name "Base Card"
|
||||
:database_id (mt/id)
|
||||
:dataset_query (mt/mbql-query venues)}]
|
||||
;; tx1 depends on card
|
||||
(let [tx1 (create-transform-with-card-source! card)
|
||||
;; tx2 depends on tx1
|
||||
tx2 (t2/insert-returning-instance! :model/Transform
|
||||
{:name "Transform 2 - depends on tx1"
|
||||
:source {:type :query
|
||||
:query {:database (mt/id)
|
||||
:type :native
|
||||
:native {:query "SELECT 1"}}}
|
||||
:target {:type "table"
|
||||
:database (mt/id)
|
||||
:schema "public"
|
||||
:name "tx2_output"}})]
|
||||
;; Create dependency: tx2 depends on tx1
|
||||
(deps/replace-dependencies! "transform" (:id tx2) {"transform" #{(:id tx1)}})
|
||||
(let [result (ws.dag/card-dependencies [(:id tx2)])]
|
||||
(is (seq result))
|
||||
(is (contains? result (:id card))))))))))
|
||||
|
||||
(deftest transforms-depend-on-cards-empty-input-test
|
||||
(mt/with-premium-features #{:workspaces :dependencies}
|
||||
(testing "empty input returns nil/empty"
|
||||
(is (empty? (ws.dag/card-dependencies []))))))
|
||||
@@ -0,0 +1,97 @@
|
||||
(ns metabase-enterprise.workspaces.promotion-test
|
||||
(:require
|
||||
[clojure.test :refer :all]
|
||||
[metabase-enterprise.workspaces.promotion :as ws.promotion]
|
||||
[metabase.test :as mt]
|
||||
[toucan2.core :as t2]))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(deftest promote-transforms-test
|
||||
(mt/with-premium-features #{:workspaces :dependencies}
|
||||
(mt/with-temp [:model/Table t1 {:schema "public", :name "table_1"}
|
||||
:model/Table t2 {:schema "public", :name "table_2"}
|
||||
:model/Transform x1 {:name "Transform 1"
|
||||
:description "Original description"
|
||||
:target {:type "table"
|
||||
:database 1
|
||||
:schema "public"
|
||||
:name "table_1"}}
|
||||
:model/Transform x2 {:name "Transform 2"
|
||||
:description "Another original"
|
||||
:target {:type "table"
|
||||
:database 1
|
||||
:schema "public"
|
||||
:name "table_2"}}
|
||||
:model/Workspace ws {:name "Test Workspace"}
|
||||
:model/Table wst1 {:schema "isolated__place", :name "public__table_1"}
|
||||
:model/Table wst2 {:schema "isolated__place", :name "public__table_2"}
|
||||
:model/WorkspaceMappingTable _ {:upstream_id (:id t1)
|
||||
:downstream_id (:id wst1)
|
||||
:workspace_id (:id ws)}
|
||||
:model/WorkspaceMappingTable _ {:upstream_id (:id t2)
|
||||
:downstream_id (:id wst2)
|
||||
:workspace_id (:id ws)}
|
||||
:model/Transform wsx1 {:name "Transform 1"
|
||||
:description "Modified description"
|
||||
:workspace_id (:id ws)
|
||||
:target {:type "table"
|
||||
:database 1
|
||||
:schema "isolated__place"
|
||||
:name "public__table_1"}}
|
||||
:model/Transform wsx2 {:name "Transform 2"
|
||||
:description "Modified description 2"
|
||||
:workspace_id (:id ws)
|
||||
:target {:type "table"
|
||||
:database 1
|
||||
:schema "isolated__place"
|
||||
:name "public__table_2"}}
|
||||
:model/WorkspaceMappingTransform _m1 {:upstream_id (:id x1)
|
||||
:downstream_id (:id wsx1)
|
||||
:workspace_id (:id ws)}
|
||||
:model/WorkspaceMappingTransform _m2 {:upstream_id (:id x2)
|
||||
:downstream_id (:id wsx2)
|
||||
:workspace_id (:id ws)}]
|
||||
|
||||
(testing "promote-transforms! updates originals and reports success"
|
||||
(let [result (ws.promotion/promote-transforms! ws)]
|
||||
(testing "promotion worked"
|
||||
(is (=? {:promoted [{:id (:id x1)}
|
||||
{:id (:id x2)}]}
|
||||
result)))
|
||||
|
||||
(testing "original transforms were updated with ws versions"
|
||||
(is (= (:description wsx1)
|
||||
(t2/select-one-fn :description :model/Transform :id (:id x1))))
|
||||
(is (= (:description wsx2)
|
||||
(t2/select-one-fn :description :model/Transform :id (:id x2))))))))))
|
||||
|
||||
(deftest promote-transforms-with-error-test
|
||||
(mt/with-premium-features #{:workspaces :dependencies}
|
||||
(mt/with-temp [:model/Table t1 {:schema "public", :name "table_1"}
|
||||
:model/Transform x1 {:name "Transform 1"}
|
||||
:model/Workspace ws {:name "Test Workspace"}
|
||||
:model/Table t2 {:schema "isolated", :name "public__table_1"}
|
||||
:model/Transform wsx1 {:name "Transform 1"
|
||||
:workspace_id (:id ws)
|
||||
:target {:type "table"
|
||||
:database 1
|
||||
:schema "isolated"
|
||||
:name "public__table_1"}}
|
||||
:model/WorkspaceMappingTable _ {:upstream_id (:id t1)
|
||||
:downstream_id (:id t2)
|
||||
:workspace_id (:id ws)}
|
||||
:model/WorkspaceMappingTransform _m {:upstream_id (:id x1)
|
||||
:downstream_id (:id wsx1)
|
||||
:workspace_id (:id ws)}]
|
||||
|
||||
(testing "promote-transforms! handles errors gracefully"
|
||||
(with-redefs [t2/update!
|
||||
(fn [& _args]
|
||||
(throw (ex-info "Execution failed" {})))]
|
||||
|
||||
(is (=? {:promoted nil
|
||||
:errors [{:id (:id wsx1)
|
||||
:error "Execution failed"}]}
|
||||
(mt/with-log-level [metabase-enterprise.workspaces.promotion :fatal]
|
||||
(ws.promotion/promote-transforms! ws)))))))))
|
||||
@@ -21,3 +21,4 @@ export * from "./transform";
|
||||
export * from "./transform-job";
|
||||
export * from "./transform-tag";
|
||||
export * from "./upload-management";
|
||||
export * from "./workspace";
|
||||
|
||||
@@ -25,6 +25,9 @@ import type {
|
||||
TransformJob,
|
||||
TransformRun,
|
||||
TransformTag,
|
||||
Workspace,
|
||||
WorkspaceContentItem,
|
||||
WorkspaceContents,
|
||||
} from "metabase-types/api";
|
||||
|
||||
export const ENTERPRISE_TAG_TYPES = [
|
||||
@@ -49,6 +52,7 @@ export const ENTERPRISE_TAG_TYPES = [
|
||||
"remote-sync-branches",
|
||||
"remote-sync-current-task",
|
||||
"python-transform-library",
|
||||
"workspace",
|
||||
"support-access-grant",
|
||||
"support-access-grant-current",
|
||||
"library-collection",
|
||||
@@ -82,6 +86,32 @@ export function invalidateTags(
|
||||
return !error ? tags : [];
|
||||
}
|
||||
|
||||
export function provideWorkspacesTags(
|
||||
workspaces: Workspace[],
|
||||
): TagDescription<EnterpriseTagType>[] {
|
||||
return [listTag("workspace"), ...workspaces.flatMap(provideWorkspaceTags)];
|
||||
}
|
||||
|
||||
export function provideWorkspaceTags(
|
||||
workspace: Workspace,
|
||||
): TagDescription<EnterpriseTagType>[] {
|
||||
return [idTag("workspace", workspace.id)];
|
||||
}
|
||||
|
||||
export function provideWorkspaceContentsTags(
|
||||
workspaceContents: WorkspaceContents,
|
||||
): TagDescription<EnterpriseTagType>[] {
|
||||
return provideWorkspaceContentItemsTags(
|
||||
workspaceContents.contents.transforms,
|
||||
);
|
||||
}
|
||||
|
||||
export function provideWorkspaceContentItemsTags(
|
||||
items: WorkspaceContentItem[],
|
||||
): TagDescription<EnterpriseTagType>[] {
|
||||
return items.flatMap((transform) => idTag("transform", transform.id));
|
||||
}
|
||||
|
||||
export function provideTransformTags(
|
||||
transform: Transform,
|
||||
): TagDescription<EnterpriseTagType>[] {
|
||||
|
||||
134
enterprise/frontend/src/metabase-enterprise/api/workspace.ts
Normal file
134
enterprise/frontend/src/metabase-enterprise/api/workspace.ts
Normal file
@@ -0,0 +1,134 @@
|
||||
import type {
|
||||
CreateWorkspaceRequest,
|
||||
TransformDownstreamMapping,
|
||||
TransformId,
|
||||
TransformUpstreamMapping,
|
||||
Workspace,
|
||||
WorkspaceContents,
|
||||
WorkspaceId,
|
||||
WorkspaceListResponse,
|
||||
WorkspaceMergeResponse,
|
||||
WorkspaceUpdateContentsRequest,
|
||||
} from "metabase-types/api";
|
||||
|
||||
import { EnterpriseApi } from "./api";
|
||||
import {
|
||||
idTag,
|
||||
invalidateTags,
|
||||
listTag,
|
||||
provideWorkspaceContentItemsTags,
|
||||
provideWorkspaceContentsTags,
|
||||
provideWorkspaceTags,
|
||||
provideWorkspacesTags,
|
||||
tag,
|
||||
} from "./tags";
|
||||
|
||||
export const workspaceApi = EnterpriseApi.injectEndpoints({
|
||||
endpoints: (builder) => ({
|
||||
getWorkspaces: builder.query<WorkspaceListResponse, void>({
|
||||
query: () => ({
|
||||
method: "GET",
|
||||
url: "/api/ee/workspace",
|
||||
}),
|
||||
providesTags: (response) =>
|
||||
response ? provideWorkspacesTags(response.items) : [],
|
||||
}),
|
||||
getWorkspace: builder.query<Workspace, WorkspaceId>({
|
||||
query: (id) => ({
|
||||
method: "GET",
|
||||
url: `/api/ee/workspace/${id}`,
|
||||
}),
|
||||
providesTags: (workspace) =>
|
||||
workspace ? provideWorkspaceTags(workspace) : [],
|
||||
}),
|
||||
createWorkspace: builder.mutation<Workspace, CreateWorkspaceRequest>({
|
||||
query: (body) => ({
|
||||
method: "POST",
|
||||
url: "/api/ee/workspace",
|
||||
body,
|
||||
}),
|
||||
invalidatesTags: (_, error) =>
|
||||
invalidateTags(error, [listTag("workspace"), listTag("transform")]),
|
||||
}),
|
||||
getWorkspaceContents: builder.query<WorkspaceContents, WorkspaceId>({
|
||||
query: (id) => ({
|
||||
method: "GET",
|
||||
url: `/api/ee/workspace/${id}`,
|
||||
}),
|
||||
transformResponse: (workspace: any) => {
|
||||
return {
|
||||
contents: workspace?.contents ?? { transforms: [] },
|
||||
} as WorkspaceContents;
|
||||
},
|
||||
providesTags: (workspaceContents) =>
|
||||
workspaceContents
|
||||
? provideWorkspaceContentsTags(workspaceContents)
|
||||
: [],
|
||||
}),
|
||||
updateWorkspaceContents: builder.mutation<
|
||||
WorkspaceContents,
|
||||
WorkspaceUpdateContentsRequest
|
||||
>({
|
||||
query: ({ id, ...body }) => ({
|
||||
method: "POST",
|
||||
url: `/api/ee/workspace/${id}/contents`,
|
||||
body,
|
||||
}),
|
||||
invalidatesTags: (_, error, { id }) =>
|
||||
invalidateTags(error, [idTag("workspace", id), tag("transform")]),
|
||||
}),
|
||||
getTransformUpstreamMapping: builder.query<
|
||||
TransformUpstreamMapping,
|
||||
TransformId
|
||||
>({
|
||||
query: (id) => ({
|
||||
method: "GET",
|
||||
url: `/api/ee/workspace/mapping/transform/${id}/upstream`,
|
||||
}),
|
||||
providesTags: (mapping) =>
|
||||
mapping?.transform
|
||||
? provideWorkspaceContentItemsTags([mapping.transform])
|
||||
: [],
|
||||
}),
|
||||
getTransformDownstreamMapping: builder.query<
|
||||
TransformDownstreamMapping,
|
||||
TransformId
|
||||
>({
|
||||
query: (id) => ({
|
||||
method: "GET",
|
||||
url: `/api/ee/workspace/mapping/transform/${id}/downstream`,
|
||||
}),
|
||||
providesTags: (mapping) =>
|
||||
mapping ? provideWorkspaceContentItemsTags(mapping.transforms) : [],
|
||||
}),
|
||||
mergeWorkspace: builder.mutation<WorkspaceMergeResponse, WorkspaceId>({
|
||||
query: (id) => ({
|
||||
method: "POST",
|
||||
url: `/api/ee/workspace/${id}/merge`,
|
||||
}),
|
||||
invalidatesTags: (_, error) =>
|
||||
invalidateTags(error, [tag("workspace"), tag("transform")]),
|
||||
}),
|
||||
archiveWorkspace: builder.mutation<Workspace, WorkspaceId>({
|
||||
query: (id) => ({
|
||||
method: "POST",
|
||||
url: `/api/ee/workspace/${id}/archive`,
|
||||
}),
|
||||
invalidatesTags: (_, error) =>
|
||||
invalidateTags(error, [tag("workspace"), tag("transform")]),
|
||||
}),
|
||||
}),
|
||||
});
|
||||
|
||||
export const {
|
||||
useGetWorkspacesQuery,
|
||||
useGetWorkspaceQuery,
|
||||
useCreateWorkspaceMutation,
|
||||
useGetWorkspaceContentsQuery,
|
||||
useUpdateWorkspaceContentsMutation,
|
||||
useLazyGetWorkspaceContentsQuery,
|
||||
useGetTransformUpstreamMappingQuery,
|
||||
useGetTransformDownstreamMappingQuery,
|
||||
useMergeWorkspaceMutation,
|
||||
useArchiveWorkspaceMutation,
|
||||
} = workspaceApi;
|
||||
@@ -1,5 +1,13 @@
|
||||
.nav {
|
||||
border-right: 1px solid var(--mb-color-border);
|
||||
transition: width 200ms ease;
|
||||
width: auto;
|
||||
min-width: 250px;
|
||||
}
|
||||
|
||||
.navCollapsed {
|
||||
width: 60px;
|
||||
min-width: 60px;
|
||||
}
|
||||
|
||||
.tab {
|
||||
@@ -14,9 +22,99 @@
|
||||
.tab,
|
||||
.toggle {
|
||||
color: var(--mb-color-text-primary);
|
||||
font-weight: 600;
|
||||
|
||||
&:hover,
|
||||
&.selected {
|
||||
color: var(--mb-color-brand);
|
||||
}
|
||||
}
|
||||
|
||||
.workspacesSection {
|
||||
border-top: 1px solid var(--mb-color-border);
|
||||
padding-top: 0.75rem;
|
||||
}
|
||||
|
||||
.workspacesSectionHeader {
|
||||
background-color: var(--mb-color-bg-secondary);
|
||||
color: var(--mb-color-text-primary);
|
||||
font-weight: 600;
|
||||
transition: background-color 150ms ease;
|
||||
|
||||
&:hover {
|
||||
background-color: var(--mb-color-brand-light);
|
||||
color: var(--mb-color-brand);
|
||||
}
|
||||
|
||||
&.selected {
|
||||
background-color: var(--mb-color-brand-light);
|
||||
color: var(--mb-color-brand);
|
||||
}
|
||||
}
|
||||
|
||||
.workspaceItem {
|
||||
background-color: var(--mb-color-bg-white);
|
||||
color: var(--mb-color-text-primary);
|
||||
cursor: pointer;
|
||||
border: 1px solid var(--mb-color-border);
|
||||
transition:
|
||||
background-color 150ms ease,
|
||||
border-color 150ms ease,
|
||||
color 150ms ease;
|
||||
|
||||
&:hover {
|
||||
background-color: var(--mb-color-bg-light);
|
||||
border-color: var(--mb-color-border-dark);
|
||||
}
|
||||
|
||||
&:hover .workspaceMenuButton {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
&.selected {
|
||||
background-color: var(--mb-color-brand-lighter);
|
||||
border-color: var(--mb-color-brand);
|
||||
color: var(--mb-color-brand);
|
||||
}
|
||||
|
||||
&.selected .workspaceMenuButton {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
&:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
}
|
||||
|
||||
.workspaceMenuButton {
|
||||
color: var(--mb-color-text-secondary);
|
||||
opacity: 0;
|
||||
transition:
|
||||
opacity 150ms ease,
|
||||
background-color 150ms ease,
|
||||
color 150ms ease;
|
||||
flex-shrink: 0;
|
||||
|
||||
&:hover {
|
||||
background-color: var(--mb-color-bg-medium);
|
||||
color: var(--mb-color-text-primary);
|
||||
}
|
||||
}
|
||||
|
||||
.newWorkspaceButton {
|
||||
background-color: transparent;
|
||||
color: var(--mb-color-text-primary);
|
||||
cursor: pointer;
|
||||
border: none;
|
||||
transition: color 150ms ease;
|
||||
|
||||
&:hover {
|
||||
color: var(--mb-color-brand);
|
||||
}
|
||||
|
||||
&:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
import cx from "classnames";
|
||||
import { type ReactNode, useMemo, useState } from "react";
|
||||
import dayjs from "dayjs";
|
||||
import { type ReactNode, useCallback, useMemo, useState } from "react";
|
||||
import { push } from "react-router-redux";
|
||||
import { t } from "ttag";
|
||||
|
||||
import { useListDatabasesQuery } from "metabase/api";
|
||||
import { ForwardRefLink } from "metabase/common/components/Link";
|
||||
import { useSelector } from "metabase/lib/redux";
|
||||
import { useDispatch, useSelector } from "metabase/lib/redux";
|
||||
import * as Urls from "metabase/lib/urls";
|
||||
import { useMetadataToasts } from "metabase/metadata/hooks";
|
||||
import {
|
||||
PLUGIN_DEPENDENCIES,
|
||||
PLUGIN_FEATURE_LEVEL_PERMISSIONS,
|
||||
@@ -12,15 +16,30 @@ import {
|
||||
} from "metabase/plugins";
|
||||
import { getLocation } from "metabase/selectors/routing";
|
||||
import {
|
||||
ActionIcon,
|
||||
Box,
|
||||
Button,
|
||||
FixedSizeIcon,
|
||||
Flex,
|
||||
Icon,
|
||||
type IconName,
|
||||
Menu,
|
||||
Modal,
|
||||
Select,
|
||||
Skeleton,
|
||||
Stack,
|
||||
Text,
|
||||
TextInput,
|
||||
Tooltip,
|
||||
UnstyledButton,
|
||||
} from "metabase/ui";
|
||||
import {
|
||||
useArchiveWorkspaceMutation,
|
||||
useCreateWorkspaceMutation,
|
||||
useGetWorkspacesQuery,
|
||||
} from "metabase-enterprise/api";
|
||||
import { DataStudioContext } from "metabase-enterprise/data-studio/common/contexts/DataStudioContext";
|
||||
import type { Database, WorkspaceId } from "metabase-types/api";
|
||||
|
||||
import S from "./DataStudioLayout.module.css";
|
||||
|
||||
@@ -31,6 +50,7 @@ type DataStudioLayoutProps = {
|
||||
export function DataStudioLayout({ children }: DataStudioLayoutProps) {
|
||||
const [isSidebarOpened, setIsSidebarOpened] = useState(false);
|
||||
const [isSidebarAvailable, setIsSidebarAvailable] = useState(false);
|
||||
const [isNavExpanded, setIsNavExpanded] = useState(true);
|
||||
const contextValue = useMemo(
|
||||
() => ({
|
||||
isSidebarOpened,
|
||||
@@ -48,6 +68,8 @@ export function DataStudioLayout({ children }: DataStudioLayoutProps) {
|
||||
isSidebarOpened={isSidebarOpened}
|
||||
isSidebarAvailable={isSidebarAvailable}
|
||||
onSidebarToggle={setIsSidebarOpened}
|
||||
isNavExpanded={isNavExpanded}
|
||||
onNavToggle={setIsNavExpanded}
|
||||
/>
|
||||
<Box h="100%" flex={1} miw={0}>
|
||||
{children}
|
||||
@@ -61,12 +83,16 @@ type DataStudioNavProps = {
|
||||
isSidebarOpened: boolean;
|
||||
isSidebarAvailable: boolean;
|
||||
onSidebarToggle: (isOpened: boolean) => void;
|
||||
isNavExpanded: boolean;
|
||||
onNavToggle: (isExpanded: boolean) => void;
|
||||
};
|
||||
|
||||
function DataStudioNav({
|
||||
isSidebarOpened,
|
||||
isSidebarAvailable,
|
||||
onSidebarToggle,
|
||||
isNavExpanded,
|
||||
onNavToggle,
|
||||
}: DataStudioNavProps) {
|
||||
const { pathname } = useSelector(getLocation);
|
||||
const canAccessDataModel = useSelector(
|
||||
@@ -76,42 +102,62 @@ function DataStudioNav({
|
||||
PLUGIN_TRANSFORMS.canAccessTransforms,
|
||||
);
|
||||
const isDataTab = pathname.startsWith(Urls.dataStudioData());
|
||||
const isTransformsTab = pathname.startsWith(Urls.transformList());
|
||||
const isWorkspacePage = pathname.startsWith(Urls.dataStudioWorkspaceList());
|
||||
const isTransformsTab =
|
||||
pathname.startsWith(Urls.transformList()) && !isWorkspacePage;
|
||||
const isModelingTab = pathname.startsWith(Urls.dataStudioModeling());
|
||||
const isDependenciesTab = pathname.startsWith(Urls.dependencyGraph());
|
||||
|
||||
return (
|
||||
<Stack className={S.nav} h="100%" p="0.75rem" justify="space-between">
|
||||
<Stack
|
||||
className={cx(S.nav, { [S.navCollapsed]: !isNavExpanded })}
|
||||
h="100%"
|
||||
p="0.75rem"
|
||||
justify="space-between"
|
||||
>
|
||||
<Stack gap="0.75rem">
|
||||
{canAccessDataModel && (
|
||||
<DataStudioTab
|
||||
label={t`Data`}
|
||||
icon="database"
|
||||
to={Urls.dataStudioData()}
|
||||
isSelected={isDataTab}
|
||||
/>
|
||||
)}
|
||||
{canAccessTransforms && (
|
||||
<DataStudioTab
|
||||
label={t`Transforms`}
|
||||
icon="transform"
|
||||
to={Urls.transformList()}
|
||||
isSelected={isTransformsTab}
|
||||
/>
|
||||
)}
|
||||
<DataStudioTab
|
||||
label={t`Modeling`}
|
||||
icon="model"
|
||||
to={Urls.dataStudioModeling()}
|
||||
isSelected={isModelingTab}
|
||||
<DataStudioNavToggle
|
||||
isNavExpanded={isNavExpanded}
|
||||
onNavToggle={onNavToggle}
|
||||
/>
|
||||
{PLUGIN_DEPENDENCIES.isEnabled && (
|
||||
<Stack gap="0.75rem">
|
||||
{canAccessDataModel && (
|
||||
<DataStudioTab
|
||||
label={t`Data`}
|
||||
icon="database"
|
||||
to={Urls.dataStudioData()}
|
||||
isSelected={isDataTab}
|
||||
isExpanded={isNavExpanded}
|
||||
/>
|
||||
)}
|
||||
{canAccessTransforms && (
|
||||
<DataStudioTab
|
||||
label={t`Transforms`}
|
||||
icon="transform"
|
||||
to={Urls.transformList()}
|
||||
isSelected={isTransformsTab}
|
||||
isExpanded={isNavExpanded}
|
||||
/>
|
||||
)}
|
||||
<DataStudioTab
|
||||
label={t`Dependency graph`}
|
||||
icon="schema"
|
||||
to={Urls.dependencyGraph()}
|
||||
isSelected={isDependenciesTab}
|
||||
label={t`Modeling`}
|
||||
icon="model"
|
||||
to={Urls.dataStudioModeling()}
|
||||
isSelected={isModelingTab}
|
||||
isExpanded={isNavExpanded}
|
||||
/>
|
||||
{PLUGIN_DEPENDENCIES.isEnabled && (
|
||||
<DataStudioTab
|
||||
label={t`Dependency graph`}
|
||||
icon="schema"
|
||||
to={Urls.dependencyGraph()}
|
||||
isSelected={isDependenciesTab}
|
||||
isExpanded={isNavExpanded}
|
||||
/>
|
||||
)}
|
||||
</Stack>
|
||||
{canAccessTransforms && (
|
||||
<WorkspacesSection isExpanded={isNavExpanded} />
|
||||
)}
|
||||
</Stack>
|
||||
{isSidebarAvailable && (
|
||||
@@ -129,23 +175,286 @@ type DataStudioTabProps = {
|
||||
icon: IconName;
|
||||
to: string;
|
||||
isSelected: boolean;
|
||||
isExpanded: boolean;
|
||||
};
|
||||
|
||||
const TOOLTIP_OPEN_DELAY = 1000;
|
||||
|
||||
function DataStudioTab({ label, icon, to, isSelected }: DataStudioTabProps) {
|
||||
function DataStudioTab({
|
||||
label,
|
||||
icon,
|
||||
to,
|
||||
isSelected,
|
||||
isExpanded,
|
||||
}: DataStudioTabProps) {
|
||||
const content = (
|
||||
<Box
|
||||
className={cx(S.tab, { [S.selected]: isSelected })}
|
||||
component={ForwardRefLink}
|
||||
to={to}
|
||||
p="0.75rem"
|
||||
bdrs="md"
|
||||
>
|
||||
<Flex
|
||||
align="center"
|
||||
gap="sm"
|
||||
justify={isExpanded ? "flex-start" : "center"}
|
||||
>
|
||||
<FixedSizeIcon name={icon} display="block" />
|
||||
{isExpanded && (
|
||||
<Text size="sm" fw={600}>
|
||||
{label}
|
||||
</Text>
|
||||
)}
|
||||
</Flex>
|
||||
</Box>
|
||||
);
|
||||
|
||||
if (!isExpanded) {
|
||||
return (
|
||||
<Tooltip label={label} position="right" openDelay={TOOLTIP_OPEN_DELAY}>
|
||||
{content}
|
||||
</Tooltip>
|
||||
);
|
||||
}
|
||||
|
||||
return content;
|
||||
}
|
||||
|
||||
type WorkspacesSectionProps = {
|
||||
isExpanded: boolean;
|
||||
};
|
||||
|
||||
function WorkspacesSection({
|
||||
isExpanded: isNavExpanded,
|
||||
}: WorkspacesSectionProps) {
|
||||
const dispatch = useDispatch();
|
||||
const [isWorkspacesExpanded, setIsWorkspacesExpanded] = useState(true);
|
||||
const [isCreateModalOpen, setIsCreateModalOpen] = useState(false);
|
||||
const [workspaceName, setWorkspaceName] = useState("");
|
||||
const [selectedDatabaseId, setSelectedDatabaseId] = useState<string | null>(
|
||||
null,
|
||||
);
|
||||
const { pathname } = useSelector(getLocation);
|
||||
const { data: workspacesData, isLoading } = useGetWorkspacesQuery();
|
||||
const { data: databaseData, isLoading: isLoadingDatabases } =
|
||||
useListDatabasesQuery({ include_analytics: true });
|
||||
const [createWorkspace, { isLoading: isCreating }] =
|
||||
useCreateWorkspaceMutation();
|
||||
|
||||
const workspaces = useMemo(
|
||||
() =>
|
||||
[...(workspacesData?.items ?? [])].sort(
|
||||
(a, b) =>
|
||||
new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime(),
|
||||
),
|
||||
[workspacesData],
|
||||
);
|
||||
|
||||
const databaseOptions = useMemo(
|
||||
() =>
|
||||
(databaseData?.data ?? []).map((db: Database) => ({
|
||||
value: String(db.id),
|
||||
label: db.name,
|
||||
})),
|
||||
[databaseData],
|
||||
);
|
||||
|
||||
const handleOpenWorkspace = useCallback(
|
||||
(workspaceId: number) => {
|
||||
dispatch(push(Urls.dataStudioWorkspace(workspaceId)));
|
||||
},
|
||||
[dispatch],
|
||||
);
|
||||
|
||||
const handleOpenCreateModal = useCallback(() => {
|
||||
setWorkspaceName(t`New workspace`);
|
||||
setSelectedDatabaseId(null);
|
||||
setIsCreateModalOpen(true);
|
||||
}, []);
|
||||
|
||||
const handleCloseCreateModal = useCallback(() => {
|
||||
setIsCreateModalOpen(false);
|
||||
setWorkspaceName("");
|
||||
setSelectedDatabaseId(null);
|
||||
}, []);
|
||||
|
||||
const { sendErrorToast, sendSuccessToast } = useMetadataToasts();
|
||||
|
||||
const handleCreateWorkspace = useCallback(async () => {
|
||||
if (!selectedDatabaseId) {
|
||||
sendErrorToast(t`Please select a database`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!workspaceName.trim()) {
|
||||
sendErrorToast(t`Please enter a workspace name`);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const workspace = await createWorkspace({
|
||||
name: workspaceName.trim(),
|
||||
database_id: Number(selectedDatabaseId),
|
||||
upstream: {},
|
||||
}).unwrap();
|
||||
handleCloseCreateModal();
|
||||
handleOpenWorkspace(workspace.id);
|
||||
} catch (error) {
|
||||
sendErrorToast(t`Failed to create workspace`);
|
||||
}
|
||||
}, [
|
||||
createWorkspace,
|
||||
selectedDatabaseId,
|
||||
workspaceName,
|
||||
handleCloseCreateModal,
|
||||
handleOpenWorkspace,
|
||||
sendErrorToast,
|
||||
]);
|
||||
|
||||
const [archiveWorkspace] = useArchiveWorkspaceMutation();
|
||||
const handleWorkspaceArchive = async (id: WorkspaceId) => {
|
||||
try {
|
||||
await archiveWorkspace(id).unwrap();
|
||||
sendSuccessToast(t`Workspace archived successfully`);
|
||||
dispatch(push(Urls.dataStudioWorkspaceList()));
|
||||
} catch (error) {
|
||||
sendErrorToast(t`Failed to archive workspace`);
|
||||
}
|
||||
};
|
||||
|
||||
const isWorkspaceListPage = pathname === Urls.dataStudioWorkspaceList();
|
||||
|
||||
if (!isNavExpanded) {
|
||||
// In collapsed mode, show only an icon button with tooltip
|
||||
return (
|
||||
<Tooltip
|
||||
label={t`Workspaces`}
|
||||
position="right"
|
||||
openDelay={TOOLTIP_OPEN_DELAY}
|
||||
>
|
||||
<UnstyledButton
|
||||
className={cx(S.tab, { [S.selected]: isWorkspaceListPage })}
|
||||
component={ForwardRefLink}
|
||||
to={Urls.dataStudioWorkspaceList()}
|
||||
p="0.75rem"
|
||||
bdrs="md"
|
||||
>
|
||||
<Flex align="center" justify="center">
|
||||
<FixedSizeIcon name="git_branch" display="block" />
|
||||
</Flex>
|
||||
</UnstyledButton>
|
||||
</Tooltip>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<Tooltip label={label} position="right" openDelay={TOOLTIP_OPEN_DELAY}>
|
||||
<Box
|
||||
className={cx(S.tab, { [S.selected]: isSelected })}
|
||||
component={ForwardRefLink}
|
||||
to={to}
|
||||
<Stack gap="0.5rem" className={S.workspacesSection}>
|
||||
<UnstyledButton
|
||||
className={cx(S.workspacesSectionHeader, {
|
||||
[S.selected]: isWorkspaceListPage,
|
||||
})}
|
||||
onClick={() => setIsWorkspacesExpanded(!isWorkspacesExpanded)}
|
||||
p="0.75rem"
|
||||
bdrs="md"
|
||||
>
|
||||
<FixedSizeIcon name={icon} display="block" />
|
||||
</Box>
|
||||
</Tooltip>
|
||||
<Flex align="center" justify="space-between" w="100%">
|
||||
<Text size="sm" fw={600}>
|
||||
{t`Workspaces`}
|
||||
</Text>
|
||||
<Icon
|
||||
name={isWorkspacesExpanded ? "chevrondown" : "chevronright"}
|
||||
size={16}
|
||||
/>
|
||||
</Flex>
|
||||
</UnstyledButton>
|
||||
|
||||
{isWorkspacesExpanded && (
|
||||
<>
|
||||
<UnstyledButton
|
||||
className={S.newWorkspaceButton}
|
||||
onClick={handleOpenCreateModal}
|
||||
disabled={isLoadingDatabases}
|
||||
p="0.5rem"
|
||||
bdrs="md"
|
||||
>
|
||||
<Flex align="center" gap="xs">
|
||||
<Icon name="add" size={16} />
|
||||
<Text size="sm" fw={500}>
|
||||
{t`New workspace`}
|
||||
</Text>
|
||||
</Flex>
|
||||
</UnstyledButton>
|
||||
<Stack gap="0.75rem" style={{ overflowY: "auto", maxHeight: "50vh" }}>
|
||||
{isLoading ? (
|
||||
<>
|
||||
<Skeleton height={80} radius="md" />
|
||||
<Skeleton height={80} radius="md" />
|
||||
</>
|
||||
) : workspaces.length === 0 ? (
|
||||
<Text c="text-secondary" size="xs" px="0.5rem">
|
||||
{t`No workspaces yet`}
|
||||
</Text>
|
||||
) : (
|
||||
workspaces.map((workspace) => {
|
||||
const isSelected =
|
||||
pathname === Urls.dataStudioWorkspace(workspace.id);
|
||||
|
||||
return (
|
||||
<WorkspaceItem
|
||||
key={workspace.id}
|
||||
workspace={workspace}
|
||||
isSelected={isSelected}
|
||||
onOpen={handleOpenWorkspace}
|
||||
archiveWorkspace={handleWorkspaceArchive}
|
||||
/>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</Stack>
|
||||
</>
|
||||
)}
|
||||
|
||||
<Modal
|
||||
opened={isCreateModalOpen}
|
||||
onClose={handleCloseCreateModal}
|
||||
title={t`Create new workspace`}
|
||||
>
|
||||
<Stack gap="md">
|
||||
<TextInput
|
||||
label={t`Workspace name`}
|
||||
placeholder={t`Enter workspace name`}
|
||||
value={workspaceName}
|
||||
onChange={(e) => setWorkspaceName(e.target.value)}
|
||||
required
|
||||
/>
|
||||
<Select
|
||||
label={t`Database`}
|
||||
placeholder={t`Select a database`}
|
||||
data={databaseOptions}
|
||||
value={selectedDatabaseId}
|
||||
onChange={setSelectedDatabaseId}
|
||||
required
|
||||
searchable
|
||||
/>
|
||||
<Flex gap="sm" justify="flex-end">
|
||||
<Button variant="subtle" onClick={handleCloseCreateModal}>
|
||||
{t`Cancel`}
|
||||
</Button>
|
||||
<Button
|
||||
variant="filled"
|
||||
onClick={handleCreateWorkspace}
|
||||
disabled={
|
||||
!selectedDatabaseId || !workspaceName.trim() || isCreating
|
||||
}
|
||||
loading={isCreating}
|
||||
>
|
||||
{t`Create`}
|
||||
</Button>
|
||||
</Flex>
|
||||
</Stack>
|
||||
</Modal>
|
||||
</Stack>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -161,6 +470,7 @@ function DataStudioSidebarToggle({
|
||||
return (
|
||||
<Tooltip
|
||||
label={isSidebarOpened ? t`Close sidebar` : t`Open sidebar`}
|
||||
position="right"
|
||||
openDelay={TOOLTIP_OPEN_DELAY}
|
||||
>
|
||||
<UnstyledButton
|
||||
@@ -176,3 +486,100 @@ function DataStudioSidebarToggle({
|
||||
</Tooltip>
|
||||
);
|
||||
}
|
||||
|
||||
type DataStudioNavToggleProps = {
|
||||
isNavExpanded: boolean;
|
||||
onNavToggle: (isExpanded: boolean) => void;
|
||||
};
|
||||
|
||||
function DataStudioNavToggle({
|
||||
isNavExpanded,
|
||||
onNavToggle,
|
||||
}: DataStudioNavToggleProps) {
|
||||
return (
|
||||
<Tooltip
|
||||
label={isNavExpanded ? t`Collapse navigation` : t`Expand navigation`}
|
||||
position="right"
|
||||
openDelay={TOOLTIP_OPEN_DELAY}
|
||||
>
|
||||
<UnstyledButton
|
||||
className={S.toggle}
|
||||
p="0.75rem"
|
||||
bdrs="md"
|
||||
onClick={() => onNavToggle(!isNavExpanded)}
|
||||
>
|
||||
<FixedSizeIcon name={isNavExpanded ? "chevronleft" : "chevronright"} />
|
||||
</UnstyledButton>
|
||||
</Tooltip>
|
||||
);
|
||||
}
|
||||
|
||||
interface WorkspaceItemProps {
|
||||
workspace: { id: number; name: string; updated_at: string };
|
||||
isSelected: boolean;
|
||||
onOpen: (workspaceId: WorkspaceId) => void;
|
||||
archiveWorkspace: (workspaceId: WorkspaceId) => Promise<void>;
|
||||
}
|
||||
|
||||
function WorkspaceItem({
|
||||
workspace,
|
||||
isSelected,
|
||||
onOpen,
|
||||
archiveWorkspace,
|
||||
}: WorkspaceItemProps) {
|
||||
const timeAgo = dayjs(workspace.updated_at).fromNow();
|
||||
const handleArchive = () => {
|
||||
archiveWorkspace(workspace.id);
|
||||
};
|
||||
|
||||
const handleDelete = () => {
|
||||
archiveWorkspace(workspace.id);
|
||||
};
|
||||
|
||||
return (
|
||||
<UnstyledButton
|
||||
className={cx(S.workspaceItem, { [S.selected]: isSelected })}
|
||||
onClick={() => onOpen(workspace.id)}
|
||||
p="0.75rem"
|
||||
bdrs="md"
|
||||
>
|
||||
<Flex align="flex-start" justify="space-between" gap="xs">
|
||||
<Stack gap="xs" style={{ flex: 1, minWidth: 0 }}>
|
||||
<Text size="sm" fw={600} truncate>
|
||||
{workspace.name}
|
||||
</Text>
|
||||
<Text size="xs" c="text-secondary" truncate>
|
||||
{t`Updated ${timeAgo}`}
|
||||
</Text>
|
||||
</Stack>
|
||||
<Menu position="right" withinPortal>
|
||||
<Menu.Target>
|
||||
<ActionIcon
|
||||
className={S.workspaceMenuButton}
|
||||
onClick={(e: React.MouseEvent) => e.stopPropagation()}
|
||||
size="sm"
|
||||
variant="subtle"
|
||||
>
|
||||
<Icon name="ellipsis" size={16} />
|
||||
</ActionIcon>
|
||||
</Menu.Target>
|
||||
<Menu.Dropdown onClick={(e: React.MouseEvent) => e.stopPropagation()}>
|
||||
<Menu.Item
|
||||
leftSection={<Icon name="archive" />}
|
||||
onClick={handleArchive}
|
||||
>
|
||||
{t`Archive`}
|
||||
</Menu.Item>
|
||||
<Menu.Divider />
|
||||
<Menu.Item
|
||||
leftSection={<Icon name="trash" />}
|
||||
onClick={handleDelete}
|
||||
>
|
||||
{t`Move to trash`}
|
||||
</Menu.Item>
|
||||
</Menu.Dropdown>
|
||||
</Menu>
|
||||
</Flex>
|
||||
</UnstyledButton>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import { t } from "ttag";
|
||||
|
||||
import { usePageTitle } from "metabase/hooks/use-page-title";
|
||||
|
||||
import { SectionLayout, SectionTitle } from "../../components/SectionLayout";
|
||||
|
||||
type WorkspacesSectionLayoutProps = {
|
||||
location: Location;
|
||||
children?: React.ReactNode;
|
||||
};
|
||||
|
||||
export function WorkspacesSectionLayout({
|
||||
children,
|
||||
}: WorkspacesSectionLayoutProps) {
|
||||
usePageTitle(t`Workspaces`);
|
||||
|
||||
return (
|
||||
<SectionLayout title={<SectionTitle title={t`Workspaces`} />}>
|
||||
{children}
|
||||
</SectionLayout>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export * from "./WorkspacesSectionLayout";
|
||||
@@ -15,12 +15,14 @@ import { DataStudioLayout } from "./app/pages/DataStudioLayout";
|
||||
import { DependenciesSectionLayout } from "./app/pages/DependenciesSectionLayout";
|
||||
import { ModelingSectionLayout } from "./app/pages/ModelingSectionLayout";
|
||||
import { TransformsSectionLayout } from "./app/pages/TransformsSectionLayout";
|
||||
import { WorkspacesSectionLayout } from "./app/pages/WorkspacesSectionLayout";
|
||||
import { getDataStudioMetadataRoutes } from "./data-model/routes";
|
||||
import { getDataStudioGlossaryRoutes } from "./glossary/routes";
|
||||
import { getDataStudioMetricRoutes } from "./metrics/routes";
|
||||
import { getDataStudioModelingRoutes } from "./modeling/routes";
|
||||
import { getDataStudioModelRoutes } from "./models/routes";
|
||||
import { getDataStudioSnippetRoutes } from "./snippets/routes";
|
||||
import { getDataStudioWorkspaceRoutes } from "./workspaces/routes";
|
||||
|
||||
export function getDataStudioRoutes(
|
||||
store: Store<State>,
|
||||
@@ -55,6 +57,9 @@ export function getDataStudioRoutes(
|
||||
{getDataStudioSnippetRoutes()}
|
||||
{getDataStudioGlossaryRoutes()}
|
||||
</Route>
|
||||
<Route path="workspaces" component={WorkspacesSectionLayout}>
|
||||
{getDataStudioWorkspaceRoutes()}
|
||||
</Route>
|
||||
{PLUGIN_DEPENDENCIES.isEnabled && (
|
||||
<Route path="dependencies" component={DependenciesSectionLayout}>
|
||||
{PLUGIN_DEPENDENCIES.getDataStudioDependencyRoutes()}
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
import { t } from "ttag";
|
||||
|
||||
import { ForwardRefLink } from "metabase/common/components/Link";
|
||||
import * as Urls from "metabase/lib/urls";
|
||||
import { Anchor, Box, Card, Group, Stack, Text, Title } from "metabase/ui";
|
||||
import { useGetWorkspacesQuery } from "metabase-enterprise/api";
|
||||
|
||||
export function WorkspaceListPage() {
|
||||
const { data: workspacesData, isLoading } = useGetWorkspacesQuery();
|
||||
|
||||
if (isLoading) {
|
||||
return (
|
||||
<Box p="lg">
|
||||
<Text>{t`Loading...`}</Text>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
const workspaces = workspacesData?.items ?? [];
|
||||
|
||||
return (
|
||||
<Box p="lg">
|
||||
<Title order={2} mb="lg">{t`Workspaces`}</Title>
|
||||
{workspaces.length === 0 ? (
|
||||
<Text c="text-secondary">{t`No workspaces yet`}</Text>
|
||||
) : (
|
||||
<Stack gap="md">
|
||||
{workspaces.map((workspace) => (
|
||||
<Card key={workspace.id} p="md" shadow="none" withBorder>
|
||||
<Group justify="space-between">
|
||||
<Anchor
|
||||
component={ForwardRefLink}
|
||||
to={Urls.dataStudioWorkspace(workspace.id)}
|
||||
fw={500}
|
||||
>
|
||||
{workspace.name}
|
||||
</Anchor>
|
||||
<Text c="text-secondary" size="sm">
|
||||
{t`Created ${new Date(workspace.created_at).toLocaleDateString()}`}
|
||||
</Text>
|
||||
</Group>
|
||||
</Card>
|
||||
))}
|
||||
</Stack>
|
||||
)}
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
import { useMemo } from "react";
|
||||
import { t } from "ttag";
|
||||
|
||||
import { checkNotNull } from "metabase/lib/types";
|
||||
import { Stack, Text } from "metabase/ui";
|
||||
import { isSameSource } from "metabase-enterprise/transforms/utils";
|
||||
import type { Transform } from "metabase-types/api";
|
||||
|
||||
import { useWorkspace } from "../WorkspaceProvider";
|
||||
|
||||
import { TransformListItem } from "./TransformListItem";
|
||||
|
||||
type CodeTabProps = {
|
||||
workspaceTransforms: Transform[];
|
||||
transforms: Transform[];
|
||||
activeTransformId?: number;
|
||||
onTransformClick: (transform: Transform) => void;
|
||||
};
|
||||
|
||||
export const CodeTab = ({
|
||||
workspaceTransforms,
|
||||
transforms,
|
||||
activeTransformId,
|
||||
onTransformClick,
|
||||
}: CodeTabProps) => {
|
||||
const { editedTransforms } = useWorkspace();
|
||||
const availableTransforms = useMemo(() => {
|
||||
return transforms.filter((transform) => {
|
||||
return (
|
||||
transform.workspace_id == null &&
|
||||
!workspaceTransforms.some((t) => t.id === transform.id)
|
||||
);
|
||||
});
|
||||
}, [workspaceTransforms, transforms]);
|
||||
|
||||
const handleTransformClick = (transform: Transform) => {
|
||||
const edited = editedTransforms.get(transform.id);
|
||||
const transformToOpen = edited
|
||||
? ({ ...transform, ...edited } as Transform)
|
||||
: transform;
|
||||
onTransformClick(transformToOpen);
|
||||
};
|
||||
|
||||
return (
|
||||
<Stack h="100%" gap={0}>
|
||||
{workspaceTransforms.length > 0 && (
|
||||
<Stack
|
||||
gap="xs"
|
||||
pb="sm"
|
||||
style={{
|
||||
borderBottom: "1px solid var(--mb-color-border)",
|
||||
}}
|
||||
>
|
||||
<Stack gap={0}>
|
||||
<Text fw={600}>{t`Workspace Transforms`}</Text>
|
||||
{workspaceTransforms.map((transform) => {
|
||||
return (
|
||||
<TransformListItem
|
||||
key={transform.id}
|
||||
name={transform.name}
|
||||
icon="pivot_table"
|
||||
fw={600}
|
||||
isActive={activeTransformId === transform.id}
|
||||
isEdited={
|
||||
editedTransforms.has(transform.id) &&
|
||||
!isSameSource(
|
||||
checkNotNull(editedTransforms.get(transform.id)?.source),
|
||||
transform.source,
|
||||
)
|
||||
}
|
||||
onClick={() => handleTransformClick(transform)}
|
||||
/>
|
||||
);
|
||||
})}
|
||||
</Stack>
|
||||
</Stack>
|
||||
)}
|
||||
<Stack py="sm" gap="xs">
|
||||
{availableTransforms.map((transform) => (
|
||||
<TransformListItem
|
||||
key={transform.id}
|
||||
name={transform.name}
|
||||
isActive={activeTransformId === transform.id}
|
||||
isEdited={editedTransforms.has(transform.id)}
|
||||
onClick={() => handleTransformClick(transform)}
|
||||
/>
|
||||
))}
|
||||
</Stack>
|
||||
</Stack>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,26 @@
|
||||
.root {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--mantine-spacing-sm);
|
||||
padding: var(--mantine-spacing-xs) var(--mantine-spacing-sm);
|
||||
border-radius: var(--mantine-radius-sm);
|
||||
cursor: pointer;
|
||||
flex-wrap: nowrap;
|
||||
|
||||
&:hover {
|
||||
background-color: var(--mb-color-bg-light);
|
||||
}
|
||||
}
|
||||
|
||||
.name {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.statusDot {
|
||||
width: 0.5rem;
|
||||
height: 0.5rem;
|
||||
border-radius: 50%;
|
||||
background-color: var(--mb-color-warning);
|
||||
flex-shrink: 0;
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import { Box, type BoxProps, Icon, type IconName, Text } from "metabase/ui";
|
||||
|
||||
import styles from "./TransformListItem.module.css";
|
||||
|
||||
type TransformListItemProps = {
|
||||
name: string;
|
||||
icon?: IconName;
|
||||
isActive?: boolean;
|
||||
isEdited?: boolean;
|
||||
onClick?: () => void;
|
||||
} & BoxProps;
|
||||
|
||||
export const TransformListItem = ({
|
||||
name,
|
||||
icon = "database",
|
||||
isActive,
|
||||
isEdited,
|
||||
onClick,
|
||||
...props
|
||||
}: TransformListItemProps) => {
|
||||
return (
|
||||
<Box className={styles.root} onClick={onClick} {...props}>
|
||||
<Icon name={icon} size={14} c="var(--mb-color-brand)" />
|
||||
<Text
|
||||
className={styles.name}
|
||||
c={isActive ? "var(--mb-color-brand)" : "text-dark"}
|
||||
truncate
|
||||
>
|
||||
{name}
|
||||
</Text>
|
||||
{isEdited && <Box className={styles.statusDot} />}
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,12 @@
|
||||
import { PLUGIN_METABOT } from "metabase/plugins";
|
||||
|
||||
export const MetabotTab = () => {
|
||||
const MetabotProvider = PLUGIN_METABOT.getMetabotProvider();
|
||||
const MetabotChat = PLUGIN_METABOT.MetabotChat;
|
||||
|
||||
return (
|
||||
<MetabotProvider>
|
||||
<MetabotChat />
|
||||
</MetabotProvider>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,30 @@
|
||||
import { t } from "ttag";
|
||||
|
||||
import { Stack, Text, Title } from "metabase/ui";
|
||||
|
||||
interface SetupTabProps {
|
||||
databaseName?: string;
|
||||
}
|
||||
|
||||
export const SetupTab = ({ databaseName }: SetupTabProps) => {
|
||||
return (
|
||||
<Stack gap="lg">
|
||||
<Stack gap="xs">
|
||||
<Title order={3}>{t`Workspace Setup`}</Title>
|
||||
<Text c="text-medium">
|
||||
{t`Configure your data warehouse connection for this workspace`}
|
||||
</Text>
|
||||
</Stack>
|
||||
|
||||
<Stack gap="xs">
|
||||
<Text fw={600}>{t`Data Warehouse`}</Text>
|
||||
<Text>
|
||||
<code>{databaseName}</code>
|
||||
</Text>
|
||||
<Text c="text-light" size="sm">
|
||||
{t`Data warehouse selection is locked after editing or timeout`}
|
||||
</Text>
|
||||
</Stack>
|
||||
</Stack>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,49 @@
|
||||
import { useState } from "react";
|
||||
import _ from "underscore";
|
||||
|
||||
import {
|
||||
TransformQueryPageEditor,
|
||||
type TransformQueryPageEditorUiState,
|
||||
} from "metabase-enterprise/transforms/pages/TransformQueryPage/TransformQueryPage";
|
||||
import type { DraftTransformSource } from "metabase-types/api";
|
||||
|
||||
type TransformEditorProps = {
|
||||
source: DraftTransformSource;
|
||||
onChange: (source: DraftTransformSource) => void;
|
||||
};
|
||||
|
||||
export function TransformEditor({ source, onChange }: TransformEditorProps) {
|
||||
const [uiState, setUiState] = useState(getInitialUiStateForTransform);
|
||||
const [uiOptions] = useState({
|
||||
canChangeDatabase: false,
|
||||
});
|
||||
|
||||
const handleSourceChange = (source: DraftTransformSource) => {
|
||||
onChange?.(source);
|
||||
};
|
||||
|
||||
return (
|
||||
<TransformQueryPageEditor
|
||||
source={source}
|
||||
isDirty={false}
|
||||
uiState={uiState}
|
||||
setUiState={setUiState}
|
||||
uiOptions={uiOptions}
|
||||
databases={[]}
|
||||
setSourceAndRejectProposed={handleSourceChange}
|
||||
acceptProposed={_.noop}
|
||||
rejectProposed={_.noop}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
function getInitialUiStateForTransform() {
|
||||
return {
|
||||
lastRunResult: null,
|
||||
lastRunQuery: null,
|
||||
selectionRange: [],
|
||||
modalSnippet: null,
|
||||
sidebarType: null,
|
||||
modalType: null,
|
||||
} as TransformQueryPageEditorUiState;
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
import { t } from "ttag";
|
||||
|
||||
import { Box, Button, Group, Icon, Stack } from "metabase/ui";
|
||||
import {
|
||||
useLazyGetTransformQuery,
|
||||
useRunTransformJobMutation,
|
||||
useRunTransformMutation,
|
||||
useUpdateTransformMutation,
|
||||
useUpdateWorkspaceContentsMutation,
|
||||
} from "metabase-enterprise/api";
|
||||
import type {
|
||||
DraftTransformSource,
|
||||
Transform,
|
||||
TransformId,
|
||||
WorkspaceId,
|
||||
} from "metabase-types/api";
|
||||
|
||||
import { TransformEditor } from "./TransformEditor";
|
||||
import { type EditedTransform, useWorkspace } from "./WorkspaceProvider";
|
||||
|
||||
interface Props {
|
||||
editedTransform: EditedTransform;
|
||||
transform: Transform;
|
||||
workspaceId: WorkspaceId;
|
||||
onChange: (source: DraftTransformSource) => void;
|
||||
onOpenTransform: (transformId: TransformId) => void;
|
||||
}
|
||||
|
||||
export const TransformTab = ({
|
||||
editedTransform,
|
||||
transform,
|
||||
workspaceId,
|
||||
onChange,
|
||||
onOpenTransform,
|
||||
}: Props) => {
|
||||
const [getTransform] = useLazyGetTransformQuery();
|
||||
const [updateTransform] = useUpdateTransformMutation();
|
||||
const [updateWorkspaceContents] = useUpdateWorkspaceContentsMutation();
|
||||
useRunTransformJobMutation();
|
||||
|
||||
const {
|
||||
addOpenedTransform,
|
||||
removeOpenedTransform,
|
||||
setActiveTransform,
|
||||
markTransformAsRun,
|
||||
} = useWorkspace();
|
||||
|
||||
const isSaved = transform.workspace_id === workspaceId;
|
||||
|
||||
const [runTransform] = useRunTransformMutation();
|
||||
|
||||
const handleRun = async () => {
|
||||
try {
|
||||
await runTransform(transform.id).unwrap();
|
||||
markTransformAsRun(transform.id);
|
||||
} catch (error) {
|
||||
console.error("Failed to run transform", error);
|
||||
}
|
||||
};
|
||||
|
||||
const handleSave = async () => {
|
||||
if (isSaved) {
|
||||
updateTransform({
|
||||
id: transform.id,
|
||||
source: editedTransform.source,
|
||||
});
|
||||
} else {
|
||||
const response = await updateWorkspaceContents({
|
||||
id: workspaceId,
|
||||
add: {
|
||||
transforms: [transform.id],
|
||||
},
|
||||
});
|
||||
|
||||
const newTransform = response.data?.contents.transforms.find(
|
||||
(t) => t.upstream_id === transform.id,
|
||||
);
|
||||
const newTransformId = newTransform?.id;
|
||||
|
||||
if (newTransformId != null) {
|
||||
// TODO: remove when backend adds contents hydration to previous request
|
||||
const newTransform = await getTransform(newTransformId).unwrap();
|
||||
|
||||
if (newTransform) {
|
||||
addOpenedTransform(newTransform);
|
||||
removeOpenedTransform(transform.id);
|
||||
setActiveTransform(newTransform);
|
||||
onOpenTransform(newTransform.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<Stack gap={0} h="100%">
|
||||
<Group
|
||||
flex="0 0 auto"
|
||||
justify="space-between"
|
||||
p="md"
|
||||
style={{ borderBottom: "1px solid var(--mb-color-border" }}
|
||||
>
|
||||
<Group>{t`Output table input?`}</Group>
|
||||
|
||||
<Group>
|
||||
{isSaved && (
|
||||
<Button
|
||||
leftSection={<Icon name="play" />}
|
||||
size="sm"
|
||||
onClick={handleRun}
|
||||
>{t`Run`}</Button>
|
||||
)}
|
||||
|
||||
<Button
|
||||
size="sm"
|
||||
variant="filled"
|
||||
onClick={handleSave}
|
||||
>{t`Save`}</Button>
|
||||
</Group>
|
||||
</Group>
|
||||
|
||||
{editedTransform && (
|
||||
<Box flex="1">
|
||||
<TransformEditor
|
||||
source={editedTransform.source}
|
||||
onChange={onChange}
|
||||
/>
|
||||
</Box>
|
||||
)}
|
||||
</Stack>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,20 @@
|
||||
.tabsPanel {
|
||||
display: flex;
|
||||
gap: 0.5rem;
|
||||
flex-wrap: nowrap;
|
||||
overflow: auto;
|
||||
scrollbar-width: none;
|
||||
-ms-overflow-style: none;
|
||||
|
||||
&::-webkit-scrollbar {
|
||||
display: none;
|
||||
}
|
||||
|
||||
&::before {
|
||||
display: none;
|
||||
}
|
||||
|
||||
& > button {
|
||||
border-radius: 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,339 @@
|
||||
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||
import { push } from "react-router-redux";
|
||||
import { t } from "ttag";
|
||||
|
||||
import { useListDatabasesQuery } from "metabase/api";
|
||||
import { useDispatch } from "metabase/lib/redux";
|
||||
import * as Urls from "metabase/lib/urls";
|
||||
import { useMetadataToasts } from "metabase/metadata/hooks";
|
||||
import { PLUGIN_METABOT } from "metabase/plugins";
|
||||
import {
|
||||
ActionIcon,
|
||||
Box,
|
||||
Button,
|
||||
Flex,
|
||||
Group,
|
||||
Icon,
|
||||
Stack,
|
||||
Tabs,
|
||||
Text,
|
||||
Title,
|
||||
} from "metabase/ui";
|
||||
import {
|
||||
useGetWorkspaceQuery,
|
||||
useListTransformsQuery,
|
||||
useMergeWorkspaceMutation,
|
||||
} from "metabase-enterprise/api";
|
||||
import type { DraftTransformSource, Transform } from "metabase-types/api";
|
||||
|
||||
import { CodeTab } from "./CodeTab/CodeTab";
|
||||
import { MetabotTab } from "./MetabotTab";
|
||||
import { SetupTab } from "./SetupTab";
|
||||
import { TransformTab } from "./TransformTab";
|
||||
import styles from "./WorkspacePage.module.css";
|
||||
import { WorkspaceProvider, useWorkspace } from "./WorkspaceProvider";
|
||||
|
||||
type WorkspacePageProps = {
|
||||
params: {
|
||||
workspaceId: string;
|
||||
};
|
||||
};
|
||||
|
||||
function WorkspacePageContent({ params }: WorkspacePageProps) {
|
||||
const id = Number(params.workspaceId);
|
||||
const dispatch = useDispatch();
|
||||
const { sendErrorToast } = useMetadataToasts();
|
||||
const isMetabotAvailable = PLUGIN_METABOT.isEnabled();
|
||||
const [tab, setTab] = useState<string>("setup");
|
||||
|
||||
const { data: databases = { data: [] } } = useListDatabasesQuery({});
|
||||
|
||||
const { data: allTransforms = [] } = useListTransformsQuery({});
|
||||
const { data: workspace, isLoading: isLoadingWorkspace } =
|
||||
useGetWorkspaceQuery(id);
|
||||
|
||||
const [mergeWorkspace, { isLoading: isMerging }] =
|
||||
useMergeWorkspaceMutation();
|
||||
|
||||
const sourceDb = databases?.data.find(
|
||||
(db) => db.id === workspace?.database_id,
|
||||
);
|
||||
|
||||
const dbTransforms = useMemo(
|
||||
() =>
|
||||
allTransforms.filter((t) => {
|
||||
if (t.source_type === "python") {
|
||||
return t.source["source-database"] === sourceDb?.id;
|
||||
}
|
||||
if (t.source_type === "native") {
|
||||
return t.source.query.database === sourceDb?.id;
|
||||
}
|
||||
return false;
|
||||
}),
|
||||
[allTransforms, sourceDb],
|
||||
);
|
||||
|
||||
const {
|
||||
openedTransforms,
|
||||
activeTransform,
|
||||
activeEditedTransform,
|
||||
setActiveTransform,
|
||||
addOpenedTransform,
|
||||
removeOpenedTransform,
|
||||
setEditedTransform,
|
||||
hasChangedAndRunTransforms,
|
||||
} = useWorkspace();
|
||||
|
||||
const workspaceTransforms = useMemo(
|
||||
() => dbTransforms.filter((transform) => transform.workspace_id === id),
|
||||
[dbTransforms, id],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (activeTransform) {
|
||||
setTab(String(activeTransform.id));
|
||||
} else {
|
||||
setTab("setup");
|
||||
}
|
||||
}, [id, activeTransform]);
|
||||
|
||||
const handleTransformChange = useCallback(
|
||||
(source: DraftTransformSource) => {
|
||||
if (activeTransform) {
|
||||
setEditedTransform(activeTransform.id, {
|
||||
name: activeTransform.name,
|
||||
source,
|
||||
target: {
|
||||
name: activeTransform.target.name,
|
||||
type: activeTransform.target.type,
|
||||
},
|
||||
});
|
||||
}
|
||||
},
|
||||
[activeTransform, setEditedTransform],
|
||||
);
|
||||
|
||||
const handleCloseClick = useCallback(
|
||||
(event: React.MouseEvent, transform: Transform, index: number) => {
|
||||
event.stopPropagation();
|
||||
|
||||
const isActive = activeTransform?.id === transform.id;
|
||||
const remaining = openedTransforms.filter(
|
||||
(item) => item.id !== transform.id,
|
||||
);
|
||||
|
||||
removeOpenedTransform(transform.id);
|
||||
|
||||
if (!isActive) {
|
||||
return;
|
||||
}
|
||||
|
||||
const fallback = remaining[index - 1] ?? remaining[index] ?? undefined;
|
||||
setActiveTransform(fallback);
|
||||
|
||||
if (fallback) {
|
||||
setTab(String(fallback.id));
|
||||
} else {
|
||||
setTab("setup");
|
||||
}
|
||||
},
|
||||
[
|
||||
activeTransform,
|
||||
removeOpenedTransform,
|
||||
setActiveTransform,
|
||||
setTab,
|
||||
openedTransforms,
|
||||
],
|
||||
);
|
||||
|
||||
const handleMergeWorkspace = useCallback(async () => {
|
||||
try {
|
||||
const response = await mergeWorkspace(id).unwrap();
|
||||
|
||||
if (response.errors && response.errors.length > 0) {
|
||||
sendErrorToast(
|
||||
t`Failed to merge workspace: ${response.errors.map((e) => e.error).join(", ")}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
dispatch(push(Urls.dataStudioWorkspaceList()));
|
||||
} catch (error) {
|
||||
sendErrorToast(t`Failed to merge workspace`);
|
||||
}
|
||||
}, [id, mergeWorkspace, sendErrorToast, dispatch]);
|
||||
|
||||
if (isLoadingWorkspace) {
|
||||
return (
|
||||
<Box p="lg">
|
||||
<Text>{t`Loading...`}</Text>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
if (!workspace) {
|
||||
return (
|
||||
<Box p="lg">
|
||||
<Text c="text-dark">{t`Workspace not found`}</Text>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<Stack h="100%" gap={0}>
|
||||
<Group
|
||||
px="lg"
|
||||
py="md"
|
||||
style={{ borderBottom: "1px solid var(--mb-color-border)" }}
|
||||
justify="space-between"
|
||||
>
|
||||
<Title order={2}>{workspace.name}</Title>
|
||||
<Button
|
||||
variant="filled"
|
||||
onClick={handleMergeWorkspace}
|
||||
loading={isMerging}
|
||||
disabled={!hasChangedAndRunTransforms()}
|
||||
size="xs"
|
||||
>
|
||||
{t`Merge`}
|
||||
</Button>
|
||||
</Group>
|
||||
<Group align="flex-start" gap={0} flex="1 1 auto" wrap="nowrap">
|
||||
<Box
|
||||
w="70%"
|
||||
h="100%"
|
||||
style={{ borderRight: "1px solid var(--mb-color-border)" }}
|
||||
pos="relative"
|
||||
>
|
||||
<Tabs
|
||||
defaultValue="setup"
|
||||
display="flex"
|
||||
h="100%"
|
||||
style={{ flexDirection: "column" }}
|
||||
value={tab}
|
||||
onChange={(tab) => {
|
||||
if (tab) {
|
||||
setTab(tab);
|
||||
}
|
||||
if (tab === "setup" || (tab === "metabot" && activeTransform)) {
|
||||
setActiveTransform(undefined);
|
||||
}
|
||||
}}
|
||||
>
|
||||
<Flex
|
||||
wrap="nowrap"
|
||||
flex="0 0 auto"
|
||||
px="md"
|
||||
style={{ borderBottom: "1px solid var(--mb-color-border)" }}
|
||||
>
|
||||
<Tabs.List className={styles.tabsPanel}>
|
||||
<Tabs.Tab value="setup">
|
||||
<Group gap="xs" wrap="nowrap">
|
||||
<Icon name="database" aria-hidden />
|
||||
{t`Setup`}
|
||||
</Group>
|
||||
</Tabs.Tab>
|
||||
{isMetabotAvailable && (
|
||||
<Tabs.Tab value="metabot">
|
||||
<Group gap="xs" wrap="nowrap">
|
||||
<Icon name="message_circle" aria-hidden />
|
||||
{t`Agent Chat`}
|
||||
</Group>
|
||||
</Tabs.Tab>
|
||||
)}
|
||||
{openedTransforms.map((transform, index) => (
|
||||
<Tabs.Tab
|
||||
key={transform.id}
|
||||
value={String(transform.id)}
|
||||
onClick={() => {
|
||||
setActiveTransform(transform);
|
||||
}}
|
||||
>
|
||||
<Group gap="xs" wrap="nowrap">
|
||||
<Icon name="pivot_table" aria-hidden />
|
||||
{transform.name}
|
||||
<ActionIcon size="1rem" p="0" ml="xs">
|
||||
<Icon
|
||||
name="close"
|
||||
size={10}
|
||||
aria-hidden
|
||||
onClick={(event) =>
|
||||
handleCloseClick(event, transform, index)
|
||||
}
|
||||
/>
|
||||
</ActionIcon>
|
||||
</Group>
|
||||
</Tabs.Tab>
|
||||
))}
|
||||
</Tabs.List>
|
||||
</Flex>
|
||||
|
||||
<Box flex={1} mih={0}>
|
||||
<Tabs.Panel value="setup" h="100%" p="md">
|
||||
<SetupTab databaseName={sourceDb?.name} />
|
||||
</Tabs.Panel>
|
||||
{isMetabotAvailable && (
|
||||
<Tabs.Panel value="metabot" h="100%">
|
||||
<MetabotTab />
|
||||
</Tabs.Panel>
|
||||
)}
|
||||
|
||||
<Tabs.Panel value={String(activeTransform?.id)} h="100%">
|
||||
{openedTransforms.length === 0 ||
|
||||
!activeTransform ||
|
||||
!activeEditedTransform ? (
|
||||
<Text c="text-medium">
|
||||
{t`Select a transform on the right.`}
|
||||
</Text>
|
||||
) : (
|
||||
<TransformTab
|
||||
transform={activeTransform}
|
||||
editedTransform={activeEditedTransform}
|
||||
workspaceId={id}
|
||||
onChange={handleTransformChange}
|
||||
onOpenTransform={(transformId) =>
|
||||
setTab(String(transformId))
|
||||
}
|
||||
/>
|
||||
)}
|
||||
</Tabs.Panel>
|
||||
</Box>
|
||||
</Tabs>
|
||||
</Box>
|
||||
<Box style={{ flex: "1 0 auto", width: "30%" }}>
|
||||
<Tabs defaultValue="code">
|
||||
<Box
|
||||
px="md"
|
||||
style={{ borderBottom: "1px solid var(--mb-color-border)" }}
|
||||
>
|
||||
<Tabs.List className={styles.tabsPanel}>
|
||||
<Tabs.Tab value="code">{t`Code`}</Tabs.Tab>
|
||||
</Tabs.List>
|
||||
</Box>
|
||||
<Tabs.Panel value="code" p="md">
|
||||
<CodeTab
|
||||
workspaceTransforms={workspaceTransforms}
|
||||
transforms={dbTransforms}
|
||||
activeTransformId={activeTransform?.id}
|
||||
onTransformClick={(transform) => {
|
||||
setTab(String(transform.id));
|
||||
addOpenedTransform(transform);
|
||||
setActiveTransform(transform);
|
||||
}}
|
||||
/>
|
||||
</Tabs.Panel>
|
||||
</Tabs>
|
||||
</Box>
|
||||
</Group>
|
||||
</Stack>
|
||||
);
|
||||
}
|
||||
|
||||
export const WorkspacePage = ({ params }: WorkspacePageProps) => {
|
||||
const workspaceId = Number(params.workspaceId);
|
||||
|
||||
return (
|
||||
<WorkspaceProvider workspaceId={workspaceId}>
|
||||
<WorkspacePageContent params={params} />
|
||||
</WorkspaceProvider>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,233 @@
|
||||
import type { ReactNode } from "react";
|
||||
import {
|
||||
createContext,
|
||||
useCallback,
|
||||
useContext,
|
||||
useMemo,
|
||||
useState,
|
||||
} from "react";
|
||||
|
||||
import type {
|
||||
DraftTransformSource,
|
||||
Transform,
|
||||
TransformTargetType,
|
||||
} from "metabase-types/api";
|
||||
|
||||
export interface EditedTransform {
|
||||
name: string;
|
||||
source: DraftTransformSource;
|
||||
target: {
|
||||
name: string;
|
||||
type: TransformTargetType;
|
||||
};
|
||||
}
|
||||
|
||||
export interface WorkspaceContextValue {
|
||||
openedTransforms: Transform[];
|
||||
activeTransform: Transform | undefined;
|
||||
activeEditedTransform: EditedTransform | undefined;
|
||||
setActiveTransform: (transform: Transform | undefined) => void;
|
||||
addOpenedTransform: (transform: Transform) => void;
|
||||
removeOpenedTransform: (transformId: number) => void;
|
||||
editedTransforms: Map<number, EditedTransform>;
|
||||
setEditedTransform: (transformId: number, data: EditedTransform) => void;
|
||||
removeEditedTransform: (transformId: number) => void;
|
||||
runTransforms: Set<number>;
|
||||
markTransformAsRun: (transformId: number) => void;
|
||||
hasChangedAndRunTransforms: () => boolean;
|
||||
}
|
||||
|
||||
const WorkspaceContext = createContext<WorkspaceContextValue | undefined>(
|
||||
undefined,
|
||||
);
|
||||
|
||||
interface WorkspaceState {
|
||||
openedTransforms: Transform[];
|
||||
activeTransform: Transform | undefined;
|
||||
editedTransforms: Map<number, EditedTransform>;
|
||||
runTransforms: Set<number>;
|
||||
}
|
||||
|
||||
interface WorkspaceProviderProps {
|
||||
children: ReactNode;
|
||||
workspaceId: number;
|
||||
}
|
||||
|
||||
const createEmptyWorkspaceState = (): WorkspaceState => ({
|
||||
openedTransforms: [],
|
||||
activeTransform: undefined,
|
||||
editedTransforms: new Map(),
|
||||
runTransforms: new Set(),
|
||||
});
|
||||
|
||||
export const WorkspaceProvider = ({
|
||||
children,
|
||||
workspaceId,
|
||||
}: WorkspaceProviderProps) => {
|
||||
const [workspaceStates, setWorkspaceStates] = useState<
|
||||
Map<number, WorkspaceState>
|
||||
>(new Map());
|
||||
|
||||
const currentState = useMemo(() => {
|
||||
const existing = workspaceStates.get(workspaceId);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
const newState = createEmptyWorkspaceState();
|
||||
setWorkspaceStates((prev) => new Map(prev).set(workspaceId, newState));
|
||||
return newState;
|
||||
}, [workspaceId, workspaceStates]);
|
||||
|
||||
const { openedTransforms, activeTransform, editedTransforms, runTransforms } =
|
||||
currentState;
|
||||
|
||||
const updateWorkspaceState = useCallback(
|
||||
(updater: (state: WorkspaceState) => WorkspaceState) => {
|
||||
setWorkspaceStates((prev) => {
|
||||
const currentState =
|
||||
prev.get(workspaceId) ?? createEmptyWorkspaceState();
|
||||
const newState = updater(currentState);
|
||||
return new Map(prev).set(workspaceId, newState);
|
||||
});
|
||||
},
|
||||
[workspaceId],
|
||||
);
|
||||
|
||||
const setActiveTransform = useCallback(
|
||||
(transform: Transform | undefined) => {
|
||||
updateWorkspaceState((state) => ({
|
||||
...state,
|
||||
activeTransform: transform,
|
||||
}));
|
||||
},
|
||||
[updateWorkspaceState],
|
||||
);
|
||||
|
||||
const addOpenedTransform = useCallback(
|
||||
(transform: Transform) => {
|
||||
updateWorkspaceState((state) => {
|
||||
const exists = state.openedTransforms.some(
|
||||
(item) => item.id === transform.id,
|
||||
);
|
||||
if (exists) {
|
||||
return state;
|
||||
}
|
||||
return {
|
||||
...state,
|
||||
openedTransforms: [...state.openedTransforms, transform],
|
||||
};
|
||||
});
|
||||
},
|
||||
[updateWorkspaceState],
|
||||
);
|
||||
|
||||
const removeOpenedTransform = useCallback(
|
||||
(transformId: number) => {
|
||||
updateWorkspaceState((state) => ({
|
||||
...state,
|
||||
openedTransforms: state.openedTransforms.filter(
|
||||
(item) => item.id !== transformId,
|
||||
),
|
||||
}));
|
||||
},
|
||||
[updateWorkspaceState],
|
||||
);
|
||||
|
||||
const setEditedTransform = useCallback(
|
||||
(transformId: number, data: EditedTransform) => {
|
||||
updateWorkspaceState((state) => {
|
||||
const newEditedTransforms = new Map(state.editedTransforms).set(
|
||||
transformId,
|
||||
data,
|
||||
);
|
||||
const newRunTransforms = new Set(state.runTransforms);
|
||||
newRunTransforms.delete(transformId);
|
||||
return {
|
||||
...state,
|
||||
editedTransforms: newEditedTransforms,
|
||||
runTransforms: newRunTransforms,
|
||||
};
|
||||
});
|
||||
},
|
||||
[updateWorkspaceState],
|
||||
);
|
||||
|
||||
const removeEditedTransform = useCallback(
|
||||
(transformId: number) => {
|
||||
updateWorkspaceState((state) => {
|
||||
const newEditedTransforms = new Map(state.editedTransforms);
|
||||
newEditedTransforms.delete(transformId);
|
||||
return {
|
||||
...state,
|
||||
editedTransforms: newEditedTransforms,
|
||||
};
|
||||
});
|
||||
},
|
||||
[updateWorkspaceState],
|
||||
);
|
||||
|
||||
const markTransformAsRun = useCallback(
|
||||
(transformId: number) => {
|
||||
updateWorkspaceState((state) => ({
|
||||
...state,
|
||||
runTransforms: new Set(state.runTransforms).add(transformId),
|
||||
}));
|
||||
},
|
||||
[updateWorkspaceState],
|
||||
);
|
||||
|
||||
const hasChangedAndRunTransforms = useCallback(() => {
|
||||
return runTransforms.size > 0;
|
||||
}, [runTransforms]);
|
||||
|
||||
const activeEditedTransform = activeTransform
|
||||
? (editedTransforms.get(activeTransform.id) ?? activeTransform)
|
||||
: activeTransform;
|
||||
|
||||
const value = useMemo(
|
||||
() => ({
|
||||
openedTransforms,
|
||||
activeTransform,
|
||||
activeEditedTransform,
|
||||
setActiveTransform,
|
||||
addOpenedTransform,
|
||||
removeOpenedTransform,
|
||||
editedTransforms,
|
||||
setEditedTransform,
|
||||
removeEditedTransform,
|
||||
runTransforms,
|
||||
markTransformAsRun,
|
||||
hasChangedAndRunTransforms,
|
||||
}),
|
||||
[
|
||||
openedTransforms,
|
||||
activeTransform,
|
||||
activeEditedTransform,
|
||||
editedTransforms,
|
||||
runTransforms,
|
||||
setActiveTransform,
|
||||
addOpenedTransform,
|
||||
removeOpenedTransform,
|
||||
setEditedTransform,
|
||||
removeEditedTransform,
|
||||
markTransformAsRun,
|
||||
hasChangedAndRunTransforms,
|
||||
],
|
||||
);
|
||||
|
||||
return (
|
||||
<WorkspaceContext.Provider value={value}>
|
||||
{children}
|
||||
</WorkspaceContext.Provider>
|
||||
);
|
||||
};
|
||||
|
||||
export const useWorkspace = (): WorkspaceContextValue => {
|
||||
const context = useContext(WorkspaceContext);
|
||||
|
||||
if (!context) {
|
||||
throw new Error("useWorkspace must be used within a WorkspaceProvider");
|
||||
}
|
||||
|
||||
return context;
|
||||
};
|
||||
@@ -0,0 +1,13 @@
|
||||
import { IndexRoute, Route } from "react-router";
|
||||
|
||||
import { WorkspaceListPage } from "./pages/WorkspaceListPage/WorkspaceListPage";
|
||||
import { WorkspacePage } from "./pages/WorkspacePage/WorkspacePage";
|
||||
|
||||
export function getDataStudioWorkspaceRoutes() {
|
||||
return (
|
||||
<>
|
||||
<IndexRoute component={WorkspaceListPage} />
|
||||
<Route path=":workspaceId" component={WorkspacePage} />
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import { tinykeys } from "tinykeys";
|
||||
|
||||
import ErrorBoundary from "metabase/ErrorBoundary";
|
||||
import { useSelector } from "metabase/lib/redux";
|
||||
import { Sidebar } from "metabase/nav/containers/MainNavbar/MainNavbar.styled";
|
||||
import { getUser } from "metabase/selectors/user";
|
||||
import type { SuggestionModel } from "metabase-enterprise/rich_text_editing/tiptap/extensions/shared/types";
|
||||
|
||||
@@ -55,7 +56,14 @@ export const MetabotAuthenticated = ({ hide, config }: MetabotProps) => {
|
||||
|
||||
return (
|
||||
<ErrorBoundary errorComponent={() => null}>
|
||||
<MetabotChat config={config} />
|
||||
<Sidebar
|
||||
isOpen={visible}
|
||||
side="right"
|
||||
width="30rem"
|
||||
aria-hidden={!visible}
|
||||
>
|
||||
<MetabotChat config={config} />
|
||||
</Sidebar>
|
||||
</ErrorBoundary>
|
||||
);
|
||||
};
|
||||
|
||||
@@ -3,7 +3,6 @@ import { useCallback, useMemo } from "react";
|
||||
import { t } from "ttag";
|
||||
|
||||
import EmptyDashboardBot from "assets/img/dashboard-empty.svg?component";
|
||||
import { Sidebar } from "metabase/nav/containers/MainNavbar/MainNavbar.styled";
|
||||
import {
|
||||
ActionIcon,
|
||||
Box,
|
||||
@@ -38,11 +37,11 @@ const defaultConfig: MetabotConfig = {
|
||||
],
|
||||
};
|
||||
|
||||
export const MetabotChat = ({
|
||||
config = defaultConfig,
|
||||
}: {
|
||||
interface Props {
|
||||
config?: MetabotConfig;
|
||||
}) => {
|
||||
}
|
||||
|
||||
export const MetabotChat = ({ config = defaultConfig }: Props) => {
|
||||
const metabot = useMetabotAgent();
|
||||
const { handleSubmitInput, handleRetryMessage, handleResetInput } =
|
||||
useMetabotChatHandlers(config);
|
||||
@@ -77,137 +76,127 @@ export const MetabotChat = ({
|
||||
}, [handleSubmitInput, metabot.prompt]);
|
||||
|
||||
return (
|
||||
<Sidebar
|
||||
isOpen={metabot.visible}
|
||||
side="right"
|
||||
width="30rem"
|
||||
aria-hidden={!metabot.visible}
|
||||
>
|
||||
<Box className={Styles.container} data-testid="metabot-chat">
|
||||
{/* header */}
|
||||
<Box ref={headerRef} className={Styles.header}>
|
||||
<Flex align-items="center">
|
||||
<Text lh={1} fz="sm" c="text-secondary">
|
||||
{metabot.profileOverride
|
||||
? t`Using profile: ${metabot.profileOverride}`
|
||||
: t`Metabot isn't perfect. Double-check results.`}
|
||||
</Text>
|
||||
</Flex>
|
||||
<Box className={Styles.container} data-testid="metabot-chat">
|
||||
{/* header */}
|
||||
<Box ref={headerRef} className={Styles.header}>
|
||||
<Flex align-items="center">
|
||||
<Text lh={1} fz="sm" c="text-secondary">
|
||||
{metabot.profileOverride
|
||||
? t`Using profile: ${metabot.profileOverride}`
|
||||
: t`Metabot isn't perfect. Double-check results.`}
|
||||
</Text>
|
||||
</Flex>
|
||||
|
||||
<Flex gap="sm">
|
||||
<Tooltip label={t`Clear conversation`} position="bottom">
|
||||
<ActionIcon
|
||||
onClick={() => metabot.resetConversation()}
|
||||
data-testid="metabot-reset-chat"
|
||||
>
|
||||
<Icon c="text-primary" name="revert" />
|
||||
</ActionIcon>
|
||||
</Tooltip>
|
||||
{!config.preventClose && (
|
||||
<ActionIcon
|
||||
onClick={handleClose}
|
||||
data-testid="metabot-close-chat"
|
||||
>
|
||||
<Icon c="text-primary" name="close" />
|
||||
</ActionIcon>
|
||||
)}
|
||||
</Flex>
|
||||
</Box>
|
||||
|
||||
{/* chat messages */}
|
||||
<Box
|
||||
ref={scrollContainerRef}
|
||||
className={Styles.messagesContainer}
|
||||
data-testid="metabot-chat-messages"
|
||||
>
|
||||
{!hasMessages && !metabot.isDoingScience && (
|
||||
<>
|
||||
{/* empty state */}
|
||||
<Flex
|
||||
h="100%"
|
||||
gap="md"
|
||||
direction="column"
|
||||
align="center"
|
||||
justify="center"
|
||||
data-testid="metabot-empty-chat-info"
|
||||
>
|
||||
<Box component={EmptyDashboardBot} w="6rem" />
|
||||
<Text c="text-light" maw="12rem" ta="center">
|
||||
{config.emptyText ??
|
||||
t`I can help you explore your metrics and models.`}
|
||||
</Text>
|
||||
</Flex>
|
||||
{!config.hideSuggestedPrompts && (
|
||||
<Stack
|
||||
gap="sm"
|
||||
className={Styles.promptSuggestionsContainer}
|
||||
data-testid="metabot-prompt-suggestions"
|
||||
>
|
||||
<>
|
||||
{suggestedPrompts.map(({ prompt }, index) => (
|
||||
<Box key={index}>
|
||||
<Button
|
||||
fz="sm"
|
||||
size="xs"
|
||||
onClick={() => handleSubmitInput(prompt)}
|
||||
className={Styles.promptSuggestionButton}
|
||||
>
|
||||
{prompt}
|
||||
</Button>
|
||||
</Box>
|
||||
))}
|
||||
</>
|
||||
</Stack>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
|
||||
{(hasMessages || metabot.isDoingScience) && (
|
||||
<Box
|
||||
className={Styles.messages}
|
||||
data-testid="metabot-chat-inner-messages"
|
||||
<Flex gap="sm">
|
||||
<Tooltip label={t`Clear conversation`} position="bottom">
|
||||
<ActionIcon
|
||||
onClick={() => metabot.resetConversation()}
|
||||
data-testid="metabot-reset-chat"
|
||||
>
|
||||
{/* conversation messages */}
|
||||
<Messages
|
||||
messages={metabot.messages}
|
||||
errorMessages={metabot.errorMessages}
|
||||
onRetryMessage={handleRetryMessage}
|
||||
isDoingScience={metabot.isDoingScience}
|
||||
showFeedbackButtons
|
||||
/>
|
||||
{/* loading */}
|
||||
{metabot.isDoingScience && (
|
||||
<MetabotThinking toolCalls={metabot.activeToolCalls} />
|
||||
)}
|
||||
{/* filler - height gets set via ref mutation */}
|
||||
<div ref={fillerRef} data-testid="metabot-message-filler" />
|
||||
{/* long convo warning */}
|
||||
{metabot.isLongConversation && <MetabotResetLongChatButton />}
|
||||
</Box>
|
||||
<Icon c="text-primary" name="revert" />
|
||||
</ActionIcon>
|
||||
</Tooltip>
|
||||
{!config.preventClose && (
|
||||
<ActionIcon onClick={handleClose} data-testid="metabot-close-chat">
|
||||
<Icon c="text-primary" name="close" />
|
||||
</ActionIcon>
|
||||
)}
|
||||
</Box>
|
||||
|
||||
<Box className={Styles.textInputContainer}>
|
||||
<Paper
|
||||
className={cx(
|
||||
Styles.inputContainer,
|
||||
metabot.isDoingScience && Styles.inputContainerLoading,
|
||||
)}
|
||||
>
|
||||
<MetabotChatEditor
|
||||
ref={metabot.promptInputRef}
|
||||
value={metabot.prompt}
|
||||
autoFocus
|
||||
isResponding={metabot.isDoingScience}
|
||||
placeholder={t`Tell me to do something, or ask a question`}
|
||||
onChange={handleEditorChange}
|
||||
onSubmit={handleEditorSubmit}
|
||||
onStop={metabot.cancelRequest}
|
||||
suggestionModels={config.suggestionModels}
|
||||
/>
|
||||
</Paper>
|
||||
</Box>
|
||||
</Flex>
|
||||
</Box>
|
||||
</Sidebar>
|
||||
|
||||
{/* chat messages */}
|
||||
<Box
|
||||
ref={scrollContainerRef}
|
||||
className={Styles.messagesContainer}
|
||||
data-testid="metabot-chat-messages"
|
||||
>
|
||||
{!hasMessages && !metabot.isDoingScience && (
|
||||
<>
|
||||
{/* empty state */}
|
||||
<Flex
|
||||
h="100%"
|
||||
gap="md"
|
||||
direction="column"
|
||||
align="center"
|
||||
justify="center"
|
||||
data-testid="metabot-empty-chat-info"
|
||||
>
|
||||
<Box component={EmptyDashboardBot} w="6rem" />
|
||||
<Text c="text-light" maw="12rem" ta="center">
|
||||
{config.emptyText ??
|
||||
t`I can help you explore your metrics and models.`}
|
||||
</Text>
|
||||
</Flex>
|
||||
{!config.hideSuggestedPrompts && (
|
||||
<Stack
|
||||
gap="sm"
|
||||
className={Styles.promptSuggestionsContainer}
|
||||
data-testid="metabot-prompt-suggestions"
|
||||
>
|
||||
<>
|
||||
{suggestedPrompts.map(({ prompt }, index) => (
|
||||
<Box key={index}>
|
||||
<Button
|
||||
fz="sm"
|
||||
size="xs"
|
||||
onClick={() => handleSubmitInput(prompt)}
|
||||
className={Styles.promptSuggestionButton}
|
||||
>
|
||||
{prompt}
|
||||
</Button>
|
||||
</Box>
|
||||
))}
|
||||
</>
|
||||
</Stack>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
|
||||
{(hasMessages || metabot.isDoingScience) && (
|
||||
<Box
|
||||
className={Styles.messages}
|
||||
data-testid="metabot-chat-inner-messages"
|
||||
>
|
||||
{/* conversation messages */}
|
||||
<Messages
|
||||
messages={metabot.messages}
|
||||
errorMessages={metabot.errorMessages}
|
||||
onRetryMessage={handleRetryMessage}
|
||||
isDoingScience={metabot.isDoingScience}
|
||||
showFeedbackButtons
|
||||
/>
|
||||
{/* loading */}
|
||||
{metabot.isDoingScience && (
|
||||
<MetabotThinking toolCalls={metabot.activeToolCalls} />
|
||||
)}
|
||||
{/* filler - height gets set via ref mutation */}
|
||||
<div ref={fillerRef} data-testid="metabot-message-filler" />
|
||||
{/* long convo warning */}
|
||||
{metabot.isLongConversation && <MetabotResetLongChatButton />}
|
||||
</Box>
|
||||
)}
|
||||
</Box>
|
||||
|
||||
<Box className={Styles.textInputContainer}>
|
||||
<Paper
|
||||
className={cx(
|
||||
Styles.inputContainer,
|
||||
metabot.isDoingScience && Styles.inputContainerLoading,
|
||||
)}
|
||||
>
|
||||
<MetabotChatEditor
|
||||
ref={metabot.promptInputRef}
|
||||
value={metabot.prompt}
|
||||
autoFocus
|
||||
isResponding={metabot.isDoingScience}
|
||||
placeholder={t`Tell me to do something, or ask a question`}
|
||||
onChange={handleEditorChange}
|
||||
onSubmit={handleEditorSubmit}
|
||||
onStop={metabot.cancelRequest}
|
||||
suggestionModels={config.suggestionModels}
|
||||
/>
|
||||
</Paper>
|
||||
</Box>
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
@@ -12,6 +12,7 @@ import { Metabot } from "./components/Metabot";
|
||||
import { MetabotAdminPage } from "./components/MetabotAdmin/MetabotAdminPage";
|
||||
import { MetabotTrialPage } from "./components/MetabotAdmin/MetabotTrialPage";
|
||||
import { MetabotAppBarButton } from "./components/MetabotAppBarButton";
|
||||
import { MetabotChat } from "./components/MetabotChat";
|
||||
import { MetabotDataStudioButton } from "./components/MetabotDataStudioButton";
|
||||
import { getMetabotQuickLinks } from "./components/MetabotQuickLinks";
|
||||
import { MetabotToggleButton } from "./components/MetabotToggleButton";
|
||||
@@ -38,6 +39,7 @@ export function initializePlugin() {
|
||||
if (hasPremiumFeature("metabot_v3")) {
|
||||
PLUGIN_METABOT.isEnabled = () => true;
|
||||
PLUGIN_METABOT.Metabot = Metabot;
|
||||
PLUGIN_METABOT.MetabotChat = MetabotChat;
|
||||
PLUGIN_METABOT.getMetabotRoutes = getMetabotQuickLinks;
|
||||
|
||||
PLUGIN_METABOT.getAdminPaths = () => [
|
||||
|
||||
@@ -30,6 +30,7 @@ import {
|
||||
|
||||
type PythonDataPickerProps = {
|
||||
database?: DatabaseId;
|
||||
canChangeDatabase?: boolean;
|
||||
tables: PythonTransformTableAliases;
|
||||
onChange: (
|
||||
database: number,
|
||||
@@ -42,6 +43,7 @@ export function PythonDataPicker({
|
||||
database,
|
||||
tables,
|
||||
onChange,
|
||||
canChangeDatabase = true,
|
||||
}: PythonDataPickerProps) {
|
||||
const [tableSelections, setTableSelections] = useState<TableSelection[]>(
|
||||
getInitialTableSelections(tables),
|
||||
@@ -150,22 +152,24 @@ export function PythonDataPicker({
|
||||
className={S.dataPicker}
|
||||
data-testid="python-data-picker"
|
||||
>
|
||||
<Box>
|
||||
<Text fw="bold">{t`Source database`}</Text>
|
||||
<Text size="sm" c="text-light" mb="sm">
|
||||
{t`Select the database that contains your source data.`}
|
||||
</Text>
|
||||
{canChangeDatabase && (
|
||||
<Box>
|
||||
<Text fw="bold">{t`Source database`}</Text>
|
||||
<Text size="sm" c="text-light" mb="sm">
|
||||
{t`Select the database that contains your source data.`}
|
||||
</Text>
|
||||
|
||||
<DatabaseDataSelector
|
||||
className={S.databaseSelector}
|
||||
selectedDatabaseId={database}
|
||||
setDatabaseFn={handleDatabaseChange}
|
||||
databases={databases?.data ?? []}
|
||||
databaseIsDisabled={(database: Database) =>
|
||||
!hasFeature(database, "transforms/python")
|
||||
}
|
||||
/>
|
||||
</Box>
|
||||
<DatabaseDataSelector
|
||||
className={S.databaseSelector}
|
||||
selectedDatabaseId={database}
|
||||
setDatabaseFn={handleDatabaseChange}
|
||||
databases={databases?.data ?? []}
|
||||
databaseIsDisabled={(database: Database) =>
|
||||
!hasFeature(database, "transforms/python")
|
||||
}
|
||||
/>
|
||||
</Box>
|
||||
)}
|
||||
{database && (
|
||||
<Box>
|
||||
<Text fw="bold">{t`Pick tables and alias them`}</Text>
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
export function PythonTransformEditor({
|
||||
source,
|
||||
proposedSource,
|
||||
uiOptions,
|
||||
isDirty,
|
||||
onChangeSource,
|
||||
onAcceptProposed,
|
||||
@@ -72,6 +73,7 @@ export function PythonTransformEditor({
|
||||
<Flex h="100%" w="100%">
|
||||
<PythonDataPicker
|
||||
database={source["source-database"]}
|
||||
canChangeDatabase={uiOptions?.canChangeDatabase}
|
||||
tables={source["source-tables"]}
|
||||
onChange={handleDataChange}
|
||||
/>
|
||||
|
||||
@@ -3,6 +3,7 @@ import { useMemo } from "react";
|
||||
import { useSelector } from "metabase/lib/redux";
|
||||
import {
|
||||
QueryEditor,
|
||||
type QueryEditorUiOptions,
|
||||
type QueryEditorUiState,
|
||||
} from "metabase/querying/editor/components/QueryEditor";
|
||||
import { getMetadata } from "metabase/selectors/metadata";
|
||||
@@ -11,9 +12,10 @@ import type { Database, QueryTransformSource } from "metabase-types/api";
|
||||
|
||||
import { getEditorOptions } from "./utils";
|
||||
|
||||
type TransformEditorProps = {
|
||||
export type TransformEditorProps = {
|
||||
source: QueryTransformSource;
|
||||
uiState: QueryEditorUiState;
|
||||
uiOptions?: QueryEditorUiOptions;
|
||||
proposedSource: QueryTransformSource | undefined;
|
||||
databases: Database[];
|
||||
onChangeSource: (source: QueryTransformSource) => void;
|
||||
@@ -27,6 +29,7 @@ export function TransformEditor({
|
||||
proposedSource,
|
||||
databases,
|
||||
uiState,
|
||||
uiOptions,
|
||||
onChangeSource,
|
||||
onChangeUiState,
|
||||
onAcceptProposed,
|
||||
@@ -44,7 +47,10 @@ export function TransformEditor({
|
||||
: undefined,
|
||||
[proposedSource, metadata],
|
||||
);
|
||||
const uiOptions = useMemo(() => getEditorOptions(databases), [databases]);
|
||||
const mergedUiOptions = useMemo(
|
||||
() => ({ ...getEditorOptions(databases), ...uiOptions }),
|
||||
[databases, uiOptions],
|
||||
);
|
||||
|
||||
const handleQueryChange = (query: Lib.Query) => {
|
||||
const newSource: QueryTransformSource = {
|
||||
@@ -60,7 +66,7 @@ export function TransformEditor({
|
||||
<QueryEditor
|
||||
query={query}
|
||||
uiState={uiState}
|
||||
uiOptions={uiOptions}
|
||||
uiOptions={mergedUiOptions}
|
||||
proposedQuery={proposedQuery}
|
||||
onChangeQuery={handleQueryChange}
|
||||
onChangeUiState={onChangeUiState}
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
import { push } from "react-router-redux";
|
||||
import { t } from "ttag";
|
||||
|
||||
import { skipToken } from "metabase/api";
|
||||
import { useDispatch } from "metabase/lib/redux";
|
||||
import * as Urls from "metabase/lib/urls";
|
||||
import { useMetadataToasts } from "metabase/metadata/hooks";
|
||||
import { Anchor, Button, Card, Group, Icon, Stack, Text } from "metabase/ui";
|
||||
import {
|
||||
useCreateWorkspaceMutation,
|
||||
useGetTransformDownstreamMappingQuery,
|
||||
useGetTransformUpstreamMappingQuery,
|
||||
useGetWorkspaceQuery,
|
||||
useMergeWorkspaceMutation,
|
||||
} from "metabase-enterprise/api";
|
||||
import type { Transform } from "metabase-types/api";
|
||||
|
||||
import { TitleSection } from "../../../components/TitleSection";
|
||||
|
||||
type WorkspaceSectionProps = {
|
||||
transform: Transform;
|
||||
};
|
||||
|
||||
export function WorkspaceSection({ transform }: WorkspaceSectionProps) {
|
||||
const dispatch = useDispatch();
|
||||
const hasWorkspace = transform.workspace_id != null;
|
||||
const [createWorkspace, { isLoading: isCreating }] =
|
||||
useCreateWorkspaceMutation();
|
||||
const [mergeWorkspace, { isLoading: isMerging }] =
|
||||
useMergeWorkspaceMutation();
|
||||
const { sendErrorToast, sendSuccessToast } = useMetadataToasts();
|
||||
|
||||
const { data: workspace } = useGetWorkspaceQuery(
|
||||
transform.workspace_id == null ? skipToken : transform.workspace_id,
|
||||
);
|
||||
|
||||
const { data: upstreamMapping, isLoading: isLoadingUpstream } =
|
||||
useGetTransformUpstreamMappingQuery(
|
||||
hasWorkspace ? transform.id : skipToken,
|
||||
);
|
||||
|
||||
const { data: downstreamMapping, isLoading: isLoadingDownstream } =
|
||||
useGetTransformDownstreamMappingQuery(
|
||||
hasWorkspace ? skipToken : transform.id,
|
||||
);
|
||||
|
||||
const isLoadingMappings = isLoadingUpstream || isLoadingDownstream;
|
||||
|
||||
const handleCheckoutClick = async () => {
|
||||
try {
|
||||
const workspace = await createWorkspace({
|
||||
name: `${transform.name} workspace`,
|
||||
upstream: {
|
||||
transforms: [transform.id],
|
||||
},
|
||||
}).unwrap();
|
||||
|
||||
const transforms = workspace.contents?.transforms;
|
||||
|
||||
if (transforms && transforms.length === 1) {
|
||||
const newTransformId = transforms[0].id;
|
||||
dispatch(push(Urls.transform(newTransformId)));
|
||||
}
|
||||
} catch (error) {
|
||||
sendErrorToast(t`Failed to create workspace`);
|
||||
}
|
||||
};
|
||||
|
||||
const handleMergeClick = async () => {
|
||||
if (!transform.workspace_id || !upstreamMapping?.transform) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await mergeWorkspace(transform.workspace_id).unwrap();
|
||||
|
||||
if (result.errors && result.errors.length > 0) {
|
||||
sendErrorToast(t`Some transforms failed to merge`);
|
||||
} else {
|
||||
sendSuccessToast(t`Successfully merged workspace`);
|
||||
dispatch(push(Urls.transform(upstreamMapping.transform.id)));
|
||||
}
|
||||
} catch (error) {
|
||||
sendErrorToast(t`Failed to merge workspace`);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<TitleSection label={t`Workspace`}>
|
||||
<Card p="md" shadow="none" withBorder>
|
||||
{hasWorkspace ? (
|
||||
<Stack gap="sm">
|
||||
<Group>
|
||||
<Icon name="folder" aria-hidden />
|
||||
<Text>
|
||||
{t`This is part of`}{" "}
|
||||
<Anchor
|
||||
href={Urls.dataStudioWorkspace(transform.workspace_id!)}
|
||||
>
|
||||
{workspace?.name ?? t`Workspace ${transform.workspace_id}`}
|
||||
</Anchor>
|
||||
</Text>
|
||||
</Group>
|
||||
{!isLoadingMappings && upstreamMapping?.transform != null && (
|
||||
<Group>
|
||||
<Icon name="arrow_left" aria-hidden />
|
||||
<Text>
|
||||
{t`Live version:`}{" "}
|
||||
<Anchor href={Urls.transform(upstreamMapping.transform.id)}>
|
||||
{upstreamMapping.transform.name}
|
||||
</Anchor>
|
||||
</Text>
|
||||
</Group>
|
||||
)}
|
||||
<Button
|
||||
leftSection={<Icon name="upload" aria-hidden />}
|
||||
onClick={handleMergeClick}
|
||||
loading={isMerging}
|
||||
disabled={!upstreamMapping?.transform}
|
||||
>
|
||||
{t`Merge`}
|
||||
</Button>
|
||||
</Stack>
|
||||
) : (
|
||||
<Stack gap="sm">
|
||||
<Group>
|
||||
<Icon name="folder" aria-hidden />
|
||||
<Text c="text-secondary">
|
||||
{t`This transform is not part of any workspace`}
|
||||
</Text>
|
||||
<Button
|
||||
leftSection={<Icon name="add" aria-hidden />}
|
||||
onClick={handleCheckoutClick}
|
||||
loading={isCreating}
|
||||
disabled={!transform.table}
|
||||
>
|
||||
{t`Check this out in a new workspace`}
|
||||
</Button>
|
||||
</Group>
|
||||
{!isLoadingMappings &&
|
||||
downstreamMapping &&
|
||||
downstreamMapping.transforms.length > 0 && (
|
||||
<Stack gap="xs">
|
||||
<Group>
|
||||
<Icon name="arrow_right" aria-hidden />
|
||||
<Text>{t`It is checked out as part of the following workspaces:`}</Text>
|
||||
</Group>
|
||||
{downstreamMapping.transforms.map((item) => (
|
||||
<Group key={item.id} ml="xl" gap="xs">
|
||||
<Text c="text-secondary">•</Text>
|
||||
<Anchor href={Urls.transform(item.id)}>
|
||||
{item.workspace.name}
|
||||
</Anchor>
|
||||
</Group>
|
||||
))}
|
||||
</Stack>
|
||||
)}
|
||||
</Stack>
|
||||
)}
|
||||
</Card>
|
||||
</TitleSection>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { WorkspaceSection } from "./WorkspaceSection";
|
||||
@@ -13,7 +13,10 @@ import {
|
||||
PLUGIN_DEPENDENCIES,
|
||||
PLUGIN_TRANSFORMS_PYTHON,
|
||||
} from "metabase/plugins";
|
||||
import { getInitialUiState } from "metabase/querying/editor/components/QueryEditor";
|
||||
import {
|
||||
getInitialUiState,
|
||||
type QueryEditorUiOptions,
|
||||
} from "metabase/querying/editor/components/QueryEditor";
|
||||
import { getMetadata } from "metabase/selectors/metadata";
|
||||
import { Stack } from "metabase/ui";
|
||||
import {
|
||||
@@ -22,9 +25,17 @@ import {
|
||||
} from "metabase-enterprise/api";
|
||||
import { PaneHeaderActions } from "metabase-enterprise/data-studio/common/components/PaneHeader";
|
||||
import * as Lib from "metabase-lib";
|
||||
import type { Database, Transform } from "metabase-types/api";
|
||||
import type {
|
||||
Database,
|
||||
DraftTransformSource,
|
||||
Transform,
|
||||
TransformSource,
|
||||
} from "metabase-types/api";
|
||||
|
||||
import { TransformEditor } from "../../components/TransformEditor";
|
||||
import {
|
||||
TransformEditor,
|
||||
type TransformEditorProps,
|
||||
} from "../../components/TransformEditor";
|
||||
import { TransformHeader } from "../../components/TransformHeader";
|
||||
import { useRegisterMetabotTransformContext } from "../../hooks/use-register-transform-metabot-context";
|
||||
import { useSourceState } from "../../hooks/use-source-state";
|
||||
@@ -171,31 +182,17 @@ function TransformQueryPageBody({
|
||||
}
|
||||
hasMenu={!isDirty}
|
||||
/>
|
||||
{source.type === "python" ? (
|
||||
<PLUGIN_TRANSFORMS_PYTHON.TransformEditor
|
||||
source={source}
|
||||
proposedSource={
|
||||
proposedSource?.type === "python" ? proposedSource : undefined
|
||||
}
|
||||
isDirty={isDirty}
|
||||
onChangeSource={setSourceAndRejectProposed}
|
||||
onAcceptProposed={acceptProposed}
|
||||
onRejectProposed={rejectProposed}
|
||||
/>
|
||||
) : (
|
||||
<TransformEditor
|
||||
source={source}
|
||||
proposedSource={
|
||||
proposedSource?.type === "query" ? proposedSource : undefined
|
||||
}
|
||||
uiState={uiState}
|
||||
databases={databases}
|
||||
onChangeSource={setSourceAndRejectProposed}
|
||||
onChangeUiState={setUiState}
|
||||
onAcceptProposed={acceptProposed}
|
||||
onRejectProposed={rejectProposed}
|
||||
/>
|
||||
)}
|
||||
<TransformQueryPageEditor
|
||||
source={source}
|
||||
proposedSource={proposedSource}
|
||||
isDirty={isDirty}
|
||||
uiState={uiState}
|
||||
databases={databases}
|
||||
setSourceAndRejectProposed={setSourceAndRejectProposed}
|
||||
setUiState={setUiState}
|
||||
acceptProposed={acceptProposed}
|
||||
rejectProposed={rejectProposed}
|
||||
/>
|
||||
</Stack>
|
||||
{isConfirmationShown && checkData != null && (
|
||||
<PLUGIN_DEPENDENCIES.CheckDependenciesModal
|
||||
@@ -213,3 +210,62 @@ function TransformQueryPageBody({
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
export type TransformQueryPageEditorUiState = ReturnType<
|
||||
typeof getInitialUiState
|
||||
>;
|
||||
|
||||
export type TransformQueryPageEditorProps = {
|
||||
source: DraftTransformSource;
|
||||
proposedSource?: DraftTransformSource;
|
||||
isDirty: boolean;
|
||||
uiState: TransformQueryPageEditorUiState;
|
||||
databases: Database[];
|
||||
setSourceAndRejectProposed: (source: DraftTransformSource) => void;
|
||||
setUiState: (uiState: TransformQueryPageEditorUiState) => void;
|
||||
acceptProposed: () => void;
|
||||
rejectProposed: () => void;
|
||||
uiOptions?: TransformEditorProps["uiOptions"];
|
||||
};
|
||||
|
||||
export function TransformQueryPageEditor({
|
||||
source,
|
||||
proposedSource,
|
||||
isDirty,
|
||||
uiState,
|
||||
|
||||
uiOptions,
|
||||
databases,
|
||||
setSourceAndRejectProposed,
|
||||
setUiState,
|
||||
acceptProposed,
|
||||
rejectProposed,
|
||||
}: TransformQueryPageEditorProps) {
|
||||
return source.type === "python" ? (
|
||||
<PLUGIN_TRANSFORMS_PYTHON.TransformEditor
|
||||
source={source}
|
||||
uiOptions={uiOptions}
|
||||
proposedSource={
|
||||
proposedSource?.type === "python" ? proposedSource : undefined
|
||||
}
|
||||
isDirty={isDirty}
|
||||
onChangeSource={setSourceAndRejectProposed}
|
||||
onAcceptProposed={acceptProposed}
|
||||
onRejectProposed={rejectProposed}
|
||||
/>
|
||||
) : (
|
||||
<TransformEditor
|
||||
source={source}
|
||||
proposedSource={
|
||||
proposedSource?.type === "query" ? proposedSource : undefined
|
||||
}
|
||||
uiState={uiState}
|
||||
uiOptions={uiOptions}
|
||||
databases={databases}
|
||||
onChangeSource={setSourceAndRejectProposed}
|
||||
onChangeUiState={setUiState}
|
||||
onAcceptProposed={acceptProposed}
|
||||
onRejectProposed={rejectProposed}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -29,6 +29,7 @@ export function getDataStudioTransformRoutes() {
|
||||
<Route path="runs" component={TransformTopNavLayout}>
|
||||
<IndexRoute component={RunListPage} />
|
||||
</Route>
|
||||
|
||||
<Route component={TransformSidebarLayout}>
|
||||
<IndexRoute component={TransformEmptyPage} />
|
||||
<Route path="jobs" component={JobEmptyPage} />
|
||||
|
||||
@@ -60,6 +60,7 @@ export * from "./util";
|
||||
export * from "./visualization";
|
||||
export * from "./visualization-settings";
|
||||
export * from "./visualizer";
|
||||
export * from "./workspace";
|
||||
|
||||
// ISO8601 timestamp
|
||||
export type ISO8601Time = string;
|
||||
|
||||
@@ -73,6 +73,7 @@ export function createMockTransform(opts?: Partial<Transform>): Transform {
|
||||
target: createMockTransformTarget(),
|
||||
created_at: "2000-01-01T00:00:00Z",
|
||||
updated_at: "2000-01-01T00:00:00Z",
|
||||
workspace_id: null,
|
||||
...opts,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -14,9 +14,11 @@ export type Transform = {
|
||||
name: string;
|
||||
description: string | null;
|
||||
source: TransformSource;
|
||||
source_type: "native" | "python" | "mbql";
|
||||
target: TransformTarget;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
workspace_id: number | null;
|
||||
|
||||
// hydrated fields
|
||||
tag_ids?: TransformTagId[];
|
||||
|
||||
80
frontend/src/metabase-types/api/workspace.ts
Normal file
80
frontend/src/metabase-types/api/workspace.ts
Normal file
@@ -0,0 +1,80 @@
|
||||
import type { CollectionId } from "./collection";
|
||||
import type { DatabaseId } from "./database";
|
||||
import type { TransformId } from "./transform";
|
||||
|
||||
export type WorkspaceId = number;
|
||||
|
||||
export type Workspace = {
|
||||
id: WorkspaceId;
|
||||
name: string;
|
||||
collection_id: CollectionId | null;
|
||||
database_id: DatabaseId | null;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
archived_at?: string | null;
|
||||
contents?: WorkspaceContents["contents"];
|
||||
};
|
||||
|
||||
export type WorkspaceItem = {
|
||||
id: WorkspaceId;
|
||||
name: string;
|
||||
};
|
||||
|
||||
export type CreateWorkspaceRequest = {
|
||||
name: string;
|
||||
database_id?: DatabaseId;
|
||||
upstream: {
|
||||
transforms?: TransformId[];
|
||||
};
|
||||
};
|
||||
|
||||
export type WorkspaceListResponse = {
|
||||
items: Workspace[];
|
||||
};
|
||||
|
||||
export type WorkspaceContentItem = WorkspaceTransformItem;
|
||||
|
||||
export type WorkspaceTransformItem = {
|
||||
type: "transform";
|
||||
id: TransformId;
|
||||
name: string;
|
||||
upstream_id: TransformId;
|
||||
workspace_id: WorkspaceId;
|
||||
};
|
||||
|
||||
export type WorkspaceContents = {
|
||||
contents: {
|
||||
transforms: WorkspaceTransformItem[];
|
||||
};
|
||||
};
|
||||
|
||||
export type TransformUpstreamMapping = {
|
||||
transform: WorkspaceTransformItem | null;
|
||||
};
|
||||
|
||||
export type DownstreamTransformInfo = {
|
||||
id: TransformId;
|
||||
name: string;
|
||||
workspace: WorkspaceItem;
|
||||
};
|
||||
|
||||
export type TransformDownstreamMapping = {
|
||||
transforms: DownstreamTransformInfo[];
|
||||
};
|
||||
|
||||
export type WorkspaceMergeResponse = {
|
||||
promoted: WorkspaceContentItem[];
|
||||
errors?: (WorkspaceContentItem & { error: string })[];
|
||||
workspace: WorkspaceItem;
|
||||
archived_at: string | null;
|
||||
};
|
||||
|
||||
export type WorkspaceUpdateContentsRequest = {
|
||||
id: WorkspaceId;
|
||||
add?: {
|
||||
transforms?: TransformId[];
|
||||
};
|
||||
remove?: {
|
||||
transforms?: TransformId[];
|
||||
};
|
||||
};
|
||||
@@ -106,3 +106,11 @@ export function dataStudioSnippetDependencies(snippetId: NativeQuerySnippetId) {
|
||||
export function newDataStudioSnippet() {
|
||||
return `${dataStudioModeling()}/snippets/new`;
|
||||
}
|
||||
|
||||
export function dataStudioWorkspaceList() {
|
||||
return `${ROOT_URL}/workspaces`;
|
||||
}
|
||||
|
||||
export function dataStudioWorkspace(workspaceId: number) {
|
||||
return `${dataStudioWorkspaceList()}/${workspaceId}`;
|
||||
}
|
||||
|
||||
@@ -54,6 +54,9 @@ type PluginMetabotType = {
|
||||
hide?: boolean;
|
||||
config?: PluginMetabotConfig;
|
||||
}) => React.ReactElement | null;
|
||||
MetabotChat: (props: {
|
||||
config?: PluginMetabotConfig;
|
||||
}) => React.ReactElement | null;
|
||||
defaultMetabotContextValue: MetabotContext;
|
||||
MetabotContext: React.Context<MetabotContext>;
|
||||
getMetabotProvider: () => ComponentType<{ children: React.ReactNode }>;
|
||||
@@ -100,6 +103,8 @@ const getDefaultPluginMetabot = (): PluginMetabotType => ({
|
||||
isEnabled: () => false,
|
||||
Metabot: (_props: { hide?: boolean; config?: PluginMetabotConfig }) =>
|
||||
null as React.ReactElement | null,
|
||||
MetabotChat: (_props: { config?: PluginMetabotConfig }) =>
|
||||
null as React.ReactElement | null,
|
||||
defaultMetabotContextValue,
|
||||
MetabotContext: React.createContext(defaultMetabotContextValue),
|
||||
getMetabotProvider: () => {
|
||||
|
||||
@@ -36,9 +36,14 @@ export type TransformsPlugin = {
|
||||
TransformPicker: ComponentType<TransformPickerProps>;
|
||||
};
|
||||
|
||||
export type PythonTransformEditorUiOptions = {
|
||||
canChangeDatabase?: boolean;
|
||||
};
|
||||
|
||||
export type PythonTransformEditorProps = {
|
||||
source: PythonTransformSourceDraft;
|
||||
proposedSource?: PythonTransformSourceDraft;
|
||||
uiOptions?: PythonTransformEditorUiOptions;
|
||||
isDirty: boolean;
|
||||
onChangeSource: (source: PythonTransformSourceDraft) => void;
|
||||
onAcceptProposed: () => void;
|
||||
|
||||
@@ -89,6 +89,7 @@ export function QueryEditor({
|
||||
modalSnippet={uiState.modalSnippet}
|
||||
nativeEditorSelectedText={selectedText}
|
||||
readOnly={uiOptions?.readOnly}
|
||||
canChangeDatabase={uiOptions?.canChangeDatabase}
|
||||
isNative={isNative}
|
||||
isRunnable={isRunnable}
|
||||
isRunning={isRunning}
|
||||
|
||||
@@ -45,6 +45,7 @@ type QueryEditorBodyProps = {
|
||||
| null;
|
||||
nativeEditorSelectedText?: string | null;
|
||||
readOnly?: boolean;
|
||||
canChangeDatabase?: boolean;
|
||||
isNative: boolean;
|
||||
isRunnable: boolean;
|
||||
isRunning: boolean;
|
||||
@@ -74,6 +75,7 @@ export function QueryEditorBody({
|
||||
modalSnippet,
|
||||
nativeEditorSelectedText,
|
||||
readOnly,
|
||||
canChangeDatabase,
|
||||
isNative,
|
||||
isRunnable,
|
||||
isRunning,
|
||||
@@ -136,6 +138,7 @@ export function QueryEditorBody({
|
||||
isInitiallyOpen
|
||||
isNativeEditorOpen
|
||||
readOnly={readOnly}
|
||||
canChangeDatabase={canChangeDatabase}
|
||||
hasParametersList={false}
|
||||
isRunnable={isRunnable}
|
||||
isRunning={isRunning}
|
||||
|
||||
@@ -45,6 +45,7 @@ export type QueryEditorUiOptions = {
|
||||
cardType?: CardType;
|
||||
cardDisplay?: CardDisplayType;
|
||||
cardVizSettings?: VisualizationSettings;
|
||||
canChangeDatabase?: boolean;
|
||||
readOnly?: boolean;
|
||||
canConvertToNative?: boolean;
|
||||
convertToNativeTitle?: string;
|
||||
|
||||
@@ -90,7 +90,6 @@
|
||||
(or (first
|
||||
(keep (fn [suffix]
|
||||
(let [file-path (str partial-file-path suffix)]
|
||||
|
||||
(cond (str/starts-with? maybe-ns "metabase.")
|
||||
;; src/metabase/...
|
||||
(let [path (str "src/" file-path)]
|
||||
@@ -98,8 +97,10 @@
|
||||
|
||||
(str/starts-with? maybe-ns "metabase-enterprise.")
|
||||
;; enterprise/backend/src/metabase_enterprise/...
|
||||
(let [path (str "enterprise/backend/" file-path)]
|
||||
(and (fs/exists? (str u/project-root-directory "/" path)) path)))))
|
||||
(or (let [path (str "enterprise/backend/src/" file-path)]
|
||||
(and (fs/exists? (str u/project-root-directory "/" path)) path))
|
||||
(let [path (str "enterprise/backend/test/" file-path)]
|
||||
(and (fs/exists? (str u/project-root-directory "/" path)) path))))))
|
||||
[".clj" ".cljc" ".bb"]))
|
||||
(throw
|
||||
(ex-info "" {:mage/error (str "Could not find a file for namespace: "
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -9,4 +9,5 @@
|
||||
[metabase.api-keys.models.api-key
|
||||
generate-key
|
||||
is-api-key-user?
|
||||
prefix])
|
||||
prefix
|
||||
create-api-key-with-new-user!])
|
||||
|
||||
@@ -203,10 +203,10 @@
|
||||
|
||||
(mu/defn create-api-key-with-new-user!
|
||||
"Create a new API key and a new user for that key at the same time."
|
||||
[{:keys [key-name group-id]} :- [:map
|
||||
{:closed true}
|
||||
[:key-name ::api-keys.schema/name]
|
||||
[:group-id {:optional true} pos-int?]]]
|
||||
[{:keys [key-name group-id]}
|
||||
:- [:map {:closed true}
|
||||
[:key-name ::api-keys.schema/name]
|
||||
[:group-id {:optional true} pos-int?]]]
|
||||
(api/checkp (not (t2/exists? :model/ApiKey :name key-name))
|
||||
"name" "An API key with this name already exists.")
|
||||
(let [unhashed-key (key-with-unique-prefix)
|
||||
@@ -218,7 +218,8 @@
|
||||
:last_name ""
|
||||
:type :api-key
|
||||
:password (str (random-uuid))})]
|
||||
(user/set-permissions-groups! user-id [(perms/all-users-group) group-id])
|
||||
(when group-id
|
||||
(user/set-permissions-groups! user-id [(perms/all-users-group) group-id]))
|
||||
(-> (t2/insert-returning-instance! :model/ApiKey
|
||||
{:user_id user-id
|
||||
:name key-name
|
||||
|
||||
@@ -1908,7 +1908,8 @@
|
||||
(serdes/fk :model/Collection)
|
||||
{:export location-path->parent-id
|
||||
:import parent-id->location-path}))
|
||||
:personal_owner_id (serdes/fk :model/User)}})
|
||||
:personal_owner_id (serdes/fk :model/User)
|
||||
:workspace_id (serdes/fk :model/Workspace)}})
|
||||
|
||||
;;; +----------------------------------------------------------------------------------------------------------------+
|
||||
;;; | Perms Checking Helper Fns |
|
||||
|
||||
@@ -108,7 +108,10 @@
|
||||
:model/User metabase.users.models.user
|
||||
:model/UserKeyValue metabase.user-key-value.models.user-key-value
|
||||
:model/UserParameterValue metabase.users.models.user-parameter-value
|
||||
:model/ViewLog metabase.view-log.models.view-log})
|
||||
:model/ViewLog metabase.view-log.models.view-log
|
||||
:model/Workspace metabase-enterprise.workspaces.models.workspace
|
||||
:model/WorkspaceMappingTable metabase-enterprise.workspaces.models.workspace-mapping
|
||||
:model/WorkspaceMappingTransform metabase-enterprise.workspaces.models.workspace-mapping})
|
||||
|
||||
;;; !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
;;; !! !!
|
||||
|
||||
@@ -1215,6 +1215,7 @@
|
||||
:document_id (serdes/fk :model/Document)
|
||||
:creator_id (serdes/fk :model/User)
|
||||
:made_public_by_id (serdes/fk :model/User)
|
||||
:workspace_id (serdes/fk :model/Workspace)
|
||||
:published_table_id (serdes/fk :model/Table)
|
||||
:dataset_query {:export serdes/export-mbql :import serdes/import-mbql}
|
||||
:parameters {:export serdes/export-parameters :import serdes/import-parameters}
|
||||
|
||||
@@ -431,6 +431,7 @@
|
||||
:table_id (serdes/fk :model/Table)
|
||||
:fk_target_field_id (serdes/fk :model/Field)
|
||||
:parent_id (serdes/fk :model/Field)
|
||||
:workspace_id (serdes/fk :model/Workspace)
|
||||
:dimensions (serdes/nested :model/Dimension :field_id opts)}})
|
||||
|
||||
(defmethod serdes/storage-path "Field" [field _]
|
||||
|
||||
@@ -490,9 +490,10 @@
|
||||
:database_require_filter :is_defective_duplicate :unique_table_helper :is_writable :data_authority
|
||||
:data_source :owner_email :owner_user_id]
|
||||
:skip [:estimated_row_count :view_count]
|
||||
:transform {:created_at (serdes/date)
|
||||
:archived_at (serdes/date)
|
||||
:transform {:created_at (serdes/date)
|
||||
:archived_at (serdes/date)
|
||||
:deactivated_at (serdes/date)
|
||||
:workspace_id (serdes/fk :model/Workspace)
|
||||
:data_layer (serdes/optional-kw)
|
||||
:db_id (serdes/fk :model/Database :name)}})
|
||||
|
||||
|
||||
@@ -54,8 +54,11 @@
|
||||
:model/TransformJobTransformTag
|
||||
:model/TransformTag
|
||||
:model/TransformTransformTag
|
||||
:model/WorkspaceMappingTable
|
||||
:model/WorkspaceMappingTransform
|
||||
:model/Undo
|
||||
:model/UserKeyValue})
|
||||
:model/UserKeyValue
|
||||
:model/Workspace})
|
||||
|
||||
(defn- all-model-names []
|
||||
(into (sorted-set)
|
||||
|
||||
@@ -141,7 +141,10 @@
|
||||
:dataset_query_metrics_v2_migration_backup
|
||||
;; `dependency_analysis_version` is an internal bookkeeping field. It doesn't affect the
|
||||
;; actual card itself, so no description is necessary.
|
||||
:dependency_analysis_version} col)
|
||||
:dependency_analysis_version
|
||||
;; this column is immutable in practice, and doesn't warrant a description
|
||||
:workspace_id}
|
||||
col)
|
||||
(testing (format "we should have a revision description for %s" col)
|
||||
(let [diff-strings (revision/diff-strings
|
||||
;; TODO -- huh? Shouldn't this be testing against `:model/Card` here???
|
||||
|
||||
Reference in New Issue
Block a user