3144 lines
198 KiB
Clojure
3144 lines
198 KiB
Clojure
(ns metabase-enterprise.serialization.v2.load-test
|
|
{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase-enterprise.serialization.v2.load-test]}}}}}}
|
|
(:require
|
|
[clojure.string :as str]
|
|
[clojure.test :refer :all]
|
|
[java-time.api :as t]
|
|
[metabase-enterprise.serialization.test-util :as ts]
|
|
[metabase-enterprise.serialization.v2.extract :as serdes.extract]
|
|
[metabase-enterprise.serialization.v2.ingest :as serdes.ingest]
|
|
[metabase-enterprise.serialization.v2.load :as serdes.load]
|
|
[metabase.actions.models :as action]
|
|
[metabase.actions.schema :as actions.schema]
|
|
[metabase.collections.models.collection :as collection]
|
|
[metabase.lib-be.core :as lib-be]
|
|
[metabase.lib.core :as lib]
|
|
[metabase.lib.metadata :as lib.metadata]
|
|
[metabase.models.serialization :as serdes]
|
|
[metabase.permissions.core :as perms]
|
|
[metabase.search.core :as search]
|
|
[metabase.test :as mt]
|
|
[metabase.util :as u]
|
|
[metabase.util.humanization :as u.humanization]
|
|
[metabase.util.json :as json]
|
|
[metabase.warehouses.models.database :as models.database]
|
|
[toucan2.core :as t2]))
|
|
|
|
;; `reindex!` below is ok in a parallel test since it's not actually executing anything.
|
|
;; Many tests here use the H2 test-data database (Card defaults), so we keep the H2 guard off
|
|
;; and re-enable H2 in the extract (production keeps it filtered).
|
|
(use-fixtures :each (fn [thunk]
|
|
(mt/with-dynamic-fn-redefs [search/reindex! (constantly nil)
|
|
models.database/assert-not-h2! (constantly nil)]
|
|
(binding [models.database/*include-h2-in-extract?* true]
|
|
(thunk)))))
|
|
|
|
(defn- no-labels [path]
|
|
(mapv #(dissoc % :label) path))
|
|
|
|
(defn- by-model [entities model-name]
|
|
(filter #(-> % :serdes/meta last :model (= model-name))
|
|
entities))
|
|
|
|
(defn- ids-by-model [entities model-name]
|
|
(->> (by-model entities model-name)
|
|
(map (comp :id last :serdes/meta))
|
|
set))
|
|
|
|
(defn- ingestion-in-memory [extractions]
|
|
(let [mapped (into {} (for [entity (vec extractions)]
|
|
[(no-labels (serdes/path entity))
|
|
entity]))]
|
|
(reify
|
|
serdes.ingest/Ingestable
|
|
(ingest-list [_]
|
|
(keys mapped))
|
|
(ingest-one [_ path]
|
|
(get mapped (no-labels path)))
|
|
(ingest-errors [_]
|
|
[]))))
|
|
|
|
;;; WARNING for test authors: [[extract/extract]] returns a lazy reducible value. To make sure you don't
|
|
;;; confound your tests with data from your dev appdb, remember to eagerly
|
|
;;; `(into [] (extract/extract ...))` in these tests.
|
|
|
|
(defn- cause-chain-messages
|
|
"Messages of `e` and every exception beneath it, skipping any that have none."
|
|
[e]
|
|
;; `keep`, not `map` - an exception with a nil message would NPE the callers' `re-find` and hide the real failure
|
|
(into [] (keep ex-message) (take-while some? (iterate ex-cause e))))
|
|
|
|
(defn- load-failure-messages!
|
|
"Loads `ingestion`, expecting it to throw, and returns the thrown exception's cause-chain messages."
|
|
[ingestion]
|
|
(try
|
|
(serdes.load/load-metabase! ingestion)
|
|
["load-metabase! unexpectedly succeeded"]
|
|
(catch Exception e
|
|
(cause-chain-messages e))))
|
|
|
|
(deftest schema-validation-opt-out-reaches-import-test
|
|
(testing (str "GHY-4241: MB_SERIALIZATION_SKIP_SCHEMA_VALIDATION has to travel env var -> Setting -> the binding "
|
|
"in load-metabase! -> import-mbql. Nothing else covers that chain, so dropping the binding would "
|
|
"silently disable the opt-out.")
|
|
(let [extracted (atom nil)]
|
|
(mt/with-empty-h2-app-db!
|
|
(let [db (ts/create! :model/Database :name "my-db")
|
|
coll (ts/create! :model/Collection :name "Some collection")
|
|
card (ts/create! :model/Card
|
|
:name "Native with a variable"
|
|
:collection_id (:id coll)
|
|
:dataset_query {:database (:id db)
|
|
:type :native
|
|
:native {:template-tags {"id" {:id "e2d15f07-37b3-01fc-3944-2ff860a5eb46"
|
|
:name "id"
|
|
:display-name "ID"
|
|
:type :number}}
|
|
:query "SELECT 1 WHERE x = {{id}}"}})]
|
|
(reset! extracted {:db (serdes/extract-one "Database" {} db)
|
|
:coll (serdes/extract-one "Collection" {} coll)
|
|
:card (serdes/extract-one "Card" {} card)})))
|
|
;; a tag type this version has no representation for - what an export from a newer Metabase that introduced
|
|
;; one would look like
|
|
(let [{:keys [db coll card]} @extracted
|
|
bad-card (assoc-in card [:dataset_query :stages 0 :template-tags]
|
|
{"id" {:type :tag-type-from-the-future :name "id" :display-name "ID" :id "abc-123"}})
|
|
ingestion #(ingestion-in-memory [db coll bad-card])
|
|
ours? #(some (partial re-find #"does not match this Metabase's query schema") %)]
|
|
(testing "by default the schema check is what refuses the import"
|
|
(mt/with-empty-h2-app-db!
|
|
(is (ours? (load-failure-messages! (ingestion))))))
|
|
(testing "with the opt-out set the schema check is skipped, so the import fails downstream instead"
|
|
(mt/with-empty-h2-app-db!
|
|
(mt/with-temp-env-var-value! [mb-serialization-skip-schema-validation "true"]
|
|
(let [messages (load-failure-messages! (ingestion))]
|
|
(is (some (partial re-find #"Invalid input.*:template-tags") messages))
|
|
(is (not (ours? messages)))))))))))
|
|
|
|
(deftest http-action-dashcards-are-dropped-test
|
|
(testing "an older export's HTTP action does not load, and only the dashboard buttons that ran it are dropped"
|
|
(let [serialized (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(let [db (ts/create! :model/Database :name "my-db")
|
|
coll (ts/create! :model/Collection :name "Actions")
|
|
model (ts/create! :model/Card
|
|
:name "A model"
|
|
:type :model
|
|
:collection_id (:id coll)
|
|
:database_id (:id db)
|
|
:dataset_query {:database (:id db)
|
|
:type :native
|
|
:native {:query "SELECT 1"}})
|
|
action (ts/create! :model/Action :name "Old HTTP" :type :query :model_id (:id model))
|
|
_ (ts/create! :model/QueryAction
|
|
:action_id (:id action)
|
|
:dataset_query {:database (:id db) :type :native :native {:query "UPDATE t SET x = 1"}})
|
|
dashboard (ts/create! :model/Dashboard :name "Buttons" :collection_id (:id coll))]
|
|
(ts/create! :model/DashboardCard :dashboard_id (:id dashboard) :action_id (:id action))
|
|
(ts/create! :model/DashboardCard :dashboard_id (:id dashboard) :card_id (:id model))
|
|
(reset! serialized (into [] (serdes.extract/extract {})))))
|
|
(ts/with-db dest-db
|
|
(serdes.load/load-metabase! (ingestion-in-memory
|
|
(for [entity @serialized]
|
|
(cond-> entity
|
|
(= "Action" (-> entity :serdes/meta last :model))
|
|
(-> (assoc :type "http") (dissoc :query))))))
|
|
(let [dashboard-id (t2/select-one-pk :model/Dashboard :name "Buttons")]
|
|
(is (not (t2/exists? :model/Action :name "Old HTTP")))
|
|
(is (some? dashboard-id))
|
|
(is (=? [{:card_id pos-int? :action_id nil}]
|
|
(t2/select :model/DashboardCard :dashboard_id dashboard-id)))))))))
|
|
|
|
(deftest load-basics-test
|
|
(testing "a simple, fresh collection is imported"
|
|
(let [serialized (atom nil)
|
|
eid1 "0123456789abcdef_0123"]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(testing "extraction succeeds"
|
|
(ts/with-db source-db
|
|
(ts/create! :model/Collection :name "Basic Collection" :entity_id eid1)
|
|
(reset! serialized (into [] (serdes.extract/extract {})))
|
|
(is (some (fn [{[{:keys [model id]}] :serdes/meta}]
|
|
(and (= model "Collection") (= id eid1)))
|
|
@serialized))))
|
|
(testing "loading into an empty database succeeds"
|
|
(ts/with-db dest-db
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
;; the trash is serialized and loaded, so restrict to nil type
|
|
(let [colls (t2/select :model/Collection :type nil)]
|
|
(is (= 1 (count colls)))
|
|
(is (= "Basic Collection" (:name (first colls))))
|
|
(is (= eid1 (:entity_id (first colls)))))))
|
|
(testing "loading again into the same database does not duplicate"
|
|
(ts/with-db dest-db
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
(let [colls (t2/select :model/Collection :type nil)]
|
|
(is (= 1 (count colls)))
|
|
(is (= "Basic Collection" (:name (first colls))))
|
|
(is (= eid1 (:entity_id (first colls)))))))))))
|
|
|
|
(deftest user-settings-round-trip-test
|
|
(testing "Table settings, Field settings and Dimensions round-trip as entities of their own"
|
|
(let [serialized (atom nil)
|
|
extract (fn [model-name filter-column ids]
|
|
(serdes/extract-all model-name {:filter-column filter-column :filter-ids ids}))]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(let [db (ts/create! :model/Database :name "my-db")
|
|
table (ts/create! :model/Table :name "customers" :db_id (:id db))
|
|
age (ts/create! :model/Field :name "age" :table_id (:id table))
|
|
email (ts/create! :model/Field :name "email" :table_id (:id table))]
|
|
(t2/insert! :model/TableUserSettings {:table_id (:id table) :display_name "Renamed"})
|
|
(t2/insert! :model/FieldUserSettings {:field_id (:id age) :description "edited"})
|
|
(ts/create! :model/Dimension :field_id (:id email) :name "Email" :type :internal)
|
|
(reset! serialized
|
|
(-> [(ts/extract-one "Database" (:id db))
|
|
(ts/extract-one "Table" (:id table))
|
|
(ts/extract-one "Field" (:id age))
|
|
(ts/extract-one "Field" (:id email))]
|
|
(into (extract "TableUserSettings" :table_id [(:id table)]))
|
|
(into (extract "FieldUserSettings" :field_id [(:id age)]))
|
|
(into (extract "Dimension" :field_id [(:id email)]))))))
|
|
(testing "each is extracted on its own, with nothing nested"
|
|
(is (=? {:display_name "Renamed"} (first (by-model @serialized "TableUserSettings"))))
|
|
(is (not (contains? (first (by-model @serialized "TableUserSettings")) :fields)))
|
|
(is (=? [{:description "edited"}] (by-model @serialized "FieldUserSettings")))
|
|
(is (=? [{:name "Email"}] (by-model @serialized "Dimension")))
|
|
(is (not-any? #(contains? % :dimensions) (by-model @serialized "Field"))))
|
|
(ts/with-db dest-db
|
|
(let [db (ts/create! :model/Database :name "my-db")
|
|
table (ts/create! :model/Table :name "customers" :db_id (:id db))
|
|
age (ts/create! :model/Field :name "age" :table_id (:id table))
|
|
email (ts/create! :model/Field :name "email" :table_id (:id table))]
|
|
(ts/create! :model/Dimension :field_id (:id email) :name "Local" :type :internal)
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
(is (= "Renamed" (t2/select-one-fn :display_name :model/TableUserSettings :table_id (:id table))))
|
|
(is (= "edited" (t2/select-one-fn :description :model/FieldUserSettings :field_id (:id age))))
|
|
(testing "the imported Dimension replaces the Field's local one"
|
|
(is (= ["Email"] (t2/select-fn-vec :name :model/Dimension :field_id (:id email)))))))))))
|
|
|
|
(deftest legacy-field-dimensions-import-test
|
|
(testing "a Field file written before Dimensions got files of their own still carries them"
|
|
(mt/with-empty-h2-app-db!
|
|
(let [db (ts/create! :model/Database :name "my-db")
|
|
table (ts/create! :model/Table :name "customers" :db_id (:id db))
|
|
email (ts/create! :model/Field :name "email" :table_id (:id table))
|
|
dimension (dissoc (ts/extract-one "Dimension" (:entity_id (ts/create! :model/Dimension :field_id (:id email)
|
|
:name "Email" :type :internal)))
|
|
:serdes/meta)
|
|
field (ts/extract-one "Field" (:id email))
|
|
load! (fn [entity] (serdes.load/load-metabase! (ingestion-in-memory [entity])))]
|
|
(t2/delete! :model/Dimension :field_id (:id email))
|
|
(load! (assoc field :dimensions [dimension]))
|
|
(is (= "Email" (t2/select-one-fn :name :model/Dimension :field_id (:id email))))
|
|
(testing "an empty list deletes them"
|
|
(load! (assoc field :dimensions []))
|
|
(is (not (t2/exists? :model/Dimension :field_id (:id email)))))))))
|
|
|
|
(deftest legacy-table-user-settings-fields-import-test
|
|
(testing "a settings file written before Field settings got files of their own replaces its Table's Field settings"
|
|
(mt/with-empty-h2-app-db!
|
|
(let [db (ts/create! :model/Database :name "my-db")
|
|
table (ts/create! :model/Table :name "customers" :db_id (:id db))
|
|
age (ts/create! :model/Field :name "age" :table_id (:id table))
|
|
email (ts/create! :model/Field :name "email" :table_id (:id table))
|
|
_ (t2/insert! :model/FieldUserSettings {:field_id (:id age) :description "edited"})
|
|
legacy {:serdes/meta [{:model "Database" :id "my-db"}
|
|
{:model "Table" :id "customers"}
|
|
{:model "TableUserSettings" :id "1"}]
|
|
:fields [(ts/extract-one "FieldUserSettings" (:id age))]}]
|
|
(t2/delete! :model/FieldUserSettings :field_id (:id age))
|
|
(t2/insert! :model/FieldUserSettings {:field_id (:id email) :description "stale"})
|
|
(serdes.load/load-metabase! (ingestion-in-memory [legacy]))
|
|
(is (= "edited" (t2/select-one-fn :description :model/FieldUserSettings :field_id (:id age))))
|
|
(is (not (t2/exists? :model/FieldUserSettings :field_id (:id email))))))))
|
|
|
|
(deftest field-user-settings-import-clears-unset-values-test
|
|
(testing "importing a Field's settings over an existing row clears the user values its file leaves out"
|
|
(mt/with-empty-h2-app-db!
|
|
(let [db (ts/create! :model/Database :name "my-db")
|
|
table (ts/create! :model/Table :name "customers" :db_id (:id db))
|
|
email (ts/create! :model/Field :name "email" :table_id (:id table))
|
|
_ (t2/insert! :model/FieldUserSettings {:field_id (:id email) :description "edited"})
|
|
settings (ts/extract-one "FieldUserSettings" (:id email))]
|
|
(t2/update! :model/FieldUserSettings (:id email) {:display_name "Local"})
|
|
(serdes.load/load-metabase! (ingestion-in-memory [settings]))
|
|
(is (=? {:description "edited" :display_name nil}
|
|
(t2/select-one :model/FieldUserSettings :field_id (:id email))))))))
|
|
|
|
(deftest nested-field-dimension-import-test
|
|
(testing "a Dimension of a nested Field in a Table without a schema imports onto that Field"
|
|
(mt/with-empty-h2-app-db!
|
|
(let [db (ts/create! :model/Database :name "my-db")
|
|
table (ts/create! :model/Table :name "orders" :db_id (:id db))
|
|
customer (ts/create! :model/Field :name "customer" :table_id (:id table))
|
|
tier (ts/create! :model/Field :name "tier" :table_id (:id table) :parent_id (:id customer))
|
|
dimension (ts/create! :model/Dimension :field_id (:id tier) :name "Tier" :type :internal)
|
|
extracted (ts/extract-one "Dimension" (:entity_id dimension))
|
|
tables (t2/count :model/Table)]
|
|
(t2/delete! :model/Dimension :field_id (:id tier))
|
|
(serdes.load/load-metabase! (ingestion-in-memory [extracted]))
|
|
(is (= [(:id tier)] (t2/select-fn-vec :field_id :model/Dimension :entity_id (:entity_id dimension))))
|
|
(is (= tables (t2/count :model/Table)))))))
|
|
|
|
(deftest escape-continue-on-error-roundtrip-test
|
|
(testing "archive exported past escape analysis imports under continue-on-error without crashing (#74622)"
|
|
(let [serialized (atom nil)
|
|
coll-eid (atom nil)
|
|
clean-eid (atom nil)
|
|
dash-eid (atom nil)
|
|
escaped-eid (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(let [db (ts/create! :model/Database :name "rt-db")
|
|
table (ts/create! :model/Table :name "t" :db_id (:id db))
|
|
user (ts/create! :model/User :first_name "A" :last_name "B" :email "a@b.example")
|
|
target (ts/create! :model/Collection :name "Target")
|
|
outside (ts/create! :model/Collection :name "Outside (not exported)")
|
|
q {:type :query :database (:id db) :query {:source-table (:id table)}}
|
|
clean (ts/create! :model/Card :name "Clean" :collection_id (:id target)
|
|
:database_id (:id db) :table_id (:id table)
|
|
:query_type :query :dataset_query q :creator_id (:id user))
|
|
;; lives outside the target collection, so it "escapes"
|
|
escaped (ts/create! :model/Card :name "Escaped" :collection_id (:id outside)
|
|
:database_id (:id db) :table_id (:id table)
|
|
:query_type :query :dataset_query q :creator_id (:id user))
|
|
dash (ts/create! :model/Dashboard :name "Dash" :collection_id (:id target) :creator_id (:id user))]
|
|
(ts/create! :model/DashboardCard :dashboard_id (:id dash) :card_id (:id clean))
|
|
(ts/create! :model/DashboardCard :dashboard_id (:id dash) :card_id (:id escaped))
|
|
(reset! coll-eid (:entity_id target))
|
|
(reset! clean-eid (:entity_id clean))
|
|
(reset! dash-eid (:entity_id dash))
|
|
(reset! escaped-eid (:entity_id escaped))
|
|
(reset! serialized
|
|
(into [] (serdes.extract/extract {:targets [["Collection" (:id target)]]
|
|
:no-settings true
|
|
:continue-on-error true})))))
|
|
(testing "export keeps the collection, clean card and dashboard, leaves the escaped card out"
|
|
(is (contains? (ids-by-model @serialized "Collection") @coll-eid))
|
|
(is (contains? (ids-by-model @serialized "Card") @clean-eid))
|
|
(is (not (contains? (ids-by-model @serialized "Card") @escaped-eid)))
|
|
(is (contains? (ids-by-model @serialized "Dashboard") @dash-eid)))
|
|
;; The dashboard carries a dashcard pointing at the escaped card, which isn't in the archive. Under
|
|
;; continue-on-error the import skips that dashboard (load-metabase! wraps each entity in try/catch)
|
|
;; rather than aborting, and everything else lands. We do NOT promise a strict (default) import works.
|
|
(testing "import under continue-on-error doesn't crash; the dangling dashboard is skipped, the rest lands"
|
|
(ts/with-db dest-db
|
|
(let [report (serdes.load/load-metabase! (ingestion-in-memory @serialized) {:continue-on-error true})]
|
|
(is (some? (t2/select-one :model/Collection :entity_id @coll-eid)) "target collection imported")
|
|
(is (some? (t2/select-one :model/Card :entity_id @clean-eid)) "clean card imported")
|
|
(is (nil? (t2/select-one :model/Card :entity_id @escaped-eid)) "escaped card not imported")
|
|
(is (nil? (t2/select-one :model/Dashboard :entity_id @dash-eid))
|
|
"the dashboard that needs the escaped card is skipped, not imported with a dangling ref")
|
|
(is (seq (:errors report)) "the skipped dashboard is recorded as an import error"))))))))
|
|
|
|
(deftest deserialization-nested-collections-test
|
|
(testing "with a three-level nesting of collections"
|
|
(let [serialized (atom nil)
|
|
parent (atom nil)
|
|
child (atom nil)
|
|
grandchild (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(testing "serialization of the three collections"
|
|
(ts/with-db source-db
|
|
(reset! parent (ts/create! :model/Collection :name "Parent Collection" :location "/"))
|
|
(reset! child (ts/create! :model/Collection
|
|
:name "Child Collection"
|
|
:location (format "/%d/" (:id @parent))))
|
|
(reset! grandchild (ts/create! :model/Collection
|
|
:name "Grandchild Collection"
|
|
:location (format "/%d/%d/" (:id @parent) (:id @child))))
|
|
(reset! serialized (into [] (serdes.extract/extract {})))))
|
|
(testing "deserialization into a database that already has the parent, but with a different ID"
|
|
(ts/with-db dest-db
|
|
(ts/create! :model/Collection :name "Unrelated Collection")
|
|
(ts/create! :model/Collection :name "Parent Collection" :location "/" :entity_id (:entity_id @parent))
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
(let [parent-dest (t2/select-one :model/Collection :entity_id (:entity_id @parent))
|
|
child-dest (t2/select-one :model/Collection :entity_id (:entity_id @child))
|
|
grandchild-dest (t2/select-one :model/Collection :entity_id (:entity_id @grandchild))]
|
|
(is (some? parent-dest))
|
|
(is (some? child-dest))
|
|
(is (some? grandchild-dest))
|
|
(is (not= (:id parent-dest) (:id @parent)) "should have different primary keys")
|
|
(is (= 4 (t2/count :model/Collection :type nil)))
|
|
(is (= "/"
|
|
(:location parent-dest)))
|
|
(is (= (format "/%d/" (:id parent-dest))
|
|
(:location child-dest)))
|
|
(is (= (format "/%d/%d/" (:id parent-dest) (:id child-dest))
|
|
(:location grandchild-dest))))))))))
|
|
|
|
(deftest deserialization-database-table-field-test
|
|
(testing "databases, tables and fields are nested in namespaces"
|
|
(let [serialized (atom nil)
|
|
db1s (atom nil)
|
|
db1d (atom nil)
|
|
db2s (atom nil)
|
|
db2d (atom nil)
|
|
t1s (atom nil)
|
|
t2s (atom nil)
|
|
f1s (atom nil)
|
|
f2s (atom nil)
|
|
f3s (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(testing "serializing the two databases"
|
|
(ts/with-db source-db
|
|
(reset! db1s (ts/create! :model/Database :name "db1"))
|
|
(reset! t1s (ts/create! :model/Table :name "posts" :db_id (:id @db1s)))
|
|
(reset! db2s (ts/create! :model/Database :name "db2"))
|
|
(reset! t2s (ts/create! :model/Table :name "posts" :db_id (:id @db2s))) ; Deliberately the same name!
|
|
(reset! f1s (ts/create! :model/Field :name "Target Field" :table_id (:id @t1s)))
|
|
(reset! f2s (ts/create! :model/Field :name "Foreign Key" :table_id (:id @t2s) :fk_target_field_id (:id @f1s)))
|
|
(reset! f3s (ts/create! :model/Field :name "Nested Field" :table_id (:id @t1s) :parent_id (:id @f1s)))
|
|
(reset! serialized (into [] (serdes.extract/extract {})))))
|
|
(testing "serialization of databases is based on the :name"
|
|
(is (= #{(:name @db1s) (:name @db2s) "test-data (h2)"} ; TODO I'm not sure where the `test-data` one comes from.
|
|
(ids-by-model @serialized "Database"))))
|
|
(testing "tables reference their databases by name"
|
|
(is (= #{(:name @db1s) (:name @db2s) "test-data (h2)"}
|
|
(->> @serialized
|
|
(filter #(-> % :serdes/meta last :model (= "Table")))
|
|
(map :db_id)
|
|
set))))
|
|
(testing "foreign key references are serialized as a field path"
|
|
(is (= ["db1" nil "posts" "Target Field"]
|
|
(->> @serialized
|
|
(u/seek #(and (-> % :serdes/meta last :model (= "Field"))
|
|
(-> % :name (= "Foreign Key"))))
|
|
:fk_target_field_id))))
|
|
(testing "Parent field references are serialized as a field path"
|
|
(is (= ["db1" nil "posts" "Target Field"]
|
|
(->> @serialized
|
|
(u/seek #(and (-> % :serdes/meta last :model (= "Field"))
|
|
(-> % :name (= "Nested Field"))))
|
|
:parent_id))))
|
|
(testing "deserialization works properly, keeping the same-named tables apart"
|
|
(ts/with-db dest-db
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
(reset! db1d (t2/select-one :model/Database :name (:name @db1s)))
|
|
(reset! db2d (t2/select-one :model/Database :name (:name @db2s)))
|
|
(is (= 3 (t2/count :model/Database)))
|
|
(is (every? #(= "complete" (:initial_sync_status %)) (t2/select :model/Database)))
|
|
(is (= #{"db1" "db2" "test-data (h2)"}
|
|
(t2/select-fn-set :name :model/Database)))
|
|
(is (= #{(:id @db1d) (:id @db2d)}
|
|
(t2/select-fn-set :db_id :model/Table :name "posts")))
|
|
(is (t2/exists? :model/Table :name "posts" :db_id (:id @db1d)))
|
|
(is (t2/exists? :model/Table :name "posts" :db_id (:id @db2d)))
|
|
(is (= (t2/select-one-fn :id :model/Field :name (:name @f1s))
|
|
(t2/select-one-fn :fk_target_field_id :model/Field :name (:name @f2s))))
|
|
(is (= (t2/select-one-fn :id :model/Field :name (:name @f1s))
|
|
(t2/select-one-fn :parent_id :model/Field :name (:name @f3s))))))))))
|
|
|
|
(deftest card-dataset-query-test
|
|
;; Card.dataset_query is a JSON-encoded MBQL query, which contain database, table, and field IDs - these need to be
|
|
;; converted to a portable form and read back in.
|
|
;; This test has a database, table and fields, that exist on both sides with different IDs, and expects a card that
|
|
;; references those fields to be correctly loaded with the dest IDs.
|
|
(testing "embedded MBQL in Card :dataset-query is portable"
|
|
(let [serialized (atom nil)
|
|
coll1s (atom nil)
|
|
db1s (atom nil)
|
|
table1s (atom nil)
|
|
field1s (atom nil)
|
|
card1s (atom nil)
|
|
user1s (atom nil)
|
|
db1d (atom nil)
|
|
table1d (atom nil)
|
|
field1d (atom nil)
|
|
user1d (atom nil)
|
|
card1d (atom nil)
|
|
db2d (atom nil)
|
|
table2d (atom nil)
|
|
field2d (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(testing "serializing the original database, table, field and card"
|
|
(ts/with-db source-db
|
|
(reset! coll1s (ts/create! :model/Collection :name "pop! minis"))
|
|
(reset! db1s (ts/create! :model/Database :name "my-db"))
|
|
(reset! table1s (ts/create! :model/Table :name "customers" :db_id (:id @db1s)))
|
|
(reset! field1s (ts/create! :model/Field :name "age" :table_id (:id @table1s)))
|
|
(reset! user1s (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on"))
|
|
(reset! card1s (ts/create! :model/Card
|
|
:database_id (:id @db1s)
|
|
:table_id (:id @table1s)
|
|
:collection_id (:id @coll1s)
|
|
:creator_id (:id @user1s)
|
|
:query_type :query
|
|
:name "Example Card"
|
|
:dataset_query {:type :query
|
|
:query {:source-table (:id @table1s)
|
|
:filter [:>= [:field (:id @field1s) nil] 18]
|
|
:aggregation [[:count]]}
|
|
:database (:id @db1s)}
|
|
:display :line))
|
|
(reset! serialized (into [] (serdes.extract/extract {})))))
|
|
(testing "the serialized form is as desired"
|
|
(let [card (first (by-model @serialized "Card"))]
|
|
(is (=? {:lib/type :mbql/query
|
|
:stages [{:source-table ["my-db" nil "customers"]
|
|
:filters [[:>= {} [:field {} ["my-db" nil "customers" "age"]] 18]]
|
|
:aggregation [[:count {}]]}]
|
|
:database "my-db"}
|
|
(:dataset_query card)))))
|
|
(testing "deserializing adjusts the IDs properly"
|
|
(ts/with-db dest-db
|
|
;; A different database and tables, so the IDs don't match.
|
|
(reset! db2d (ts/create! :model/Database :name "other-db"))
|
|
(reset! table2d (ts/create! :model/Table :name "orders" :db_id (:id @db2d)))
|
|
(reset! field2d (ts/create! :model/Field :name "subtotal" :table_id (:id @table2d)))
|
|
(reset! user1d (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on"))
|
|
;; Load the serialized content.
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
;; Fetch the relevant bits
|
|
(reset! db1d (t2/select-one :model/Database :name "my-db"))
|
|
(reset! table1d (t2/select-one :model/Table :name "customers"))
|
|
(reset! field1d (t2/select-one :model/Field :table_id (:id @table1d) :name "age"))
|
|
(reset! card1d (t2/select-one :model/Card :name "Example Card"))
|
|
(testing "the main Database, Table, and Field have different IDs now"
|
|
(is (not= (:id @db1s) (:id @db1d)))
|
|
(is (not= (:id @table1s) (:id @table1d)))
|
|
(is (not= (:id @field1s) (:id @field1d))))
|
|
(is (not= (:dataset_query @card1s)
|
|
(:dataset_query @card1d)))
|
|
(testing "the Card's query is based on the new Database, Table, and Field IDs"
|
|
(is (=? {:lib/type :mbql/query
|
|
:stages [{:source-table (:id @table1d)
|
|
:filters [[:>= {}
|
|
[:field {} (:id @field1d)]
|
|
18]]
|
|
:aggregation [[:count {}]]}]
|
|
:database (:id @db1d)}
|
|
(:dataset_query @card1d))))))))))
|
|
|
|
(deftest model-result-metadata-id-roundtrips-test
|
|
(testing "GHY-3946: deserializing a model re-derives its result_metadata :id from the query when the underlying Fields are already present, so a filter mapped to the model column keeps its value dropdown; overrides are preserved"
|
|
(let [serialized (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(testing "serialize a model whose CATEGORY column carries a user override"
|
|
(ts/with-db source-db
|
|
(let [coll (ts/create! :model/Collection :name "models")
|
|
db (ts/create! :model/Database :name "my-db")
|
|
table (ts/create! :model/Table :name "products" :db_id (:id db))
|
|
field (ts/create! :model/Field :name "category" :table_id (:id table)
|
|
:base_type :type/Text :semantic_type :type/Category
|
|
:has_field_values :list)]
|
|
(ts/create! :model/Card
|
|
:type :model
|
|
:name "Products Model"
|
|
:collection_id (:id coll)
|
|
:database_id (:id db)
|
|
:table_id (:id table)
|
|
:query_type :query
|
|
:dataset_query {:type :query
|
|
:query {:source-table (:id table)}
|
|
:database (:id db)}
|
|
:result_metadata [{:name "category" :display_name "Custom Category"
|
|
:base_type :type/Text :semantic_type :type/Category
|
|
:id (:id field) :field_ref [:field (:id field) nil]
|
|
:table_id (:id table)}])
|
|
(reset! serialized (into [] (serdes.extract/extract {}))))))
|
|
(testing "the serialized model column keeps its user override"
|
|
(let [card (first (filter #(= "Products Model" (:name %)) (by-model @serialized "Card")))
|
|
cat (first (filter #(= "category" (:name %)) (:result_metadata card)))]
|
|
(is (some? cat))
|
|
(is (= "Custom Category" (:display_name cat)))))
|
|
(testing "loading re-derives the local :id when the data model already exists on the destination"
|
|
(ts/with-db dest-db
|
|
;; the data model is already present on the destination (and at different IDs than the source)
|
|
(ts/create! :model/Database :name "decoy")
|
|
(let [db (ts/create! :model/Database :name "my-db")
|
|
table (ts/create! :model/Table :name "products" :db_id (:id db))
|
|
field (ts/create! :model/Field :name "category" :table_id (:id table)
|
|
:base_type :type/Text :semantic_type :type/Category
|
|
:has_field_values :list)]
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
(let [dest-card (t2/select-one :model/Card :name "Products Model")
|
|
cat (first (filter #(= "category" (:name %)) (:result_metadata dest-card)))]
|
|
(is (number? (:id cat))
|
|
"model column kept a numeric :id link after the round-trip")
|
|
(is (= (:id field) (:id cat))
|
|
"model column :id points at the dest Field id")
|
|
(is (= "Custom Category" (:display_name cat))
|
|
"user override on the column survived the round-trip")))))))))
|
|
|
|
(deftest card-with-unexported-table-and-field-test
|
|
(testing "a Card referencing a Table/Field absent from the bundle still loads by synthesizing inactive rows"
|
|
(let [serialized (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(testing "serializing a database, table, field and a card that references them"
|
|
(ts/with-db source-db
|
|
(let [coll (ts/create! :model/Collection :name "pop! minis")
|
|
db (ts/create! :model/Database :name "my-db")
|
|
table (ts/create! :model/Table :name "customers" :db_id (:id db))
|
|
field (ts/create! :model/Field :name "age" :table_id (:id table) :base_type :type/Integer)
|
|
_user (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on")
|
|
mp (lib-be/application-database-metadata-provider (:id db))
|
|
query (-> (lib/query mp (lib.metadata/table mp (:id table)))
|
|
(lib/filter (lib/>= (lib.metadata/field mp (:id field)) 18))
|
|
(lib/aggregate (lib/count)))]
|
|
(ts/create! :model/Card
|
|
:database_id (:id db)
|
|
:table_id (:id table)
|
|
:collection_id (:id coll)
|
|
:query_type :query
|
|
:name "Example Card"
|
|
:dataset_query query
|
|
:display :line)
|
|
(reset! serialized (into [] (remove #(#{"Table" "Field"} (-> % :serdes/meta last :model))
|
|
(serdes.extract/extract {})))))))
|
|
(testing "the bundle has the Card but no Table/Field"
|
|
(is (seq (by-model @serialized "Card")))
|
|
(is (empty? (by-model @serialized "Table")))
|
|
(is (empty? (by-model @serialized "Field"))))
|
|
(testing "deserializing synthesizes inactive Table and Field for the dangling references"
|
|
(ts/with-db dest-db
|
|
(ts/create! :model/Database :name "my-db")
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
(let [db (t2/select-one :model/Database :name "my-db")
|
|
table (t2/select-one :model/Table :name "customers" :db_id (:id db))
|
|
field (and table (t2/select-one :model/Field :name "age" :table_id (:id table)))
|
|
card (t2/select-one :model/Card :name "Example Card")
|
|
query (:dataset_query card)]
|
|
(is (=? {:active false} table))
|
|
(is (=? {:active false} field))
|
|
(is (= (:id db) (lib/database-id query)))
|
|
(is (= (:id table) (lib/primary-source-table-id query))))))))))
|
|
|
|
(deftest segment-test
|
|
;; Segment.definition is a JSON-encoded MBQL query, which contain database, table, and field IDs - these need to be
|
|
;; converted to a portable form and read back in.
|
|
;; This test has a database, table and fields, that exist on both sides with different IDs, and expects a segment that
|
|
;; references those fields to be correctly loaded with the dest IDs.
|
|
(testing "embedded MBQL in Segment :definition is portable"
|
|
(let [serialized (atom nil)
|
|
coll1s (atom nil)
|
|
db1s (atom nil)
|
|
table1s (atom nil)
|
|
field1s (atom nil)
|
|
seg1s (atom nil)
|
|
user1s (atom nil)
|
|
db1d (atom nil)
|
|
table1d (atom nil)
|
|
field1d (atom nil)
|
|
user1d (atom nil)
|
|
seg1d (atom nil)
|
|
db2d (atom nil)
|
|
table2d (atom nil)
|
|
field2d (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(testing "serializing the original database, table, field and card"
|
|
(ts/with-db source-db
|
|
(reset! coll1s (ts/create! :model/Collection :name "pop! minis"))
|
|
(reset! db1s (ts/create! :model/Database :name "my-db"))
|
|
(reset! table1s (ts/create! :model/Table :name "customers" :db_id (:id @db1s)))
|
|
(reset! field1s (ts/create! :model/Field :name "age" :table_id (:id @table1s)))
|
|
(reset! user1s (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on"))
|
|
(reset! seg1s (ts/create! :model/Segment :table_id (:id @table1s) :name "Minors"
|
|
:definition {:source-table (:id @table1s)
|
|
:filter [:< [:field (:id @field1s) nil] 18]}
|
|
:creator_id (:id @user1s)))
|
|
(reset! serialized (into [] (serdes.extract/extract {})))))
|
|
(testing "exported form is properly converted"
|
|
(is (=? {:database "my-db"
|
|
:stages [{:filters [[:< {} [:field {} ["my-db" nil "customers" "age"]] 18]]
|
|
:source-table ["my-db" nil "customers"]}]
|
|
:lib/type :mbql/query}
|
|
(-> @serialized
|
|
(by-model "Segment")
|
|
first
|
|
:definition))))
|
|
(testing "deserializing adjusts the IDs properly"
|
|
(ts/with-db dest-db
|
|
;; A different database and tables, so the IDs don't match.
|
|
(reset! db2d (ts/create! :model/Database :name "other-db"))
|
|
(reset! table2d (ts/create! :model/Table :name "orders" :db_id (:id @db2d)))
|
|
(reset! field2d (ts/create! :model/Field :name "subtotal" :table_id (:id @table2d)))
|
|
(reset! user1d (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on"))
|
|
;; Load the serialized content.
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
;; Fetch the relevant bits
|
|
(reset! db1d (t2/select-one :model/Database :name "my-db"))
|
|
(reset! table1d (t2/select-one :model/Table :name "customers"))
|
|
(reset! field1d (t2/select-one :model/Field :table_id (:id @table1d) :name "age"))
|
|
(reset! seg1d (t2/select-one :model/Segment :name "Minors"))
|
|
(testing "the main Database, Table, and Field have different IDs now"
|
|
(is (not= (:id @db1s) (:id @db1d)))
|
|
(is (not= (:id @table1s) (:id @table1d)))
|
|
(is (not= (:id @field1s) (:id @field1d))))
|
|
(is (not= (:definition @seg1s)
|
|
(:definition @seg1d)))
|
|
(testing "the Segment's definition is based on the new Database, Table, and Field IDs"
|
|
(is (=? {:lib/type :mbql/query
|
|
:stages [{:source-table (:id @table1d)
|
|
:filters [[:< {} [:field {} (:id @field1d)] 18]]}]
|
|
:database (:id @db1d)}
|
|
(:definition @seg1d))))))))))
|
|
|
|
(defn- mbql5-query
|
|
"Create a simple MBQL 5 query for the given database and table."
|
|
[db-id table-id]
|
|
(let [metadata-provider (lib-be/application-database-metadata-provider db-id)
|
|
table (lib.metadata/table metadata-provider table-id)]
|
|
(lib/query metadata-provider table)))
|
|
|
|
(defn- mbql5-segment-definition
|
|
"Create a simple MBQL 5 segment definition with a filter."
|
|
[db-id table-id field-id]
|
|
(let [metadata-provider (lib-be/application-database-metadata-provider db-id)
|
|
table (lib.metadata/table metadata-provider table-id)
|
|
query (lib/query metadata-provider table)
|
|
field (lib.metadata/field metadata-provider field-id)]
|
|
(lib/filter query (lib/< field 18))))
|
|
|
|
(defn- mbql5-measure-definition
|
|
"Create an MBQL 5 measure definition with a sum aggregation."
|
|
[db-id table-id field-id]
|
|
(let [metadata-provider (lib-be/application-database-metadata-provider db-id)
|
|
table (lib.metadata/table metadata-provider table-id)
|
|
query (lib/query metadata-provider table)
|
|
field (lib.metadata/field metadata-provider field-id)]
|
|
(lib/aggregate query (lib/sum field))))
|
|
|
|
(deftest measure-test
|
|
;; Measure.definition is a JSON-encoded MBQL query, which contains database, table, and field IDs - these need to be
|
|
;; converted to a portable form and read back in.
|
|
;; This test has a database, table and fields, that exist on both sides with different IDs, and expects a measure that
|
|
;; references those fields to be correctly loaded with the dest IDs.
|
|
(testing "embedded MBQL in Measure :definition is portable"
|
|
(let [serialized (atom nil)
|
|
db1s (atom nil)
|
|
table1s (atom nil)
|
|
field1s (atom nil)
|
|
msr1s (atom nil)
|
|
user1s (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(testing "serializing the original database, table, field and measure"
|
|
(ts/with-db source-db
|
|
(reset! db1s (ts/create! :model/Database :name "my-db"))
|
|
(reset! table1s (ts/create! :model/Table :name "sales" :db_id (:id @db1s)))
|
|
(reset! field1s (ts/create! :model/Field :name "amount" :table_id (:id @table1s)))
|
|
(reset! user1s (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on"))
|
|
(reset! msr1s (ts/create! :model/Measure :table_id (:id @table1s) :name "Total Sales"
|
|
:definition (mbql5-measure-definition (:id @db1s) (:id @table1s) (:id @field1s))
|
|
:creator_id (:id @user1s)))
|
|
(reset! serialized (into [] (serdes.extract/extract {})))))
|
|
(testing "exported form is properly converted"
|
|
(is (=? {:database "my-db"
|
|
:lib/type :mbql/query
|
|
:stages [{:aggregation [[:sum {} [:field {} ["my-db" nil "sales" "amount"]]]]
|
|
:source-table ["my-db" nil "sales"]}]}
|
|
(-> @serialized
|
|
(by-model "Measure")
|
|
first
|
|
:definition))))
|
|
(testing "deserializing adjusts the IDs properly"
|
|
(ts/with-db dest-db
|
|
;; A different database and tables, so the IDs don't match.
|
|
(let [db2d (ts/create! :model/Database :name "other-db")
|
|
table2d (ts/create! :model/Table :name "orders" :db_id (:id db2d))
|
|
_ (ts/create! :model/Field :name "subtotal" :table_id (:id table2d))
|
|
_ (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on")]
|
|
;; Load the serialized content.
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
;; Fetch the relevant bits
|
|
(let [db1d (t2/select-one :model/Database :name "my-db")
|
|
table1d (t2/select-one :model/Table :name "sales")
|
|
field1d (t2/select-one :model/Field :table_id (:id table1d) :name "amount")
|
|
msr1d (t2/select-one :model/Measure :name "Total Sales")]
|
|
(testing "the main Database, Table, and Field have different IDs now"
|
|
(is (pos-int? (:id db1d)))
|
|
(is (not= (:id @db1s) (:id db1d)))
|
|
(is (pos-int? (:id table1d)))
|
|
(is (not= (:id @table1s) (:id table1d)))
|
|
(is (pos-int? (:id field1d)))
|
|
(is (not= (:id @field1s) (:id field1d))))
|
|
(is (not= (:definition @msr1s)
|
|
(:definition msr1d)))
|
|
(testing "the Measure's definition is based on the new Database, Table, and Field IDs"
|
|
(is (=? {:lib/type :mbql/query
|
|
:stages [{:source-table (:id table1d)
|
|
:aggregation [[:sum {} [:field {} (:id field1d)]]]}]
|
|
:database (:id db1d)}
|
|
(:definition msr1d))))))))))))
|
|
|
|
(deftest measure-referencing-measure-test
|
|
;; Test that a measure referencing another measure can be serialized and deserialized correctly.
|
|
;; The :measure clause in the definition should use entity_id for portability.
|
|
(testing "measure referencing another measure round-trip"
|
|
(let [serialized (atom nil)
|
|
db1s (atom nil)
|
|
table1s (atom nil)
|
|
field1s (atom nil)
|
|
msr1s (atom nil)
|
|
msr2s (atom nil)
|
|
user1s (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(testing "serializing measures where one references the other"
|
|
(ts/with-db source-db
|
|
(reset! db1s (ts/create! :model/Database :name "my-db"))
|
|
(reset! table1s (ts/create! :model/Table :name "sales" :db_id (:id @db1s)))
|
|
(reset! field1s (ts/create! :model/Field :name "amount" :table_id (:id @table1s)))
|
|
(reset! user1s (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on"))
|
|
;; Create base measure: sum(amount)
|
|
(reset! msr1s (ts/create! :model/Measure :table_id (:id @table1s) :name "Total Sales"
|
|
:definition (mbql5-measure-definition (:id @db1s) (:id @table1s) (:id @field1s))
|
|
:creator_id (:id @user1s)))
|
|
;; Create derived measure: base_measure * 2
|
|
(let [mp (lib-be/application-database-metadata-provider (:id @db1s))
|
|
table (lib.metadata/table mp (:id @table1s))
|
|
query (lib/query mp table)
|
|
base-measure (lib.metadata/measure mp (:id @msr1s))
|
|
derived-definition (lib/aggregate query (lib/* base-measure 2))]
|
|
(reset! msr2s (ts/create! :model/Measure :table_id (:id @table1s) :name "Double Sales"
|
|
:definition derived-definition
|
|
:creator_id (:id @user1s))))
|
|
(reset! serialized (into [] (serdes.extract/extract {})))))
|
|
(testing "exported form has measure reference with entity_id"
|
|
(let [derived-measure (first (filter #(= "Double Sales" (:name %)) (by-model @serialized "Measure")))]
|
|
(is (=? {:definition {:stages [{:aggregation [[:* {} [:measure {} (:entity_id @msr1s)] 2]]}]}}
|
|
derived-measure))
|
|
(is (= #{[{:id "my-db", :model "Database"}]
|
|
[{:id (:entity_id @msr1s), :model "Measure"}]}
|
|
(serdes/mbql-deps false (:definition derived-measure))))))
|
|
(testing "deserializing adjusts the measure IDs properly"
|
|
(ts/with-db dest-db
|
|
(ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on")
|
|
;; Load the serialized content
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
;; Fetch the relevant bits
|
|
(let [db1d (t2/select-one :model/Database :name "my-db")
|
|
table1d (t2/select-one :model/Table :name "sales")
|
|
;; field1d (t2/select-one :model/Field :table_id (:id table1d) :name "amount")
|
|
msr1d (t2/select-one :model/Measure :name "Total Sales")
|
|
msr2d (t2/select-one :model/Measure :name "Double Sales")]
|
|
(testing "both measures were loaded"
|
|
(is (some? msr1d))
|
|
(is (some? msr2d)))
|
|
(testing "the derived measure's definition references the base measure by new ID"
|
|
(is (=? {:lib/type :mbql/query
|
|
:stages [{:source-table (:id table1d)
|
|
:aggregation [[:* {} [:measure {} (:id msr1d)] 2]]}]
|
|
:database (:id db1d)}
|
|
(:definition msr2d)))))))))))
|
|
|
|
(deftest measure-referencing-segment-test
|
|
;; Test that a measure referencing a segment can be serialized and deserialized correctly.
|
|
;; The :segment clause in the definition should use entity_id for portability.
|
|
(testing "measure referencing a segment round-trip"
|
|
(let [serialized (atom nil)
|
|
db1s (atom nil)
|
|
table1s (atom nil)
|
|
field1s (atom nil)
|
|
seg1s (atom nil)
|
|
msr1s (atom nil)
|
|
user1s (atom nil)
|
|
db1d (atom nil)
|
|
table1d (atom nil)
|
|
field1d (atom nil)
|
|
seg1d (atom nil)
|
|
msr1d (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(testing "serializing measure that references a segment"
|
|
(ts/with-db source-db
|
|
(reset! db1s (ts/create! :model/Database :name "my-db"))
|
|
(reset! table1s (ts/create! :model/Table :name "products" :db_id (:id @db1s)))
|
|
(reset! field1s (ts/create! :model/Field :name "price" :table_id (:id @table1s)))
|
|
(reset! user1s (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on"))
|
|
;; Create segment: price > 100
|
|
(reset! seg1s (ts/create! :model/Segment :table_id (:id @table1s) :name "Expensive"
|
|
:definition {:source-table (:id @table1s)
|
|
:filter [:> [:field (:id @field1s) nil] 100]}
|
|
:creator_id (:id @user1s)))
|
|
;; Create measure: count-where(segment)
|
|
(let [mp (lib-be/application-database-metadata-provider (:id @db1s))
|
|
table (lib.metadata/table mp (:id @table1s))
|
|
query (lib/query mp table)
|
|
segment (lib.metadata/segment mp (:id @seg1s))
|
|
measure-definition (lib/aggregate query (lib/count-where segment))]
|
|
(reset! msr1s (ts/create! :model/Measure :table_id (:id @table1s) :name "Expensive Count"
|
|
:definition measure-definition
|
|
:creator_id (:id @user1s))))
|
|
(reset! serialized (into [] (serdes.extract/extract {})))))
|
|
(testing "exported form has segment reference with entity_id"
|
|
(let [measure (first (by-model @serialized "Measure"))]
|
|
(is (=? {:definition {:stages [{:aggregation [[:count-where {} [:segment {} (:entity_id @seg1s)]]]}]}}
|
|
measure))))
|
|
(testing "deserializing adjusts the segment IDs properly"
|
|
(ts/with-db dest-db
|
|
(reset! user1s (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on"))
|
|
;; Load the serialized content
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
;; Fetch the relevant bits
|
|
(reset! db1d (t2/select-one :model/Database :name "my-db"))
|
|
(reset! table1d (t2/select-one :model/Table :name "products"))
|
|
(reset! field1d (t2/select-one :model/Field :table_id (:id @table1d) :name "price"))
|
|
(reset! seg1d (t2/select-one :model/Segment :name "Expensive"))
|
|
(reset! msr1d (t2/select-one :model/Measure :name "Expensive Count"))
|
|
(testing "segment and measure were loaded"
|
|
(is (some? @seg1d))
|
|
(is (some? @msr1d)))
|
|
(testing "the measure's definition references the segment by new ID"
|
|
(is (=? {:lib/type :mbql/query
|
|
:stages [{:source-table (:id @table1d)
|
|
:aggregation [[:count-where {} [:segment {} (:id @seg1d)]]]}]
|
|
:database (:id @db1d)}
|
|
(:definition @msr1d))))))))))
|
|
|
|
;; full serdes round-trip: source graph, export, load, then cross-check remapped IDs -- one indivisible flow
|
|
#_{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]}
|
|
(deftest dashboard-card-test
|
|
;; DashboardCard.parameter_mappings and Card.parameter_mappings are JSON-encoded lists of parameter maps, which
|
|
;; contain field IDs - these need to be converted to a portable form and read back in.
|
|
;; DashboardCard.visualization_settings contains JSON with several places where IDs are embedded.
|
|
;; This test has a database, table and fields, that exist on both sides with different IDs, and expects a Card and
|
|
;; DashboardCard to be correctly loaded with the dest IDs.
|
|
(testing "parameter_mappings are portable"
|
|
(let [serialized (atom nil)
|
|
coll1s (atom nil)
|
|
db1s (atom nil)
|
|
table1s (atom nil)
|
|
field1s (atom nil)
|
|
field2s (atom nil)
|
|
field3s (atom nil)
|
|
field4s (atom nil)
|
|
dash1s (atom nil)
|
|
dash2s (atom nil)
|
|
tab2s (atom nil)
|
|
card1s (atom nil)
|
|
dashcard1s (atom nil)
|
|
user1s (atom nil)
|
|
db1d (atom nil)
|
|
table1d (atom nil)
|
|
field1d (atom nil)
|
|
field2d (atom nil)
|
|
user1d (atom nil)
|
|
dash1d (atom nil)
|
|
tab2d (atom nil)
|
|
card1d (atom nil)
|
|
dashcard1d (atom nil)
|
|
db2d (atom nil)
|
|
table2d (atom nil)
|
|
field3d (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(testing "serializing the original database, table, field and card"
|
|
(ts/with-db source-db
|
|
(reset! coll1s (ts/create! :model/Collection :name "pop! minis"))
|
|
(reset! db1s (ts/create! :model/Database :name "my-db"))
|
|
(reset! table1s (ts/create! :model/Table :name "orders" :db_id (:id @db1s)))
|
|
(reset! field1s (ts/create! :model/Field :name "subtotal" :table_id (:id @table1s)))
|
|
(reset! field2s (ts/create! :model/Field :name "invoice" :table_id (:id @table1s)))
|
|
(reset! field3s (ts/create! :model/Field :name "discount" :table_id (:id @table1s)))
|
|
(reset! field4s (ts/create! :model/Field :name "quantity" :table_id (:id @table1s)))
|
|
(reset! user1s (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on"))
|
|
(reset! dash1s (ts/create! :model/Dashboard :name "My Dashboard" :collection_id (:id @coll1s) :creator_id (:id @user1s)))
|
|
(reset! dash2s (ts/create! :model/Dashboard :name "Linked dashboard" :collection_id (:id @coll1s) :creator_id (:id @user1s)))
|
|
(reset! tab2s (ts/create! :model/DashboardTab :name "Tab for dash2" :dashboard_id (:id @dash2s) :position 0))
|
|
(let [columns [{:name "SOME_FIELD"
|
|
:fieldRef [:field (:id @field1s) nil]
|
|
:enabled true}
|
|
{:name "sum"
|
|
:fieldRef [:field "sum" {:base-type :type/Float}]
|
|
:enabled true}
|
|
{:name "count"
|
|
:fieldRef [:field "count" {:base-type :type/BigInteger}]
|
|
:enabled true}
|
|
{:name "Average order total"
|
|
:fieldRef [:field "Average order total" {:base-type :type/Float}]
|
|
:enabled true}]
|
|
mapping-id (json/encode [:dimension [:field (:id @field2s) {:source-field (:id @field1s)}]])
|
|
mapping-dimension [:dimension [:field (:id @field2s) {:source-field (:id @field1s)}]]]
|
|
(reset! card1s (ts/create! :model/Card :name "The Card" :database_id (:id @db1s) :table_id (:id @table1s)
|
|
:collection_id (:id @coll1s) :creator_id (:id @user1s)
|
|
:visualization_settings
|
|
{:table.pivot_column "SOURCE"
|
|
:table.cell_column "sum"
|
|
:table.columns columns
|
|
:column_settings {(json/encode [:ref [:field (:id @field2s) nil]]) {:column_title "Locus"}}}
|
|
:parameter_mappings [{:parameter_id "12345678"
|
|
:target [:dimension [:field (:id @field1s) {:source-field (:id @field2s)}]]}]))
|
|
(reset! dashcard1s (ts/create! :model/DashboardCard :dashboard_id (:id @dash1s) :card_id (:id @card1s)
|
|
:visualization_settings
|
|
{:table.pivot_column "SOURCE"
|
|
:table.cell_column "sum"
|
|
:table.columns columns
|
|
:column_settings
|
|
{(json/encode [:ref [:field (:id @field1s) nil]])
|
|
{:click_behavior {:type "link"
|
|
:linkType "dashboard"
|
|
:targetId (:id @dash2s)
|
|
:tabId (:id @tab2s)}}
|
|
(json/encode [:ref [:field (:id @field2s) nil]])
|
|
{:column_title "Locus"
|
|
:click_behavior
|
|
{:type "link"
|
|
:linkType "question"
|
|
:targetId (:id @card1s)
|
|
:parameterMapping
|
|
{mapping-id {:id mapping-id
|
|
:source {:type "column" :id "Category_ID" :name "Category ID"}
|
|
:target {:type "dimension" :id mapping-id :dimension mapping-dimension}}}}}
|
|
(json/encode [:ref [:field (:id @field3s) nil]])
|
|
{:click_behavior
|
|
{:type "link"
|
|
:linkType "question"
|
|
:targetId (:id @card1s)
|
|
:parameterMapping
|
|
{"qweqwe" {:id "qweqwe"
|
|
:source {:id "DISCOUNT" :name "Discount" :type "column"}
|
|
:target {:id "amount_between" :type "variable"}}}}}
|
|
(json/encode [:ref [:field (:id @field4s) nil]])
|
|
{:click_behavior
|
|
{:type "link"
|
|
:linkType "url"
|
|
:linkTemplate "https://example.com/order/{{QUANTITY}}"}}}
|
|
:click_behavior {:type "link"
|
|
:linkType "question"
|
|
:targetId (:id @card1s)}}
|
|
:parameter_mappings [{:parameter_id "deadbeef"
|
|
:card_id (:id @card1s)
|
|
:target [:dimension [:field (:id @field1s) {:source-field (:id @field2s)}]]}])))
|
|
(reset! serialized (into [] (serdes.extract/extract {})))
|
|
(let [card (-> @serialized (by-model "Card") first)
|
|
dash (-> @serialized (by-model "Dashboard") first)]
|
|
(testing "exported :parameter_mappings are properly converted"
|
|
(is (= [{:parameter_id "12345678"
|
|
:target [:dimension [:field ["my-db" nil "orders" "subtotal"]
|
|
{:source-field ["my-db" nil "orders" "invoice"]}]]}]
|
|
(:parameter_mappings card)))
|
|
(is (=? [{:parameter_mappings [{:parameter_id "deadbeef"
|
|
:card_id (:entity_id @card1s)
|
|
:target [:dimension [:field ["my-db" nil "orders" "subtotal"]
|
|
{:source-field ["my-db" nil "orders" "invoice"]}]]}]}]
|
|
(:dashcards dash))))
|
|
(testing "exported :visualization_settings are properly converted"
|
|
(let [exp-card {:table.pivot_column "SOURCE"
|
|
:table.cell_column "sum"
|
|
:table.columns
|
|
[{:name "SOME_FIELD"
|
|
:fieldRef [:field ["my-db" nil "orders" "subtotal"] nil]
|
|
:enabled true}
|
|
{:name "sum"
|
|
:fieldRef [:field "sum" {:base-type :type/Float}]
|
|
:enabled true}
|
|
{:name "count"
|
|
:fieldRef [:field "count" {:base-type :type/BigInteger}]
|
|
:enabled true}
|
|
{:name "Average order total"
|
|
:fieldRef [:field "Average order total" {:base-type :type/Float}]
|
|
:enabled true}]
|
|
:column_settings
|
|
{"[\"ref\",[\"field\",[\"my-db\",null,\"orders\",\"invoice\"],null]]" {:column_title "Locus"}}}
|
|
dimension [:dimension [:field ["my-db" nil "orders" "invoice"] {:source-field ["my-db" nil "orders" "subtotal"]}]]
|
|
dimension-id (json/encode [:dimension [:field [:my-db nil :orders :invoice] {:source-field [:my-db nil :orders :subtotal]}]])
|
|
exp-dashcard (-> exp-card
|
|
(assoc :click_behavior {:type "link"
|
|
:linkType "question"
|
|
:targetId (:entity_id @card1s)})
|
|
(assoc-in [:column_settings
|
|
(json/encode [:ref [:field [:my-db nil :orders :subtotal] nil]])
|
|
:click_behavior]
|
|
{:type "link"
|
|
:linkType "dashboard"
|
|
:targetId (:entity_id @dash2s)
|
|
:tabId [(:entity_id @dash2s) (:entity_id @tab2s)]})
|
|
(assoc-in [:column_settings
|
|
(json/encode [:ref [:field [:my-db nil :orders :invoice] nil]])
|
|
:click_behavior]
|
|
{:type "link"
|
|
:linkType "question"
|
|
:targetId (:entity_id @card1s)
|
|
:parameterMapping
|
|
{dimension-id
|
|
{:id dimension-id
|
|
:source {:type "column" :id "Category_ID" :name "Category ID"}
|
|
:target {:type "dimension" :id dimension-id :dimension dimension}}}})
|
|
(assoc-in [:column_settings
|
|
(json/encode [:ref [:field [:my-db nil :orders :discount] nil]])
|
|
:click_behavior]
|
|
{:type "link"
|
|
:linkType "question"
|
|
:targetId (:entity_id @card1s)
|
|
:parameterMapping
|
|
{"qweqwe" {:id "qweqwe"
|
|
:source {:id "DISCOUNT" :name "Discount" :type "column"}
|
|
:target {:id "amount_between" :type "variable"}}}})
|
|
(assoc-in [:column_settings
|
|
(json/encode [:ref [:field [:my-db nil :orders :quantity] nil]])
|
|
:click_behavior]
|
|
{:type "link"
|
|
:linkType "url"
|
|
:linkTemplate "https://example.com/order/{{QUANTITY}}"}))]
|
|
(is (= exp-card
|
|
(:visualization_settings card)))
|
|
(is (= exp-dashcard
|
|
(-> dash :dashcards first :visualization_settings))))))))
|
|
(testing "deserializing adjusts the IDs properly"
|
|
(ts/with-db dest-db
|
|
;; A different database and tables, so the IDs don't match.
|
|
(reset! db2d (ts/create! :model/Database :name "other-db"))
|
|
(reset! table2d (ts/create! :model/Table :name "customers" :db_id (:id @db2d)))
|
|
(reset! field3d (ts/create! :model/Field :name "age" :table_id (:id @table2d)))
|
|
(ts/create! :model/Field :name "name" :table_id (:id @table2d))
|
|
(ts/create! :model/Field :name "address" :table_id (:id @table2d))
|
|
(reset! user1d (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on"))
|
|
;; Load the serialized content.
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
;; Fetch the relevant bits
|
|
(reset! db1d (t2/select-one :model/Database :name "my-db"))
|
|
(reset! table1d (t2/select-one :model/Table :name "orders"))
|
|
(reset! field1d (t2/select-one :model/Field :table_id (:id @table1d) :name "subtotal"))
|
|
(reset! field2d (t2/select-one :model/Field :table_id (:id @table1d) :name "invoice"))
|
|
(reset! dash1d (t2/select-one :model/Dashboard :name "My Dashboard"))
|
|
(reset! tab2d (t2/select-one :model/DashboardTab :name "Tab for dash2"))
|
|
(reset! card1d (t2/select-one :model/Card :name "The Card"))
|
|
(reset! dashcard1d (t2/select-one :model/DashboardCard :card_id (:id @card1d) :dashboard_id (:id @dash1d)))
|
|
(testing "the main Database, Table, and Field have different IDs now"
|
|
(is (not= (:id @db1s) (:id @db1d)))
|
|
(is (not= (:id @table1s) (:id @table1d)))
|
|
(is (not= (:id @field1s) (:id @field1d)))
|
|
(is (not= (:id @field2s) (:id @field2d))))
|
|
(is (not= (:parameter_mappings @dashcard1s)
|
|
(:parameter_mappings @dashcard1d)))
|
|
(is (not= (:parameter_mappings @card1s)
|
|
(:parameter_mappings @card1d)))
|
|
(testing "Card.parameter_mappings are based on the new Field IDs"
|
|
(is (= [{:parameter_id "12345678"
|
|
:target [:dimension [:field (:id @field1d) {:source-field (:id @field2d)}]]}]
|
|
(:parameter_mappings @card1d))))
|
|
(testing "DashboardCard.parameter_mappings are based on the new Field IDs"
|
|
(is (= [{:parameter_id "deadbeef"
|
|
:card_id (:id @card1d)
|
|
:target [:dimension [:field (:id @field1d) {:source-field (:id @field2d)}]]}]
|
|
(:parameter_mappings @dashcard1d))))
|
|
(testing "DashboardTab was exported even though he wasn't mentioned by click_behavior"
|
|
(is (= (:entity_id @tab2s) (:entity_id @tab2d)))
|
|
(let [settings (-> @dashcard1d :visualization_settings :column_settings)
|
|
setting (get settings (json/encode [:ref [:field (:id @field1d) nil]]))]
|
|
(is (= (:id @tab2d)
|
|
(-> setting :click_behavior :tabId)))))))))))
|
|
|
|
(deftest timelines-test
|
|
(testing "timelines"
|
|
(let [serialized (atom nil)
|
|
coll1s (atom nil)
|
|
user1s (atom nil)
|
|
timeline1s (atom nil)
|
|
event1s (atom nil)
|
|
event2s (atom nil)
|
|
timeline2s (atom nil)
|
|
event3s (atom nil)
|
|
|
|
coll1d (atom nil)
|
|
user1d (atom nil)
|
|
timeline1d (atom nil)
|
|
timeline2d (atom nil)
|
|
eventsT1 (atom nil)
|
|
eventsT2 (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(testing "serialize correctly"
|
|
(ts/with-db source-db
|
|
(reset! coll1s (ts/create! :model/Collection :name "col1"))
|
|
(reset! user1s (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on"))
|
|
(reset! timeline1s (ts/create! :model/Timeline :name "Some events" :creator_id (:id @user1s)
|
|
:collection_id (:id @coll1s)))
|
|
(reset! event1s (ts/create! :model/TimelineEvent :name "First thing" :timeline_id (:id @timeline1s)
|
|
:creator_id (:id @user1s) :timezone "America/New_York"
|
|
:timestamp (t/local-date 2022 11 3)))
|
|
(reset! event2s (ts/create! :model/TimelineEvent :name "Second thing" :timeline_id (:id @timeline1s)
|
|
:creator_id (:id @user1s) :timezone "America/New_York"
|
|
:timestamp (t/local-date 2022 11 8)))
|
|
(reset! timeline2s (ts/create! :model/Timeline :name "More events" :creator_id (:id @user1s)
|
|
:collection_id (:id @coll1s)))
|
|
(reset! event3s (ts/create! :model/TimelineEvent :name "Different event" :timeline_id (:id @timeline2s)
|
|
:creator_id (:id @user1s) :timezone "America/New_York"
|
|
:time_matters true :timestamp (t/offset-date-time 2022 10 31 19 00 00)))
|
|
(testing "expecting 3 events"
|
|
(is (= 3 (t2/count :model/TimelineEvent))))
|
|
(reset! serialized (into [] (serdes.extract/extract {})))
|
|
(let [timelines (by-model @serialized "Timeline")
|
|
timeline1 (first (filter #(= (:entity_id %) (:entity_id @timeline1s)) timelines))
|
|
timeline2 (first (filter #(= (:entity_id %) (:entity_id @timeline2s)) timelines))]
|
|
(testing "with inline :events"
|
|
(is (malli= [:map
|
|
[:serdes/meta [:= [{:model "Timeline"
|
|
:id (:entity_id timeline1)
|
|
:label "some_events"}]]]
|
|
[:collection_id [:= (:entity_id @coll1s)]]
|
|
[:name [:= "Some events"]]
|
|
[:creator_id [:= "tom@bost.on"]]
|
|
[:created_at :string]
|
|
[:entity_id [:= (:entity_id timeline1)]]
|
|
[:description {:optional true} [:maybe :string]]
|
|
[:events [:sequential
|
|
[:map
|
|
[:timezone :string]
|
|
[:time_matters :boolean]
|
|
[:name :string]
|
|
[:archived {:optional true} :boolean]
|
|
[:description {:optional true} [:maybe :string]]
|
|
[:creator_id :string]
|
|
[:created_at :string]
|
|
[:timestamp :string]
|
|
[:icon {:optional true} [:maybe :string]]]]]
|
|
[:icon {:optional true} [:maybe :string]]]
|
|
timeline1))
|
|
(is (= 2 (-> timeline1 :events count)))
|
|
(is (= 1 (-> timeline2 :events count)))))))
|
|
(testing "deserializing merges events properly"
|
|
(ts/with-db dest-db
|
|
;; The collection, timeline 1 and event 2 already exist. Event 1, plus timeline 2 and its event 3, are new.
|
|
(reset! user1d (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on"))
|
|
(reset! coll1d (ts/create! :model/Collection :name "col1" :entity_id (:entity_id @coll1s)))
|
|
(reset! timeline1d (ts/create! :model/Timeline :name "Some events" :creator_id (:id @user1s)
|
|
:entity_id (:entity_id @timeline1s)
|
|
:collection_id (:id @coll1d)))
|
|
(ts/create! :model/TimelineEvent :name "Second thing with different name" :timeline_id (:id @timeline1s)
|
|
:timestamp (:timestamp @event2s)
|
|
:creator_id (:id @user1s) :timezone "America/New_York")
|
|
;; Load the serialized content.
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
;; Fetch the relevant bits
|
|
(reset! timeline2d (t2/select-one :model/Timeline :entity_id (:entity_id @timeline2s)))
|
|
(reset! eventsT1 (t2/select :model/TimelineEvent :timeline_id (:id @timeline1d)))
|
|
(reset! eventsT2 (t2/select :model/TimelineEvent :timeline_id (:id @timeline2d)))
|
|
(testing "no duplication - there are two timelines with the right event counts"
|
|
(is (some? @timeline2d))
|
|
(is (= 2 (count @eventsT1)))
|
|
(is (= 1 (count @eventsT2))))
|
|
(testing "resulting events match up"
|
|
(let [[event1 event2] (sort-by :timestamp @eventsT1)]
|
|
(is (= (:timestamp @event1s) (:timestamp event1)))
|
|
(is (= (:timestamp @event2s) (:timestamp event2)))
|
|
(is (= (:timestamp @event3s)
|
|
(:timestamp (first @eventsT2))))
|
|
(is (= (:name @event2s)
|
|
(:name event2))
|
|
"existing event name should be updated")))))))))
|
|
|
|
(deftest users-test
|
|
;; Users are serialized as their email address. If a corresponding user is found during deserialization, its ID is
|
|
;; used. However, if no such user exists, a new one is created with mostly blank fields.
|
|
(testing "existing users are found and used; missing users are created on the fly"
|
|
(let [serialized (atom nil)
|
|
user1s (atom nil)
|
|
user2s (atom nil)
|
|
dash1s (atom nil)
|
|
dash2s (atom nil)
|
|
user1d (atom nil)
|
|
dash1d (atom nil)
|
|
dash2d (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(testing "serializing the original entities"
|
|
(ts/with-db source-db
|
|
(reset! user1s (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on"))
|
|
(reset! user2s (ts/create! :model/User :first_name "Neil" :last_name "Peart" :email "neil@rush.yyz"))
|
|
(reset! dash1s (ts/create! :model/Dashboard :name "My Dashboard" :creator_id (:id @user1s)))
|
|
(reset! dash2s (ts/create! :model/Dashboard :name "Linked dashboard" :creator_id (:id @user2s)))
|
|
(reset! serialized (into [] (serdes.extract/extract {})))))
|
|
(testing "deserializing finds the matching user and synthesizes the missing one"
|
|
(ts/with-db dest-db
|
|
;; Create another random user to change the user IDs.
|
|
(ts/create! :model/User :first_name "Gideon" :last_name "Nav" :email "griddle@ninth.tomb")
|
|
;; Likewise, create some other dashboards.
|
|
(ts/create! :model/Dashboard :name "Other dashboard A")
|
|
(ts/create! :model/Dashboard :name "Other dashboard B")
|
|
(ts/create! :model/Dashboard :name "Other dashboard C")
|
|
(reset! user1d (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on"))
|
|
;; Load the serialized content.
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
(reset! dash1d (t2/select-one :model/Dashboard :name "My Dashboard"))
|
|
(reset! dash2d (t2/select-one :model/Dashboard :name "Linked dashboard"))
|
|
(testing "the Dashboards and Users have different IDs now"
|
|
(is (not= (:id @dash1s) (:id @dash1d)))
|
|
(is (not= (:id @dash2s) (:id @dash2d)))
|
|
(is (not= (:id @user1s) (:id @user1d))))
|
|
(testing "both existing User and the new one are set up properly"
|
|
(is (= (:id @user1d) (:creator_id @dash1d)))
|
|
(let [user2d-id (:creator_id @dash2d)
|
|
user2d (t2/select-one [:model/User :email :is_active] :id user2d-id)]
|
|
(is (= (:email @user2s) (:email user2d)))
|
|
(is (false? (:is_active user2d)))))))))))
|
|
|
|
(deftest field-values-test
|
|
;; FieldValues are a bit special - they map 1-1 with Fields but are a separate table serialized separately.
|
|
;; The main special thing to test here is that the custom load-find-local correctly finds an existing FieldValues.
|
|
;; This test creates:
|
|
;; - in src: a database, table, and two fields each with field values.
|
|
;; - in dst: a different database, table and field, to fiddle the IDs; plus the same database, table, both fields, but
|
|
;; only one FieldValues. The existing and new FieldValues should both work correctly.
|
|
;; Another thing tested here is that the :field_id is properly reconstructed from the serdes path.
|
|
(testing "FieldValues are portable"
|
|
(let [serialized (atom nil)
|
|
db1s (atom nil)
|
|
table1s (atom nil)
|
|
field1s (atom nil)
|
|
field2s (atom nil)
|
|
fv1s (atom nil)
|
|
fv2s (atom nil)
|
|
|
|
db1d (atom nil)
|
|
table1d (atom nil)
|
|
field1d (atom nil)
|
|
field2d (atom nil)
|
|
fv1d (atom nil)
|
|
fv2d (atom nil)
|
|
db2d (atom nil)
|
|
table2d (atom nil)
|
|
field3d (atom nil)]
|
|
(testing "serializing the original database, table, field and fieldvalues"
|
|
(mt/with-empty-h2-app-db!
|
|
(reset! db1s (ts/create! :model/Database :name "my-db"))
|
|
(reset! table1s (ts/create! :model/Table :name "CUSTOMERS" :db_id (:id @db1s)))
|
|
(reset! field1s (ts/create! :model/Field :name "STATE" :table_id (:id @table1s)))
|
|
(reset! field2s (ts/create! :model/Field :name "CATEGORY" :table_id (:id @table1s)))
|
|
(reset! fv1s (ts/create! :model/FieldValues :field_id (:id @field1s) :values ["AZ" "CA" "NY" "TX"]))
|
|
(reset! fv2s (ts/create! :model/FieldValues :field_id (:id @field2s)
|
|
:values ["CONSTRUCTION" "DAYLIGHTING" "DELIVERY" "HAULING"]))
|
|
(reset! serialized (into [] (serdes.extract/extract {:include-field-values true})))
|
|
(testing "the expected fields are serialized"
|
|
(is (= 1
|
|
(->> @serialized
|
|
(filter #(= (:serdes/meta %)
|
|
[{:model "Database" :id "test-data (h2)"}
|
|
{:model "Schema" :id "PUBLIC"}
|
|
{:model "Table" :id "VENUES"}
|
|
{:model "Field" :id "NAME"}]))
|
|
count))))
|
|
(testing "FieldValues are serialized under their fields, with their own ID always 0"
|
|
(let [fvs (by-model @serialized "FieldValues")]
|
|
(is (= #{[{:model "Database" :id "my-db"}
|
|
{:model "Table" :id "CUSTOMERS"}
|
|
{:model "Field" :id "STATE"}
|
|
{:model "FieldValues" :id "0"}]
|
|
[{:model "Database" :id "my-db"}
|
|
{:model "Table" :id "CUSTOMERS"}
|
|
{:model "Field" :id "CATEGORY"}
|
|
{:model "FieldValues" :id "0"}]}
|
|
(->> fvs
|
|
(map serdes/path)
|
|
(filter #(-> % first :id (= "my-db")))
|
|
set)))))))
|
|
(testing "deserializing finds existing FieldValues properly"
|
|
(mt/with-empty-h2-app-db!
|
|
;; A different database and tables, so the IDs don't match.
|
|
(reset! db2d (ts/create! :model/Database :name "other-db"))
|
|
(reset! table2d (ts/create! :model/Table :name "ORDERS" :db_id (:id @db2d)))
|
|
(reset! field3d (ts/create! :model/Field :name "SUBTOTAL" :table_id (:id @table2d)))
|
|
(ts/create! :model/Field :name "DISCOUNT" :table_id (:id @table2d))
|
|
(ts/create! :model/Field :name "UNITS" :table_id (:id @table2d))
|
|
;; Now the database, table, fields and *one* of the FieldValues from the src side.
|
|
(reset! db1d (ts/create! :model/Database :name "my-db"))
|
|
(reset! table1d (ts/create! :model/Table :name "CUSTOMERS" :db_id (:id @db1d)))
|
|
(reset! field1d (ts/create! :model/Field :name "STATE" :table_id (:id @table1d)))
|
|
(reset! field2d (ts/create! :model/Field :name "CATEGORY" :table_id (:id @table1d)))
|
|
;; The :values are different here; they should get overwritten by the update.
|
|
(reset! fv1d (ts/create! :model/FieldValues :field_id (:id @field1d) :values ["WA" "NC" "NM" "WI"]))
|
|
;; Load the serialized content.
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
;; Fetch the relevant bits
|
|
(reset! fv1d (t2/select-one :model/FieldValues :field_id (:id @field1d)))
|
|
(reset! fv2d (t2/select-one :model/FieldValues :field_id (:id @field2d)))
|
|
(testing "the main Database, Table, and Field have different IDs now"
|
|
(is (not= (:id @db1s) (:id @db1d)))
|
|
(is (not= (:id @table1s) (:id @table1d)))
|
|
(is (not= (:id @field1s) (:id @field1d)))
|
|
(is (not= (:id @field2s) (:id @field2d))))
|
|
(testing "there are 2 FieldValues defined under fields of table1d"
|
|
(let [fields (t2/select-pks-set :model/Field :table_id (:id @table1d))]
|
|
(is (= 2 (t2/count :model/FieldValues :field_id [:in fields])))))
|
|
(testing "existing FieldValues are properly found and updated"
|
|
(is (= (set (:values @fv1s)) (set (:values @fv1d)))))
|
|
(testing "new FieldValues are properly added"
|
|
(is (= (dissoc @fv2s :id :field_id :created_at :updated_at)
|
|
(dissoc @fv2d :id :field_id :created_at :updated_at)))))))))
|
|
|
|
(deftest table-publishing-round-trip-test
|
|
;; Tests that is_published and collection_id survive a round-trip through serialization.
|
|
;; The collection_id FK transform converts DB id to entity_id and back.
|
|
(testing "published table with collection_id survives round-trip"
|
|
(let [serialized (atom nil)
|
|
db1s (atom nil)
|
|
table1s (atom nil)
|
|
coll1s (atom nil)
|
|
table2s (atom nil)
|
|
|
|
db1d (atom nil)
|
|
table1d (atom nil)
|
|
coll1d (atom nil)
|
|
table2d (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(testing "serializing published tables"
|
|
(ts/with-db source-db
|
|
(reset! db1s (ts/create! :model/Database :name "my-db"))
|
|
(reset! coll1s (ts/create! :model/Collection :name "Publishing Collection"))
|
|
(reset! table1s (ts/create! :model/Table :name "published_table" :db_id (:id @db1s)
|
|
:is_published true :collection_id (:id @coll1s)))
|
|
(reset! table2s (ts/create! :model/Table :name "unpublished_table" :db_id (:id @db1s)
|
|
:is_published false))
|
|
(reset! serialized (into [] (serdes.extract/extract {})))))
|
|
(testing "serialized form has collection entity_id, not DB id"
|
|
(let [pub-table (->> @serialized
|
|
(filter #(and (-> % :serdes/meta last :model (= "Table"))
|
|
(= (:name %) "published_table")))
|
|
first)]
|
|
(is (true? (:is_published pub-table)))
|
|
(is (= (:entity_id @coll1s) (:collection_id pub-table)))))
|
|
(testing "deserializing restores collection_id correctly"
|
|
(ts/with-db dest-db
|
|
;; Load the serialized content
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
;; Fetch the relevant bits
|
|
(reset! db1d (t2/select-one :model/Database :name "my-db"))
|
|
(reset! coll1d (t2/select-one :model/Collection :name "Publishing Collection"))
|
|
(reset! table1d (t2/select-one :model/Table :name "published_table" :db_id (:id @db1d)))
|
|
(reset! table2d (t2/select-one :model/Table :name "unpublished_table" :db_id (:id @db1d)))
|
|
(testing "published table has correct is_published and collection_id"
|
|
(is (true? (:is_published @table1d)))
|
|
(is (= (:id @coll1d) (:collection_id @table1d))))
|
|
(testing "unpublished table has is_published=false and no collection"
|
|
(is (false? (:is_published @table2d)))
|
|
(is (nil? (:collection_id @table2d))))))))))
|
|
|
|
(deftest bare-import-test
|
|
;; If the dependencies of an entity exist in the receiving database, they don't need to be in the export.
|
|
;; This tests that such an import will succeed, and that it still fails when the dependency is not found in
|
|
;; either location.
|
|
(let [db1s (atom nil)
|
|
table1s (atom nil)]
|
|
(testing "loading a bare card"
|
|
(mt/with-empty-h2-app-db!
|
|
(reset! db1s (ts/create! :model/Database :name "my-db"))
|
|
(reset! table1s (ts/create! :model/Table :name "CUSTOMERS" :db_id (:id @db1s)))
|
|
(ts/create! :model/Field :name "STATE" :table_id (:id @table1s))
|
|
(ts/create! :model/User :first_name "Geddy" :last_name "Lee" :email "glee@rush.yyz")
|
|
(testing "depending on existing values works"
|
|
(let [ingestion (ingestion-in-memory [{:serdes/meta [{:model "Card" :id "0123456789abcdef_0123"}]
|
|
:created_at (t/instant)
|
|
:creator_id "glee@rush.yyz"
|
|
:database_id "my-db"
|
|
:dataset_query {:database "my-db"
|
|
:type :query
|
|
:query {:source-table ["my-db" nil "CUSTOMERS"]}}
|
|
:display :table
|
|
:entity_id "0123456789abcdef_0123"
|
|
:name "Some card"
|
|
:table_id ["my-db" nil "CUSTOMERS"]
|
|
:visualization_settings {}}])]
|
|
(is (some? (serdes.load/load-metabase! ingestion)))))
|
|
(testing "depending on nonexisting values fails"
|
|
(let [ingestion (ingestion-in-memory [{:serdes/meta [{:model "Card" :id "0123456789abcdef_0123"}]
|
|
:created_at (t/instant)
|
|
:creator_id "glee@rush.yyz"
|
|
:database_id "bad-db"
|
|
:dataset_query {:database "bad-db"
|
|
:type :query
|
|
:query {:source-table ["bad-db" nil "CUSTOMERS"]}}
|
|
:display :table
|
|
:entity_id "0123456789abcdef_0123"
|
|
:name "Some card"
|
|
:table_id ["bad-db" nil "CUSTOMERS"]
|
|
:visualization_settings {}}])
|
|
e (try
|
|
(serdes.load/load-metabase! ingestion)
|
|
nil
|
|
(catch clojure.lang.ExceptionInfo e e))]
|
|
(is (some? e))
|
|
(is (re-find #"was not found" (ex-message e)))
|
|
(testing "the not-found error names the entity holding the dangling reference (GHY-3992)"
|
|
(is (= {:model "Database" :id "bad-db"}
|
|
(select-keys (ex-data e) [:model :id])))
|
|
(is (= {:model "Card" :id "0123456789abcdef_0123" :name "Some card"}
|
|
(:referrer (ex-data e)))))
|
|
;; attaching the referrer must not nest the original beneath itself: error reporting renders the
|
|
;; whole cause chain, and a self-nested exception prints the same message twice
|
|
(testing "attaching the referrer does not duplicate the error through the cause chain"
|
|
(is (nil? (ex-cause e))))))
|
|
;; `result-metadata-deps` derives deps only from each entry's `:field_ref`, never from its `:table_id`.
|
|
;; So a result_metadata `:table_id` naming an absent database gets past dependency resolution and only
|
|
;; fails inside `serdes/load-one!`, where `import-table-fk` raises ::database-not-found. (The Card's own
|
|
;; top-level `table_id` cannot be used here: `populate-query-fields` recomputes it from `dataset_query`.)
|
|
(testing "a result_metadata table_id naming an absent database names the database and the card (GHY-3992)"
|
|
(let [ingestion (ingestion-in-memory [{:serdes/meta [{:model "Card" :id "0123456789abcdef_0123"}]
|
|
:created_at (t/instant)
|
|
:creator_id "glee@rush.yyz"
|
|
:database_id "my-db"
|
|
:dataset_query {:database "my-db"
|
|
:type :query
|
|
:query {:source-table ["my-db" nil "CUSTOMERS"]}}
|
|
:display :table
|
|
:entity_id "0123456789abcdef_0123"
|
|
:name "Some card"
|
|
:result_metadata [{:name "STATE"
|
|
:base_type :type/Text
|
|
:table_id ["absent-db" nil "CUSTOMERS"]}]
|
|
:table_id ["my-db" nil "CUSTOMERS"]
|
|
:visualization_settings {}}])
|
|
e (try
|
|
(serdes.load/load-metabase! ingestion)
|
|
nil
|
|
(catch clojure.lang.ExceptionInfo e e))
|
|
root (->> (iterate ex-cause e)
|
|
(take-while some?)
|
|
(some #(when (= :metabase.models.serialization.resolve.db/database-not-found
|
|
(:error (ex-data %)))
|
|
%)))]
|
|
(is (some? e))
|
|
(testing "the root cause carries the missing database name"
|
|
(is (some? root))
|
|
(is (= "absent-db" (:db-name (ex-data root)))))
|
|
(testing "the outer load-failure names the card that failed to load"
|
|
(is (= {:model "Card" :id "0123456789abcdef_0123" :name "Some card"}
|
|
(:entity (ex-data e)))))))))))
|
|
|
|
(deftest card-with-snippet-test
|
|
(let [db1s (atom nil)
|
|
table1s (atom nil)
|
|
snippet1s (atom nil)
|
|
card1s (atom nil)
|
|
extracted (atom nil)]
|
|
(testing "snippets referenced by native cards must be deserialized"
|
|
(mt/with-empty-h2-app-db!
|
|
(reset! db1s (ts/create! :model/Database :name "my-db"))
|
|
(reset! table1s (ts/create! :model/Table :name "CUSTOMERS" :db_id (:id @db1s)))
|
|
(reset! snippet1s (ts/create! :model/NativeQuerySnippet :name "some snippet"))
|
|
(reset! card1s (ts/create! :model/Card
|
|
:name "the query"
|
|
:dataset_query {:database (:id @db1s)
|
|
:type :native
|
|
:native {:template-tags {"snippet: things"
|
|
{:id "e2d15f07-37b3-01fc-3944-2ff860a5eb46"
|
|
:name "snippet: filtered data"
|
|
:display-name "Snippet: Filtered Data"
|
|
:type :snippet
|
|
:snippet-name "filtered data"
|
|
:snippet-id (:id @snippet1s)}}
|
|
:query "SELECT 1;"}}))
|
|
(ts/create! :model/User :first_name "Geddy" :last_name "Lee" :email "glee@rush.yyz")
|
|
(testing "on extraction"
|
|
(reset! extracted (serdes/extract-one "Card" {} @card1s))
|
|
(is (=? {:stages [{:lib/type :mbql.stage/native
|
|
:template-tags [{:name "snippet: things", :snippet-id (:entity_id @snippet1s)}]}]}
|
|
(:dataset_query @extracted))))
|
|
(testing "when loading"
|
|
(let [new-eid (u/generate-nano-id)
|
|
ingestion (ingestion-in-memory [(assoc @extracted :entity_id new-eid)])]
|
|
(is (some? (serdes.load/load-metabase! ingestion)))
|
|
(is (= (:id @snippet1s)
|
|
(-> (t2/select-one :model/Card :entity_id new-eid)
|
|
:dataset_query
|
|
lib/all-template-tags-map
|
|
(get "snippet: things")
|
|
:snippet-id)))))))))
|
|
|
|
(deftest snippet-with-unique-name
|
|
(testing "Snippets with the same name should be replaced/removed on deserialization"
|
|
(mt/with-empty-h2-app-db!
|
|
(let [unique-name "some snippet"
|
|
snippet (ts/create! :model/NativeQuerySnippet :name unique-name)
|
|
id1 (u/generate-nano-id)
|
|
id2 (u/generate-nano-id)
|
|
load! #(serdes.load/load-metabase!
|
|
(ingestion-in-memory [(serdes/extract-one "NativeQuerySnippet" {} %)]))]
|
|
(testing "setup is correct"
|
|
(is (= (:entity_id snippet)
|
|
(t2/select-one-fn :entity_id :model/NativeQuerySnippet :name unique-name))))
|
|
(testing "loading snippet with same name will get it renamed"
|
|
(load! (assoc snippet :entity_id id1))
|
|
(testing "old snippet is in place"
|
|
(is (= (:entity_id snippet)
|
|
(t2/select-one-fn :entity_id :model/NativeQuerySnippet :name unique-name))))
|
|
(testing "new one got new name"
|
|
(is (= (str unique-name " (copy)")
|
|
(t2/select-one-fn :name :model/NativeQuerySnippet :entity_id id1)))))
|
|
(testing "can handle multiple name conflicts"
|
|
(load! (assoc snippet :entity_id id2))
|
|
(is (= (str unique-name " (copy) (copy)")
|
|
(t2/select-one-fn :name :model/NativeQuerySnippet :entity_id id2))))
|
|
(testing "will still update original one"
|
|
(load! (assoc snippet :content "11 = 11"))
|
|
(is (=? {:name unique-name
|
|
:content "11 = 11"}
|
|
(t2/select-one :model/NativeQuerySnippet :entity_id (:entity_id snippet)))))))))
|
|
|
|
(deftest snippet-template-tags-import-test
|
|
(testing "Template tags import preserves nil, empty, and populated values"
|
|
(testing "Missing template_tags field -> {} when selected"
|
|
(mt/with-empty-h2-app-db!
|
|
(let [snippet-data {:serdes/meta [{:model "NativeQuerySnippet"
|
|
:id "test-entity-1"
|
|
:label "test_snippet_1"}]
|
|
:name "no-tags-field"
|
|
:content "WHERE id = {{id}}"
|
|
:creator_id "test@example.com"
|
|
:entity_id "test-entity-1"}
|
|
ingestion (ingestion-in-memory [snippet-data])]
|
|
(serdes.load/load-metabase! ingestion)
|
|
(let [template-tags (t2/select-one-fn :template_tags :model/NativeQuerySnippet :entity_id "test-entity-1")]
|
|
;; Toucan hooks compute the template-tags:
|
|
(is (=? {"id" {:type :text
|
|
:display-name "ID"
|
|
:name "id"}}
|
|
template-tags))))))
|
|
(testing "Empty map template_tags -> preserved as empty map"
|
|
(mt/with-empty-h2-app-db!
|
|
(let [snippet-data {:serdes/meta [{:model "NativeQuerySnippet"
|
|
:id "test-entity-2"
|
|
:label "test_snippet_2"}]
|
|
:name "empty-tags"
|
|
:content "SELECT 1"
|
|
:template_tags {}
|
|
:creator_id "test@example.com"
|
|
:entity_id "test-entity-2"}
|
|
ingestion (ingestion-in-memory [snippet-data])]
|
|
(serdes.load/load-metabase! ingestion)
|
|
(let [template-tags (t2/select-one-fn :template_tags :model/NativeQuerySnippet :entity_id "test-entity-2")]
|
|
(is (= {} template-tags))))))
|
|
(testing "Snippet template tags get preserved rather than recalculated"
|
|
(mt/with-empty-h2-app-db!
|
|
(let [snippet-data {:serdes/meta [{:model "NativeQuerySnippet"
|
|
:id "test-entity-3"
|
|
:label "test_snippet_3"}]
|
|
:name "with-tags"
|
|
:content "WHERE id = {{snippet: id}}"
|
|
:template_tags {"id" {:type :snippet
|
|
:name "snippet: id"
|
|
:snippet-name "id"
|
|
;; Definitely not calculated that way:
|
|
:display-name "Snippet: WOOP"}}
|
|
:creator_id "test@example.com"
|
|
:entity_id "test-entity-3"}
|
|
ingestion (ingestion-in-memory [snippet-data])]
|
|
(serdes.load/load-metabase! ingestion)
|
|
(let [template-tags (t2/select-one-fn :template_tags :model/NativeQuerySnippet :entity_id "test-entity-3")]
|
|
(is (=? {"snippet: id" {:type :snippet
|
|
:name "snippet: id"
|
|
:display-name "Snippet: WOOP"}}
|
|
template-tags))))))))
|
|
|
|
(deftest load-native-card-template-tags-remap-test
|
|
(testing "card template tag names, display names, and the native text follow :card-id to the local card id (#77516)"
|
|
(let [serialized (atom nil)
|
|
target1s (atom nil)
|
|
target-eid (atom nil)
|
|
native-eid (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(let [db (ts/create! :model/Database :name "tag-db")
|
|
table (ts/create! :model/Table :name "customers" :db_id (:id db))
|
|
user (ts/create! :model/User :first_name "Tag" :last_name "User" :email "tag@user.example")
|
|
coll (ts/create! :model/Collection :name "Tag Collection")
|
|
target (ts/create! :model/Card
|
|
:name "BH Population Model"
|
|
:collection_id (:id coll)
|
|
:creator_id (:id user)
|
|
:database_id (:id db)
|
|
:table_id (:id table)
|
|
:dataset_query {:type :query
|
|
:database (:id db)
|
|
:query {:source-table (:id table)}})
|
|
tag-name (str "#" (:id target) "-bh-population-model")]
|
|
(ts/create! :model/Card
|
|
:name "Native Referencing Card"
|
|
:collection_id (:id coll)
|
|
:creator_id (:id user)
|
|
:database_id (:id db)
|
|
:query_type :native
|
|
:entity_id "nativecardtagseid0001"
|
|
:dataset_query {:type :native
|
|
:database (:id db)
|
|
:native {:query (format "SELECT * FROM {{%s}}" tag-name)
|
|
:template-tags
|
|
{tag-name {:id "203a5ecb-24b1-4d5b-8bee-b3520d21fa89"
|
|
:name tag-name
|
|
:display-name (u.humanization/name->human-readable-name :simple tag-name)
|
|
:type "card"
|
|
:card-id (:id target)}}}})
|
|
(reset! target1s target)
|
|
(reset! target-eid (:entity_id target))
|
|
(reset! native-eid "nativecardtagseid0001")
|
|
(reset! serialized (into [] (serdes.extract/extract {:no-settings true})))))
|
|
(ts/with-db dest-db
|
|
;; pre-create some cards so the referenced card can't land on its source id
|
|
(let [db (ts/create! :model/Database :name "other-db")
|
|
table (ts/create! :model/Table :name "orders" :db_id (:id db))
|
|
user (ts/create! :model/User :first_name "Tag" :last_name "User" :email "tag@user.example")]
|
|
(dotimes [i 3]
|
|
(ts/create! :model/Card
|
|
:name (str "Filler " i)
|
|
:creator_id (:id user)
|
|
:database_id (:id db)
|
|
:table_id (:id table)
|
|
:dataset_query {:type :query
|
|
:database (:id db)
|
|
:query {:source-table (:id table)}})))
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
(let [target-id (t2/select-one-fn :id :model/Card :entity_id @target-eid)
|
|
new-tag-name (str "#" target-id "-bh-population-model")
|
|
stored-query #(t2/select-one-fn :dataset_query :model/Card :entity_id @native-eid)
|
|
query (stored-query)]
|
|
(is (some? target-id))
|
|
(is (not= (:id @target1s) target-id)
|
|
"the referenced card must land on a different id for this test to prove anything")
|
|
(is (= (format "SELECT * FROM {{%s}}" new-tag-name)
|
|
(-> query :stages first :native)))
|
|
(is (=? [{:type :card
|
|
:name new-tag-name
|
|
:display-name (u.humanization/name->human-readable-name :simple new-tag-name)
|
|
:card-id target-id}]
|
|
(-> query :stages first :template-tags)))
|
|
(testing "re-importing over the existing card (the update path) repairs the same way"
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
(is (= query (stored-query))))))))))
|
|
|
|
(deftest load-action-test
|
|
(let [serialized (atom nil)
|
|
eid (u/generate-nano-id)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(testing "extraction succeeds"
|
|
(ts/with-db source-db
|
|
(let [db (ts/create! :model/Database :name "my-db")
|
|
card (ts/create! :model/Card
|
|
:name "the query"
|
|
:query_type :native
|
|
:type :model
|
|
:database_id (:id db)
|
|
:dataset_query {:database (:id db)
|
|
:type :native
|
|
:native {:query "select 1"}})
|
|
_action-id (action/insert! (lib/normalize ::actions.schema/action.for-insert
|
|
{:entity_id eid
|
|
:name "the action"
|
|
:model_id (:id card)
|
|
:type :query
|
|
:dataset_query (mt/mbql-query users {:limit 1})
|
|
:database_id (:id db)}))]
|
|
(reset! serialized (into [] (serdes.extract/extract {:no-settings true})))
|
|
(let [action-serialized (first (filter (fn [{[{:keys [model id]}] :serdes/meta}]
|
|
(and (= model "Action") (= id eid)))
|
|
@serialized))]
|
|
(is (some? action-serialized))
|
|
(testing ":type should be a string"
|
|
(is (string? (:type action-serialized))))))))
|
|
(testing "loading succeeds"
|
|
(ts/with-db dest-db
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
(let [action (action/select-action :entity_id eid)]
|
|
(is (some? action))
|
|
(is (some? (:dataset_query action)))
|
|
(testing ":type should be a keyword again"
|
|
(is (keyword? (:type action))))))))))
|
|
|
|
(deftest remove-dashcards-test
|
|
(let [serialized (atom nil)
|
|
dash1s (atom nil)
|
|
dash1d (atom nil)
|
|
dashcard1s (atom nil)
|
|
dashcard2d (atom nil)
|
|
tab1s (atom nil)
|
|
tab2d (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(testing "Serializing the original database"
|
|
(ts/with-db source-db
|
|
(reset! dash1s (ts/create! :model/Dashboard :name "My Dashboard"))
|
|
(reset! tab1s (ts/create! :model/DashboardTab :name "Tab 1" :dashboard_id (:id @dash1s)))
|
|
(reset! dashcard1s (ts/create! :model/DashboardCard :dashboard_id (:id @dash1s) :dashboard_tab_id (:id tab1s)))
|
|
(reset! serialized (into [] (serdes.extract/extract {:no-settings true})))))
|
|
(testing "New dashcard will be removed on load"
|
|
(ts/with-db dest-db
|
|
(reset! dash1d (ts/create! :model/Dashboard :name "Weird Name" :entity_id (:entity_id @dash1s)))
|
|
;; A dashcard to be removed since it does not exist in serialized data
|
|
(reset! dashcard2d (ts/create! :model/DashboardCard :dashboard_id (:id @dash1d)))
|
|
(reset! tab2d (ts/create! :model/DashboardTab :name "Tab 2" :dashboard_id (:id @dash1d)))
|
|
;; Load the serialized content.
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
(reset! dash1d (-> (t2/select-one :model/Dashboard :name "My Dashboard")
|
|
(t2/hydrate :dashcards)
|
|
(t2/hydrate :tabs)))
|
|
(testing "Dashboard has correct number of dashcards"
|
|
(is (= 1
|
|
(count (:dashcards @dash1d))))
|
|
(is (= (:entity_id @dashcard1s)
|
|
(get-in @dash1d [:dashcards 0 :entity_id])))
|
|
(is (not= (:entity_id @dashcard1s)
|
|
(:entity_id @dashcard2d))))
|
|
(testing "Dashboard has correct number of tabs"
|
|
(is (= 1
|
|
(count (:tabs @dash1d))))
|
|
(is (= (:entity_id @tab1s)
|
|
(get-in @dash1d [:tabs 0 :entity_id])))
|
|
(is (not= (:entity_id @tab1s)
|
|
(:entity_id @tab2d)))))))))
|
|
|
|
(deftest dashcard-series-test
|
|
(ts/with-dbs [source-db dest-db]
|
|
(testing "Dashcard series are updated and deleted correctly"
|
|
(ts/with-db source-db
|
|
(let [dash1s (ts/create! :model/Dashboard :name "My Dashboard")
|
|
tab1s (ts/create! :model/DashboardTab :name "Tab 1" :dashboard_id (:id dash1s))
|
|
card1s (ts/create! :model/Card :name "The Card")
|
|
series-card1s (ts/create! :model/Card :name "The Series Card 1")
|
|
series-card2s (ts/create! :model/Card :name "The Series Card 2")
|
|
series-card3s (ts/create! :model/Card :name "The Series Card 3")
|
|
dashcard1s (ts/create! :model/DashboardCard :card_id (:id card1s) :dashboard_id (:id dash1s) :dashboard_tab_id (:id tab1s))
|
|
series1s (ts/create! :model/DashboardCardSeries :dashboardcard_id (:id dashcard1s) :card_id (:id series-card1s) :position 0)
|
|
series2s (ts/create! :model/DashboardCardSeries :dashboardcard_id (:id dashcard1s) :card_id (:id series-card2s) :position 1)
|
|
series3s (ts/create! :model/DashboardCardSeries :dashboardcard_id (:id dashcard1s) :card_id (:id series-card3s) :position 2)
|
|
extract1 (into [] (serdes.extract/extract {:no-settings true}))]
|
|
(ts/with-db dest-db
|
|
(serdes.load/load-metabase! (ingestion-in-memory extract1))
|
|
(ts/with-db source-db
|
|
;; delete the 1st series and update the 3rd series to have position 0, and the 2nd series to have position 1
|
|
(t2/delete! :model/DashboardCardSeries (:id series1s))
|
|
(t2/update! :model/DashboardCardSeries (:id series3s) {:position 0})
|
|
(t2/update! :model/DashboardCardSeries (:id series2s) {:position 1})
|
|
(let [extract2 (into [] (serdes.extract/extract {:no-settings true :no-data-model true}))]
|
|
(ts/with-db dest-db
|
|
(let [series-card2d (t2/select-one :model/Card :entity_id (:entity_id series-card2s))
|
|
series-card3d (t2/select-one :model/Card :entity_id (:entity_id series-card3s))
|
|
;; we deleted the card that corresponds to `series1s`, so a shortcut is to get the one with position=0
|
|
series-to-be-deleted (t2/select-one :model/DashboardCardSeries :position 0)]
|
|
(testing "Sense check: there are 3 series for the dashboard card initially"
|
|
(is (= 3
|
|
(t2/count :model/DashboardCardSeries :dashboardcard_id (:dashboardcard_id series-to-be-deleted)))))
|
|
(serdes.load/load-metabase! (ingestion-in-memory extract2))
|
|
(let [dash1d (-> (t2/select-one :model/Dashboard :name "My Dashboard")
|
|
(t2/hydrate [:dashcards :series]))]
|
|
(testing "Dashboard cards have the same entity ID"
|
|
(is (= (:entity_id dashcard1s)
|
|
(get-in dash1d [:dashcards 0 :entity_id]))))
|
|
(testing "The dashboard's series is updated"
|
|
(is (=? [{:id (:id series-card3d)}
|
|
{:id (:id series-card2d)}]
|
|
(get-in dash1d [:dashcards 0 :series]))))
|
|
(testing "Dashboard card series are correctly updated/deleted in the database"
|
|
(is (=? [{:position 0
|
|
:card_id (:id series-card3d)}
|
|
{:position 1
|
|
:card_id (:id series-card2d)}]
|
|
(->> (t2/select :model/DashboardCardSeries :dashboardcard_id (:dashboardcard_id series-to-be-deleted))
|
|
(sort-by :position))))))))))))))))
|
|
|
|
(deftest dashcard-series-multi-test
|
|
(ts/with-dbs [source-db dest-db]
|
|
(testing "Dashcard series works correctly with one card in multiple series"
|
|
(ts/with-db source-db
|
|
(mt/with-temp [:model/Dashboard dash {:name "Dashboard"}
|
|
:model/Card c1 {:name "Card 1"}
|
|
:model/Card c2 {:name "Card 2"}
|
|
:model/Card sc {:name "Series Card"}
|
|
:model/DashboardCard dc1 {:card_id (:id c1) :dashboard_id (:id dash)}
|
|
:model/DashboardCard dc2 {:card_id (:id c2) :dashboard_id (:id dash)}
|
|
:model/DashboardCardSeries _s1 {:dashboardcard_id (:id dc1) :card_id (:id sc) :position 0}
|
|
:model/DashboardCardSeries _s2 {:dashboardcard_id (:id dc2) :card_id (:id sc) :position 0}]
|
|
(let [extract (into [] (serdes.extract/extract {:no-settings true}))]
|
|
(ts/with-db dest-db
|
|
(serdes.load/load-metabase! (ingestion-in-memory extract))
|
|
(testing "Both series get imported even though they point at the same card"
|
|
(is (= 2
|
|
(t2/count :model/DashboardCardSeries)))))))))))
|
|
|
|
(deftest extraneous-keys-test
|
|
(let [serialized (atom nil)
|
|
eid (u/generate-nano-id)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(testing "Sprinkle the source database with a variety of different models"
|
|
(ts/with-db source-db
|
|
(let [db (ts/create! :model/Database :name "my-db")
|
|
card (ts/create! :model/Card
|
|
:name "the query"
|
|
:query_type :native
|
|
:type :model
|
|
:database_id (:id db)
|
|
:dataset_query {:database (:id db)
|
|
:type :native
|
|
:native {:query "wow"}})
|
|
parent (ts/create! :model/Collection :name "Parent Collection" :location "/")
|
|
_child (ts/create! :model/Collection
|
|
:name "Child Collection"
|
|
:location (format "/%d/" (:id parent)))
|
|
_action-id (action/insert! {:entity_id eid
|
|
:name "the action"
|
|
:model_id (:id card)
|
|
:type :query
|
|
:dataset_query {}
|
|
:database_id (:id db)})]
|
|
(reset! serialized
|
|
(->> (serdes.extract/extract {})
|
|
;; add an extra key to *every serialized model*
|
|
(map #(assoc % :my-extraneous-keeeeeey "foobar!!!!"))
|
|
(into []))))))
|
|
(testing "The extraneous keys do not interfere with loading"
|
|
(ts/with-db dest-db
|
|
(is (serdes.load/load-metabase! (ingestion-in-memory @serialized))))))))
|
|
|
|
(deftest tx-test
|
|
(mt/with-empty-h2-app-db!
|
|
(let [coll (ts/create! :model/Collection :name "coll")
|
|
card (ts/create! :model/Card :name "card" :collection_id (:id coll))
|
|
serialized (atom {})]
|
|
(reset! serialized (->> (serdes.extract/extract {:no-settings true
|
|
:no-data-model true
|
|
:targets [["Collection" (:id coll)]]})
|
|
vec))
|
|
(testing "Load completes successfully"
|
|
(t2/update! :model/Card {:id (:id card)} {:name (str "qwe_" (:name card))})
|
|
(is (serdes.load/load-metabase! (ingestion-in-memory @serialized)))
|
|
(is (= (:name card)
|
|
(t2/select-one-fn :name :model/Card :id (:id card)))))
|
|
(testing "Partial load commits successful entities; failed entity does not persist"
|
|
(t2/update! :model/Collection {:id (:id coll)} {:name (str "qwe_" (:name coll))})
|
|
(t2/update! :model/Card {:id (:id card)} {:name (str "qwe_" (:name card))})
|
|
(let [load-update! serdes/load-update!]
|
|
(with-redefs [serdes/load-update! (fn [model adjusted local]
|
|
;; Collection is loaded first, Card fails
|
|
(if (= model "Card")
|
|
(throw (ex-info "oops, error" {}))
|
|
(load-update! model adjusted local)))]
|
|
(is (thrown? clojure.lang.ExceptionInfo
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))))
|
|
;; Collection loaded successfully in its own transaction — committed despite Card failure
|
|
(is (= (:name coll)
|
|
(t2/select-one-fn :name :model/Collection :id (:id coll))))
|
|
;; Card failed — retains its pre-load value
|
|
(is (= (str "qwe_" (:name card))
|
|
(t2/select-one-fn :name :model/Card :id (:id card))))))))))
|
|
|
|
(deftest transient-db-error-retry-test
|
|
(testing "Import survives a transient deadlock on a single entity (issue #74412)"
|
|
(mt/with-empty-h2-app-db!
|
|
(let [coll (ts/create! :model/Collection :name "coll")
|
|
card (ts/create! :model/Card :name "card" :collection_id (:id coll))
|
|
serialized (atom {})]
|
|
(reset! serialized (->> (serdes.extract/extract {:no-settings true
|
|
:no-data-model true
|
|
:targets [["Collection" (:id coll)]]})
|
|
vec))
|
|
(testing "A transient deadlock on one entity is retried and the import succeeds"
|
|
(t2/update! :model/Card {:id (:id card)} {:name "pre-retry"})
|
|
(let [call-count (atom 0)
|
|
load-update! serdes/load-update!]
|
|
(with-redefs [serdes/load-update! (fn [model adjusted local]
|
|
(when (= model "Card")
|
|
(swap! call-count inc)
|
|
(when (= 1 @call-count)
|
|
;; Simulate a deadlock: H2 error code 40001, PG SQL state 40P01.
|
|
;; Use H2 codes since tests run against H2 appdb.
|
|
(throw (ex-info "load-update! failed"
|
|
{}
|
|
(java.sql.SQLException.
|
|
"Deadlock detected"
|
|
"40001" ; SQL state
|
|
40001))))) ; H2 error code
|
|
(load-update! model adjusted local))]
|
|
(is (serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
"Import should succeed after retrying the deadlocked entity")
|
|
(is (= (:name card)
|
|
(t2/select-one-fn :name :model/Card :id (:id card)))
|
|
"Card should be updated to the serialized value after retry")
|
|
(is (= 2 @call-count)
|
|
"Card load-update! should have been called twice (first attempt deadlocked, second succeeded)"))))
|
|
(testing "Non-transient errors propagate immediately without retry"
|
|
(let [call-count (atom 0)
|
|
load-update! serdes/load-update!]
|
|
(with-redefs [serdes/load-update! (fn [model adjusted local]
|
|
(when (= model "Card")
|
|
(swap! call-count inc)
|
|
(throw (ex-info "constraint violation" {})))
|
|
(load-update! model adjusted local))]
|
|
(is (thrown? clojure.lang.ExceptionInfo
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized)))
|
|
"Non-transient error should propagate")
|
|
(is (= 1 @call-count)
|
|
"Should not retry non-transient errors"))))
|
|
(testing "Successful entities are committed even when a later entity fails"
|
|
(t2/update! :model/Collection {:id (:id coll)} {:name "pre-import"})
|
|
(t2/update! :model/Card {:id (:id card)} {:name "pre-import"})
|
|
(let [load-update! serdes/load-update!]
|
|
(with-redefs [serdes/load-update! (fn [model adjusted local]
|
|
;; Collection loads first and succeeds; Card fails
|
|
(if (= model "Card")
|
|
(throw (ex-info "oops" {}))
|
|
(load-update! model adjusted local)))]
|
|
(is (thrown? clojure.lang.ExceptionInfo
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))))
|
|
;; With per-entity transactions, the Collection commit survives the Card failure.
|
|
;; On master (single transaction), the Collection update is rolled back.
|
|
(is (= (:name coll)
|
|
(t2/select-one-fn :name :model/Collection :id (:id coll)))
|
|
"Collection should be committed despite later Card failure")
|
|
(is (= "pre-import"
|
|
(t2/select-one-fn :name :model/Card :id (:id card)))
|
|
"Card should retain its pre-import value"))))))))
|
|
|
|
(deftest path-error-data-handles-lookup-failure-test
|
|
(testing "path-error-data returns a well-formed map even when serdes/load-find-local throws
|
|
(e.g. because the outer load transaction is already poisoned by the real failure).
|
|
This is what lets the caller's ex-info still chain the original cause instead of
|
|
being replaced by a `current transaction is aborted` error from the enrichment lookup."
|
|
(with-redefs [serdes/load-find-local (fn [_] (throw (ex-info "current transaction is aborted" {})))]
|
|
(let [path [{:model "Card" :id "some-entity-id"}]
|
|
result (#'serdes.load/path-error-data
|
|
:metabase-enterprise.serialization.v2.load/load-failure
|
|
#{}
|
|
path)]
|
|
(is (nil? (:local-id result))
|
|
"local-id is nil when the lookup fails")
|
|
(is (= "Card" (:model result)))
|
|
(is (= :metabase-enterprise.serialization.v2.load/load-failure (:error result)))
|
|
(is (= [{:model "Card" :id "some-entity-id"}] (:path result)))))))
|
|
|
|
(deftest circular-links-test
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(let [coll (ts/create! :model/Collection :name "coll")
|
|
card (ts/create! :model/Card :name "card" :collection_id (:id coll))
|
|
dash1 (ts/create! :model/Dashboard :name "dash1" :collection_id (:id coll))
|
|
dash2 (ts/create! :model/Dashboard :name "dash2" :collection_id (:id coll))
|
|
dash3 (ts/create! :model/Dashboard :name "dash3" :collection_id (:id coll))
|
|
dc1 (ts/create! :model/DashboardCard :dashboard_id (:id dash1) :card_id (:id card)
|
|
:visualization_settings {:click_behavior {:type "link"
|
|
:linkType "dashboard"
|
|
:targetId (:id dash2)}})
|
|
dc2 (ts/create! :model/DashboardCard :dashboard_id (:id dash2) :card_id (:id card)
|
|
:visualization_settings {:click_behavior {:type "link"
|
|
:linkType "dashboard"
|
|
:targetId (:id dash3)}})
|
|
dc3 (ts/create! :model/DashboardCard :dashboard_id (:id dash2) :card_id (:id card)
|
|
:visualization_settings {:click_behavior {:type "link"
|
|
:linkType "dashboard"
|
|
:targetId (:id dash1)}})
|
|
card-2 (ts/create! :model/Card :name "card-2" :dashboard_id (:id dash1))
|
|
_dc2-1 (ts/create! :model/DashboardCard :dashboard_id (:id dash1) :card_id (:id card-2))
|
|
ser (into [] (serdes.extract/extract {:no-settings true
|
|
:no-data-model false}))
|
|
select-target #(-> % :visualization_settings :click_behavior :targetId)]
|
|
;; loaded to the same db so we know ids won't be corrupted
|
|
(testing "Dashcard ids won't be changed by loading on top of them"
|
|
(is (serdes.load/load-metabase! (ingestion-in-memory ser)))
|
|
(is (= (:id dc1)
|
|
(t2/select-one-fn :id :model/DashboardCard :entity_id (:entity_id dc1))))
|
|
(is (= (:id dc2)
|
|
(t2/select-one-fn :id :model/DashboardCard :entity_id (:entity_id dc2))))
|
|
(is (= (:id dc3)
|
|
(t2/select-one-fn :id :model/DashboardCard :entity_id (:entity_id dc3))))
|
|
(is (= (:id dash2)
|
|
(t2/select-one-fn select-target :model/DashboardCard :entity_id (:entity_id dc1))))
|
|
(is (= (:id dash3)
|
|
(t2/select-one-fn select-target :model/DashboardCard :entity_id (:entity_id dc2))))
|
|
(is (= (:id dash1)
|
|
(t2/select-one-fn select-target :model/DashboardCard :entity_id (:entity_id dc3)))))
|
|
;; loaded to a different db so we know it works for sure instead of relying on data being present in the
|
|
;; database
|
|
(ts/with-db dest-db
|
|
(is (serdes.load/load-metabase! (ingestion-in-memory ser)))
|
|
(testing "Circular dependencies work for Dashboard Questions as well"
|
|
(is (= (t2/select-one-fn :id :model/Dashboard :entity_id (:entity_id dash1))
|
|
(t2/select-one-fn :dashboard_id :model/Card :entity_id (:entity_id card-2))))))))))
|
|
|
|
(defn- card-sourced-param
|
|
"A category parameter whose dropdown values come from `card-id`'s results."
|
|
[card-id]
|
|
{:id "abc"
|
|
:type "category"
|
|
:name "CATEGORY"
|
|
:slug "category"
|
|
:values_source_type "card"
|
|
:values_source_config {:card_id card-id}})
|
|
|
|
(deftest self-referencing-parameter-card-test
|
|
(testing "a card whose parameter sources dropdown values from the card itself can be loaded (#73133)"
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(let [coll (ts/create! :model/Collection :name "coll")
|
|
card (ts/create! :model/Card :name "self-ref card" :collection_id (:id coll))
|
|
_ (t2/update! :model/Card (:id card)
|
|
{:parameters [(card-sourced-param (:id card))]})
|
|
ser (into [] (serdes.extract/extract {:no-settings true
|
|
:no-data-model false}))]
|
|
(testing "loading on top of the existing card"
|
|
(is (serdes.load/load-metabase! (ingestion-in-memory ser)))
|
|
(is (= (:id card)
|
|
(-> (t2/select-one :model/Card :entity_id (:entity_id card))
|
|
:parameters first :values_source_config :card_id))))
|
|
(testing "loading into an empty database"
|
|
(ts/with-db dest-db
|
|
(is (serdes.load/load-metabase! (ingestion-in-memory ser)))
|
|
(let [new-card (t2/select-one :model/Card :entity_id (:entity_id card))]
|
|
(is (= (:id new-card)
|
|
(-> new-card :parameters first :values_source_config :card_id)))))))))))
|
|
|
|
(deftest mutually-referencing-parameter-cards-test
|
|
(testing "two cards whose parameters source dropdown values from each other can be loaded (#73133)"
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(let [coll (ts/create! :model/Collection :name "coll")
|
|
card-a (ts/create! :model/Card :name "card a" :collection_id (:id coll))
|
|
card-b (ts/create! :model/Card :name "card b" :collection_id (:id coll))
|
|
_ (t2/update! :model/Card (:id card-a) {:parameters [(card-sourced-param (:id card-b))]})
|
|
_ (t2/update! :model/Card (:id card-b) {:parameters [(card-sourced-param (:id card-a))]})
|
|
ser (into [] (serdes.extract/extract {:no-settings true
|
|
:no-data-model false}))]
|
|
(testing "loading on top of the existing cards"
|
|
(is (serdes.load/load-metabase! (ingestion-in-memory ser)))
|
|
(is (= (:id card-b)
|
|
(-> (t2/select-one :model/Card :entity_id (:entity_id card-a))
|
|
:parameters first :values_source_config :card_id)))
|
|
(is (= (:id card-a)
|
|
(-> (t2/select-one :model/Card :entity_id (:entity_id card-b))
|
|
:parameters first :values_source_config :card_id))))
|
|
(testing "loading into an empty database"
|
|
(ts/with-db dest-db
|
|
(is (serdes.load/load-metabase! (ingestion-in-memory ser)))
|
|
(let [new-a (t2/select-one :model/Card :entity_id (:entity_id card-a))
|
|
new-b (t2/select-one :model/Card :entity_id (:entity_id card-b))]
|
|
(is (= (:id new-b)
|
|
(-> new-a :parameters first :values_source_config :card_id)))
|
|
(is (= (:id new-a)
|
|
(-> new-b :parameters first :values_source_config :card_id)))))))))))
|
|
|
|
(deftest dashboard-parameter-sourcing-own-dashboard-question-test
|
|
(testing "a dashboard whose parameter sources dropdown values from a dashboard question on that same dashboard can be loaded (#73133)"
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(let [coll (ts/create! :model/Collection :name "coll")
|
|
dash (ts/create! :model/Dashboard :name "dash" :collection_id (:id coll))
|
|
card (ts/create! :model/Card :name "dq card" :dashboard_id (:id dash))
|
|
_ (ts/create! :model/DashboardCard :dashboard_id (:id dash) :card_id (:id card))
|
|
_ (t2/update! :model/Dashboard (:id dash)
|
|
{:parameters [(card-sourced-param (:id card))]})
|
|
ser (into [] (serdes.extract/extract {:no-settings true
|
|
:no-data-model false}))]
|
|
(testing "loading on top of the existing dashboard"
|
|
(is (serdes.load/load-metabase! (ingestion-in-memory ser)))
|
|
(is (= (:id card)
|
|
(-> (t2/select-one :model/Dashboard :entity_id (:entity_id dash))
|
|
:parameters first :values_source_config :card_id))))
|
|
(testing "loading into an empty database"
|
|
(ts/with-db dest-db
|
|
(is (serdes.load/load-metabase! (ingestion-in-memory ser)))
|
|
(let [new-dash (t2/select-one :model/Dashboard :entity_id (:entity_id dash))
|
|
new-card (t2/select-one :model/Card :entity_id (:entity_id card))]
|
|
(is (= (:id new-card)
|
|
(-> new-dash :parameters first :values_source_config :card_id)))
|
|
(is (= (:id new-dash)
|
|
(:dashboard_id new-card)))))))))))
|
|
|
|
(deftest continue-on-error-test
|
|
(let [change-ser (fn [ser changes] ;; kind of like left-join, but right side is indexed
|
|
(vec (for [entity ser]
|
|
(merge entity (get changes (:entity_id entity))))))
|
|
logs-extract (fn [re logs]
|
|
(keep #(not-empty (rest (re-find re %)))
|
|
(map :message logs)))]
|
|
(mt/with-empty-h2-app-db!
|
|
(mt/with-temp [:model/Collection coll {:name "coll"}
|
|
:model/Card c1 {:name "card1" :collection_id (:id coll)}
|
|
:model/Card c2 {:name "card2" :collection_id (:id coll)}
|
|
:model/Card _c3 {:name "card3" :collection_id (:id coll)}]
|
|
(testing "It's possible to skip a few errors during extract"
|
|
(let [extract-one (mt/original-fn #'serdes/extract-one)]
|
|
(mt/with-dynamic-fn-redefs [serdes/extract-one (fn [model-name opts instance]
|
|
(if (= (:entity_id instance) (:entity_id c1))
|
|
(throw (ex-info "Skip me" {}))
|
|
(extract-one model-name opts instance)))]
|
|
(mt/with-log-messages-for-level [messages [metabase.models.serialization :warn]]
|
|
(let [ser (vec (serdes.extract/extract {:no-settings true
|
|
:no-data-model true
|
|
:no-transforms true
|
|
:continue-on-error true}))
|
|
{errors true
|
|
others false} (group-by #(instance? Exception %) ser)]
|
|
(is (= 1 (count errors)))
|
|
(is (= 3 (count others))))
|
|
(is (= [["Card" (str (:id c1))]]
|
|
(logs-extract #"Skipping (\w+) (\d+)"
|
|
(messages))))))))
|
|
(testing "It's possible to skip a few errors during load"
|
|
(let [ser (vec (serdes.extract/extract {:no-settings true
|
|
:no-data-model true
|
|
:no-transforms true}))
|
|
changed (change-ser ser {(:entity_id c2) {:collection_id "does-not-exist"}})]
|
|
(mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]
|
|
(let [report (serdes.load/load-metabase! (ingestion-in-memory changed) {:continue-on-error true})]
|
|
(is (= 1 (count (:errors report))))
|
|
(is (= 3 (count (:seen report)))))
|
|
(let [log-msgs (logs-extract #"Skipping deserialization error: (.*)" (messages))]
|
|
(is (= 1 (count log-msgs)))
|
|
(is (str/includes? (ffirst log-msgs) "Collection 'does-not-exist' was not found"))))))))))
|
|
|
|
(deftest with-dbs-works-as-expected-test
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(mt/with-temp
|
|
[:model/Card _ {:name "MY CARD"}]
|
|
(testing "card is available in the source db"
|
|
(is (some? (t2/select-one :model/Card :name "MY CARD"))))
|
|
(ts/with-db dest-db
|
|
(testing "card should not be available in the dest db"
|
|
(is (nil? (t2/select-one :model/Card :name "MY CARD")))))))))
|
|
|
|
(deftest database-test
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(mt/with-temp [:model/Database _ {:name "My Database"
|
|
:details {:some "secret"}}]
|
|
(testing "without :include-database-secrets"
|
|
(let [extracted (vec (serdes.extract/extract {:no-settings true}))
|
|
dbs (filterv #(= "Database" (:model (last (serdes/path %)))) extracted)]
|
|
(is (= 1 (count dbs)))
|
|
(is (not-any? :details dbs))
|
|
(ts/with-db dest-db
|
|
(testing "loading still works even if there are no details"
|
|
(serdes.load/load-metabase! (ingestion-in-memory extracted))
|
|
(is (= {}
|
|
(t2/select-one-fn :details :model/Database)))
|
|
(testing "If we did not export details - it won't override existing data"
|
|
(t2/update! :model/Database {:details {:other "secret"}})
|
|
(serdes.load/load-metabase! (ingestion-in-memory extracted))
|
|
(is (= {:other "secret"}
|
|
(t2/select-one-fn :details :model/Database)))))))))
|
|
(mt/with-temp [:model/Database _ {:name "My Database"
|
|
:details {:some "secret"}}]
|
|
(testing "connection details are never exported, even when :include-database-secrets is requested"
|
|
(let [extracted (vec (serdes.extract/extract {:no-settings true :include-database-secrets true}))
|
|
dbs (filterv #(= "Database" (:model (last (serdes/path %)))) extracted)]
|
|
(is (= 1 (count dbs)))
|
|
(is (not-any? :details dbs))))))))
|
|
|
|
(deftest unique-dimensions-test
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(mt/with-temp [:model/Dimension d1 {:name "Some Dimension"
|
|
:field_id (mt/id :venues :price)
|
|
:type "internal"}]
|
|
(let [ser (vec (serdes.extract/extract {:no-settings true}))]
|
|
(ts/with-db dest-db
|
|
(mt/with-temp [:model/Dimension d2 {:name "Absolutely Other Dimension"
|
|
:field_id (mt/id :venues :price)
|
|
:type "internal"}]
|
|
(serdes.load/load-metabase! (ingestion-in-memory ser))
|
|
(is (= (:entity_id d1)
|
|
(t2/select-one-fn :entity_id :model/Dimension :field_id (mt/id :venues :price))))
|
|
(is (= nil
|
|
(t2/select-one :model/Dimension :entity_id (:entity_id d2)))))))))))
|
|
|
|
(deftest identically-named-fields-test
|
|
(mt/with-empty-h2-app-db!
|
|
(let [db (ts/create! :model/Database :name "mydb")
|
|
t (ts/create! :model/Table :name "table" :db_id (:id db))
|
|
f1 (ts/create! :model/Field :name "field" :table_id (:id t))
|
|
;; name is the same, but parent_id is different
|
|
f2 (ts/create! :model/Field :name "field" :table_id (:id t) :parent_id (:id f1))
|
|
f3 (ts/create! :model/Field :name "field" :table_id (:id t) :parent_id (:id f2)
|
|
:description "desc")
|
|
|
|
ser (into [] (serdes.extract/extract {}))]
|
|
(is (=? {:parent_id ["mydb" nil "table" "field"]
|
|
:serdes/meta [{:model "Database" :id "mydb"}
|
|
{:model "Table" :id "table"}
|
|
{:model "Field" :id "field"}
|
|
{:model "Field" :id "field"}]}
|
|
(ts/extract-one "Field" (:id f2))))
|
|
(is (=? {:parent_id ["mydb" nil "table" "field" "field"]
|
|
:serdes/meta [{:model "Database" :id "mydb"}
|
|
{:model "Table" :id "table"}
|
|
{:model "Field" :id "field"}
|
|
{:model "Field" :id "field"}
|
|
{:model "Field" :id "field"}]}
|
|
(ts/extract-one "Field" (:id f3))))
|
|
(t2/update! :model/Field (:id f3) {:description "some new one"})
|
|
(is (serdes.load/load-metabase! (ingestion-in-memory ser)))
|
|
(is (= "desc"
|
|
(t2/select-one-fn :description :model/Field (:id f3)))))))
|
|
|
|
(deftest field-data-sensitivity-round-trip-test
|
|
(testing "data_sensitivity travels through export and import on both Field and FieldUserSettings"
|
|
(let [serialized (atom nil)
|
|
in-db? (fn [entity] (-> entity :serdes/meta first :id (= "labeled-db")))
|
|
field-ser (fn [entities field-name]
|
|
(u/seek #(and (in-db? %)
|
|
(-> % :serdes/meta last :model (= "Field"))
|
|
(= field-name (:name %)))
|
|
entities))]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(let [db (ts/create! :model/Database :name "labeled-db")
|
|
table (ts/create! :model/Table :name "CONTACTS" :db_id (:id db))
|
|
email (ts/create! :model/Field :name "CONTACT_EMAIL" :table_id (:id table) :data_sensitivity :PII)
|
|
_ (ts/create! :model/Field :name "CONTACT_ROW" :table_id (:id table))
|
|
_ (ts/create! :model/FieldUserSettings :field_id (:id email) :data_sensitivity :PII)]
|
|
(reset! serialized (into [] (serdes.extract/extract {})))
|
|
(testing "the labeled Field and its FieldUserSettings both export the label"
|
|
(is (= :PII (:data_sensitivity (field-ser @serialized "CONTACT_EMAIL"))))
|
|
(is (= :PII (-> (u/seek #(and (in-db? %) (-> % :serdes/meta last :model (= "FieldUserSettings")))
|
|
@serialized)
|
|
:data_sensitivity))))
|
|
(testing "the unlabeled Field exports no data_sensitivity key"
|
|
(is (not (contains? (field-ser @serialized "CONTACT_ROW") :data_sensitivity))))))
|
|
(ts/with-db dest-db
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
(let [db-id (t2/select-one-fn :id :model/Database :name "labeled-db")
|
|
table-id (t2/select-one-fn :id :model/Table :name "CONTACTS" :db_id db-id)
|
|
email-id (t2/select-one-fn :id :model/Field :name "CONTACT_EMAIL" :table_id table-id)
|
|
row-id (t2/select-one-fn :id :model/Field :name "CONTACT_ROW" :table_id table-id)]
|
|
(testing "the label imports back to the keyword on the Field"
|
|
(is (= :PII (t2/select-one-fn :data_sensitivity :model/Field :id email-id))))
|
|
(testing "the label imports back to the keyword on the FieldUserSettings mirror"
|
|
(is (= :PII (t2/select-one-fn :data_sensitivity :model/FieldUserSettings :field_id email-id))))
|
|
(testing "the unlabeled field stays nil with no mirror row"
|
|
(is (nil? (t2/select-one-fn :data_sensitivity :model/Field :id row-id)))
|
|
(is (not (t2/exists? :model/FieldUserSettings :field_id row-id))))))))))
|
|
|
|
(deftest blank-eid-creates-new-entity-test
|
|
(mt/with-empty-h2-app-db!
|
|
(let [coll (ts/create! :model/Collection :name "mycoll")
|
|
[coll-ser] (serdes.extract/extract {:targets [["Collection" (:id coll)]]})
|
|
new-coll (assoc coll-ser :entity_id nil)
|
|
coll-count (fn [] (t2/count :model/Collection :name "mycoll"))]
|
|
(serdes.load/load-metabase! (ingestion-in-memory [new-coll]))
|
|
(is (= 2 (coll-count)))
|
|
(serdes.load/load-metabase! (ingestion-in-memory [new-coll]))
|
|
(is (= 3 (coll-count)))
|
|
(testing "absent :entity_id also works"
|
|
(serdes.load/load-metabase! (ingestion-in-memory [(dissoc coll-ser :entity_id)]))
|
|
(is (= 4 (coll-count)))))))
|
|
|
|
(deftest import-published-table-with-existing-database-test
|
|
(testing "Importing a published table works when database already exists on target"
|
|
(let [serialized (atom nil)
|
|
coll-eid (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
;; Export from source (with no-data-model, so Database is not in export)
|
|
(testing "export published table without database"
|
|
(ts/with-db source-db
|
|
(let [db (ts/create! :model/Database :name "shared-db")
|
|
coll (ts/create! :model/Collection :name "My Collection")
|
|
_ (ts/create! :model/Table
|
|
:name "published_table"
|
|
:db_id (:id db)
|
|
:is_published true
|
|
:collection_id (:id coll)
|
|
:description "A published table")]
|
|
(reset! coll-eid (:entity_id coll))
|
|
(reset! serialized
|
|
(into [] (serdes.extract/extract
|
|
{:targets [["Collection" (:id coll)]]
|
|
:no-data-model true
|
|
:no-settings true}))))))
|
|
(testing "serialized data contains table but not database"
|
|
(is (some #(= "Table" (-> % :serdes/meta last :model)) @serialized))
|
|
(is (not-any? #(= "Database" (-> % :serdes/meta last :model)) @serialized)))
|
|
;; Import to destination (where database already exists)
|
|
(testing "import succeeds when database exists on target"
|
|
(ts/with-db dest-db
|
|
;; Pre-create the database on target
|
|
(let [target-db (ts/create! :model/Database :name "shared-db")]
|
|
;; Load the serialized content
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
;; Verify the table was imported correctly
|
|
(let [imported-table (t2/select-one :model/Table :name "published_table")
|
|
imported-coll (t2/select-one :model/Collection :entity_id @coll-eid)]
|
|
(testing "table exists with correct properties"
|
|
(is (some? imported-table))
|
|
(is (= "A published table" (:description imported-table)))
|
|
(is (true? (:is_published imported-table))))
|
|
(testing "table is linked to correct database"
|
|
(is (= (:id target-db) (:db_id imported-table))))
|
|
(testing "table is linked to imported collection"
|
|
(is (= (:id imported-coll) (:collection_id imported-table))))))))))))
|
|
|
|
(deftest import-published-table-without-database-fails-test
|
|
(testing "Importing a published table fails when database doesn't exist on target"
|
|
(let [serialized (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(testing "export published table"
|
|
(ts/with-db source-db
|
|
(let [db (ts/create! :model/Database :name "source-only-db")
|
|
coll (ts/create! :model/Collection :name "My Collection")
|
|
_ (ts/create! :model/Table
|
|
:name "published_table"
|
|
:db_id (:id db)
|
|
:is_published true
|
|
:collection_id (:id coll))]
|
|
(reset! serialized
|
|
(into [] (serdes.extract/extract
|
|
{:targets [["Collection" (:id coll)]]
|
|
:no-data-model true
|
|
:no-settings true}))))))
|
|
(testing "import fails when database doesn't exist"
|
|
(ts/with-db dest-db
|
|
;; Don't create the database - import should fail
|
|
(is (thrown-with-msg?
|
|
Exception
|
|
#"source-only-db|not found|Failed"
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))))))))))
|
|
|
|
(deftest segment-minimal-required-properties-test
|
|
(testing "Segment deserialized with only: entity_id, name, definition, creator_id"
|
|
(let [serialized (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(let [db (ts/create! :model/Database :name "my-db")
|
|
table (ts/create! :model/Table :name "customers" :db_id (:id db))
|
|
field (ts/create! :model/Field :name "age" :table_id (:id table))
|
|
user (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on")
|
|
_segment (ts/create! :model/Segment :table_id (:id table) :name "Minors"
|
|
:definition (mbql5-segment-definition (:id db) (:id table) (:id field))
|
|
:creator_id (:id user))]
|
|
(reset! serialized (into [] (serdes.extract/extract {})))))
|
|
(let [minimal (mapv (fn [entity]
|
|
(if (= "Segment" (-> entity :serdes/meta last :model))
|
|
(select-keys entity [:serdes/meta :entity_id :name :definition :creator_id])
|
|
entity))
|
|
@serialized)]
|
|
(ts/with-db dest-db
|
|
(ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on")
|
|
(serdes.load/load-metabase! (ingestion-in-memory minimal))
|
|
(let [segment (t2/select-one :model/Segment :name "Minors")
|
|
table (t2/select-one :model/Table :name "customers")]
|
|
(is (some? segment))
|
|
(is (= (:id table) (:table_id segment))))))))))
|
|
|
|
(deftest measure-minimal-required-properties-test
|
|
(testing "Measure deserialized with only: entity_id, name, definition, creator_id"
|
|
(let [serialized (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(let [db (ts/create! :model/Database :name "my-db")
|
|
table (ts/create! :model/Table :name "sales" :db_id (:id db))
|
|
field (ts/create! :model/Field :name "amount" :table_id (:id table))
|
|
user (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on")
|
|
_measure (ts/create! :model/Measure :table_id (:id table) :name "Total Sales"
|
|
:definition (mbql5-measure-definition (:id db) (:id table) (:id field))
|
|
:creator_id (:id user))]
|
|
(reset! serialized (into [] (serdes.extract/extract {})))))
|
|
(let [minimal (mapv (fn [entity]
|
|
(if (= "Measure" (-> entity :serdes/meta last :model))
|
|
(select-keys entity [:serdes/meta :entity_id :name :definition :creator_id])
|
|
entity))
|
|
@serialized)]
|
|
(ts/with-db dest-db
|
|
(ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on")
|
|
(serdes.load/load-metabase! (ingestion-in-memory minimal))
|
|
(let [measure (t2/select-one :model/Measure :name "Total Sales")
|
|
table (t2/select-one :model/Table :name "sales")]
|
|
(is (some? measure))
|
|
(is (= (:id table) (:table_id measure))))))))))
|
|
|
|
(deftest card-minimal-required-properties-test
|
|
(testing "Card deserialized with only: entity_id, name, display, dataset_query, visualization_settings, creator_id"
|
|
(let [serialized (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(let [db (ts/create! :model/Database :name "my-db")
|
|
table (ts/create! :model/Table :name "customers" :db_id (:id db))
|
|
user (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on")
|
|
_card (ts/create! :model/Card
|
|
:collection_id nil
|
|
:creator_id (:id user)
|
|
:name "Example Card"
|
|
:dataset_query (mbql5-query (:id db) (:id table))
|
|
:display :line)]
|
|
(reset! serialized (into [] (serdes.extract/extract {})))))
|
|
(let [minimal (mapv (fn [entity]
|
|
(if (= "Card" (-> entity :serdes/meta last :model))
|
|
(select-keys entity [:serdes/meta :entity_id :name :display :dataset_query
|
|
:visualization_settings :creator_id])
|
|
entity))
|
|
@serialized)]
|
|
(ts/with-db dest-db
|
|
(ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on")
|
|
(serdes.load/load-metabase! (ingestion-in-memory minimal))
|
|
(let [card (t2/select-one :model/Card :name "Example Card")
|
|
db (t2/select-one :model/Database :name "my-db")
|
|
table (t2/select-one :model/Table :name "customers")]
|
|
(is (some? card))
|
|
(is (= (:id db) (:database_id card)))
|
|
(is (= (:id table) (:table_id card)))
|
|
(is (nil? (:source_card_id card)))
|
|
(is (= :query (:query_type card))))))))))
|
|
|
|
(deftest transform-minimal-required-properties-test
|
|
(testing "Transform deserialized with only: serdes/meta, entity_id, name, source, target"
|
|
(mt/with-premium-features #{:transforms-basic}
|
|
(let [serialized (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(t2/delete! :model/TransformTag)
|
|
(let [db (ts/create! :model/Database :name "my-db")
|
|
table (ts/create! :model/Table :name "customers" :db_id (:id db))
|
|
coll (ts/create! :model/Collection :name "Transform Collection" :namespace :transforms)
|
|
_transform (ts/create! :model/Transform
|
|
:name "Test Transform"
|
|
:description "A test transform"
|
|
:collection_id (:id coll)
|
|
:source {:query (mbql5-query (:id db) (:id table))
|
|
:type "query"}
|
|
:target {:database (:id db)
|
|
:type "table"
|
|
:schema "public"
|
|
:name "target_table"})]
|
|
(reset! serialized (into [] (serdes.extract/extract {})))))
|
|
(let [minimal (mapv (fn [entity]
|
|
(if (= "Transform" (-> entity :serdes/meta last :model))
|
|
(select-keys entity [:serdes/meta :entity_id :name
|
|
:source :target])
|
|
entity))
|
|
@serialized)]
|
|
(ts/with-db dest-db
|
|
(t2/delete! :model/TransformTag)
|
|
(serdes.load/load-metabase! (ingestion-in-memory minimal))
|
|
(let [transform (t2/select-one :model/Transform :name "Test Transform")
|
|
db (t2/select-one :model/Database :name "my-db")]
|
|
(is (some? transform))
|
|
(is (= (:id db) (:source_database_id transform)))))))))))
|
|
|
|
(deftest transform-checkpoint-field-remap-test
|
|
(testing "checkpoint-filter-field-id survives serdes into an instance with different field IDs (GDGT-2906)"
|
|
(mt/with-premium-features #{:transforms-basic}
|
|
(let [serialized (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(t2/delete! :model/TransformTag)
|
|
(let [db (ts/create! :model/Database :name "my-db")
|
|
table (ts/create! :model/Table :name "customers" :db_id (:id db))
|
|
field (ts/create! :model/Field :name "updated_at" :table_id (:id table)
|
|
:base_type :type/DateTime)
|
|
_ (ts/create! :model/Transform
|
|
:name "Checkpoint Transform"
|
|
:source {:type "query"
|
|
:query (mbql5-query (:id db) (:id table))
|
|
:source-incremental-strategy {:type "checkpoint"
|
|
:checkpoint-filter-field-id (:id field)
|
|
:lookback {:value 4 :unit "day"}}}
|
|
:target {:database (:id db)
|
|
:type "table-incremental"
|
|
:schema "public"
|
|
:name "target_table"
|
|
:target-incremental-strategy {:type "append"}})]
|
|
(reset! serialized (into [] (serdes.extract/extract {})))))
|
|
(testing "the export contains a portable field ref, not a numeric ID"
|
|
(let [transform (first (filter #(= "Transform" (-> % :serdes/meta last :model)) @serialized))]
|
|
(is (= ["my-db" nil "customers" "updated_at"]
|
|
(get-in transform [:source :source-incremental-strategy :checkpoint-filter-field-id])))))
|
|
(ts/with-db dest-db
|
|
(t2/delete! :model/TransformTag)
|
|
;; shift the ID sequences so a raw numeric ID could not accidentally resolve
|
|
(let [db (ts/create! :model/Database :name "padding-db")
|
|
table (ts/create! :model/Table :name "padding" :db_id (:id db))]
|
|
(ts/create! :model/Field :name "padding_field" :table_id (:id table)))
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
(let [transform (t2/select-one :model/Transform :name "Checkpoint Transform")
|
|
field (t2/select-one :model/Field :name "updated_at")]
|
|
(is (some? transform))
|
|
(is (some? field))
|
|
(is (= (:id field)
|
|
(get-in transform [:source :source-incremental-strategy :checkpoint-filter-field-id]))
|
|
"the imported checkpoint field id points at the destination's field row")
|
|
(testing "the lookback config rides along unchanged"
|
|
(is (= {:value 4 :unit "day"}
|
|
(get-in transform [:source :source-incremental-strategy :lookback])))))))))))
|
|
|
|
(deftest transform-dangling-checkpoint-field-export-test
|
|
(testing "a checkpoint field id whose field no longer exists exports as-is and doesn't fail the import (GDGT-2906)"
|
|
(mt/with-premium-features #{:transforms-basic}
|
|
(let [serialized (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(t2/delete! :model/TransformTag)
|
|
(let [db (ts/create! :model/Database :name "my-db")
|
|
table (ts/create! :model/Table :name "customers" :db_id (:id db))
|
|
_ (ts/create! :model/Transform
|
|
:name "Stale Checkpoint Transform"
|
|
:source {:type "query"
|
|
:query (mbql5-query (:id db) (:id table))
|
|
:source-incremental-strategy {:type "checkpoint"
|
|
:checkpoint-filter-field-id 999999999}}
|
|
:target {:database (:id db)
|
|
:type "table-incremental"
|
|
:schema "public"
|
|
:name "target_table"
|
|
:target-incremental-strategy {:type "append"}})]
|
|
(reset! serialized (into [] (serdes.extract/extract {})))))
|
|
(testing "the dangling id is exported as a raw number, not a broken field ref"
|
|
(let [transform (first (filter #(= "Transform" (-> % :serdes/meta last :model)) @serialized))]
|
|
(is (= 999999999
|
|
(get-in transform [:source :source-incremental-strategy :checkpoint-filter-field-id])))))
|
|
(ts/with-db dest-db
|
|
(t2/delete! :model/TransformTag)
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
(is (= 999999999
|
|
(get-in (t2/select-one :model/Transform :name "Stale Checkpoint Transform")
|
|
[:source :source-incremental-strategy :checkpoint-filter-field-id]))
|
|
"the stale numeric id survives the import unchanged")))))))
|
|
|
|
(deftest transform-with-deleted-source-database-load-test
|
|
(testing "An orphaned transform (source database was deleted before export) round-trips through serdes as a tombstone (GDGT-2447)"
|
|
(mt/with-premium-features #{:transforms-basic}
|
|
(let [serialized (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(t2/delete! :model/TransformTag)
|
|
(let [db (ts/create! :model/Database :name "soon-to-be-deleted")
|
|
table (ts/create! :model/Table :name "customers" :db_id (:id db))
|
|
coll (ts/create! :model/Collection :name "Transform Collection" :namespace :transforms)
|
|
_ (ts/create! :model/Transform
|
|
:name "Orphan Transform"
|
|
:description "Transform whose source DB will be deleted"
|
|
:collection_id (:id coll)
|
|
:source {:query (mbql5-query (:id db) (:id table))
|
|
:type "query"}
|
|
:target {:database (:id db)
|
|
:type "table"
|
|
:schema "public"
|
|
:name "orphan_target"})]
|
|
;; Deleting the database cascade-SET-NULLs source_database_id (FK action)
|
|
;; AND cascade-deletes the Table (Field FKs) — the transform survives as a tombstone.
|
|
(t2/delete! :model/Database :name "soon-to-be-deleted")
|
|
(reset! serialized (into [] (serdes.extract/extract {})))))
|
|
(let [minimal (mapv (fn [entity]
|
|
(if (= "Transform" (-> entity :serdes/meta last :model))
|
|
(select-keys entity [:serdes/meta :entity_id :name
|
|
:source :target])
|
|
entity))
|
|
@serialized)]
|
|
(testing "the extracted transform carries the :serdes/unresolved flag and verbatim body"
|
|
(let [extracted (first (filter #(= "Transform" (-> % :serdes/meta last :model)) minimal))]
|
|
(is (some? extracted))
|
|
(is (true? (get-in extracted [:source :serdes/unresolved])))
|
|
;; native query text is preserved verbatim
|
|
(is (some? (get-in extracted [:source :query])))))
|
|
(ts/with-db dest-db
|
|
(t2/delete! :model/TransformTag)
|
|
(serdes.load/load-metabase! (ingestion-in-memory minimal))
|
|
(let [transform (t2/select-one :model/Transform :name "Orphan Transform")]
|
|
(testing "transform round-trips and ends up as a tombstone in the destination instance"
|
|
(is (some? transform))
|
|
(is (nil? (:source_database_id transform))
|
|
"source_database_id should be nil — there's no DB to bind to")
|
|
(is (not (contains? (:source transform) :serdes/unresolved))
|
|
"the :serdes/unresolved marker should have been stripped on import")
|
|
(is (some? (get-in transform [:source :query]))
|
|
"the query body should be preserved as a breadcrumb"))))))))))
|
|
|
|
(deftest table-created-by-transform-load-test
|
|
(testing "Table created by a Transform can be imported via serialization (GDGT-2444)"
|
|
(mt/with-premium-features #{:transforms-basic}
|
|
(let [serialized (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(t2/delete! :model/TransformTag)
|
|
(let [db (ts/create! :model/Database :name "my-db")
|
|
transform (ts/create! :model/Transform
|
|
:name "Hello Transform"
|
|
:source {:query {:database (:id db)
|
|
:type "native"
|
|
:native {:query "select 'hello' message"}}
|
|
:type "query"}
|
|
:target {:database (:id db)
|
|
:type "table"
|
|
:schema "public"
|
|
:name "hello_transforms_world"})]
|
|
(ts/create! :model/Table
|
|
:name "hello_transforms_world"
|
|
:db_id (:id db)
|
|
:schema "public"
|
|
:transform_id (:id transform))
|
|
(reset! serialized (into [] (serdes.extract/extract {})))))
|
|
(ts/with-db dest-db
|
|
(t2/delete! :model/TransformTag)
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
(let [transform (t2/select-one :model/Transform :name "Hello Transform")
|
|
table (t2/select-one :model/Table :name "hello_transforms_world")]
|
|
(is (some? transform))
|
|
(is (some? table))
|
|
(is (= (:id transform) (:transform_id table))))))))))
|
|
|
|
(deftest dashboard-minimal-required-properties-test
|
|
(testing "Dashboard deserialized with only: serdes/meta, entity_id, name, creator_id"
|
|
(let [serialized (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(let [_dash (ts/create! :model/Dashboard :name "Test Dashboard")]
|
|
(reset! serialized (into [] (serdes.extract/extract {})))))
|
|
(let [minimal (mapv (fn [entity]
|
|
(if (= "Dashboard" (-> entity :serdes/meta last :model))
|
|
(select-keys entity [:serdes/meta :entity_id :name :creator_id])
|
|
entity))
|
|
@serialized)]
|
|
(ts/with-db dest-db
|
|
(serdes.load/load-metabase! (ingestion-in-memory minimal))
|
|
(let [dashboard (t2/select-one :model/Dashboard :name "Test Dashboard")]
|
|
(is (some? dashboard)))))))))
|
|
|
|
(deftest dashboard-with-dashcard-minimal-required-properties-test
|
|
(testing "DashboardCard minimal required properties: serdes/meta, entity_id, row, col, size_x, size_y"
|
|
(let [serialized (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(let [dash (ts/create! :model/Dashboard :name "Test Dashboard")
|
|
_dc (ts/create! :model/DashboardCard :dashboard_id (:id dash))]
|
|
(reset! serialized (into [] (serdes.extract/extract {})))))
|
|
(let [minimal (mapv (fn [entity]
|
|
(let [model (-> entity :serdes/meta last :model)]
|
|
(case model
|
|
"Dashboard" (-> (select-keys entity [:serdes/meta :entity_id :name :creator_id
|
|
:dashcards])
|
|
(update :dashcards
|
|
(fn [dcs]
|
|
(mapv #(select-keys % [:serdes/meta :entity_id
|
|
:row :col :size_x :size_y])
|
|
dcs))))
|
|
entity)))
|
|
@serialized)]
|
|
(ts/with-db dest-db
|
|
(serdes.load/load-metabase! (ingestion-in-memory minimal))
|
|
(let [dashboard (t2/select-one :model/Dashboard :name "Test Dashboard")
|
|
dashcards (t2/select :model/DashboardCard :dashboard_id (:id dashboard))]
|
|
(is (some? dashboard))
|
|
(is (= 1 (count dashcards))))))))))
|
|
|
|
(deftest dashcard-series-minimal-required-properties-test
|
|
(testing "DashboardCardSeries minimal required properties: card_id, position"
|
|
(let [serialized (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(let [dash (ts/create! :model/Dashboard :name "Test Dashboard")
|
|
card (ts/create! :model/Card :name "Main Card")
|
|
series-card (ts/create! :model/Card :name "Series Card")
|
|
dc (ts/create! :model/DashboardCard :dashboard_id (:id dash) :card_id (:id card))
|
|
_series (ts/create! :model/DashboardCardSeries :dashboardcard_id (:id dc)
|
|
:card_id (:id series-card) :position 0)]
|
|
(reset! serialized (into [] (serdes.extract/extract {})))))
|
|
(let [minimal (mapv (fn [entity]
|
|
(let [model (-> entity :serdes/meta last :model)]
|
|
(case model
|
|
"Dashboard" (-> (select-keys entity [:serdes/meta :entity_id :name :creator_id
|
|
:dashcards])
|
|
(update :dashcards
|
|
(fn [dcs]
|
|
(mapv (fn [dc]
|
|
(-> (select-keys dc [:serdes/meta :entity_id
|
|
:row :col :size_x :size_y
|
|
:card_id :series])
|
|
(update :series
|
|
(fn [ss]
|
|
(mapv #(select-keys % [:card_id :position])
|
|
ss)))))
|
|
dcs))))
|
|
entity)))
|
|
@serialized)]
|
|
(ts/with-db dest-db
|
|
(serdes.load/load-metabase! (ingestion-in-memory minimal))
|
|
(let [dashboard (t2/select-one :model/Dashboard :name "Test Dashboard")
|
|
dashcards (t2/select :model/DashboardCard :dashboard_id (:id dashboard))
|
|
series (t2/select :model/DashboardCardSeries :dashboardcard_id (:id (first dashcards)))]
|
|
(is (some? dashboard))
|
|
(is (= 1 (count dashcards)))
|
|
(is (= 1 (count series))))))))))
|
|
|
|
(deftest channel-roundtrip-test
|
|
(testing "a channel is serialized and deserialized correctly"
|
|
(let [serialized (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(testing "extraction succeeds"
|
|
(ts/with-db source-db
|
|
(ts/create! :model/Channel :name "Test Email Channel"
|
|
:type :channel/email
|
|
:details {}
|
|
:description "A test email channel")
|
|
(reset! serialized (into [] (serdes.extract/extract {})))
|
|
(is (some (fn [{[{:keys [model id]}] :serdes/meta}]
|
|
(and (= model "Channel") (= id "Test Email Channel")))
|
|
@serialized))))
|
|
(testing "loading into an empty database succeeds"
|
|
(ts/with-db dest-db
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
(let [channels (t2/select :model/Channel :name "Test Email Channel")]
|
|
(is (= 1 (count channels)))
|
|
(is (= "Test Email Channel" (:name (first channels))))
|
|
(is (= :channel/email (:type (first channels))))
|
|
(is (= "A test email channel" (:description (first channels)))))))
|
|
(testing "loading again does not duplicate"
|
|
(ts/with-db dest-db
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
(is (= 1 (t2/count :model/Channel :name "Test Email Channel")))))))))
|
|
|
|
(deftest metabot-roundtrip-test
|
|
(testing "a metabot is serialized and deserialized correctly"
|
|
(let [serialized (atom nil)
|
|
metabot-eid (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(testing "extraction succeeds"
|
|
(ts/with-db source-db
|
|
(let [mb (ts/create! :model/Metabot :name "Test Bot"
|
|
:description "A test metabot"
|
|
:use_verified_content false)]
|
|
(reset! metabot-eid (:entity_id mb)))
|
|
(reset! serialized (into [] (serdes.extract/extract {:include-metabot true})))
|
|
(is (some (fn [{[{:keys [model id]}] :serdes/meta}]
|
|
(and (= model "Metabot") (= id @metabot-eid)))
|
|
@serialized))))
|
|
(testing "loading into an empty database succeeds"
|
|
(ts/with-db dest-db
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
(let [metabots (t2/select :model/Metabot :entity_id @metabot-eid)]
|
|
(is (= 1 (count metabots)))
|
|
(is (= "Test Bot" (:name (first metabots))))
|
|
(is (= "A test metabot" (:description (first metabots)))))))
|
|
(testing "loading again does not duplicate"
|
|
(ts/with-db dest-db
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
(is (= 1 (t2/count :model/Metabot :entity_id @metabot-eid)))))))))
|
|
|
|
(deftest channel-minimal-required-properties-test
|
|
(testing "Channel deserialized with only: name, type, details"
|
|
(let [serialized (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(ts/create! :model/Channel :name "Minimal Channel"
|
|
:type :channel/email
|
|
:details {}
|
|
:description "Some description")
|
|
(reset! serialized (into [] (serdes.extract/extract {}))))
|
|
(let [minimal (mapv (fn [entity]
|
|
(if (= "Channel" (-> entity :serdes/meta last :model))
|
|
(select-keys entity [:serdes/meta :name :type :details])
|
|
entity))
|
|
@serialized)]
|
|
(ts/with-db dest-db
|
|
(serdes.load/load-metabase! (ingestion-in-memory minimal))
|
|
(let [channel (t2/select-one :model/Channel :name "Minimal Channel")]
|
|
(is (some? channel))
|
|
(is (= :channel/email (:type channel))))))))))
|
|
|
|
(deftest metabot-minimal-required-properties-test
|
|
(testing "Metabot deserialized with only: entity_id, name"
|
|
(let [serialized (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(ts/create! :model/Metabot :name "Minimal Bot"
|
|
:description "Some description"
|
|
:use_verified_content false)
|
|
(reset! serialized (into [] (serdes.extract/extract {:include-metabot true}))))
|
|
(let [minimal (mapv (fn [entity]
|
|
(if (= "Metabot" (-> entity :serdes/meta last :model))
|
|
(select-keys entity [:serdes/meta :entity_id :name])
|
|
entity))
|
|
@serialized)]
|
|
(ts/with-db dest-db
|
|
(serdes.load/load-metabase! (ingestion-in-memory minimal))
|
|
(let [metabot (t2/select-one :model/Metabot :name "Minimal Bot")]
|
|
(is (some? metabot))
|
|
(is (= "Minimal Bot" (:name metabot))))))))))
|
|
|
|
;;; ===========================================================================
|
|
;;; Round-trip tests for stripped fields
|
|
;;;
|
|
;;; Export now omits table_id, database_id, query_type, source_card_id from
|
|
;;; Cards, and table_id from Segments/Measures. These tests verify the full
|
|
;;; export → import round-trip backfills the fields correctly.
|
|
;;; ===========================================================================
|
|
|
|
(deftest card-stripped-fields-round-trip-test
|
|
(testing "Card round-trip: stripped fields (table_id, database_id, query_type) are backfilled on import"
|
|
(let [serialized (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(let [db (ts/create! :model/Database :name "my-db")
|
|
table (ts/create! :model/Table :name "orders" :db_id (:id db))
|
|
_ (ts/create! :model/Field :name "id" :table_id (:id table))
|
|
user (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on")
|
|
_card (ts/create! :model/Card
|
|
:collection_id nil
|
|
:creator_id (:id user)
|
|
:name "Round Trip Card"
|
|
:dataset_query (mbql5-query (:id db) (:id table))
|
|
:display :table)]
|
|
(reset! serialized (into [] (serdes.extract/extract {})))
|
|
;; Verify the fields are actually stripped from export
|
|
(let [card-ser (first (filter #(= "Card" (-> % :serdes/meta last :model)) @serialized))]
|
|
(is (not (contains? card-ser :table_id)) "table_id should be stripped from export")
|
|
(is (not (contains? card-ser :database_id)) "database_id should be stripped from export")
|
|
(is (not (contains? card-ser :query_type)) "query_type should be stripped from export"))))
|
|
(ts/with-db dest-db
|
|
(ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on")
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
(let [card (t2/select-one :model/Card :name "Round Trip Card")
|
|
db (t2/select-one :model/Database :name "my-db")
|
|
table (t2/select-one :model/Table :name "orders")]
|
|
(is (some? card))
|
|
(is (= (:id db) (:database_id card)) "database_id backfilled from query")
|
|
(is (= (:id table) (:table_id card)) "table_id backfilled from query")
|
|
(is (= :query (:query_type card)) "query_type backfilled from query")
|
|
(is (nil? (:source_card_id card)) "source_card_id nil for table-based query")))))))
|
|
|
|
(deftest segment-stripped-fields-round-trip-test
|
|
(testing "Segment round-trip: stripped table_id is backfilled from definition on import"
|
|
(let [serialized (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(let [db (ts/create! :model/Database :name "my-db")
|
|
table (ts/create! :model/Table :name "customers" :db_id (:id db))
|
|
field (ts/create! :model/Field :name "age" :table_id (:id table))
|
|
user (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on")
|
|
_segment (ts/create! :model/Segment :table_id (:id table) :name "Minors"
|
|
:definition (mbql5-segment-definition (:id db) (:id table) (:id field))
|
|
:creator_id (:id user))]
|
|
(reset! serialized (into [] (serdes.extract/extract {})))
|
|
(let [seg-ser (first (filter #(= "Segment" (-> % :serdes/meta last :model)) @serialized))]
|
|
(is (not (contains? seg-ser :table_id)) "table_id should be stripped from export"))))
|
|
(ts/with-db dest-db
|
|
(ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on")
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
(let [segment (t2/select-one :model/Segment :name "Minors")
|
|
table (t2/select-one :model/Table :name "customers")]
|
|
(is (some? segment))
|
|
(is (= (:id table) (:table_id segment)) "table_id backfilled from definition")))))))
|
|
|
|
(deftest measure-stripped-fields-round-trip-test
|
|
(testing "Measure round-trip: stripped table_id is backfilled from definition on import"
|
|
(let [serialized (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(let [db (ts/create! :model/Database :name "my-db")
|
|
table (ts/create! :model/Table :name "sales" :db_id (:id db))
|
|
field (ts/create! :model/Field :name "amount" :table_id (:id table))
|
|
user (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on")
|
|
_measure (ts/create! :model/Measure :table_id (:id table) :name "Total Sales"
|
|
:definition (mbql5-measure-definition (:id db) (:id table) (:id field))
|
|
:creator_id (:id user))]
|
|
(reset! serialized (into [] (serdes.extract/extract {})))
|
|
(let [msr-ser (first (filter #(= "Measure" (-> % :serdes/meta last :model)) @serialized))]
|
|
(is (not (contains? msr-ser :table_id)) "table_id should be stripped from export"))))
|
|
(ts/with-db dest-db
|
|
(ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on")
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
(let [measure (t2/select-one :model/Measure :name "Total Sales")
|
|
table (t2/select-one :model/Table :name "sales")]
|
|
(is (some? measure))
|
|
(is (= (:id table) (:table_id measure)) "table_id backfilled from definition")))))))
|
|
|
|
;;; ---------------------------------------------------------------------------
|
|
;;; Keep-side round-trips: fields that cannot be derived are exported and then
|
|
;;; correctly resolved on import.
|
|
;;; ---------------------------------------------------------------------------
|
|
|
|
(deftest card-kept-fields-round-trip-test
|
|
(testing "Card round-trip: database_id is exported and resolved when dataset_query is empty"
|
|
(let [serialized (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(let [db (ts/create! :model/Database :name "my-db")
|
|
user (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on")
|
|
;; Empty dataset_query — populate-query-fields does not run, so database_id
|
|
;; comes only from the explicit field we pass.
|
|
_card (ts/create! :model/Card
|
|
:collection_id nil
|
|
:creator_id (:id user)
|
|
:name "Empty Query Card"
|
|
:database_id (:id db)
|
|
:dataset_query {}
|
|
:display :table)]
|
|
(reset! serialized (into [] (serdes.extract/extract {})))
|
|
(let [card-ser (first (filter #(= "Card" (-> % :serdes/meta last :model)) @serialized))]
|
|
(is (contains? card-ser :database_id) "database_id exported — not derivable from empty query"))))
|
|
(ts/with-db dest-db
|
|
(ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on")
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
(let [card (t2/select-one :model/Card :name "Empty Query Card")
|
|
db (t2/select-one :model/Database :name "my-db")]
|
|
(is (some? card))
|
|
(is (= (:id db) (:database_id card)) "database_id resolved from exported portable ref")))))))
|
|
|
|
(deftest card-broken-query-exports-kept-fields-test
|
|
(testing "Card with a non-empty but broken dataset_query still exports database_id"
|
|
(ts/with-dbs [source-db _dest-db]
|
|
(ts/with-db source-db
|
|
(let [db (ts/create! :model/Database :name "my-db")
|
|
user (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "tom@bost.on")
|
|
card (ts/create! :model/Card
|
|
:creator_id (:id user)
|
|
:name "Broken Query Card"
|
|
:database_id (:id db)
|
|
:dataset_query {}
|
|
:display :table)]
|
|
;; Corrupt dataset_query to a non-empty but structurally broken value, bypassing the
|
|
;; model hooks. This simulates a card whose query became malformed after migrations.
|
|
(t2/query {:update :report_card
|
|
:set {:dataset_query (json/encode {:broken true})}
|
|
:where [:= :id (:id card)]})
|
|
(let [serialized (into [] (serdes.extract/extract {}))
|
|
card-ser (first (filter #(= "Card" (-> % :serdes/meta last :model)) serialized))]
|
|
(is (not (contains? card-ser :table_id)) "table_id always skipped for cards — re-derived on import")
|
|
(is (contains? card-ser :database_id) "database_id exported — not derivable from broken query")))))))
|
|
|
|
(deftest library-subcollection-round-trip-test
|
|
(testing "Library subcollection structure and types are preserved through serdes round-trip"
|
|
(let [serialized (atom nil)
|
|
library (atom nil)
|
|
data-root (atom nil)
|
|
metrics-root (atom nil)
|
|
data-sub (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(testing "extraction of library hierarchy"
|
|
(ts/with-db source-db
|
|
(reset! library (ts/create! :model/Collection
|
|
:name "Library"
|
|
:type collection/library-collection-type
|
|
:location "/"
|
|
:entity_id @#'collection/library-entity-id))
|
|
(reset! data-root (ts/create! :model/Collection
|
|
:name "Data"
|
|
:type collection/library-data-collection-type
|
|
:location (format "/%d/" (:id @library))
|
|
:entity_id @#'collection/library-data-entity-id))
|
|
(reset! metrics-root (ts/create! :model/Collection
|
|
:name "Metrics"
|
|
:type collection/library-metrics-collection-type
|
|
:location (format "/%d/" (:id @library))
|
|
:entity_id @#'collection/library-metrics-entity-id))
|
|
(reset! data-sub (ts/create! :model/Collection
|
|
:name "Sales Tables"
|
|
:type collection/library-data-collection-type
|
|
:location (format "/%d/%d/" (:id @library) (:id @data-root))))
|
|
(reset! serialized (into [] (serdes.extract/extract {})))
|
|
(is (some (fn [{[{:keys [model id]}] :serdes/meta}]
|
|
(and (= model "Collection") (= id (:entity_id @library))))
|
|
@serialized))))
|
|
(testing "loading into destination preserves library structure"
|
|
(ts/with-db dest-db
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
(let [lib-dest (t2/select-one :model/Collection :entity_id (:entity_id @library))
|
|
data-dest (t2/select-one :model/Collection :entity_id (:entity_id @data-root))
|
|
metrics-dest (t2/select-one :model/Collection :entity_id (:entity_id @metrics-root))
|
|
sub-dest (t2/select-one :model/Collection :entity_id (:entity_id @data-sub))]
|
|
(testing "all collections exist"
|
|
(is (some? lib-dest))
|
|
(is (some? data-dest))
|
|
(is (some? metrics-dest))
|
|
(is (some? sub-dest)))
|
|
(testing "types are preserved"
|
|
(is (= collection/library-collection-type (:type lib-dest)))
|
|
(is (= collection/library-data-collection-type (:type data-dest)))
|
|
(is (= collection/library-metrics-collection-type (:type metrics-dest)))
|
|
(is (= collection/library-data-collection-type (:type sub-dest))))
|
|
(testing "parent-child hierarchy is correct"
|
|
(is (= "/" (:location lib-dest)))
|
|
(is (= (format "/%d/" (:id lib-dest)) (:location data-dest)))
|
|
(is (= (format "/%d/" (:id lib-dest)) (:location metrics-dest)))
|
|
(is (= (format "/%d/%d/" (:id lib-dest) (:id data-dest)) (:location sub-dest)))))))))))
|
|
|
|
(deftest library-import-preserves-existing-permissions-test
|
|
(testing "Importing library subcollections does not overwrite existing permissions on destination"
|
|
(let [serialized (atom nil)
|
|
eid-data "testdataeid0000000000"
|
|
eid-sub "testsubeid00000000000"]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(testing "extract from source"
|
|
(ts/with-db source-db
|
|
(let [data-coll (ts/create! :model/Collection
|
|
:name "Data"
|
|
:type collection/library-data-collection-type
|
|
:location "/"
|
|
:entity_id eid-data)
|
|
_sub-coll (ts/create! :model/Collection
|
|
:name "Sales Tables"
|
|
:type collection/library-data-collection-type
|
|
:location (format "/%d/" (:id data-coll))
|
|
:entity_id eid-sub)]
|
|
(reset! serialized (into [] (serdes.extract/extract {}))))))
|
|
(testing "pre-populate dest with same collections and custom permissions"
|
|
(ts/with-db dest-db
|
|
(let [data-dest (ts/create! :model/Collection
|
|
:name "Data"
|
|
:type collection/library-data-collection-type
|
|
:location "/"
|
|
:entity_id eid-data)
|
|
sub-dest (ts/create! :model/Collection
|
|
:name "Sales Tables"
|
|
:type collection/library-data-collection-type
|
|
:location (format "/%d/" (:id data-dest))
|
|
:entity_id eid-sub)
|
|
group (ts/create! :model/PermissionsGroup :name "Custom Group")]
|
|
(perms/grant-collection-readwrite-permissions! group data-dest)
|
|
(perms/grant-collection-read-permissions! group sub-dest)
|
|
(let [perms-before (set (map #(select-keys % [:group_id :object])
|
|
(t2/select :model/Permissions
|
|
:object [:like (format "/collection/%d/%%" (:id data-dest))])))]
|
|
(testing "custom permissions exist before import"
|
|
(is (seq perms-before)))
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
(let [data-after (t2/select-one :model/Collection :entity_id eid-data)
|
|
perms-after (set (map #(select-keys % [:group_id :object])
|
|
(t2/select :model/Permissions
|
|
:object [:like (format "/collection/%d/%%" (:id data-after))])))]
|
|
(testing "collection still exists with same ID"
|
|
(is (= (:id data-dest) (:id data-after))))
|
|
(testing "permissions are unchanged after import"
|
|
(is (= perms-before perms-after))))))))))))
|
|
|
|
(deftest glossary-round-trip-test
|
|
(let [serialized (atom nil)
|
|
eid (atom nil)]
|
|
(ts/with-dbs [source-db dest-db]
|
|
(ts/with-db source-db
|
|
(let [entry (ts/create! :model/Glossary :term "ARR" :definition "Annual recurring revenue")]
|
|
(reset! eid (:entity_id entry))
|
|
(reset! serialized (into [] (serdes.extract/extract {:no-settings true})))))
|
|
(testing "the export is keyed on entity_id"
|
|
(is (contains? (ids-by-model @serialized "Glossary") @eid)))
|
|
(testing "importing into an empty app db reproduces the term, definition and entity_id"
|
|
(ts/with-db dest-db
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
(is (=? [{:term "ARR" :definition "Annual recurring revenue" :entity_id @eid}]
|
|
(t2/select :model/Glossary)))))
|
|
(testing "importing again updates in place rather than duplicating"
|
|
(ts/with-db dest-db
|
|
(serdes.load/load-metabase! (ingestion-in-memory @serialized))
|
|
(is (= 1 (t2/count :model/Glossary))))))))
|
|
|
|
(deftest glossary-load-find-local-test
|
|
(mt/with-temp [:model/Glossary entry {:term "ARR" :definition "Annual recurring revenue"}]
|
|
(testing "an entity_id path finds the row"
|
|
(is (= (:id entry) (:id (serdes/load-find-local [{:model "Glossary" :id (:entity_id entry)}])))))
|
|
(testing "a term-keyed path from a pre-entity_id export finds the row"
|
|
(is (= (:id entry) (:id (serdes/load-find-local [{:model "Glossary" :id "ARR"}])))))
|
|
(testing "a term that is itself 21 nano-id characters still finds the row by term"
|
|
(mt/with-temp [:model/Glossary shaped {:term "CustomerLifetimeValue" :definition "x"}]
|
|
(is (= (:id shaped) (:id (serdes/load-find-local [{:model "Glossary" :id "CustomerLifetimeValue"}]))))))
|
|
(testing "an unknown term finds nothing"
|
|
(is (nil? (serdes/load-find-local [{:model "Glossary" :id "No such term"}]))))))
|
|
|
|
(deftest glossary-import-matches-existing-term-test
|
|
(let [glossary-file (fn [id entity]
|
|
(merge {:serdes/meta [{:model "Glossary" :id id}]
|
|
:term "ARR"
|
|
:definition "Annual recurring revenue (imported)"
|
|
:creator_id "crowberto@metabase.com"
|
|
:created_at "2026-09-11T00:00:00Z"
|
|
:updated_at "2026-09-11T00:00:00Z"}
|
|
entity))]
|
|
(testing "a term-keyed file exported before entity_id existed updates the same-term row and keeps its entity_id"
|
|
(mt/with-empty-h2-app-db!
|
|
(let [{local-eid :entity_id} (ts/create! :model/Glossary :term "ARR" :definition "local")]
|
|
(serdes.load/load-metabase! (ingestion-in-memory [(glossary-file "ARR" nil)]))
|
|
(is (=? [{:term "ARR" :definition "Annual recurring revenue (imported)" :entity_id local-eid}]
|
|
(t2/select :model/Glossary))))))
|
|
(testing "a file whose term exists locally under another entity_id updates that row in place and adopts the file's entity_id"
|
|
(mt/with-empty-h2-app-db!
|
|
(let [file-eid "glossaryfileeid000001"
|
|
{local-id :id} (ts/create! :model/Glossary :term "ARR" :definition "local")]
|
|
(serdes.load/load-metabase! (ingestion-in-memory [(glossary-file file-eid {:entity_id file-eid})]))
|
|
(is (=? [{:id local-id :term "ARR" :definition "Annual recurring revenue (imported)" :entity_id file-eid}]
|
|
(t2/select :model/Glossary))))))
|
|
(testing "a file whose entity_id and term are both new inserts a row"
|
|
(mt/with-empty-h2-app-db!
|
|
(let [file-eid "glossaryfileeid000002"]
|
|
(serdes.load/load-metabase! (ingestion-in-memory [(glossary-file file-eid {:entity_id file-eid})]))
|
|
(is (=? [{:term "ARR" :entity_id file-eid}]
|
|
(t2/select :model/Glossary))))))))
|