Kimirock/metabase
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
9050b176d6263a2e23b4b10aebff25568c5f2e1b
metabase/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
Alexander Polyankin 9050b176d6 Move Dimensions to FieldUserSettings in serdes (#83652)
2026-10-06 02:26:31 +00:00

2934 lines
186 KiB
Clojure
Raw Blame History

(ns metabase-enterprise.serialization.v2.extract-test
{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase-enterprise.serialization.v2.extract-test]}
metabase.test.data/query {:namespaces [metabase-enterprise.serialization.v2.extract-test]}
metabase.test.data/run-mbql-query {:namespaces [metabase-enterprise.serialization.v2.extract-test]}}}}}}
(:require
[clojure.set :as set]
[clojure.string :as str]
[clojure.test :refer :all]
[java-time.api :as t]
[metabase-enterprise.serialization.test-util :as ts]
[metabase-enterprise.serialization.v2.extract :as extract]
[metabase-enterprise.serialization.v2.round-trip-test :as round-trip-test]
[metabase.actions.models :as action]
[metabase.audit-app.core :as audit]
[metabase.collections.test-utils :refer [personal-collection]]
[metabase.core.core :as mbc]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.models.serialization :as serdes]
[metabase.query-processor.preprocess :as qp.preprocess]
[metabase.query-processor.test :as qp]
[metabase.search.test-util :as search.tu]
[metabase.test :as mt]
[metabase.util :as u]
[metabase.util.json :as json]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
(comment
;; Use this spell in your test body to add the given fixtures to the round trip baseline.
(round-trip-test/add-to-baseline!))
(use-fixtures :each (fn [f] (search.tu/with-index-disabled (f))))
(defn- by-model [model-name extraction]
(->> extraction
(into [])
(filter #(= model-name ((comp :model last :serdes/meta) %)))))
(defn- ids-by-model [model-name extraction]
(->> (by-model model-name extraction)
(map (comp :id last :serdes/meta))
set))
(defn- extract-aborts!
"Realize `(extract/extract opts)`, asserting it aborts with the escape-analysis error (#75176).
Escape warnings are logged before the abort, so callers can still assert on the logged messages."
[opts]
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"incomplete export"
(into [] (extract/extract opts)))))
(deftest fundamentals-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Collection
{coll-id :id
coll-eid :entity_id
coll-slug :slug}
{:name "Some Collection"}
:model/Collection
{child-id :id
child-eid :entity_id
child-slug :slug}
{:name "Nested Collection"
:location (format "/%s/" coll-id)}
:model/User
{mark-id :id}
{:first_name "Mark"
:last_name "Knopfler"
:email "mark@direstrai.ts"}]
(testing "a top-level collection is extracted correctly"
(let [ser (serdes/extract-one "Collection" {} (t2/select-one :model/Collection :id coll-id))]
(is (=? {:serdes/meta [{:model "Collection" :id coll-eid :label coll-slug}]}
ser))
(is (not (contains? ser :personal_owner_id)))
(is (not (contains? ser :parent_id)))
(is (not (contains? ser :location)))
(is (not (contains? ser :id)))))
(testing "a nested collection is extracted with the right parent_id"
(let [ser (serdes/extract-one "Collection" {} (t2/select-one :model/Collection :id child-id))]
(is (=? {:serdes/meta [{:model "Collection" :id child-eid :label child-slug}]
:parent_id coll-eid}
ser))
(is (not (contains? ser :personal_owner_id)))
(is (not (contains? ser :location)))
(is (not (contains? ser :id)))))
(testing "personal collections are extracted with email as key"
(let [pc (personal-collection mark-id)
ser (serdes/extract-one "Collection" {} pc)]
(is (=? {:serdes/meta [{:model "Collection" :id (:entity_id pc) :label (:slug pc)}]
:personal_owner_id "mark@direstrai.ts"}
ser))
(is (not (contains? ser :parent_id)))
(is (not (contains? ser :location)))
(is (not (contains? ser :id)))))
(testing "overall extraction returns the expected set"
(testing "no user specified"
(is (= #{coll-eid child-eid}
(ids-by-model "Collection" (extract/extract nil)))))
(testing "valid user specified"
(is (= #{coll-eid child-eid (:entity_id (personal-collection mark-id))}
(ids-by-model "Collection" (extract/extract {:user-id mark-id})))))
(testing "invalid user specified"
(is (= #{coll-eid child-eid}
(ids-by-model "Collection" (extract/extract {:user-id 218921})))))))))
;; dozens of extraction cases share one hand-built dashboard/card graph; splitting duplicates the fixture
#_{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]}
(deftest dashboard-and-cards-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Collection
{coll-id :id
coll-eid :entity_id}
{:name "Some Collection"}
:model/User
{mark-id :id}
{:first_name "Mark"
:last_name "Knopfler"
:email "mark@direstrai.ts"}
:model/User
{dave-id :id}
{:first_name "David"
:last_name "Knopfler"
:email "david@direstrai.ts"}
:model/Database
{db-id :id}
{:name "My Database"}
:model/Table
{no-schema-id :id}
{:name "Schemaless Table" :db_id db-id}
:model/Field
{field-id :id}
{:name "Some Field" :table_id no-schema-id}
:model/Table
{schema-id :id}
{:name "Schema'd Table"
:db_id db-id
:schema "PUBLIC"}
:model/Field
{field2-id :id}
{:name "Other Field" :table_id schema-id}
:model/Card
{c1-id :id
c1-eid :entity_id}
{:name "Some Question"
:database_id db-id
:table_id no-schema-id
:collection_id coll-id
:creator_id mark-id
:dataset_query {:query {:source-table no-schema-id
:filter [:>= [:field field-id nil] 18]
:aggregation [[:count]]}
:type :query
:database db-id}}
:model/Card
{model-id :id}
{:name "Some Model"
:database_id db-id
:table_id no-schema-id
:collection_id coll-id
:creator_id mark-id
:type :model
:dataset_query {:query {:source-table no-schema-id
:filter [:>= [:field field-id nil] 18]
:aggregation [[:count]]}
:type :query
:database db-id}}
:model/Card
{c2-id :id
c2-eid :entity_id}
{:name "Second Question"
:database_id db-id
:table_id schema-id
:collection_id coll-id
:creator_id mark-id
:parameter_mappings
[{:parameter_id "deadbeef"
:card_id c1-id
:target [:dimension [:field field-id
{:source-field field2-id}]]}]}
:model/Card
{c3-id :id
c3-eid :entity_id}
{:name "Third Question"
:database_id db-id
:table_id schema-id
:collection_id coll-id
:creator_id mark-id
:visualization_settings
{:table.pivot_column "SOURCE"
:table.cell_column "sum"
:table.columns
[{:name "SOME_FIELD"
:fieldRef [:field field-id nil]
:enabled true}
{:name "OTHER_FIELD"
:fieldRef [:field field2-id nil]
:enabled true}
{:name "sum"
:fieldRef [:field "sum" {:base-type :type/Float}]
:enabled true}
{:name "count"
:fieldRef [:field "count" {:base-type :type/BigInteger}]
:enabled true}
{:name "Average order total"
:fieldRef [:field "Average order total" {:base-type :type/Float}]
:enabled true}]
:column_settings
{(str "[\"ref\",[\"field\"," field2-id ",null]]") {:column_title "Locus"}}}}
:model/Card {c4-id :id
c4-eid :entity_id} {:name "Referenced Question"
:database_id db-id
:table_id no-schema-id
:collection_id coll-id
:creator_id mark-id
:dataset_query
{:query {:source-table no-schema-id
:filter [:>= [:field field-id nil] 18]}
:database db-id
:type :query}}
:model/Card
{c5-id :id
c5-eid :entity_id}
{:name "Dependent Question"
:database_id db-id
:table_id no-schema-id
:collection_id coll-id
:creator_id mark-id
:dataset_query
{:query {:source-table (str "card__" c4-id)
:aggregation [[:count]]}
:type :query
:database db-id}}
:model/Action
{action-id :id
action-eid :entity_id}
{:name "Some action"
:type :query
:model_id model-id}
:model/Dashboard
{dash-id :id
dash-eid :entity_id}
{:name "Shared Dashboard"
:collection_id coll-id
:creator_id mark-id
:parameters []}
:model/Dashboard
{other-dash-id :id
other-dash :entity_id}
{:name "Dave's Dash"
:collection_id (:id (personal-collection dave-id))
:creator_id mark-id
:parameters []}
:model/Dashboard
{param-dash-id :id
param-dash :entity_id}
{:name "Dave's Dash with parameters"
:collection_id (:id (personal-collection dave-id))
:creator_id mark-id
:parameters [{:id "abc"
:type "category"
:name "CATEGORY"
:values_source_type :card
;; card_id is in a different collection with dashboard's collection
:values_source_config {:card_id c1-id
:value_field [:field field-id nil]}}]}
:model/DashboardCard
_
{:card_id c1-id
:dashboard_id dash-id
:inline_parameters ["12345678"]
:parameter_mappings
[{:parameter_id "12345678"
:card_id c1-id
:target [:dimension [:field field-id
{:source-field field2-id}]]}]}
:model/DashboardCard
_
{:card_id c2-id
:dashboard_id other-dash-id
:visualization_settings
{:table.pivot_column "SOURCE"
:table.cell_column "sum"
:table.columns
[{:name "SOME_FIELD"
:fieldRef [:field field-id nil]
:enabled true}
{:name "sum"
:fieldRef [:field "sum" {:base-type :type/Float}]
:enabled true}
{:name "count"
:fieldRef [:field "count" {:base-type :type/BigInteger}]
:enabled true}
{:name "Average order total"
:fieldRef [:field "Average order total" {:base-type :type/Float}]
:enabled true}]
:column_settings
{(str "[\"ref\",[\"field\"," field2-id ",null]]") {:column_title "Locus"}}}}
:model/DashboardCard
_
{:action_id action-id
:dashboard_id other-dash-id}]
(testing "table and database are extracted as [db schema table] triples"
(let [ser (serdes/extract-one "Card" {} (t2/select-one :model/Card :id c1-id))]
(is (=? {:serdes/meta [{:model "Card" :id c1-eid :label "some_question"}]
:creator_id "mark@direstrai.ts"
:collection_id coll-eid
:dataset_query {:stages [{:source-table ["My Database" nil "Schemaless Table"]
:filters [[:>= {}
[:field {} ["My Database" nil "Schemaless Table" "Some Field"]]
18]]
:aggregation [[:count {}]]}]
:database "My Database"}
:created_at string?}
ser))
(is (not (contains? ser :id)))
(testing "cards depend on their Collection and the query's Database (not the Tables/Fields it references)"
(is (= #{[{:model "Database" :id "My Database"}]
[{:model "Collection" :id coll-eid}]}
(set (serdes/deserialization-dependencies ser))))))
(let [ser (serdes/extract-one "Card" {} (t2/select-one :model/Card :id c2-id))]
(is (=? {:serdes/meta [{:model "Card" :id c2-eid :label "second_question"}]
:creator_id "mark@direstrai.ts"
:collection_id coll-eid
:dataset_query {}
:parameter_mappings [{:parameter_id "deadbeef"
:card_id c1-eid
:target [:dimension [:field ["My Database" nil "Schemaless Table" "Some Field"]
{:source-field ["My Database" "PUBLIC" "Schema'd Table" "Other Field"]}]]}]
:created_at string?}
ser))
(is (not (contains? ser :id)))
(is (not (contains? ser :table_id)) "table_id always skipped for cards — re-derived on import")
(is (contains? ser :database_id) "database_id kept when query is empty")
(testing "cards depend on their Database (kept because query is empty), Collection, and parameter_mappings card refs (not the Fields)"
(is (= #{[{:model "Database" :id "My Database"}]
[{:model "Collection" :id coll-eid}]
[{:model "Card" :id c1-eid}]}
(set (serdes/deserialization-dependencies ser))))))
(let [ser (serdes/extract-one "Card" {} (t2/select-one :model/Card :id c3-id))]
(is (=? {:serdes/meta [{:model "Card" :id c3-eid :label "third_question"}]
:creator_id "mark@direstrai.ts"
:collection_id coll-eid
:dataset_query {}
:visualization_settings
{:table.pivot_column "SOURCE"
:table.cell_column "sum"
:table.columns
[{:name "SOME_FIELD"
:fieldRef [:field ["My Database" nil "Schemaless Table" "Some Field"] nil]
:enabled true}
{:name "OTHER_FIELD"
:fieldRef [:field ["My Database" "PUBLIC" "Schema'd Table" "Other Field"] nil]
:enabled true}
{:name "sum"
:fieldRef [:field "sum" {:base-type :type/Float}]
:enabled true}
{:name "count"
:fieldRef [:field "count" {:base-type :type/BigInteger}]
:enabled true}
{:name "Average order total"
:fieldRef [:field "Average order total" {:base-type :type/Float}]
:enabled true}]
:column_settings
{"[\"ref\",[\"field\",[\"My Database\",\"PUBLIC\",\"Schema'd Table\",\"Other Field\"],null]]" {:column_title "Locus"}}}
:created_at string?}
ser))
(is (not (contains? ser :id)))
(testing "cards depend on their Database (kept, query empty) and Collection (not the Fields in visualization_settings)"
(is (= #{[{:model "Database" :id "My Database"}]
[{:model "Collection" :id coll-eid}]}
(set (serdes/deserialization-dependencies ser)))))))
(testing "Cards can be based on other cards"
(let [ser (serdes/extract-one "Card" {} (t2/select-one :model/Card :id c5-id))]
(is (=? {:serdes/meta [{:model "Card" :id c5-eid :label "dependent_question"}]
:creator_id "mark@direstrai.ts"
:collection_id coll-eid
:dataset_query {:stages [{:source-card c4-eid
:aggregation [[:count {}]]}]
:database "My Database"}
:created_at string?}
ser))
(is (not (contains? ser :id)))
(is (not (contains? ser :table_id)) "table_id stripped")
(is (not (contains? ser :database_id)) "database_id stripped — derivable from query")
(testing "and depend on their Collection, Database (from query), and the upstream Card"
(is (= #{[{:model "Database" :id "My Database"}]
[{:model "Collection" :id coll-eid}]
[{:model "Card" :id c4-eid}]}
(set (serdes/deserialization-dependencies ser)))))))
(testing "Dashboards include their Dashcards"
(let [ser (ts/extract-one "Dashboard" other-dash-id)]
(is (=? {:serdes/meta [{:model "Dashboard" :id other-dash :label "dave_s_dash"}]
:entity_id other-dash
:dashcards
[{:visualization_settings {:table.pivot_column "SOURCE"
:table.cell_column "sum"
:table.columns
[{:name "SOME_FIELD"
:fieldRef [:field ["My Database" nil "Schemaless Table" "Some Field"] nil]
:enabled true}
{:name "sum"
:fieldRef [:field "sum" {:base-type :type/Float}]
:enabled true}
{:name "count"
:fieldRef [:field "count" {:base-type :type/BigInteger}]
:enabled true}
{:name "Average order total"
:fieldRef [:field "Average order total" {:base-type :type/Float}]
:enabled true}]
:column_settings
{"[\"ref\",[\"field\",[\"My Database\",\"PUBLIC\",\"Schema'd Table\",\"Other Field\"],null]]" {:column_title "Locus"}}}
:created_at string?}
{:action_id action-eid}]
:created_at string?}
ser))
(is (not (contains? ser :id)))
(testing "and depend on all referenced cards and actions, plus the Database of fields referenced in visualization_settings"
(is (= #{[{:model "Card" :id c2-eid}]
[{:model "Action" :id action-eid}]
[{:model "Database" :id "My Database"}]
[{:model "Collection" :id (:entity_id (personal-collection dave-id))}]}
(set (serdes/deserialization-dependencies ser)))))))
(testing "Dashboards with parameters where the source is a card"
(let [ser (ts/extract-one "Dashboard" param-dash-id)]
(is (=? {:parameters
[{:id "abc"
:name "CATEGORY"
:type :category
:values_source_config {:card_id c1-eid
:value_field [:field
["My Database" nil "Schemaless Table" "Some Field"]
nil]},
:values_source_type :card}]}
ser))
(is (= #{[{:model "Collection" :id (:entity_id (personal-collection dave-id))}]
[{:model "Card" :id c1-eid}]
;; the parameter's value_field references a Field, but only its Database is a dependency
[{:model "Database", :id "My Database"}]}
(set (serdes/deserialization-dependencies ser))))))
(testing "Cards with parameters where the source is a card"
(let [ser (ts/extract-one "Dashboard" param-dash-id)]
(is (=? {:parameters
[{:id "abc"
:name "CATEGORY"
:type :category
:values_source_config {:card_id c1-eid
:value_field [:field
["My Database" nil "Schemaless Table" "Some Field"]
nil]},
:values_source_type :card}]}
ser))
(is (= #{[{:model "Collection" :id (:entity_id (personal-collection dave-id))}]
[{:model "Card" :id c1-eid}]
;; the parameter's value_field references a Field, but only its Database is a dependency
[{:model "Database", :id "My Database"}]}
(set (serdes/deserialization-dependencies ser))))))
(testing "collection filtering based on :user option"
(testing "only unowned collections are returned with no user"
(is (= ["Some Collection"]
(->> (serdes/extract-all "Collection" {:collection-set #{coll-id}})
(into [])
(map :name)))))
(testing "unowned collections and the personal one with a user"
(is (= #{coll-eid (:entity_id (personal-collection mark-id))}
(->> {:collection-set (#'extract/collection-set-for-user mark-id)}
(serdes/extract-all "Collection")
(ids-by-model "Collection"))))
(is (= #{coll-eid (:entity_id (personal-collection dave-id))}
(->> {:collection-set (#'extract/collection-set-for-user dave-id)}
(serdes/extract-all "Collection")
(ids-by-model "Collection"))))))
(testing "dashboards are filtered based on :user"
(testing "dashboards in unowned collections are always returned"
(is (= #{dash-eid}
(->> {:collection-set #{coll-id}}
(serdes/extract-all "Dashboard")
(ids-by-model "Dashboard"))))
(is (= #{dash-eid}
(->> {:collection-set (#'extract/collection-set-for-user mark-id)}
(serdes/extract-all "Dashboard")
(ids-by-model "Dashboard")))))
(testing "dashboards in personal collections are returned for the :user"
(is (= #{dash-eid other-dash param-dash}
(->> {:collection-set (#'extract/collection-set-for-user dave-id)}
(serdes/extract-all "Dashboard")
(ids-by-model "Dashboard")))))))))
(deftest dashboard-card-series-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc
[:model/Collection {coll-id :id, coll-eid :entity_id} {:name "Some Collection"}
:model/Card {c1-id :id, c1-eid :entity_id} {:name "Some Question", :collection_id coll-id}
:model/Card {c2-id :id, c2-eid :entity_id} {:name "Series Question A", :collection_id coll-id}
:model/Card {c3-id :id, c3-eid :entity_id} {:name "Series Question B", :collection_id coll-id}
:model/Dashboard {dash-id :id, dash-eid :entity_id} {:name "Shared Dashboard", :collection_id coll-id}
:model/DashboardCard {dc1-id :id, dc1-eid :entity_id} {:card_id c1-id, :dashboard_id dash-id}
:model/DashboardCard {dc2-eid :entity_id} {:card_id c1-id, :dashboard_id dash-id}
:model/DashboardCardSeries _ {:card_id c3-id, :dashboardcard_id dc1-id, :position 1}
:model/DashboardCardSeries _ {:card_id c2-id, :dashboardcard_id dc1-id, :position 0}]
(testing "Inlined dashcards include their series' card entity IDs"
(let [ser (t2/with-call-count [q]
(u/prog1 (ts/extract-one "Dashboard" dash-id)
(is (< (q) 13))))]
(is (=? {:entity_id dash-eid
:dashcards [{:entity_id dc1-eid
:series (mt/exactly=? [{:card_id c2-eid :position 0}
{:card_id c3-eid :position 1}])}
{:entity_id dc2-eid}]}
ser))
(testing "and depend on all referenced cards, including cards from dashboard cards' series"
(is (= #{[{:model "Card" :id c1-eid}]
[{:model "Card" :id c2-eid}]
[{:model "Card" :id c3-eid}]
[{:model "Collection" :id coll-eid}]}
(set (serdes/deserialization-dependencies ser))))))))))
(deftest dimensions-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [;; Simple case: a singular field, no human-readable field.
:model/Database {db-id :id} {:name "My Database"}
:model/Table {no-schema-id :id} {:name "Schemaless Table" :db_id db-id}
:model/Field {email-id :id} {:name "email" :table_id no-schema-id}
:model/Dimension {dim1-eid :entity_id} {:name "Vanilla Dimension"
:field_id email-id
:type "internal"
:created_at (t/minus (t/offset-date-time)
(t/days 3))}
;; Advanced case: Dimension capturing a foreign relationship.
;; The parent field (Orders.customer_id) is the foreign key.
;; Dimension.field_id (Customers.id) is the foreign ID field;
;; Dimension.human_readable_field_id (Customers.name) is what we want to render.
:model/Table {customers :id} {:name "Customers"
:db_id db-id
:schema "PUBLIC"}
:model/Field {cust-id :id} {:name "id" :table_id customers}
:model/Field {cust-name :id} {:name "name" :table_id customers}
:model/Table {orders :id} {:name "Orders"
:db_id db-id
:schema "PUBLIC"}
:model/Field {fk-id :id} {:name "customer_id"
:table_id orders
:fk_target_field_id cust-id}
:model/Dimension _ {:name "Customer Name"
:type "external"
:field_id fk-id
:human_readable_field_id cust-name}]
(testing "a dimension without a foreign key is its own entity under its Field\n"
(let [ser (ts/extract-one "Dimension" dim1-eid)]
(is (malli= [:map
[:serdes/meta [:= [{:model "Database", :id "My Database"}
{:model "Table", :id "Schemaless Table"}
{:model "Field", :id "email"}
{:model "Dimension", :id dim1-eid}]]]
[:created_at :string]
[:human_readable_field_id {:optional true} [:maybe [:sequential [:maybe :string]]]]]
ser))
(is (not (contains? ser :id)))
(is (not (contains? ser :field_id)) ":field_id is dropped; it's implied by the path")
(testing "depend only on the Database; the Table and Field are synthesized on import if missing"
(is (= #{[{:model "Database" :id "My Database"}]}
(set (serdes/deserialization-dependencies ser)))))))
(testing "a foreign key dimension is its own entity under its Field"
(let [ser (ts/extract-one "Dimension" (t2/select-one-fn :entity_id :model/Dimension :field_id fk-id))]
(is (=? {:serdes/meta [{:model "Database" :id "My Database"}
{:model "Schema" :id "PUBLIC"}
{:model "Table" :id "Orders"}
{:model "Field" :id "customer_id"}
{:model "Dimension" :id string?}]
:human_readable_field_id ["My Database" "PUBLIC" "Customers" "name"]
:created_at string?}
ser))
(testing "depend only on the Database; the FK target and human-readable Fields are synthesized on import if missing"
(is (= #{[{:model "Database" :id "My Database"}]}
(set (serdes/deserialization-dependencies ser)))))))
(testing "Fields carry no dimensions"
(is (not (contains? (ts/extract-one "Field" fk-id) :dimensions)))))))
(deftest native-query-snippets-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/User {ann-id :id} {:first_name "Ann"
:last_name "Wilson"
:email "ann@heart.band"}
:model/Collection {coll-id :id
coll-eid :entity_id} {:name "Shared Collection"
:personal_owner_id nil
:namespace :snippets}
:model/Collection {coll2-id :id} {:name "Card Collection"
:personal_owner_id nil}
:model/NativeQuerySnippet {s1-id :id
s1-eid :entity_id} {:name "Snippet 1"
:collection_id coll-id
:creator_id ann-id}
:model/NativeQuerySnippet {s2-id :id
s2-eid :entity_id} {:name "Snippet 2"
:collection_id nil
:creator_id ann-id}
:model/Card {card-id :id} {:name "Card using Snippet 1"
:creator_id ann-id
:collection_id coll2-id
:dataset_query
(mt/native-query
{:query "select {{snippet}}"
:template-tags {"snippet"
{:display-name "Snippet 1",
:id s1-eid,
:name "snippet",
:snippet-id s1-id,
:snippet-name "snip",
:type :snippet}}})}]
(testing "native query snippets"
(testing "can belong to :snippets collections"
(let [ser (serdes/extract-one "NativeQuerySnippet" {} (t2/select-one :model/NativeQuerySnippet :id s1-id))]
(is (=? {:serdes/meta [{:model "NativeQuerySnippet"
:id s1-eid
:label "snippet_1"}]
:collection_id coll-eid
:creator_id "ann@heart.band"
:created_at string?}
ser))
(is (not (contains? ser :id)))
(testing "and depend on the Collection"
(is (= #{[{:model "Collection" :id coll-eid}]}
(set (serdes/deserialization-dependencies ser)))))
(testing "and will bring collection to extraction"
(is (= {["Collection" coll-id] {"NativeQuerySnippet" s1-id}}
(serdes/required "NativeQuerySnippet" s1-id))))))
(testing "or can be outside collections"
(let [ser (serdes/extract-one "NativeQuerySnippet" {} (t2/select-one :model/NativeQuerySnippet :id s2-id))]
(is (malli= [:map
[:serdes/meta [:= [{:model "NativeQuerySnippet"
:id s2-eid
:label "snippet_2"}]]]
[:creator_id [:= "ann@heart.band"]]
[:created_at :string]
[:collection_id {:optional true} :nil]]
ser))
(is (not (contains? ser :id)))
(testing "and has no deps"
(is (empty? (serdes/deserialization-dependencies ser))))))
(testing "Snippet collection is exported when snippet is exported as a card dep (#51901)"
(is (= {["Collection" coll2-id] nil
["Card" card-id] {"Collection" coll2-id}
["NativeQuerySnippet" s1-id] {"Card" card-id}
["Collection" coll-id] {"NativeQuerySnippet" s1-id}}
(#'extract/resolve-targets {:targets [["Collection" coll2-id]]} nil))))))))
(deftest resolve-targets-skip-archived-test
(testing "resolve-targets with skip-archived handles archived collections and nested items"
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Collection {parent-id :id} {:name "Parent Collection"}
:model/Collection {archived-child-id :id} {:name "Archived Child Collection"
:archived true
:location (format "/%d/" parent-id)}
:model/Collection {active-child-id :id} {:name "Active Child Collection"
:archived false
:location (format "/%d/" parent-id)}
:model/Card {card-in-archived-id :id} {:name "Card in archived collection"
:collection_id archived-child-id}
:model/Card {card-in-active-id :id} {:name "Card in active collection"
:collection_id active-child-id}]
(testing "archived child collections and their contents are excluded when skip-archived: true"
(let [targets-with-skip (#'extract/resolve-targets {:targets [["Collection" parent-id]]
:skip-archived true} nil)]
(is (contains? targets-with-skip ["Collection" parent-id]))
(is (contains? targets-with-skip ["Collection" active-child-id]))
(is (not (contains? targets-with-skip ["Collection" archived-child-id])))
(is (contains? targets-with-skip ["Card" card-in-active-id]))
(is (not (contains? targets-with-skip ["Card" card-in-archived-id]))
"cards in archived collections should not be included")))
(testing "all collections and cards are included when skip-archived: false"
(let [targets-without-skip (#'extract/resolve-targets {:targets [["Collection" parent-id]]
:skip-archived false} nil)]
(is (contains? targets-without-skip ["Collection" parent-id]))
(is (contains? targets-without-skip ["Collection" active-child-id]))
(is (contains? targets-without-skip ["Collection" archived-child-id]))
(is (contains? targets-without-skip ["Card" card-in-active-id]))
(is (contains? targets-without-skip ["Card" card-in-archived-id]))))))))
(defn- resolve-targets-ex
"Returns the ExceptionInfo thrown by `resolve-targets` for `targets`, or nil if it did not throw."
[targets]
(try
(#'extract/resolve-targets {:targets targets} nil)
nil
(catch clojure.lang.ExceptionInfo e e)))
(deftest resolve-targets-missing-id-test
(testing "a target id that does not exist is rejected as client input, not left to fail deep in extraction"
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Collection {coll-id :id} {:name "Real Collection"}
:model/Card {card-id :id} {:name "Real Card"
:collection_id coll-id}]
(let [missing-id Integer/MAX_VALUE]
(testing "nonexistent Collection id"
(let [e (resolve-targets-ex [["Collection" missing-id]])]
(is (some? e))
(is (re-find #"Could not find Collection with ID" (ex-message e)))
(is (re-find (re-pattern (str missing-id)) (ex-message e))
"the offending id is named so the user can correct it")
(is (= {:status-code 400 :model "Collection" :id missing-id}
(select-keys (ex-data e) [:status-code :model :id]))
"carries a :status-code so the API layer renders a 4xx instead of a server error")))
(testing "nonexistent id of a model other than Collection"
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Could not find Card with ID"
(#'extract/resolve-targets {:targets [["Card" missing-id]]} nil))))
(testing "nil id, which a caller can produce by parsing a non-numeric id"
;; without this check nil reaches `serdes/descendants`, which happily queries
;; `collection_id IS NULL` and exports root-level content instead of failing
(let [e (resolve-targets-ex [["Collection" nil]])]
(is (some? e))
(is (= 400 (:status-code (ex-data e))))))
(testing "ids that do exist still resolve"
(let [targets (#'extract/resolve-targets {:targets [["Collection" coll-id]]} nil)]
(is (contains? targets ["Collection" coll-id]))
(is (contains? targets ["Card" card-id])))))))))
(deftest extract-skip-archived-test
(testing "extract with skip-archived: true excludes archived items from final extraction"
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Collection {coll-id :id} {:name "Test Collection"}
:model/Card {active-card-id :id} {:name "Active Card"
:archived false
:collection_id coll-id}
:model/Card {archived-card-id :id} {:name "Archived Card"
:archived true
:collection_id coll-id}]
(testing "archived cards are excluded from extraction with skip-archived: true"
(let [extraction (extract/extract {:targets [["Collection" coll-id]]
:skip-archived true})
card-ids (into #{} (map (comp :id last :serdes/meta)) (by-model "Card" extraction))]
(is (contains? card-ids (:entity_id (t2/select-one :model/Card :id active-card-id))))
(is (not (contains? card-ids (:entity_id (t2/select-one :model/Card :id archived-card-id)))))))
(testing "archived cards are included in extraction with skip-archived: false"
(let [extraction (extract/extract {:targets [["Collection" coll-id]]
:skip-archived false})
card-ids (into #{} (map (comp :id last :serdes/meta)) (by-model "Card" extraction))]
(is (contains? card-ids (:entity_id (t2/select-one :model/Card :id active-card-id))))
(is (contains? card-ids (:entity_id (t2/select-one :model/Card :id archived-card-id))))))))))
(deftest timelines-and-events-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/User
{ann-id :id}
{:first_name "Ann"
:last_name "Wilson"
:email "ann@heart.band"}
:model/Collection
{coll-id :id
coll-eid :entity_id}
{:name "Shared Collection"
:personal_owner_id nil}
:model/Timeline
{empty-id :id
empty-eid :entity_id}
{:name "Empty Timeline"
:collection_id coll-id
:creator_id ann-id}
:model/Timeline
{line-id :id
line-eid :entity_id}
{:name "Populated Timeline"
:collection_id coll-id
:creator_id ann-id}
:model/TimelineEvent
_
{:name "First Event"
:creator_id ann-id
:timestamp #t "2020-04-11T00:00Z"
:timeline_id line-id}]
(testing "timelines"
(testing "with no events"
(let [ser (ts/extract-one "Timeline" empty-id)]
(is (=? {:serdes/meta [{:model "Timeline" :id empty-eid :label "empty_timeline"}]
:collection_id coll-eid
:creator_id "ann@heart.band"
:created_at string?}
ser))
(is (not (contains? ser :id)))
(testing "depend on the Collection"
(is (= #{[{:model "Collection" :id coll-eid}]}
(set (serdes/deserialization-dependencies ser)))))))
(testing "with events"
(let [ser (ts/extract-one "Timeline" line-id)]
(is (=? {:serdes/meta [{:model "Timeline" :id line-eid :label "populated_timeline"}]
:collection_id coll-eid
:creator_id "ann@heart.band"
:created_at string?
:events [{:timestamp "2020-04-11T00:00:00Z"
:creator_id "ann@heart.band"
:created_at string?}]}
ser))
(is (not (contains? ser :id)))
(is (not (contains? (-> ser :events first) :id)))
(testing "depend on the Collection"
(is (= #{[{:model "Collection" :id coll-eid}]}
(set (serdes/deserialization-dependencies ser)))))))))))
(deftest segments-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/User {ann-id :id} {:first_name "Ann"
:last_name "Wilson"
:email "ann@heart.band"}
:model/Database {db-id :id} {:name "My Database"}
:model/Table {no-schema-id :id} {:name "Schemaless Table" :db_id db-id}
:model/Field {field-id :id} {:name "Some Field" :table_id no-schema-id}
:model/Segment
{s1-id :id
s1-eid :entity_id}
{:name "My Segment"
:creator_id ann-id
:table_id no-schema-id
:definition {:source-table no-schema-id
:filter [:< [:field field-id nil] 18]}}]
(testing "segment"
(let [ser (serdes/extract-one "Segment" {} (t2/select-one :model/Segment :id s1-id))]
(is (=? {:serdes/meta [{:model "Segment" :id s1-eid :label "my_segment"}]
:creator_id "ann@heart.band"
:definition {:database "My Database",
:lib/type :mbql/query
:stages [{:source-table ["My Database" nil "Schemaless Table"],
:filters [[:< {}
[:field {} ["My Database" nil "Schemaless Table" "Some Field"]]
18]]}]}
:created_at string?}
ser))
(is (not (contains? ser :id)))
(testing "depend only on the Database; the Table/Fields from the definition are not dependencies"
(is (= #{[{:model "Database" :id "My Database"}]}
(set (serdes/deserialization-dependencies ser))))))))))
(defn- mbql5-measure-definition
"Create an MBQL5 measure definition with a sum aggregation."
[db-id table-id field-id]
(let [metadata-provider (lib-be/application-database-metadata-provider db-id)
table (lib.metadata/table metadata-provider table-id)
query (lib/query metadata-provider table)
field (lib.metadata/field metadata-provider field-id)]
(lib/aggregate query (lib/sum field))))
(deftest measures-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/User {ann-id :id} {:first_name "Ann"
:last_name "Wilson"
:email "ann@heart.band"}
:model/Database {db-id :id} {:name "My Database"}
:model/Table {no-schema-id :id} {:name "Schemaless Table" :db_id db-id}
:model/Field {field-id :id} {:name "Some Field" :table_id no-schema-id}]
(let [definition (mbql5-measure-definition db-id no-schema-id field-id)]
(ts/with-temp-dpc [:model/Measure
{m1-id :id
m1-eid :entity_id}
{:name "My Measure"
:creator_id ann-id
:table_id no-schema-id
:definition definition}]
;; Uncomment to regenerate measure baseline: (round-trip-test/add-to-baseline!)
(testing "measure"
(let [ser (serdes/extract-one "Measure" {} (t2/select-one :model/Measure :id m1-id))]
(is (=? {:serdes/meta [{:model "Measure" :id m1-eid :label "my_measure"}]
:creator_id "ann@heart.band"
:definition {:database "My Database"
:lib/type :mbql/query
:stages [{:source-table ["My Database" nil "Schemaless Table"]
:aggregation [[:sum {} [:field {} ["My Database" nil "Schemaless Table" "Some Field"]]]]}]}
:created_at string?}
ser))
(is (not (contains? ser :id)))
(testing "depend only on the Database; the Table/Fields from the definition are not dependencies"
(is (= #{[{:model "Database" :id "My Database"}]}
(set (serdes/deserialization-dependencies ser))))))))))))
(deftest measure-referencing-measure-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/User {ann-id :id} {:first_name "Ann"
:last_name "Wilson"
:email "ann@heart.band"}
:model/Database {db-id :id} {:name "My Database"}
:model/Table {table-id :id} {:name "My Table" :db_id db-id}
:model/Field {field-id :id} {:name "Amount" :table_id table-id}]
(let [base-definition (mbql5-measure-definition db-id table-id field-id)]
(ts/with-temp-dpc [:model/Measure
{m1-id :id
m1-eid :entity_id}
{:name "Base Measure"
:creator_id ann-id
:table_id table-id
:definition base-definition}]
;; Create a measure that references the base measure
(let [metadata-provider (lib-be/application-database-metadata-provider db-id)
table (lib.metadata/table metadata-provider table-id)
query (lib/query metadata-provider table)
measure-ref (lib.metadata/measure metadata-provider m1-id)
derived-definition (lib/aggregate query (lib/* measure-ref 2))]
(ts/with-temp-dpc [:model/Measure
{m2-id :id
m2-eid :entity_id}
{:name "Derived Measure"
:creator_id ann-id
:table_id table-id
:definition derived-definition}]
(testing "measure referencing another measure"
(let [ser (serdes/extract-one "Measure" {} (t2/select-one :model/Measure :id m2-id))]
(is (=? {:serdes/meta [{:model "Measure" :id m2-eid :label "derived_measure"}]
:creator_id "ann@heart.band"
:definition {:database "My Database"
:lib/type :mbql/query
:stages [{:source-table ["My Database" nil "My Table"]
:aggregation [[:* {} [:measure {} m1-eid] 2]]}]}}
ser))
(testing "depends on the referenced Measure"
(is (contains? (set (serdes/deserialization-dependencies ser))
[{:model "Measure" :id m1-eid}]))))))))))))
(deftest measure-referencing-segment-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/User {ann-id :id} {:first_name "Ann"
:last_name "Wilson"
:email "ann@heart.band"}
:model/Database {db-id :id} {:name "My Database"}
:model/Table {table-id :id} {:name "My Table" :db_id db-id}
:model/Field {field-id :id} {:name "Price" :table_id table-id}
:model/Segment
{seg-id :id
seg-eid :entity_id}
{:name "Expensive Items"
:creator_id ann-id
:table_id table-id
:definition {:source-table table-id
:filter [:> [:field field-id nil] 100]}}]
;; Create a measure that uses count-where with a segment reference
(let [metadata-provider (lib-be/application-database-metadata-provider db-id)
table (lib.metadata/table metadata-provider table-id)
query (lib/query metadata-provider table)
segment-ref (lib.metadata/segment metadata-provider seg-id)
measure-definition (lib/aggregate query (lib/count-where segment-ref))]
(ts/with-temp-dpc [:model/Measure
{m-id :id
m-eid :entity_id}
{:name "Expensive Item Count"
:creator_id ann-id
:table_id table-id
:definition measure-definition}]
(testing "measure referencing a segment"
(let [ser (serdes/extract-one "Measure" {} (t2/select-one :model/Measure :id m-id))]
(is (=? {:serdes/meta [{:model "Measure" :id m-eid :label "expensive_item_count"}]
:creator_id "ann@heart.band"
:definition {:database "My Database"
:lib/type :mbql/query
:stages [{:source-table ["My Database" nil "My Table"]
:aggregation [[:count-where {} [:segment {} seg-eid]]]}]}}
ser))
(testing "depends on the referenced Segment"
(is (contains? (set (serdes/deserialization-dependencies ser))
[{:model "Segment" :id seg-eid}]))))))))))
(deftest table-publishing-serdes-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Database {db-id :id} {:name "My Database"}
:model/Table {table-id :id} {:name "Schemaless Table" :db_id db-id}
:model/Collection {coll-id :id
coll-eid :entity_id} {:name "Publishing Collection"}
:model/Table {pub-table-id :id} {:name "Published Table"
:db_id db-id
:is_published true
:collection_id coll-id}
:model/Table {unpub-table-id :id} {:name "Unpublished Table"
:db_id db-id
:is_published false}]
(testing "published table with collection_id"
(let [ser (ts/extract-one "Table" pub-table-id)]
(testing "is_published is included in extraction"
(is (true? (:is_published ser))))
(testing "collection_id is transformed to entity_id"
(is (= coll-eid (:collection_id ser))))
(testing "depends on the collection"
(is (contains? (set (serdes/deserialization-dependencies ser))
[{:model "Collection" :id coll-eid}])))))
(testing "unpublished table without collection_id"
(let [ser (ts/extract-one "Table" unpub-table-id)]
(testing "is_published is omitted (default false)"
(is (not (contains? ser :is_published))))
(testing "collection_id is nil"
(is (nil? (:collection_id ser))))
(testing "does not depend on any collection"
(is (not (some #(= "Collection" (:model (first %)))
(serdes/deserialization-dependencies ser)))))))
(testing "regular table without publishing fields set"
(let [ser (ts/extract-one "Table" table-id)]
(testing "is_published is omitted (default false)"
(is (not (contains? ser :is_published))))
(testing "collection_id is nil"
(is (nil? (:collection_id ser)))))))))
(deftest implicit-action-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/User {ann-id :id} {:first_name "Ann"
:last_name "Wilson"
:email "ann@heart.band"}
:model/Database {db-id :id :as db} {:name "My Database"}]
(mt/with-db db
(mt/with-actions [{card-id-1 :id
card-eid-1 :entity_id}
{:name "Source question"
:database_id db-id
:type :model
:query_type :native
:dataset_query (mt/native-query {:query "select 1"})
:creator_id ann-id}
{:keys [action-id]}
{:name "My Action"
:type :implicit
:kind "row/update"
:creator_id ann-id
:model_id card-id-1}]
(let [action (action/select-action :id action-id)]
(testing "implicit action"
(let [ser (ts/extract-one "Action" action-id)]
(is (=? {:serdes/meta [{:model "Action" :id (:entity_id action) :label "my_action"}]
:creator_id "ann@heart.band"
:type "implicit"
:created_at string?
:model_id card-eid-1
:implicit [{:kind :row/update}]}
ser))
(is (not (contains? ser :id)))
(testing "depends on the Model"
(is (= #{[{:model "Card" :id card-eid-1}]}
(set (serdes/deserialization-dependencies ser)))))))))))))
(deftest collection-export-includes-model-actions-test
(testing "GHY-4722: exporting a collection exports the actions of its models, though a model doesn't reference them"
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Database {db-id :id} {:name "My Database"}
:model/Collection {coll-id :id} {:name "Models"}
:model/Card {model-id :id} {:name "Model"
:type :model
:database_id db-id
:collection_id coll-id
:dataset_query {:database db-id
:type :native
:native {:query "select 1"}}}
:model/Action {action-id :id} {:name "My Action" :type :implicit :model_id model-id}
:model/Action {archived-id :id} {:name "Old Action" :type :implicit :model_id model-id
:archived true}]
(let [eid (fn [id] (t2/select-one-fn :entity_id :model/Action :id id))]
(is (= #{(eid action-id) (eid archived-id)}
(ids-by-model "Action" (extract/extract {:targets [["Collection" coll-id]]
:no-data-model true}))))
(testing "with :skip-archived, the archived action is left out"
(is (= #{(eid action-id)}
(ids-by-model "Action" (extract/extract {:targets [["Collection" coll-id]]
:no-data-model true
:skip-archived true}))))))))))
(deftest query-action-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/User {ann-id :id} {:first_name "Ann"
:last_name "Wilson"
:email "ann@heart.band"}
:model/Database {db-id :id :as db} {:name "My Database"}]
(mt/with-db db
(mt/with-actions [{card-id-1 :id
card-eid-1 :entity_id}
{:name "Source question"
:database_id db-id
:type :model
:query_type :native
:dataset_query (mt/native-query {:query "select 1"})
:creator_id ann-id}
{:keys [action-id]}
{:name "My Action"
:type :query
:dataset_query {:type "native", :native {:query "select 1"}, :database db-id}
:database_id db-id
:creator_id ann-id
:model_id card-id-1}]
(let [action (action/select-action :id action-id)]
(testing "action"
(let [ser (ts/extract-one "Action" action-id)]
(is (=? {:serdes/meta [{:model "Action"
:id (:entity_id action)
:label "my_action"}]
:type "query"
:creator_id "ann@heart.band"
:created_at string?
:query [{:dataset_query {:database "My Database"
:lib/type :mbql/query
:stages [{:lib/type :mbql.stage/native
:native "select 1"}]}}]
:model_id card-eid-1}
ser))
(is (not (contains? ser :id)))
(testing "depends on the Model and Database"
(is (= #{[{:model "Database" :id "My Database"}]
[{:model "Card" :id card-eid-1}]}
(set (serdes/deserialization-dependencies ser)))))))))))))
(deftest field-values-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Database {db-id :id} {:name "My Database"}
:model/Table {no-schema-id :id} {:name "Schemaless Table" :db_id db-id}
:model/Field {field-id :id} {:name "Some Field"
:table_id no-schema-id
:fingerprint {:global {:distinct-count 75 :nil% 0.0}
:type {:type/Text {:percent-json 0.0
:percent-url 0.0
:percent-email 0.0
:percent-state 0.0
:average-length 8.333333333333334}}}}
:model/FieldValues
{fv-id :id
values :values}
{:field_id field-id
:hash_key nil
:has_more_values false
:type :full
:human_readable_values []
:values ["Artisan" "Asian" "BBQ" "Bakery" "Bar" "Brewery" "Burger" "Coffee Shop"
"Diner" "Indian" "Italian" "Japanese" "Mexican" "Middle Eastern" "Pizza"
"Seafood" "Steakhouse" "Tea Room" "Winery"]}]
(testing "field values"
(let [ser (serdes/extract-one "FieldValues" {} (t2/select-one :model/FieldValues :id fv-id))]
(is (=? {:serdes/meta [{:model "Database" :id "My Database"}
{:model "Table" :id "Schemaless Table"}
{:model "Field" :id "Some Field"}
{:model "FieldValues" :id "0"}] ; Always 0.
:created_at string?
:values values}
ser))
(is (not (contains? ser :id)))
(is (not (contains? ser :field_id))
":field_id is dropped; its implied by the path")
(testing "depend only on the Database; the parent Field is synthesized on import if missing"
(is (= #{[{:model "Database" :id "My Database"}]}
(set (serdes/deserialization-dependencies ser)))))))
(testing "extract-metabase behavior"
(testing "without :include-field-values"
(is (= #{}
(ids-by-model "FieldValues" (extract/extract {})))))
(testing "with :include-field-values true"
(let [models (->> {:include-field-values true} extract/extract (map (comp :model last :serdes/meta)))]
(is (= 1
(t2/count :model/FieldValues)
(count (filter #{"FieldValues"} models))))))))))
(deftest field-user-settings-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Database {db-id :id} {:name "My Database"}
:model/Table {no-schema-id :id} {:name "Schemaless Table" :db_id db-id}
:model/Field {field-id :id} {:name "Some Field"
:table_id no-schema-id
:data_sensitivity :PII}
:model/Field {plain-id :id} {:name "Plain Field" :table_id no-schema-id}
:model/FieldUserSettings {description :description}
{:field_id field-id
:description "Some custom Description"
:data_sensitivity :PII}]
(testing "field values"
(let [ser (serdes/extract-one "FieldUserSettings" {} (t2/select-one :model/FieldUserSettings :field_id field-id))]
(is (=? {:serdes/meta [{:model "Database" :id "My Database"}
{:model "Table" :id "Schemaless Table"}
{:model "Field" :id "Some Field"}
{:model "FieldUserSettings" :id "1"}] ; Always 1.
:created_at string?
:description description
:data_sensitivity :PII}
ser))
(is (not (contains? ser :field_id))
":field_id is dropped; its implied by the path")))
(testing "data_sensitivity on the Field itself"
(is (= :PII (:data_sensitivity (ts/extract-one "Field" field-id)))
"a labeled field exports the keyword as-is")
(is (not (contains? (ts/extract-one "Field" plain-id) :data_sensitivity))
"an unlabeled field exports no key, so nil never reaches the YAML"))
(testing "extract-metabase behavior"
(let [entities (into [] (extract/extract {}))
models (map (comp :model last :serdes/meta) entities)]
(is (= 1 (count (filter #{"FieldUserSettings"} models)))
"written as its own entity, only for the Field that has settings")
(is (empty? (filter #{"TableUserSettings"} models))
"no TableUserSettings for a Table without a settings row"))))))
(deftest table-descendants-user-settings-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Database {db-id :id} {:name "DB"}
:model/Table {table-id :id} {:name "T" :db_id db-id}
:model/Field {f1-id :id} {:name "F1" :table_id table-id}
:model/Field {f2-id :id} {:name "F2" :table_id table-id}
:model/Field {f3-id :id} {:name "F3" :table_id table-id}]
(testing "every Field is a descendant, and none has user settings to speak of yet"
(is (= #{["Field" f1-id] ["Field" f2-id] ["Field" f3-id]}
(set (keys (serdes/descendants "Table" table-id {}))))))
(testing "a Field's settings and Dimension are descendants of their own"
(t2/insert! :model/FieldUserSettings {:field_id f2-id :description "edited"})
(let [dimension-id (t2/insert-returning-pk! :model/Dimension {:field_id f3-id :name "F3" :type :internal})]
(is (= #{["Field" f1-id] ["Field" f2-id] ["Field" f3-id]
["FieldUserSettings" f2-id] ["Dimension" dimension-id]}
(set (keys (serdes/descendants "Table" table-id {}))))))
(t2/delete! :model/FieldUserSettings :field_id f2-id)
(t2/delete! :model/Dimension :field_id f3-id))
(testing "a Table with user settings appears as TableUserSettings"
(t2/insert! :model/TableUserSettings {:table_id table-id :display_name "Renamed"})
(is (contains? (set (keys (serdes/descendants "Table" table-id {})))
["TableUserSettings" table-id])))
(testing "Field is a leaf node in the descendants graph"
(is (empty? (serdes/descendants "Field" f1-id {})))))))
(deftest table-user-settings-extract-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Database {db-id :id} {:name "DB"}
:model/Table {table-id :id} {:name "T" :db_id db-id}
:model/Table {other-id :id} {:name "Unedited T" :db_id db-id}
:model/Field {f1-id :id} {:name "F1" :table_id table-id}
:model/FieldUserSettings _ {:field_id f1-id :description "edited"}
:model/TableUserSettings _ {:table_id table-id :display_name "Renamed"}]
(testing "a Table's settings carry only its own values, not its Fields'"
(is (=? [{:display_name "Renamed"
:serdes/meta [{:model "Database" :id "DB"}
{:model "Table" :id "T"}
{:model "TableUserSettings" :id "1"}]}]
(into [] (serdes/extract-all "TableUserSettings" {:filter-column :table_id
:filter-ids [table-id]}))))
(is (not (contains? (ts/extract-one "TableUserSettings" table-id) :fields))))
(testing "a Table with no settings row yields nothing"
(is (empty? (into [] (serdes/extract-all "TableUserSettings" {:filter-column :table_id
:filter-ids [other-id]}))))))))
(deftest collection-export-includes-user-settings-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Database {db-id :id} {:name "DB"}
:model/Collection {coll-id :id} {:name "Library" :type "library-data"}
:model/Table {table-id :id} {:name "T" :db_id db-id
:is_published true
:collection_id coll-id}
:model/Field _ {:name "F1" :table_id table-id}
:model/Field {f2-id :id} {:name "F2" :table_id table-id}
:model/FieldUserSettings _ {:field_id f2-id :description "curated"}
:model/TableUserSettings _ {:table_id table-id :display_name "Renamed"}]
(testing "targeting the collection exports the Table and its Fields, each beside its user settings"
(let [entities (into [] (extract/extract {:targets [["Collection" coll-id]]
:no-data-model true}))
by-model (group-by (comp :model last :serdes/meta) entities)]
(is (some #(= "T" (:name %)) (by-model "Table")))
(is (= #{"F1" "F2"}
(set (map :name (by-model "Field")))))
(is (= 1 (count (by-model "TableUserSettings"))))
(is (= #{"F2"}
(set (map #(-> % :serdes/meta (nth 2) :id) (by-model "FieldUserSettings"))))
"only the field the user edited has settings to export"))))))
(deftest cards-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc
[:model/User {mark-id :id} {:first_name "Mark"
:last_name "Knopfler"
:email "mark@direstrai.ts"}
:model/Database {db-id :id} {:name "My Database"}
:model/Table {table-id :id} {:name "Schemaless Table" :db_id db-id}
:model/Field {field-id :id} {:name "A Field" :table_id table-id}
:model/Collection {coll-id-1 :id} {:name "1st collection"}
:model/Collection
{coll-id-2 :id
coll-eid-2 :entity_id}
{:name "2nd collection"}
:model/Card
{card-id-1 :id
card-eid-1 :entity_id}
{:name "Source question"
:database_id db-id
:table_id table-id
:collection_id coll-id-1
:creator_id mark-id}
:model/Card
{card-id-2 :id}
{:name "Card 2"
:database_id db-id
:table_id table-id
:collection_id coll-id-2
:creator_id mark-id
:parameters [{:id "abc"
:type "category"
:name "CATEGORY"
:values_source_type "card"
;; card_id is in a different collection with dashboard's collection
:values_source_config {:card_id card-id-1
:value_field [:field field-id nil]}}]}]
(testing "Cards with parameter's source is another question"
(let [ser (serdes/extract-one "Card" {} (t2/select-one :model/Card :id card-id-2))]
(is (= [{:id "abc",
:type :category,
:name "CATEGORY",
:position 0,
:values_source_type :card
:values_source_config {:card_id card-eid-1,
:value_field [:field ["My Database" nil "Schemaless Table" "A Field"] nil]}}]
(:parameters ser)))
;; The parameter's value_field references a Field, but Tables/Fields are not dependencies — only their
;; Database is.
(is (= #{[{:model "Database" :id "My Database"}]
[{:model "Collection" :id coll-eid-2}]
[{:model "Card" :id card-eid-1}]}
(set (serdes/deserialization-dependencies ser))))))
(testing "Nullable transformations are omitted"
(let [ser (serdes/extract-one "Card" {} (t2/select-one :model/Card :id card-id-2))]
(is (not (contains? ser :made_public_by_id))))))))
(deftest parameters-with-deleted-source-card-test
(testing (str "A parameter whose values-source Card is gone — e.g. it was hard-deleted along with its Database — "
"exports with the source dropped rather than knocking the whole entity out of the export (GHY-3259)")
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc
[:model/Collection {coll-id :id} {:name "Dropdowns"}
;; non-H2 engine so the Database isn't filtered out of the export before we delete it
:model/Database {db-id :id} {:name "Doomed Database" :engine :postgres}
:model/Card {source-id :id} {:name "Dropdown List Options"
:database_id db-id
:collection_id coll-id}
:model/Card _ {:name "Card with dropdown"
:collection_id coll-id
:parameters [{:id "abc"
:type "category"
:name "CATEGORY"
:values_source_type "card"
:values_source_config {:card_id source-id}}]}
:model/Dashboard _ {:name "Dashboard with dropdown"
:collection_id coll-id
:parameters [{:id "def"
:type "category"
:name "CATEGORY"
:values_source_type "card"
:values_source_config {:card_id source-id}}]}]
;; deleting a Database hard-deletes its Cards, leaving the parameters above pointing at a row that is gone
(t2/delete! :model/Database :id db-id)
(is (not (t2/exists? :model/Card :id source-id))
"deleting the Database should have taken its Card with it")
(let [ser (into [] (extract/extract {:targets [["Collection" coll-id]]
:no-settings true
:no-data-model true}))
by-model (group-by (comp :model last :serdes/meta) (remove #(instance? Exception %) ser))]
(is (empty? (filter #(instance? Exception %) ser))
"nothing should be skipped because of the dangling reference")
(is (= [{:id "abc" :type :category :name "CATEGORY" :position 0}]
(:parameters (first (get by-model "Card")))))
(is (= [{:id "def" :type :category :name "CATEGORY" :position 0}]
(:parameters (first (get by-model "Dashboard"))))))))))
;; the selective-serialization cases all walk one shared entity graph; splitting duplicates the fixture
#_{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]}
(deftest selective-serialization-basic-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/User {mark-id :id} {:first_name "Mark"
:last_name "Knopfler"
:email "mark@direstrai.ts"}
:model/Collection {coll1-id :id
coll1-eid :entity_id} {:name "Some Collection"}
:model/Collection {coll2-id :id
coll2-eid :entity_id} {:name "Nested Collection"
:location (str "/" coll1-id "/")}
:model/Collection {coll3-id :id
coll3-eid :entity_id} {:name "Grandchild Collection"
:location (str "/" coll1-id "/" coll2-id "/")}
:model/Database {db-id :id} {:name "My Database"}
:model/Table {no-schema-id :id} {:name "Schemaless Table" :db_id db-id}
:model/Field _ {:name "Some Field" :table_id no-schema-id}
:model/Table {schema-id :id} {:name "Schema'd Table"
:db_id db-id
:schema "PUBLIC"}
:model/Field {field-id :id} {:name "Other Field" :table_id schema-id}
:model/Field {field-id2 :id} {:name "Field To Click 1" :table_id schema-id}
:model/Field {field-id3 :id} {:name "Field To Click 2" :table_id schema-id}
;; One dashboard and three cards in each of the three collections:
;; Two cards contained in the dashboard and one freestanding.
:model/Dashboard {dash1-id :id
dash1-eid :entity_id} {:name "Dashboard 1"
:collection_id coll1-id
:creator_id mark-id}
:model/Card {c1-1-id :id
c1-1-eid :entity_id} {:name "Question 1-1"
:database_id db-id
:table_id no-schema-id
:collection_id coll1-id
:creator_id mark-id}
:model/Card {c1-2-id :id
c1-2-eid :entity_id} {:name "Question 1-2"
:database_id db-id
:table_id schema-id
:collection_id coll1-id
:creator_id mark-id}
:model/Card {c1-3-eid :entity_id} {:name "Question 1-3"
:database_id db-id
:table_id schema-id
:collection_id coll1-id
:creator_id mark-id}
:model/DashboardCard _ {:card_id c1-1-id
:dashboard_id dash1-id}
:model/DashboardCard _ {:card_id c1-2-id
:dashboard_id dash1-id}
;; Second dashboard, in the middle collection.
:model/Dashboard {dash2-id :id
dash2-eid :entity_id} {:name "Dashboard 2"
:collection_id coll2-id
:creator_id mark-id}
:model/Card {c2-1-id :id
c2-1-eid :entity_id} {:name "Question 2-1"
:database_id db-id
:table_id no-schema-id
:collection_id coll2-id
:creator_id mark-id}
:model/Card {c2-2-id :id
c2-2-eid :entity_id} {:name "Question 2-2"
:database_id db-id
:table_id schema-id
:collection_id coll2-id
:creator_id mark-id}
:model/Card {c2-3-eid :entity_id} {:name "Question 2-3"
:database_id db-id
:table_id schema-id
:collection_id coll2-id
:creator_id mark-id}
:model/DashboardCard _ {:card_id c2-1-id
:dashboard_id dash2-id}
:model/DashboardCard _ {:card_id c2-2-id
:dashboard_id dash2-id}
;; Third dashboard, in the grandchild collection.
:model/Dashboard {dash3-id :id
dash3-eid :entity_id} {:name "Dashboard 3"
:collection_id coll3-id
:creator_id mark-id}
:model/Card {c3-1-id :id
c3-1-eid :entity_id} {:name "Question 3-1"
:database_id db-id
:table_id no-schema-id
:collection_id coll3-id
:creator_id mark-id}
:model/Card {c3-2-id :id
c3-2-eid :entity_id} {:name "Question 3-2"
:database_id db-id
:table_id schema-id
:collection_id coll3-id
:creator_id mark-id}
:model/Card {c3-3-eid :entity_id} {:name "Question 3-3"
:database_id db-id
:table_id schema-id
:collection_id coll3-id
:creator_id mark-id}
:model/DashboardCard _ {:card_id c3-1-id
:dashboard_id dash3-id}
:model/DashboardCard _ {:card_id c3-2-id
:dashboard_id dash3-id}
;; Fourth dashboard where its parameter's source is another card
:model/Collection {coll4-id :id
_coll4-eid :entity_id} {:name "Forth collection"}
:model/Card {c4-id :id
c4-eid :entity_id} {:name "Question 4-1"
:database_id db-id
:table_id no-schema-id
:collection_id coll4-id
:creator_id mark-id
:parameters [{:id "abc"
:type "category"
:name "CATEGORY"
:values_source_type "card"
;; card_id is in a different collection with dashboard's collection
:values_source_config {:card_id c1-1-id
:value_field [:field field-id nil]}}]}
:model/Dashboard {dash4-id :id
dash4-eid :entity_id} {:name "Dashboard 4"
:collection_id coll4-id
:creator_id mark-id
:parameters [{:id "def"
:type "category"
:name "CATEGORY"
:values_source_type "card"
;; card_id is in a different collection with dashboard's collection
:values_source_config {:card_id c1-2-id
:value_field [:field field-id nil]}}]}
:model/DashboardCard _ {:card_id c4-id
:dashboard_id dash4-id}
;; Fifth dashboard which has :click_behavior defined.
:model/Collection {coll5-id :id} {:name "Fifth collection"}
:model/Dashboard {clickdash-id :id
clickdash-eid :entity_id} {:name "Dashboard with click behavior"
:collection_id coll5-id
:creator_id mark-id}
:model/DashboardCard _ {:card_id c3-1-id
:dashboard_id clickdash-id
:visualization_settings
;; Top-level click behavior for the card.
(let [dimension [:dimension [:field "something" {:base-type "type/Text"}]]
mapping-id (json/encode dimension)]
{:click_behavior {:type "link"
:linkType "question"
:targetId c3-2-id
:parameterMapping
{mapping-id {:id mapping-id
:source {:type "column"
:id "whatever"
:name "Just to serialize"}
:target {:type "dimension"
:id mapping-id
:dimension dimension}}}}})}
;;; stress-test that exporting various visualization_settings does not break
:model/DashboardCard _ {:card_id c3-1-id
:dashboard_id clickdash-id
:visualization_settings
{:column_settings
{(str "[\"ref\",[\"field\"," field-id ",null]]")
{:click_behavior
{:type "link"
:linkType "dashboard"
:targetId dash4-id}}
(str "[\"ref\",[\"field\"," field-id2 ",null]]")
{:click_behavior
{:type "crossfilter"
:parameterMapping
{"abcdef" {:id "abcdef"
:source {:type "column"
:id field-id2
:name "Field To Click 1"}
:target {:type "parameter"
:id "abcdef"}}}}}
(str "[\"ref\",[\"field\"," field-id3 ",null]]")
(let [mapping-id (format "[\"dimension\",[\"fk->\",[\"field\",%d,null],[\"field\",%d,null]]]" field-id3 field-id)
dimension [:dimension [:field field-id {:source-field field-id3}]]]
{:click_behavior
{:type "link"
:linkType "question"
:targetId c4-id
:parameterMapping
{mapping-id {:id mapping-id
:source {:type "column"
:id "Category_ID"
:name "Category ID"}
:target {:type "dimension"
:id mapping-id
:dimension dimension}}}}})}}}]
(testing "selecting a collection includes settings metabot and data model by default"
(is (= #{"Card" "Collection" "Dashboard" "Database" "PythonLibrary" "Setting" "TransformTag" "TransformJob"}
(->> (extract/extract {:targets [["Collection" coll1-id]]})
(map (comp :model first serdes/path))
set))))
(testing "selecting a dashboard gets all cards its dashcards depend on"
(testing "grandparent dashboard"
(is (= #{[{:model "Dashboard" :id dash1-eid :label "dashboard_1"}]
[{:model "Card" :id c1-1-eid :label "question_1_1"}]
[{:model "Card" :id c1-2-eid :label "question_1_2"}]}
(->> (extract/extract {:targets [["Dashboard" dash1-id]] :no-settings true :no-data-model true :no-transforms true})
(map serdes/path)
set))))
(testing "middle dashboard"
(is (= #{[{:model "Dashboard" :id dash2-eid :label "dashboard_2"}]
[{:model "Card" :id c2-1-eid :label "question_2_1"}]
[{:model "Card" :id c2-2-eid :label "question_2_2"}]}
(->> (extract/extract {:targets [["Dashboard" dash2-id]] :no-settings true :no-data-model true :no-transforms true})
(map serdes/path)
set))))
(testing "grandchild dashboard"
(is (= #{[{:model "Dashboard" :id dash3-eid :label "dashboard_3"}]
[{:model "Card" :id c3-1-eid :label "question_3_1"}]
[{:model "Card" :id c3-2-eid :label "question_3_2"}]}
(->> (extract/extract {:targets [["Dashboard" dash3-id]] :no-settings true :no-data-model true :no-transforms true})
(map serdes/path)
set))))
(testing "a dashboard that has parameter source is another card"
(is (=? #{[{:model "Dashboard" :id dash4-eid :label "dashboard_4"}]
[{:model "Card" :id c4-eid :label "question_4_1"}]
;; card that parameter on dashboard linked to
[{:model "Card" :id c1-1-eid :label "question_1_1"}]
;; card that the card on dashboard linked to
[{:model "Card" :id c1-2-eid :label "question_1_2"}]}
(->> (extract/extract {:targets [["Dashboard" dash4-id]] :no-settings true :no-data-model true :no-transforms true})
(map serdes/path)
set)))))
(testing "selecting a dashboard gets any dashboards or cards it links to when clicked"
(is (=? #{[{:model "Dashboard" :id clickdash-eid :label "dashboard_with_click_behavior"}]
[{:model "Card" :id c3-1-eid :label "question_3_1"}] ; Visualized card
[{:model "Dashboard" :id dash4-eid :label "dashboard_4"}] ; Linked dashboard
[{:model "Card" :id c3-2-eid :label "question_3_2"}] ; Linked card
[{:model "Card" :id c4-eid :label "question_4_1"}] ; Transitive via dash4
[{:model "Card" :id c1-1-eid :label "question_1_1"}] ; Linked by c4
[{:model "Card" :id c1-2-eid :label "question_1_2"}]} ; Linked by dash4
(->> (extract/extract {:targets [["Dashboard" clickdash-id]] :no-settings true :no-data-model true :no-transforms true})
(map serdes/path)
set))))
(testing "selecting a collection gets all its contents"
(let [grandchild-paths #{[{:model "Collection" :id coll1-eid :label "some_collection"}]
[{:model "Collection" :id coll2-eid :label "nested_collection"}]
[{:model "Collection" :id coll3-eid :label "grandchild_collection"}]
[{:model "Dashboard" :id dash3-eid :label "dashboard_3"}]
[{:model "Card" :id c3-1-eid :label "question_3_1"}]
[{:model "Card" :id c3-2-eid :label "question_3_2"}]
[{:model "Card" :id c3-3-eid :label "question_3_3"}]}
middle-paths #{[{:model "Collection" :id coll1-eid :label "some_collection"}]
[{:model "Collection" :id coll2-eid :label "nested_collection"}]
[{:model "Dashboard" :id dash2-eid :label "dashboard_2"}]
[{:model "Card" :id c2-1-eid :label "question_2_1"}]
[{:model "Card" :id c2-2-eid :label "question_2_2"}]
[{:model "Card" :id c2-3-eid :label "question_2_3"}]}
grandparent-paths #{[{:model "Collection" :id coll1-eid :label "some_collection"}]
[{:model "Dashboard" :id dash1-eid :label "dashboard_1"}]
[{:model "Card" :id c1-1-eid :label "question_1_1"}]
[{:model "Card" :id c1-2-eid :label "question_1_2"}]
[{:model "Card" :id c1-3-eid :label "question_1_3"}]}]
(testing "grandchild collection has all its own contents"
(is (= grandchild-paths ; Includes the third card not found in the collection
(->> (extract/extract {:targets [["Collection" coll3-id]] :no-settings true :no-data-model true :no-transforms true})
(map serdes/path)
set))))
(testing "middle collection has all its own plus the grandchild and its contents"
(is (= (set/union middle-paths grandchild-paths)
(->> (extract/extract {:targets [["Collection" coll2-id]] :no-settings true :no-data-model true :no-transforms true})
(map serdes/path)
set))))
(testing "grandparent collection has all its own plus the grandchild and middle collections with contents"
(is (= (set/union grandparent-paths middle-paths grandchild-paths)
(->> (extract/extract {:targets [["Collection" coll1-id]] :no-settings true :no-data-model true :no-transforms true})
(map serdes/path)
set))))
(testing "depending on data from personal collections results in errors"
(mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]
(extract-aborts! {:targets [["Collection" coll4-id]] :no-settings true :no-data-model true :no-transforms true})
(let [msgs (into #{}
(map :message)
(messages))]
(is (some #(re-find #"not included in the export" %) msgs)
"the export warns that a referenced personal-collection card is not included")))))))))
(deftest click-behavior-references-to-deleted-cards
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/User {mark-id :id} {:first_name "Mark"
:last_name "Knopfler"
:email "mark@direstrai.ts"}
:model/Collection {coll-id :id
coll-eid :entity_id} {:name "Some Collection"}
:model/Database {db-id :id} {:name "My Database"}
:model/Table {no-schema-id :id} {:name "Schemaless Table" :db_id db-id}
:model/Field _ {:name "Some Field" :table_id no-schema-id}
:model/Table {schema-id :id} {:name "Schema'd Table"
:db_id db-id
:schema "PUBLIC"}
:model/Field {field-id :id} {:name "Other Field" :table_id schema-id}
:model/Field {field-id3 :id} {:name "Field To Click 2" :table_id schema-id}
:model/Card {card-id :id
card-eid :entity_id} {:name "A Normal Question"
:database_id db-id
:table_id no-schema-id
:collection_id coll-id
:creator_id mark-id}
:model/Card {deleted-card-id :id} {:collection_id coll-id}
:model/Dashboard {deleted-dash-id :id} {:collection_id coll-id}
:model/Dashboard {clickdash-id :id
clickdash-eid :entity_id} {:name "Dashboard"
:collection_id coll-id
:creator_id mark-id}
:model/DashboardCard _ {:card_id card-id
:dashboard_id clickdash-id
:visualization_settings
;; links to a (soon-to-be) deleted card
{:click_behavior {:type "link"
:linkType "question"
:targetId deleted-card-id}}}
;;; stress-test that exporting various visualization_settings does not break
:model/DashboardCard _ {:card_id card-id
:dashboard_id clickdash-id
:visualization_settings
{:column_settings
{(str "[\"ref\",[\"field\"," field-id ",null]]")
{:click_behavior
{:type "link"
:linkType "dashboard"
:targetId deleted-dash-id}}
(str "[\"ref\",[\"field\"," field-id3 ",null]]")
{:click_behavior
{:type "link"
:linkType "question"
:targetId deleted-card-id}}}}}]
(t2/delete! :model/Card deleted-card-id)
(t2/delete! :model/Dashboard deleted-dash-id)
(testing "the references to deleted cards and dashboards are ignored"
(is (= #{[{:model "Dashboard" :id clickdash-eid :label "dashboard"}]
[{:model "Collection" :id coll-eid :label "some_collection"}]
[{:model "Card" :id card-eid :label "a_normal_question"}]}
(->> {:targets [["Collection" coll-id]]
:no-settings true :no-data-model true :no-transforms true}
extract/extract
(map serdes/path)
(into #{})))))
(testing "the click behavior looks sane"
(is (= #{{:column_settings nil}
{:column_settings {"[\"ref\",[\"field\",[\"My Database\",\"PUBLIC\",\"Schema'd Table\",\"Other Field\"],null]]" {}
"[\"ref\",[\"field\",[\"My Database\",\"PUBLIC\",\"Schema'd Table\",\"Field To Click 2\"],null]]" {}}}}
(->> {:targets [["Collection" coll-id]]
:no-settings true :no-data-model true :no-transforms true}
extract/extract
(filter #(= (:entity_id %) clickdash-eid))
first
:dashcards
(map :visualization_settings)
(into #{}))))))))
(deftest field-references-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Database {db-id :id} {:name "My Database"}
:model/Table {no-schema-id :id} {:name "Schemaless Table" :db_id db-id}
:model/Field {some-field-id :id} {:name "Some Field" :table_id no-schema-id}
:model/Table {schema-id :id} {:name "Schema'd Table"
:db_id db-id
:schema "PUBLIC"}
:model/Field {other-field-id :id} {:name "Other Field" :table_id schema-id}
:model/Field {fk-id :id} {:name "Foreign Key"
:table_id schema-id
:fk_target_field_id some-field-id}
:model/Field {nested-id :id} {:name "Nested Field"
:table_id schema-id
:parent_id other-field-id}]
(testing "fields that reference foreign keys are properly exported as Field references"
(is (= ["My Database" nil "Schemaless Table" "Some Field"]
(:fk_target_field_id (ts/extract-one "Field" fk-id)))))
(testing "Fields that reference parents are properly exported as Field references"
(is (= ["My Database" "PUBLIC" "Schema'd Table" "Other Field"]
(:parent_id (ts/extract-one "Field" nested-id))))
(is (= [{:model "Database", :id "My Database"}
{:model "Schema", :id "PUBLIC"}
{:model "Table", :id "Schema'd Table"}
{:model "Field", :id "Other Field"}
{:model "Field", :id "Nested Field"}]
(:serdes/meta (ts/extract-one "Field" nested-id))))))))
(deftest recursive-colls-test
(mt/with-empty-h2-app-db!
(mt/with-temp [:model/Collection {parent-id :id
parent-eid :entity_id} {:name "Top-Level Collection"}
:model/Collection {middle-id :id
middle-eid :entity_id} {:name "Nested Collection"
:location (format "/%s/" parent-id)}
:model/Collection {nested-id :id
nested-eid :entity_id} {:name "Nested Collection"
:location (format "/%s/%s/" parent-id middle-id)}
:model/Card _ {:name "Card To Skip"
:collection_id parent-id}
:model/Card {ncard-eid :entity_id} {:name "Card To Export"
:collection_id nested-id}]
(let [ser (extract/extract {:targets [["Collection" nested-id]]
:no-settings true
:no-data-model true})]
(is (= #{parent-eid middle-eid nested-eid}
(ids-by-model "Collection" ser)))
(is (= #{ncard-eid}
(ids-by-model "Card" ser)))))))
(deftest skip-analytics-collections-test
(testing "Collections in 'analytics' namespace should not be extracted, see #37453"
(mt/with-empty-h2-app-db!
(mbc/ensure-audit-db-installed!)
(testing "sanity check that the audit collection exists"
(is (some? (audit/default-audit-collection)))
(is (some? (audit/default-custom-reports-collection))))
(let [ser (extract/extract {:no-settings true
:no-data-model true})]
(is (= #{} (ids-by-model "Collection" ser)))))))
(deftest skip-h2-databases-test
(testing "H2 databases must not be extracted because import rejects them (see GHY-3633)"
(mt/with-empty-h2-app-db!
(mt/with-temp [:model/Database _h2-db {:name "H2 DB" :engine :h2}
:model/Database _non-h2 {:name "Postgres DB" :engine :postgres}]
(let [ser (extract/extract {:no-settings true})]
(is (= #{"Postgres DB"} (ids-by-model "Database" ser))
"Only non-H2 databases should appear in the extract"))))))
(deftest xray-of-analytics-model-export-test
(testing "X-rays of analytics models can be exported without errors"
(mt/with-empty-h2-app-db!
(mbc/ensure-audit-db-installed!)
(testing "sanity check that the analytics content exists"
(is (some? (audit/default-audit-collection)))
;; Find the Query log model (entity_id: QOtZaiTLf2FDD4AT6Oinb)
(let [query-log-card (t2/select-one :model/Card :entity_id "QOtZaiTLf2FDD4AT6Oinb")]
(is (some? query-log-card) "Query log model should exist in analytics")
(when query-log-card
;; Create a card that depends on the analytics model (simulating an x-ray)
(mt/with-temp [:model/Collection {coll-id :id coll-eid :entity_id} {:name "Test Collection"}
:model/Card {_xray-card-id :id xray-card-eid :entity_id}
{:name "X-ray of Query log"
:collection_id coll-id
:database_id (:database_id query-log-card)
:dataset_query {:type :query
:database (:database_id query-log-card)
:query {:source-table (str "card__" (:id query-log-card))}}}]
(testing "Export should succeed without warnings about analytics dependencies"
(mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]
(let [ser (extract/extract {:targets [["Collection" coll-id]]
:no-settings true
:no-data-model true})]
(is (contains? (ids-by-model "Card" ser) xray-card-eid)
"X-ray card should be exported")
(is (contains? (ids-by-model "Collection" ser) coll-eid)
"Collection should be exported")
(is (not (contains? (ids-by-model "Card" ser) "QOtZaiTLf2FDD4AT6Oinb"))
"Analytics model should NOT be exported")
(let [warn-msgs (into #{} (map :message) (messages))]
(is (not (some #(str/starts-with? % "Failed to export") warn-msgs))
(str "Should not have export warnings, got: " warn-msgs)))))))))))))
(deftest entity-id-in-targets-test
(mt/with-temp [:model/Collection c {:name "Top-Level Collection"}]
(testing "Conversion from eid to id works"
(is (= (:id c)
(serdes/eid->id "Collection" (:entity_id c)))))
(testing "Extracting by entity id works"
(let [ser (extract/extract {:targets [["Collection" (:entity_id c)]]
:no-settings true
:no-data-model true})]
(is (= #{(:entity_id c)}
(ids-by-model "Collection" ser)))))))
(deftest extract-nested-test
(testing "extract-nested working"
(mt/with-temp [:model/Dashboard d {:name "Top Dash"}
:model/Card c1 {:name "Some Card"}
:model/Card c2 {:name "Some Inner Card"}
:model/DashboardCard dc1 {:dashboard_id (:id d) :card_id (:id c1)}
:model/DashboardCardSeries s {:dashboardcard_id (:id dc1) :card_id (:id c2)}]
(let [spec (serdes/make-spec "DashboardCard" nil)]
(is (= {(:id dc1) [s]}
(#'serdes/transform->nested (-> spec :transform :series) {} [dc1])))
(is (=? (assoc dc1 :series [s])
(u/rfirst (serdes/extract-query "DashboardCard" {:filter-column :id :filter-ids [(:id dc1)]})))))
(let [spec (serdes/make-spec "Dashboard" nil)]
(is (= {(:id d) [(assoc dc1 :series [s])]}
(#'serdes/transform->nested (-> spec :transform :dashcards) {} [d])))
(is (=? (assoc d
:dashcards [(assoc dc1 :series [s])]
:tabs nil)
(u/rfirst (serdes/extract-query "Dashboard" {:filter-column :id :filter-ids [(:id d)]}))))))))
(deftest extract-nested-efficient-test
(testing "extract-nested is efficient"
(mt/with-temp [:model/Dashboard d1 {:name "Top Dash 1"}
:model/Dashboard d2 {:name "Top Dash 2"}
:model/Card c1 {:name "Some Card"}
:model/Card c2 {:name "Some Inner Card"}
:model/Card c3 {:name "Card for Dash 2"}
:model/DashboardCard dc1 {:dashboard_id (:id d1) :card_id (:id c1)}
:model/DashboardCard dc2 {:dashboard_id (:id d2) :card_id (:id c2)}
:model/DashboardCard dc3 {:dashboard_id (:id d2) :card_id (:id c3)}
:model/DashboardCardSeries s {:dashboardcard_id (:id dc1) :card_id (:id c2)}]
(t2/with-call-count [qc]
;; Use sets to ensure order independence of `extract-query` result.
(is (=? #{(assoc d1
:dashcards [(assoc dc1 :series [s])]
:tabs nil)
(assoc d2
:dashcards [(assoc dc2 :series nil)
(assoc dc3 :series nil)]
:tabs nil)}
(into #{} (map (fn [dashboard]
(update dashboard :dashcards #(sort-by :id %))))
(serdes/extract-query "Dashboard" {:filter-column :id :filter-ids [(:id d1) (:id d2)]}))))
;; 1 per dashboard/dashcard/series/tabs
(is (= 4 (qc)))))))
(deftest extract-nested-partitioned-test
(testing "extract-nested will partition stuff by 100s"
(mt/with-empty-h2-app-db!
(let [d (ts/create! :model/Dashboard {:name "Dash"})
c1 (ts/create! :model/Card {:name "Card"})
dcs (vec (for [_ (range 7)]
(ts/create! :model/DashboardCard {:dashboard_id (:id d)
:card_id (:id c1)})))]
(t2/with-call-count [qc]
(is (=? [(assoc d :dashcards dcs)]
(into [] (serdes/extract-query "Dashboard" {:batch-limit 5
:filter-column :id
:filter-ids [(:id d)]}))))
;; query count breakdown:
;; - 1 for dashboard
;; - 1 for tabs, there are none
;; - 1 for dashcards, there are 7
;; - 2 for series (7 dashcards / 5 -> 2 batches)
(is (= 5 (qc))))))))
(deftest result-metadata-test
(testing "model Card extraction portablizes :fk_target_field_id in :result_metadata"
(mt/with-temp [:model/Card c {:type :model :dataset_query (mt/query venues)}]
(let [res (qp/process-query
(qp/userland-query
(:dataset_query c)
{:card-id (:id c)}))]
(when-not (= (:status res) :completed)
(throw (ex-info "Query failed" res)))
(let [ser (serdes/extract-one "Card" nil (t2/select-one :model/Card (:id c)))]
(is (=? {:fk_target_field_id [string? "PUBLIC" "CATEGORIES" "ID"]}
(->> (:result_metadata ser)
(u/seek #(and (= (:name %) "CATEGORY_ID")
(= (:display_name %) "Category ID")))))))))))
(deftest model-preserved-keys-extract-test
(testing "model Card preserved-key overrides survive extract"
(let [base-cols (qp.preprocess/query->expected-cols (mt/query venues))
overridden (mapv (fn [col]
(cond-> (assoc col
:display_name (str "Custom " (:name col))
:visibility_type :normal
:description "user-set")
(= "CATEGORY_ID" (:name col)) (assoc :semantic_type :type/Category)))
base-cols)]
(mt/with-temp
[:model/Card {card-id :id} {:type :model
:dataset_query (mt/query venues)
:result_metadata overridden}]
(let [extracted (serdes/extract-one "Card" nil (t2/select-one :model/Card card-id))
by-name (u/index-by :name (:result_metadata extracted))]
(is (every? #(re-matches #"Custom .*" (:display_name %))
(:result_metadata extracted)))
(is (= "user-set" (:description (get by-name "ID"))))
(is (= :type/Category (:semantic_type (get by-name "CATEGORY_ID"))))
(is (vector? (:fk_target_field_id (get by-name "CATEGORY_ID")))))))))
(deftest native-model-preserves-column-types-extract-test
(testing "native model Card extraction preserves structural column types (GHY-4043)"
;; Native model columns can't be re-derived from the query at import time without executing
;; the SQL, so :base_type/:effective_type must survive extract — otherwise filters on the
;; read-only target instance collapse to Is empty / Not empty.
(mt/with-temp
[:model/Card {card-id :id}
{:type :model
:dataset_query (mt/native-query
{:query "SELECT project_id, STRING_AGG(region_name, ', ') AS names FROM project_regions GROUP BY project_id"})
:result_metadata [{:name "project_id"
:display_name "project_id"
:base_type :type/Integer
:effective_type :type/Integer
:semantic_type :type/PK}
{:name "names"
:display_name "names"
:base_type :type/Text
:effective_type :type/Text
:semantic_type :type/Title}]}]
(let [extracted (serdes/extract-one "Card" nil (t2/select-one :model/Card card-id))
by-name (u/index-by :name (:result_metadata extracted))]
(testing "physical passthrough column keeps its types"
(is (= :type/Integer (:base_type (get by-name "project_id"))))
(is (= :type/Integer (:effective_type (get by-name "project_id")))))
(testing "aggregated projection column keeps its types and soft keys"
(is (= :type/Text (:base_type (get by-name "names"))))
(is (= :type/Text (:effective_type (get by-name "names"))))
(is (= :type/Title (:semantic_type (get by-name "names")))))))))
(deftest extract-single-collection-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc
[:model/Collection {coll-id :id} {:name "Top-Level Collection"}
:model/Dashboard {dash-id :id
dash-eid :entity_id} {:name "Top Dash"
:collection_id coll-id}
:model/Card {card-id-1 :id
card-eid-1 :entity_id} {:name "Some Card"
:collection_id coll-id}
:model/Card {card-id-2 :id
card-eid-2 :entity_id} {:name "Some Inner Card"
:collection_id coll-id}
:model/DashboardTab {tab-id-1 :id
tab-eid-1 :entity_id} {:dashboard_id dash-id
:name "Tab 1"}
:model/DashboardCard _ {:dashboard_id dash-id
:dashboard_tab_id tab-id-1
:card_id card-id-1}
:model/DashboardTab {tab-id-2 :id
tab-eid-2 :entity_id} {:dashboard_id dash-id
:name "Tab 2"}
:model/DashboardCard _ {:dashboard_id dash-id
:dashboard_tab_id tab-id-2
:card_id card-id-2}]
(let [extraction (extract/extract {:targets [["Collection" coll-id]] :no-settings true :no-data-model true})]
(is (=? [{:name "Top Dash"
:dashcards [{:dashboard_tab_id [dash-eid tab-eid-1]
:card_id card-eid-1}
{:dashboard_tab_id [dash-eid tab-eid-2]
:card_id card-eid-2}]
:tabs [{:name "Tab 1"}
{:name "Tab 2"}]}]
(by-model "Dashboard" extraction)))))))
(deftest metabot-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc
[:model/Card {model-id :id
model-eid :entity_id} {:name "AI Model"
:type :model}
:model/Collection {coll-id :id
coll-eid :entity_id} {:name "Metabot Collection"}
:model/Metabot {metabot-id :id
metabot-eid :entity_id} {:name "Test Metabot"
:description "A test metabot"
:use_verified_content false
:collection_id coll-id}
:model/MetabotPrompt {metabot-prompt-eid :entity_id} {:metabot_id metabot-id
:prompt "A sample prompt"
:model :model
:card_id model-id}]
(testing "metabot extraction"
(let [ser (ts/extract-one "Metabot" metabot-id)]
(is (=? {:serdes/meta [{:model "Metabot" :id metabot-eid}]
:name "Test Metabot"
:description "A test metabot"
:entity_id metabot-eid
:collection_id coll-eid
:prompts [{:prompt "A sample prompt"
:model "model"
:entity_id metabot-prompt-eid
:card_id model-eid
:serdes/meta [{:model "Metabot" :id metabot-eid}
{:model "MetabotPrompt" :id metabot-prompt-eid}]
:created_at string?}]
:created_at string?}
ser))
(is (not (contains? ser :id)))
(is (not (contains? ser :use_verified_content)))
(testing "metabot depends on its model entities and collection"
(is (= #{[{:model "Card" :id model-eid}]
[{:model "Collection" :id coll-eid}]}
(set (serdes/deserialization-dependencies ser)))))
(testing "metabot storage-path uses top-level metabots directory"
(is (= [{:label "metabots"} {:label "Test Metabot" :key metabot-eid}]
(serdes/storage-path ser {})))))))))
(deftest metabot-collection-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc
[:model/Collection {model-id :id} {:name "AI Model"}
:model/Card {card-id :id
card-eid :entity_id} {:name "AI Model"
:type :model
:collection_id model-id}
:model/Collection {coll-id :id
coll-eid :entity_id} {:name "Metabot Collection"}
:model/Metabot {metabot-id :id
metabot-eid :entity_id} {:name "Test Metabot"
:description "A test metabot"
:use_verified_content false
:collection_id coll-id}
:model/MetabotPrompt {metabot-prompt-eid :entity_id} {:metabot_id metabot-id
:prompt "A sample prompt"
:model :model
:card_id card-id}]
(testing "metabot extraction"
(let [ser (ts/extract-one "Metabot" metabot-id)]
(is (=? {:serdes/meta [{:model "Metabot" :id metabot-eid}]
:name "Test Metabot"
:description "A test metabot"
:entity_id metabot-eid
:collection_id coll-eid
:prompts [{:prompt "A sample prompt"
:model "model"
:entity_id metabot-prompt-eid
:card_id card-eid
:serdes/meta [{:model "Metabot" :id metabot-eid}
{:model "MetabotPrompt" :id metabot-prompt-eid}]
:created_at string?}]
:created_at string?}
ser))
(is (not (contains? ser :id)))
(is (not (contains? ser :use_verified_content)))
(testing "metabot depends on its prompts' cards and its collection"
(is (= #{[{:model "Card" :id card-eid}]
[{:model "Collection" :id coll-eid}]}
(set (serdes/deserialization-dependencies ser))))))))))
(deftest document-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Collection collection {}
:model/User user {:first_name "Mark"
:last_name "Knopfler"
:email "mark@direstrai.ts"}
:model/Document document {:name "Test Document"
:document {}
:content_type "application/json+vnd.prose-mirror"
:collection_id (:id collection)
:creator_id (:id user)}
:model/Card card {:name "Dependent Card"
:document_id (u/the-id document)}
:model/Card linked-card {:name "Linked Card"}
:model/Dashboard dashboard {:name "Smart Linked Dashboard"}
:model/Table table {:name "linked_table"}]
(t2/update! :model/Document :id (u/the-id document) {:document {:type "doc"
:content [{:type "cardEmbed"
:attrs {"id" (u/the-id card)}}
{:type "smartLink"
:attrs {"entityId" (u/the-id linked-card)
"model" "card"}}
{:type "smartLink"
:attrs {"entityId" (u/the-id table)
"model" "table"}}
{:type "smartLink"
:attrs {"entityId" (u/the-id dashboard)
"model" "dashboard"}}]}})
(testing "document extraction"
(let [ser (ts/extract-one "Document" (u/the-id document))]
(is (=? {:serdes/meta [{:model "Document" :id (:entity_id document)}]
:name "Test Document"
:entity_id (:entity_id document)
:document {:type "doc"
:content [{:type "cardEmbed"
:attrs {"id" [{:model "Card" :id (:entity_id card)}]}}
{:type "smartLink"
:attrs {"entityId" [{:model "Card" :id (:entity_id linked-card)}]
"model" "card"}}
{:type "smartLink"
:attrs {"entityId" (serdes/generate-path "Table" table)
"model" "table"}}
{:type "smartLink"
:attrs {"entityId" [{:model "Dashboard" :id (:entity_id dashboard)}]
"model" "dashboard"}}]}
:creator_id (:email user)
:collection_id (:entity_id collection)
:content_type "application/json+vnd.prose-mirror"
:updated_at string?
:created_at string?}
ser))
(is (not (contains? ser :archived)))
(is (not (contains? ser :archived_directly)))
(is (not (contains? ser :id)))
(testing "depends on its collection, cardEmbeds and smarkLinks "
(is (= #{[{:model "Collection" :id (:entity_id collection)}]
[{:model "Card" :id (:entity_id card)}]
[{:model "Dashboard" :id (:entity_id dashboard)}]
[{:model "Card" :id (:entity_id linked-card)}]
(serdes/generate-path "Table" table)}
(set (serdes/deserialization-dependencies ser))))))))))
(deftest visualizer-dashboard-card-settings-test
(testing "visualizer settings transform entity IDs <-> card IDs"
(let [card-entity-id "WcMlLFNVcy0iO49mKW3WH"
card-id 621]
(mt/with-dynamic-fn-redefs [serdes/*import-fk* (fn [_entity-id _model]
card-id)
serdes/*export-fk* (fn [_card-id _model]
card-entity-id)]
(testing "transforms sourceId in column mappings"
(let [input {:visualization
{:columnValuesMapping
{:COLUMN_1 [{:sourceId (str "card:" card-entity-id)
:originalName "CREATED_AT"
:name "COLUMN_1"}]
:DIMENSION [(str "$_card:" card-entity-id "_name")]}}}
expected {:visualization
{:columnValuesMapping
{:COLUMN_1 [{:sourceId (str "card:" card-id)
:originalName "CREATED_AT"
:name "COLUMN_1"}]
:DIMENSION [(str "$_card:" card-id "_name")]}}}
result (serdes/import-visualizer-settings input)]
(is (= expected result))))
(testing "transforms sourceId in column mappings"
(let [input {:visualization
{:columnValuesMapping
{:COLUMN_1 [{:sourceId (str "card:" card-id)
:originalName "CREATED_AT"
:name "COLUMN_1"}]
:DIMENSION [(str "$_card:" card-id "_name")]}}}
expected {:visualization
{:columnValuesMapping
{:COLUMN_1 [{:sourceId (str "card:" card-entity-id)
:originalName "CREATED_AT"
:name "COLUMN_1"}]
:DIMENSION [(str "$_card:" card-entity-id "_name")]}}}
result (serdes/export-visualizer-settings input)]
(is (= expected result))))))))
(deftest glossary-test
(testing "Glossary entries are keyed on entity_id and carry the term"
(mt/with-temp [:model/Glossary {eid :entity_id} {:term "foobar"
:definition "It's foobar2000 actually"}]
(let [ser (serdes/extract-one "Glossary" {} (t2/select-one :model/Glossary :term "foobar"))]
(is (=? {:serdes/meta [{:model "Glossary" :id eid}]
:entity_id eid
:term "foobar"
:definition "It's foobar2000 actually"}
ser))))))
(deftest transform-tag-extraction-test
(testing "TransformTag extraction and serialization"
(mt/with-premium-features #{:transforms-basic}
(mt/with-empty-h2-app-db!
;; Delete the existing built-in transform tags
(t2/delete! :model/TransformTag)
(ts/with-temp-dpc [;; Create built-in tags
:model/TransformTag
{hourly-tag-id :id
hourly-tag-eid :entity_id}
{:name "hourly" :built_in_type "hourly"}
:model/TransformTag
{daily-tag-eid :entity_id}
{:name "daily" :built_in_type "daily"}
;; Create custom tag
:model/TransformTag
{custom-tag-id :id
custom-tag-eid :entity_id}
{:name "custom-etl"}]
(testing "built-in tags extract correctly"
(let [ser (serdes/extract-one "TransformTag" {} (t2/hydrate (t2/select-one :model/TransformTag :id hourly-tag-id) :tags))]
(is (=? {:serdes/meta [{:model "TransformTag"
:id hourly-tag-eid}]
:name "hourly"
:built_in_type "hourly"
:created_at string?}
ser))
(is (not (contains? ser :id)))
(is (empty? (serdes/deserialization-dependencies ser)))))
(testing "custom tags extract correctly"
(let [ser (serdes/extract-one "TransformTag" {} (t2/hydrate (t2/select-one :model/TransformTag :id custom-tag-id) :tags))]
(is (=? {:serdes/meta [{:model "TransformTag"
:id custom-tag-eid}]
:name "custom-etl"
:created_at string?}
ser))
(is (not (contains? ser :built_in_type)))
(is (not (contains? ser :id)))
(is (empty? (serdes/deserialization-dependencies ser)))))
(testing "all transform tags are extracted"
(is (= #{hourly-tag-eid daily-tag-eid custom-tag-eid}
(ids-by-model "TransformTag" (extract/extract {}))))))))))
(deftest transform-extraction-test
(testing "Transform extraction and serialization"
(mt/with-premium-features #{:transforms-basic}
(mt/with-empty-h2-app-db!
;; Delete the existing built-in transform tags
(t2/delete! :model/TransformTag)
(ts/with-temp-dpc [:model/Database
{db-id :id}
{:name "My Database"}
:model/Table
{table-id :id}
{:name "Schemaless Table" :db_id db-id}
:model/Field
{_field-id :id}
{:name "Some Field" :table_id table-id}
:model/Collection
{coll-id :id coll-eid :entity_id}
{:name "Transform Collection"
:namespace :transforms}
;; Create tags for associations
:model/TransformTag
{hourly-tag-id :id
hourly-tag-eid :entity_id}
{:name "hourly" :built_in_type "hourly" :entity_id "hourlyhourlyhourlyxxx"}
:model/TransformTag
{daily-tag-id :id
daily-tag-eid :entity_id}
{:name "daily" :built_in_type "daily" :entity_id "dailydailydailydailyx"}
:model/TransformTag
{custom-tag-id :id
custom-tag-eid :entity_id}
{:name "custom-etl" :entity_id "custometlcustometlcus"}
;; Create Transform
:model/Transform
{transform-id :id
transform-eid :entity_id}
{:name "Test Transform"
:description "A test transform for serialization"
:entity_id "2HzIFwJ6720JAx07UMavl"
:collection_id coll-id
:source {:query {:database db-id
:type "query"
:query {:source-table table-id}}
:type "query"}
:target {:database db-id
:type "table"
:schema "public"
:name "target_table"}}
;; Python transform with source-tables
:model/Transform
{python-transform-id :id
python-transform-eid :entity_id}
{:name "Python ETL"
:description "A python transform"
:entity_id "pythonEtlPythonEtlxxx"
:collection_id coll-id
:source_database_id db-id
:source {:type "python"
:source-database db-id
:source-tables [{:alias "orders"
:table_id table-id}]
:body "df = ctx.source.orders"}
:target {:database db-id
:type "table"
:schema "public"
:name "target_table"}}
;; Create tag associations with specific positions
:model/TransformTransformTag
{}
{:transform_id transform-id
:tag_id hourly-tag-id
:position 0}
:model/TransformTransformTag
{}
{:transform_id transform-id
:tag_id custom-tag-id
:position 1}
:model/TransformTransformTag
{}
{:transform_id transform-id
:tag_id daily-tag-id
:position 2}]
(let [ser (serdes/extract-one "Transform" {} (t2/hydrate (t2/select-one :model/Transform :id transform-id) :tags :indexes))]
(testing "basic Transform structure"
(is (=? {:serdes/meta [{:model "Transform"
:id transform-eid}]
:name "Test Transform"
:description "A test transform for serialization"
:created_at string?}
ser))
(is (not (contains? ser :id))))
(testing "source and target MBQL export"
(is (=? {:source {:query {:database "My Database"
:lib/type :mbql/query
:stages [{:source-table ["My Database" nil "Schemaless Table"]}]}}
:target {:database "My Database" :type "table" :schema "public" :name "target_table"}}
(select-keys ser [:source :target]))))
(testing "tag associations with preserved order"
(is (= 3 (count (:tags ser))))
(let [tag-ids (map :tag_id (:tags ser))
positions (map :position (:tags ser))]
(is (= [hourly-tag-eid custom-tag-eid daily-tag-eid] tag-ids))
(is (= [0 1 2] positions))))
(testing "dependencies include collection, source database, and tags (the source Table itself is not a dependency)"
(let [deps (set (serdes/deserialization-dependencies ser))]
(is (contains? deps [{:model "Collection" :id coll-eid}]))
(is (contains? deps [{:model "Database" :id "My Database"}]))
(is (not (contains? deps [{:model "Database" :id "My Database"}
{:model "Table" :id "Schemaless Table"}])))
(is (contains? deps [{:model "TransformTag" :id hourly-tag-eid}]))
(is (contains? deps [{:model "TransformTag" :id custom-tag-eid}]))
(is (contains? deps [{:model "TransformTag" :id daily-tag-eid}])))))
(testing "python transform source-tables export"
(let [ser (serdes/extract-one "Transform" {} (t2/hydrate (t2/select-one :model/Transform :id python-transform-id) :tags :indexes))]
(is (=? {:source {:type :python
:source-database "My Database"
:source-tables [{:alias "orders"
:table_id ["My Database" nil "Schemaless Table"]
:database_id "My Database"
:schema nil
:table "Schemaless Table"}]
:body "df = ctx.source.orders"}}
ser))))
(testing "transforms are extracted"
(is (= #{transform-eid python-transform-eid}
(ids-by-model "Transform" (extract/extract {}))))))))))
(deftest transform-with-null-source-database-extract-test
(testing "A transform whose source database has been deleted serializes as a tombstone (GDGT-2447)"
(mt/with-premium-features #{:transforms-basic}
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Database {db-id :id} {:name "Soon-to-be-deleted DB"}
:model/Transform {transform-id :id transform-eid :entity_id}
{:name "Orphan Transform"
:entity_id "orphanXxxxxxxxxxxxxxx"
:source {:type "query"
:query {:database db-id
:type "native"
:native {:query "SELECT 1"}}}
:target {:database db-id
:type "table"
:schema "public"
:name "orphan_target"}}]
;; Delete the database — ON DELETE SET NULL nulls source_database_id and target_db_id.
(t2/delete! :model/Database db-id)
(let [reloaded (t2/hydrate (t2/select-one :model/Transform :id transform-id) :tags :indexes)
ser (serdes/extract-one "Transform" {} reloaded)]
(is (nil? (:source_database_id reloaded))
"Database deletion should have nulled the column")
(testing "exported entity carries a nil source_database_id"
(is (=? {:serdes/meta [{:model "Transform" :id transform-eid}]
:name "Orphan Transform"}
ser))
(is (nil? (:source_database_id ser))))
(testing "exported source is marked :serdes/unresolved with the dead :database ref nulled"
;; The :database slot is nulled because the numeric id refers to a now-deleted database —
;; keeping it would make the destination instance's before-insert hook bind
;; source_database_id to a stale id and crash the FK.
(is (true? (get-in ser [:source :serdes/unresolved])))
(is (nil? (get-in ser [:source :query :database]))))
(testing "the native SQL body is preserved verbatim as a breadcrumb"
;; `lib-be/normalize-query` rewrites the raw query to MBQL5 (`:stages [...]`) at read time,
;; so the native text now lives under `:stages [0] :native`.
(is (= "SELECT 1" (get-in ser [:source :query :stages 0 :native]))))
(testing "Transform/dependencies does not emit a Database dep when source_database_id is nil"
(is (not-any? #(some (fn [{:keys [model]}] (= "Database" model)) %)
(serdes/deserialization-dependencies (assoc reloaded :tags [])))))))))))
(deftest table-with-transform-id-dependency-test
(testing "Table created by a Transform declares the Transform as a serdes dependency (GDGT-2444)"
(mt/with-premium-features #{:transforms-basic}
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Database
{db-id :id}
{:name "My Database"}
:model/Transform
{transform-id :id
transform-eid :entity_id}
{:name "Transform Creating Table"
:entity_id "transformCreatingTblx"
:source {:query {:database db-id
:type "native"
:native {:query "select 1 as x"}}
:type "query"}
:target {:database db-id
:type "table"
:schema "public"
:name "transform_output"}}
:model/Table
{table-id :id}
{:name "transform_output"
:db_id db-id
:schema "public"
:transform_id transform-id}]
(let [ser (ts/extract-one "Table" table-id)]
(testing "transform_id is transformed to entity_id"
(is (= transform-eid (:transform_id ser))))
(testing "depends on the transform"
(is (contains? (set (serdes/deserialization-dependencies ser))
[{:model "Transform" :id transform-eid}])))))))))
(deftest transform-job-extraction-test
(testing "TransformJob extraction and serialization"
(mt/with-premium-features #{:transforms-basic}
(mt/with-empty-h2-app-db!
;; Delete the existing built-in transform tags and jobs
(t2/delete! :model/TransformTag)
(t2/delete! :model/TransformJob)
(ts/with-temp-dpc [;; Create tags for job associations
:model/TransformTag
{hourly-tag-id :id
hourly-tag-eid :entity_id}
{:name "hourly" :built_in_type "hourly"}
:model/TransformTag
{daily-tag-id :id
daily-tag-eid :entity_id}
{:name "daily" :built_in_type "daily"}
:model/TransformTag
{custom-tag-id :id
custom-tag-eid :entity_id}
{:name "custom-etl"}
;; Create built-in TransformJob
:model/TransformJob
{hourly-job-id :id
hourly-job-eid :entity_id}
{:name "Hourly job"
:description "Executes transforms tagged with 'hourly' every hour"
:schedule "0 0 * * * ? *"
:built_in_type "hourly"}
;; Create custom TransformJob
:model/TransformJob
{custom-job-id :id
custom-job-eid :entity_id}
{:name "Custom ETL Job"
:description "Custom data processing job"
:schedule "0 0 2 * * ? *"}
;; Create job-tag associations
:model/TransformJobTransformTag
{}
{:job_id hourly-job-id
:tag_id hourly-tag-id
:position 0}
:model/TransformJobTransformTag
{}
{:job_id custom-job-id
:tag_id custom-tag-id
:position 0}
:model/TransformJobTransformTag
{}
{:job_id custom-job-id
:tag_id daily-tag-id
:position 1}]
(testing "built-in job extracts correctly"
(let [ser (serdes/extract-one "TransformJob" {} (t2/hydrate (t2/select-one :model/TransformJob :id hourly-job-id) :job_tags))]
(is (=? {:serdes/meta [{:model "TransformJob"
:id hourly-job-eid}]
:name "Hourly job"
:description "Executes transforms tagged with 'hourly' every hour"
:schedule "0 0 * * * ? *"
:built_in_type "hourly"
:created_at string?}
ser))
(is (not (contains? ser :id)))
(testing "job has associated tags"
(is (= 1 (count (:job_tags ser))))
(is (= hourly-tag-eid (-> ser :job_tags first :tag_id)))
(is (= 0 (-> ser :job_tags first :position))))
(testing "dependencies include referenced tags"
(is (= #{[{:model "TransformTag" :id hourly-tag-eid}]}
(set (serdes/deserialization-dependencies ser)))))))
(testing "custom job extracts correctly"
(let [ser (serdes/extract-one "TransformJob" {} (t2/hydrate (t2/select-one :model/TransformJob :id custom-job-id) :job_tags))]
(is (=? {:serdes/meta [{:model "TransformJob"
:id custom-job-eid}]
:name "Custom ETL Job"
:description "Custom data processing job"
:schedule "0 0 2 * * ? *"
:created_at string?}
ser))
(is (not (contains? ser :built_in_type)))
(is (not (contains? ser :id)))
(testing "job has multiple associated tags in correct order"
(is (= 2 (count (:job_tags ser))))
(let [tag-ids (map :tag_id (:job_tags ser))
positions (map :position (:job_tags ser))]
(is (= [custom-tag-eid daily-tag-eid] tag-ids))
(is (= [0 1] positions))))
(testing "dependencies include all referenced tags"
(is (= #{[{:model "TransformTag" :id custom-tag-eid}]
[{:model "TransformTag" :id daily-tag-eid}]}
(set (serdes/deserialization-dependencies ser)))))))
(testing "all transform jobs are extracted"
(is (= #{hourly-job-eid custom-job-eid}
(ids-by-model "TransformJob" (extract/extract {}))))))))))
(deftest collection-export-includes-published-tables-test
(testing "Exporting a collection includes published tables without requiring data_model flag"
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Database {db-id :id} {:name "My Database"}
:model/Collection {coll-id :id} {:name "Export Collection"}
:model/Table _ {:name "Published Table"
:db_id db-id
:is_published true
:collection_id coll-id}
:model/Table _ {:name "Unpublished Table"
:db_id db-id
:is_published false}
:model/Table _ {:name "Other Table"
:db_id db-id
:is_published true
:collection_id nil}]
(let [opts {:targets [["Collection" coll-id]]
:no-data-model true
:no-settings true}
extracted (into [] (extract/extract opts))
table-names (->> extracted
(filter #(= "Table" (-> % :serdes/meta last :model)))
(map :name)
set)]
(testing "published table in target collection is exported"
(is (contains? table-names "Published Table")))
(testing "unpublished table is not exported"
(is (not (contains? table-names "Unpublished Table"))))
(testing "published table in other collection is not exported"
(is (not (contains? table-names "Other Table"))))
(testing "database is not exported (no-data-model: true)"
(is (empty? (filter #(= "Database" (-> % :serdes/meta last :model)) extracted)))))))))
(deftest collection-export-includes-fields-and-segments-test
(testing "Exporting a collection with published tables includes their Fields and Segments"
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Database {db-id :id} {:name "My Database"}
:model/Collection {coll-id :id} {:name "Export Collection"}
:model/Table {table-id :id} {:name "Published Table"
:db_id db-id
:is_published true
:collection_id coll-id}
:model/Field {field1-id :id} {:name "Field One"
:table_id table-id
:base_type :type/Integer}
:model/Field {field2-id :id} {:name "Field Two"
:table_id table-id
:base_type :type/Text}
:model/Segment {segment-id :id} {:name "Test Segment"
:table_id table-id
:definition {}}
;; Create another table with fields that should NOT be exported
:model/Table {other-table-id :id} {:name "Other Table"
:db_id db-id
:is_published false}
:model/Field _ {:name "Other Field"
:table_id other-table-id
:base_type :type/Integer}]
(let [opts {:targets [["Collection" coll-id]]
:no-data-model true
:no-settings true}
extracted (into [] (extract/extract opts))
field-ids (->> extracted
(filter #(= "Field" (-> % :serdes/meta last :model)))
(map #(t2/select-one-pk :model/Field
:table_id (t2/select-one-pk :model/Table :name (-> % :serdes/meta butlast last :id))
:name (-> % :serdes/meta last :id)))
set)
segment-ids (->> extracted
(filter #(= "Segment" (-> % :serdes/meta last :model)))
(map #(:id (t2/select-one :model/Segment :entity_id (:entity_id %))))
set)]
(testing "fields from published table are exported"
(is (contains? field-ids field1-id))
(is (contains? field-ids field2-id)))
(testing "segments from published table are exported"
(is (contains? segment-ids segment-id)))
(testing "fields from non-published table are NOT exported"
(is (= 2 (count field-ids)))))))))
(deftest python-library-test
(mt/with-empty-h2-app-db!
;; Delete any pre-existing python_library entries from migrations
(t2/delete! :model/PythonLibrary)
(ts/with-temp-dpc [:model/PythonLibrary {lib-id :id lib-entity-id :entity_id} {:path "common"
:source "def helper():\n return 42"}]
(testing "python library extraction"
(let [ser (serdes/extract-one "PythonLibrary" {} (t2/select-one :model/PythonLibrary :id lib-id))]
(is (=? {:serdes/meta [{:model "PythonLibrary"
:id lib-entity-id}]
:path "common.py"
:source "def helper():\n return 42"
:entity_id lib-entity-id
:created_at string?}
ser))
(is (not (contains? ser :id)))
(testing "has no dependencies"
(is (empty? (serdes/deserialization-dependencies ser)))))))))
(deftest custom-viz-plugin-test
(mt/with-empty-h2-app-db!
(t2/delete! :model/CustomVizPlugin)
(ts/with-temp-dpc [:model/CustomVizPlugin {plugin-id :id} {:display_name "Test Plugin"
:identifier "test-plugin"
:status :active
:manifest "{}"
:bundle (.getBytes "pretend tgz bytes" "UTF-8")
:bundle_hash "deadbeef"}]
;; Uncomment to regenerate baseline:
;; (round-trip-test/add-to-baseline!)
(testing "custom viz plugin extraction"
(let [ser (serdes/extract-one "CustomVizPlugin" {} (t2/select-one :model/CustomVizPlugin :id plugin-id))]
(is (=? {:serdes/meta [{:model "CustomVizPlugin"
:id "test-plugin"}]
:display_name "Test Plugin"
:identifier "test-plugin"
:manifest {}
:bundle_hash "deadbeef"
:bundle string?
:created_at string?}
ser))
(is (not (contains? ser :id)))
(is (not (contains? ser :status)))
(testing "has no dependencies"
(is (empty? (serdes/deserialization-dependencies ser)))))))))
(deftest ^:parallel export-parameters-sorts-by-id-test
(let [params [{:id "zebra" :name "Z param" :type :category}
{:id "alpha" :name "A param" :type :category}
{:id "middle" :name "M param" :type :category}]
result (serdes/export-parameters params)]
(testing "export sorts parameters by :id for stable diffs"
(is (= ["alpha" "middle" "zebra"]
(map :id result))))
(testing "export stamps :position from original array index"
(is (= [1 2 0]
(map :position result))))))
(deftest ^:parallel export-parameters-empty-input-test
(is (= [] (serdes/export-parameters []))))
(deftest ^:parallel export-parameters-nil-input-test
(is (= [] (serdes/export-parameters nil))))
(deftest ^:parallel export-parameters-single-element-test
(let [params [{:id "only" :name "Only param"}]
result (serdes/export-parameters params)]
(is (= ["only"] (map :id result)))
(is (= [0] (map :position result)))))
(deftest ^:parallel export-parameters-nil-id-sorts-first-test
(let [params [{:id "beta" :name "B"}
{:id nil :name "Nil"}
{:id "alpha" :name "A"}]
result (serdes/export-parameters params)]
(is (= [nil "alpha" "beta"] (map :id result)))
(is (= [1 2 0] (map :position result)))))
(deftest ^:parallel export-parameter-mappings-sorts-by-parameter-id-test
(let [mappings [{:parameter_id "z-param" :target [:dimension [:field 1 nil]]}
{:parameter_id "a-param" :target [:dimension [:field 2 nil]]}
{:parameter_id "m-param" :target [:dimension [:field 3 nil]]}]
result (serdes/export-parameter-mappings mappings)]
(is (= ["a-param" "m-param" "z-param"]
(map :parameter_id result)))))
(deftest ^:parallel export-parameter-mappings-empty-input-test
(is (= [] (serdes/export-parameter-mappings []))))
(deftest ^:parallel export-parameter-mappings-nil-input-test
(is (= [] (serdes/export-parameter-mappings nil))))
(deftest ^:parallel export-parameter-mappings-single-element-test
(let [mappings [{:parameter_id "only" :target [:dimension [:field 1 nil]]}]
result (serdes/export-parameter-mappings mappings)]
(is (= ["only"] (map :parameter_id result)))))
(deftest dashboard-parameters-sorted-extraction-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Collection {coll-id :id} {:name "Test Collection"}
:model/Card {card-id :id} {:name "Source Card"
:collection_id coll-id}
:model/Dashboard {dash-id :id} {:name "Test Dashboard"
:collection_id coll-id
:parameters [{:id "zebra" :name "Z" :type :category}
{:id "alpha" :name "A" :type :category}
{:id "beta" :name "B" :type :category}]}
:model/DashboardCard _ {:dashboard_id dash-id
:card_id card-id
:parameter_mappings [{:parameter_id "zebra"
:card_id card-id
:target [:dimension [:field 1 nil]]}
{:parameter_id "alpha"
:card_id card-id
:target [:dimension [:field 2 nil]]}]}]
(let [ser (ts/extract-one "Dashboard" dash-id)]
(testing "parameters are sorted by :id"
(is (= ["alpha" "beta" "zebra"]
(map :id (:parameters ser)))))
(testing "positions preserve original display order"
(is (= [1 2 0]
(map :position (:parameters ser)))))
(is (= ["alpha" "zebra"]
(map :parameter_id (-> ser :dashcards first :parameter_mappings))))))))
(deftest ^:parallel import-parameters-round-trip-test
(testing "export stamps position, import restores original display order"
(let [params [{:id "zebra" :name "Z param" :type :category}
{:id "alpha" :name "A param" :type :category}
{:id "middle" :name "M param" :type :category}]
exported (serdes/export-parameters params)
imported (serdes/import-parameters exported)]
(is (= ["alpha" "middle" "zebra"] (map :id exported))
"export sorts by id")
(is (= [1 2 0] (map :position exported))
"export stamps position from original array index")
(is (= ["zebra" "alpha" "middle"] (map :id imported))
"import restores original display order via position"))))
(deftest ^:parallel import-parameters-without-position-preserves-order-test
(testing "importing legacy params without :position preserves existing order"
(let [params [{:id "zebra" :name "Z param" :type :category}
{:id "alpha" :name "A param" :type :category}]
imported (serdes/import-parameters params)]
(is (= ["zebra" "alpha"] (map :id imported))))))
(deftest channel-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc
[:model/Channel {email-id :id} {:name "Email Channel"
:type :channel/email
:details {:host "smtp.example.com" :port 587}
:description "An email channel"}
:model/Channel {http-id :id} {:name "HTTP Channel"
:type :channel/http
:details {:url "https://example.com/webhook"
:method "POST"
:auth-method "none"}
:description "An HTTP channel"}]
(testing "email channel extraction"
(let [ser (ts/extract-one "Channel" email-id)]
(is (=? {:serdes/meta [{:model "Channel" :id "Email Channel"}]
:name "Email Channel"
:type :channel/email
:description "An email channel"
:details {:host "smtp.example.com" :port 587}
:created_at string?}
ser))
(is (not (contains? ser :id)))
(is (not (contains? ser :active)))))
(testing "http channel extraction"
(let [ser (ts/extract-one "Channel" http-id)]
(is (=? {:serdes/meta [{:model "Channel" :id "HTTP Channel"}]
:name "HTTP Channel"
:type :channel/http
:description "An HTTP channel"
:details {:url "https://example.com/webhook"
:method "POST"
:auth-method "none"}
:created_at string?}
ser))))
(testing "channel storage-path uses top-level channels directory"
(let [ser (ts/extract-one "Channel" email-id)]
(is (= [{:label "channels"} {:label "Email Channel" :key "Email Channel"}]
(serdes/storage-path ser {}))))
(let [ser (ts/extract-one "Channel" http-id)]
(is (= [{:label "channels"} {:label "HTTP Channel" :key "HTTP Channel"}]
(serdes/storage-path ser {}))))))))
;;; ===========================================================================
;;; Redundant field stripping tests
;;;
;;; Fields derivable from dataset_query (table_id, database_id, query_type,
;;; source_card_id) should be omitted from export.
;;; ===========================================================================
(defn- mbql5-query
"Create a simple MBQL 5 query for the given database and table."
[db-id table-id]
(let [metadata-provider (lib-be/application-database-metadata-provider db-id)
table (lib.metadata/table metadata-provider table-id)]
(lib/query metadata-provider table)))
(deftest card-export-strips-redundant-fields-test
(testing "Card export omits table_id, query_type, source_card_id when derivable from query"
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Database {db-id :id} {:name "Test DB"}
:model/Table {table-id :id} {:name "orders" :db_id db-id}
:model/Field _ {:name "id" :table_id table-id}
:model/Card {card-id :id} {:name "MBQL Card"
:dataset_query (mbql5-query db-id table-id)
:display :table}]
(let [ser (serdes/extract-one "Card" {} (t2/select-one :model/Card card-id))]
(is (not (contains? ser :table_id))
"table_id should be omitted — derivable from dataset_query")
(is (not (contains? ser :query_type))
"query_type should be omitted — derivable from dataset_query")
(is (not (contains? ser :source_card_id))
"source_card_id should be omitted — derivable from dataset_query")
(is (not (contains? ser :database_id))
"database_id should be omitted when derivable from dataset_query"))))))
(deftest card-export-keeps-database-id-when-query-empty-test
(testing "Card export keeps database_id when dataset_query has no :database key"
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Database {db-id :id} {:name "Test DB"}
:model/Card {card-id :id} {:name "Empty Query Card"
:database_id db-id
:dataset_query {}
:display :table}]
(let [ser (serdes/extract-one "Card" {} (t2/select-one :model/Card card-id))]
(is (= "Test DB" (:database_id ser))
"database_id should be kept when query is empty"))))))
(deftest card-export-deps-come-from-query-test
(testing "Card dependencies come from dataset_query, not from stripped FK fields"
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Database {db-id :id} {:name "Test DB"}
:model/Table {table-id :id} {:name "orders" :db_id db-id}
:model/Field _ {:name "id" :table_id table-id}
:model/Card {card-id :id} {:name "MBQL Card"
:dataset_query (mbql5-query db-id table-id)
:display :table}]
(let [ser (serdes/extract-one "Card" {} (t2/select-one :model/Card card-id))
deps (serdes/deserialization-dependencies ser)]
;; Database dep comes from mbql-deps on the query's :database key
(is (contains? (set deps) [{:model "Database" :id "Test DB"}])
"Database dependency should come from the query")
;; Tables/Fields are intentionally not dependencies — they're synthesized as inactive rows on import
;; if missing, and upserted otherwise.
(is (not-any? #(some (fn [step] (#{"Table" "Field"} (:model step))) %) deps)
"Table/Field should not be dependencies"))))))
(deftest segment-export-strips-table-id-test
(testing "Segment export omits table_id — derivable from definition"
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Database {db-id :id} {:name "Test DB"}
:model/Table {table-id :id} {:name "orders" :db_id db-id}
:model/Field {field-id :id} {:name "total" :table_id table-id}
:model/Segment _ {:name "Big Orders"
:table_id table-id
:definition (let [mp (lib-be/application-database-metadata-provider db-id)
table (lib.metadata/table mp table-id)
query (lib/query mp table)
field (lib.metadata/field mp field-id)]
(lib/filter query (lib/> field 100)))}]
(let [ser (first (by-model "Segment" (extract/extract {})))]
(is (some? ser))
(is (not (contains? ser :table_id))
"table_id should be omitted — derivable from definition"))))))
(deftest measure-export-strips-table-id-test
(testing "Measure export omits table_id — derivable from definition"
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Database {db-id :id} {:name "Test DB"}
:model/Table {table-id :id} {:name "orders" :db_id db-id}
:model/Field {field-id :id} {:name "total" :table_id table-id
:base_type :type/Float}
:model/Measure _ {:name "Total Sales"
:table_id table-id
:definition (let [mp (lib-be/application-database-metadata-provider db-id)
table (lib.metadata/table mp table-id)
query (lib/query mp table)
field (lib.metadata/field mp field-id)]
(lib/aggregate query (lib/sum field)))}]
(let [ser (first (by-model "Measure" (extract/extract {})))]
(is (some? ser))
(is (not (contains? ser :table_id))
"table_id should be omitted — derivable from definition"))))))
(deftest embedding-theme-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/EmbeddingTheme
{light-id :id
light-eid :entity_id}
{:name "Light"
:settings {}}
:model/EmbeddingTheme
{_dark-id :id
dark-eid :entity_id}
{:name "Dark"
:settings {}}]
(testing "embedding theme extracts correctly"
(let [ser (serdes/extract-one "EmbeddingTheme" {} (t2/select-one :model/EmbeddingTheme :id light-id))]
(is (=? {:serdes/meta [{:model "EmbeddingTheme"
:id light-eid
:label "light"}]
:entity_id light-eid
:name "Light"
:settings {}
:created_at string?
:updated_at string?}
ser))
(is (not (contains? ser :id)))
(testing "embedding themes have no dependencies"
(is (empty? (serdes/deserialization-dependencies ser))))))
(testing "all embedding themes are extracted"
(is (= #{light-eid dark-eid}
(ids-by-model "EmbeddingTheme" (extract/extract {}))))))))
(deftest no-metabase-version-stamp-test
(testing "extract does not stamp :metabase_version on entities (GHY-4013: it caused spurious remote-sync diffs)"
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Collection {coll-id :id} {:name "My Collection"}
:model/Card _ {:name "My Card" :collection_id coll-id}]
(let [extracted (into [] (extract/extract {}))
by-model (group-by (comp :model last :serdes/meta) extracted)]
(doseq [m ["Collection" "Card"]
entity (get by-model m)]
(is (not (contains? entity :metabase_version))
(str m " should not carry a :metabase_version"))))))))
Reference in New Issue View Git Blame Copy Permalink