Make tests implementable by other drivers

modules/drivers/mongo/src/metabase/driver/mongo.clj

@@ -414,6 +414,8 @@
    :name (:name table)
    :fields (-> (fetch-dbfields database table) dbfields->ftree ftree->nested-fields)})
 
+(defmethod driver/should-sync-active-subset? :mongo [_driver] true)
+
 ;; describe-table impl end
 
 (doseq [[feature supported?] {:basic-aggregations              true

resources/migrations/057_update_migrations.yaml

@@ -2459,6 +2459,20 @@ databaseChangeLog:
             constraintName: fk_transform_ref_creator_id
             onDelete: CASCADE
 
+  - changeSet:
+      id: v57.2025-11-07T11:32:41
+      author: lbrdnk
+      comment: Add active_subset column to metabase_field
+      changes:
+        - addColumn:
+            tableName: metabase_field
+            columns:
+              - column:
+                  name: active_subset
+                  remarks: Is this field path of the active subset (see the driver/should-sync-active-subset?)
+                  type: ${boolean.type}
+                  defaultValue: null
+
 # >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
 
 ########################################################################################################################

src/metabase/driver.clj

@@ -1660,3 +1660,26 @@
       (if (table-known-to-not-exist? driver e)
         false
         (throw e)))))
+
+(defmulti should-sync-active-subset?
+  "For some drivers we sync only a subset of fields available for a table. This method is meant to decide whether
+  the alternative, _active subset_, codepath should be taken, or the standard sync codepath.
+
+  # Problem
+
+  I may have happend than _active subset_ has changed during sync. That _could_ result in saved questions not working
+  as fields they use may have been deactivated.
+
+
+  # Solution
+  We've decided to address that as follows: (1) for database that sync _active subset_, we do not deactivate any
+  fields. Instead we decide what the current active subset is during sync (2) and fingerprint only those fields.
+
+  As a result, (1) the existing questions will not break as their fields are not deactivated and (2) fignerprinting
+  process should not get overloaded, possbily resulting in OOMs, because number of fields it processes will not
+  get excessively large."
+  {:added "0.57.0" :arglists '([driver])}
+  dispatch-on-initialized-driver
+  :hierarchy #'hierarchy)
+
+(defmethod should-sync-active-subset? ::driver [_driver] false)

src/metabase/sync/analyze/fingerprint.clj

@@ -189,8 +189,13 @@
   "Return a sequences of Fields belonging to `table` for which we should generate (and save) fingerprints.
    This should include NEW fields that are active and visible."
   [table :- i/TableInstance]
-  (seq (t2/select :model/Field
-                  (honeysql-for-fields-that-need-fingerprint-updating table))))
+  ;; TODO (11/9/2025 lbrdnk): Database fetching in following expressions should be optimized (e.g. passed down from
+  ;;                          upper layers).
+  (let [database (table/database table)
+        driver (driver.u/database->driver database)]
+    (seq (t2/select :model/Field
+                    (cond-> (honeysql-for-fields-that-need-fingerprint-updating table)
+                      (driver/should-sync-active-subset? driver) (update :where conj [:= :active_subset true]))))))
 
 (mu/defn fingerprint-table!
   "Generate and save fingerprints for all the Fields in `table` that have not been previously analyzed."

src/metabase/sync/field_values.clj

@@ -3,12 +3,15 @@
   (:require
    [java-time.api :as t]
    [metabase.app-db.core :as mdb]
+   [metabase.driver :as driver]
+   [metabase.driver.util :as driver.u]
    [metabase.sync.interface :as i]
    [metabase.sync.util :as sync-util]
    [metabase.util :as u]
    [metabase.util.log :as log]
    [metabase.util.malli :as mu]
    [metabase.warehouse-schema.models.field-values :as field-values]
+   [metabase.warehouse-schema.models.table :as table]
    [toucan2.core :as t2]))
 
 (mu/defn- clear-field-values-for-field!
@@ -51,7 +54,14 @@
 
 (defn- table->fields-to-scan
   [table]
-  (t2/select :model/Field :table_id (u/the-id table), :active true, :visibility_type "normal"))
+  (let [database (table/database table)
+        driver (driver.u/database->driver database)]
+    (t2/select :model/Field (cond-> {:where [:and
+                                             [:= :table_id (u/the-id table)]
+                                             [:= :active true]
+                                             [:= :visibility_type "normal"]]}
+                              (driver/should-sync-active-subset? driver) (update :where conj
+                                                                                 [:= :active_subset true])))))
 
 (mu/defn update-field-values-for-table!
   "Update the FieldValues for all Fields (as needed) for `table`."

src/metabase/sync/sync_metadata/fields.clj

@@ -63,7 +63,7 @@
   [database    :- i/DatabaseInstance
    table       :- i/TableInstance
    db-metadata :- [:set i/TableMetadataField]]
-  (+ (sync-instances/sync-instances! table db-metadata (fields.our-metadata/our-metadata table))
+  (+ (sync-instances/sync-instances! database table db-metadata (fields.our-metadata/our-metadata table))
      ;; Now that tables are synced and fields created as needed make sure field properties are in sync.
      ;; Re-fetch our metadata because there might be some things that have changed after calling
      ;; `sync-instances`

src/metabase/sync/sync_metadata/fields/active_subset.clj

@@ -0,0 +1,29 @@
+(ns metabase.sync.sync-metadata.fields.active-subset
+  (:require
+   [metabase.sync.interface :as i]
+   [metabase.sync.sync-metadata.fields.common :as common]
+   [metabase.sync.util :as sync-util]
+   [metabase.util :as u]
+   [metabase.util.log :as log]
+   [metabase.util.malli :as mu]
+   [toucan2.core :as t2]))
+
+(mu/defn- remove-field-from-active-subset!
+  [table          :- i/TableInstance
+   metabase-field :- common/TableMetadataFieldWithID]
+  ;; TODO: Can following poison logs?s
+  (log/debugf "Removing Field ''%s'' from active subset."
+              (common/field-metadata-name-for-logging table metabase-field))
+  (when (pos? (t2/update! :model/Field (u/the-id metabase-field) {:active_subset false}))
+    1))
+
+(defn adjust-active-subset!
+  "TBD Remove fields from active subset"
+  [table db-metadata our-metadata]
+  (sync-util/sum-for
+   [metabase-field our-metadata
+    :when          (not (common/matching-field-metadata metabase-field db-metadata))]
+   ;; TODO: Can following poison logs?
+    (sync-util/with-error-handling (format "Error removing %s from active subset"
+                                           (common/field-metadata-name-for-logging table metabase-field))
+      (remove-field-from-active-subset! table metabase-field))))

src/metabase/sync/sync_metadata/fields/our_metadata.clj

@@ -5,6 +5,8 @@
   comparing the differences in the two sets of Metadata."
   (:require
    [medley.core :as m]
+   [metabase.driver :as driver]
+   [metabase.driver.util :as driver.u]
    [metabase.sync.interface :as i]
    [metabase.sync.sync-metadata.fields.common :as common]
    [metabase.util :as u]
@@ -70,17 +72,26 @@
      (set (for [metabase-field (get parent-id->fields top-level-parent-id)]
             (add-nested-fields metabase-field parent-id->fields))))))
 
+;; making this not select active_subset not true would ensure the activation happens outside of "updates"
 (mu/defn- table->fields :- [:maybe [:sequential i/FieldInstance]]
   "Fetch active Fields from the Metabase application database for a given `table`."
   [table :- i/TableInstance]
-  (t2/select [:model/Field :name :database_type :base_type :effective_type :coercion_strategy :semantic_type
-              :parent_id :id :description :database_position :nfc_path
-              :database_is_auto_increment :database_required
-              :database_default :database_is_generated :database_is_nullable :database_is_pk
-              :database_partitioned :json_unfolding :position]
-             :table_id  (u/the-id table)
-             :active    true
-             {:order-by table/field-order-rule}))
+  (let [database (table/database table)
+        should-use-active-subset? (driver/should-sync-active-subset? (driver.u/database->driver database))]
+    (t2/select [:model/Field :name :database_type :base_type :effective_type :coercion_strategy :semantic_type
+                :parent_id :id :description :database_position :nfc_path
+                :database_is_auto_increment :database_required
+                :database_default :database_is_generated :database_is_nullable :database_is_pk
+                :database_partitioned :json_unfolding :position]
+               (cond-> {:where [:and
+                                [:= :table_id  (u/the-id table)]]
+                        :order-by table/field-order-rule}
+
+                 (not should-use-active-subset?)
+                 (update :where conj [:= :active true])
+
+                 should-use-active-subset?
+                 (update :where conj [:= :active_subset true])))))
 
 (mu/defn our-metadata :- [:set common/TableMetadataFieldWithID]
   "Return information we have about Fields for a `table` in the application database in (almost) exactly the same

src/metabase/sync/sync_metadata/fields/sync_instances.clj

@@ -8,9 +8,12 @@
   namespace should ignore nested fields entirely; the will be invoked with those Fields as appropriate."
   (:require
    [medley.core :as m]
+   [metabase.driver :as driver]
+   [metabase.driver.util :as driver.u]
    [metabase.lib.schema.id :as lib.schema.id]
    [metabase.models.humanization :as humanization]
    [metabase.sync.interface :as i]
+   [metabase.sync.sync-metadata.fields.active-subset :as sync-metadata.fields.active-subset]
    [metabase.sync.sync-metadata.fields.common :as common]
    [metabase.sync.sync-metadata.fields.our-metadata :as fields.our-metadata]
    [metabase.sync.util :as sync-util]
@@ -27,83 +30,102 @@
 (mu/defn- matching-inactive-fields :- [:maybe [:sequential i/FieldInstance]]
   "Return inactive Metabase Fields that match any of the Fields described by `new-field-metadatas`, if any such Fields
   exist."
-  [table               :- i/TableInstance
+  [database
+   table               :- i/TableInstance
    new-field-metadatas :- [:maybe [:sequential i/TableMetadataField]]
    parent-id           :- common/ParentID]
   (when (seq new-field-metadatas)
-    (t2/select     :model/Field
-                   :table_id    (u/the-id table)
-                   :%lower.name [:in (map common/canonical-name new-field-metadatas)]
-                   :parent_id   parent-id
-                   :active      false)))
+    (let [should-use-active-subset? (driver/should-sync-active-subset? (driver.u/database->driver database))]
+      (t2/select :model/Field
+                 (cond-> {:where
+                          [:and
+                           [:= :table_id (u/the-id table)]
+                           [:in :%lower.name (map common/canonical-name new-field-metadatas)]
+                           [:= :parent_id parent-id]]}
+
+                   (not should-use-active-subset?)
+                   (update :where conj [:= :active false])
+
+                   should-use-active-subset?
+                   (update :where conj [:or
+                                        [:= :active_subset false]
+                                        [:= :active_subset nil]]))))))
 
 (mu/defn- insert-new-fields! :- [:maybe [:sequential ::lib.schema.id/field]]
   "Insert new Field rows for for all the Fields described by `new-field-metadatas`. Returns IDs of newly inserted
   Fields."
-  [table               :- i/TableInstance
+  [database
+   table               :- i/TableInstance
    new-field-metadatas :- [:maybe [:sequential i/TableMetadataField]]
    parent-id           :- common/ParentID]
   (when (seq new-field-metadatas)
-    (t2/insert-returning-pks! :model/Field
-                              (for [{:keys [base-type coercion-strategy database-is-auto-increment database-partitioned database-position
-                                            database-is-generated database-is-nullable database-default pk?
-                                            database-required database-type effective-type field-comment json-unfolding nfc-path visibility-type]
-                                     field-name :name :as field} (sort-by :database-position new-field-metadatas)
-                                    :let [semantic-type (common/semantic-type field)
-                                          has-field-values (when (sync-util/can-be-list? base-type semantic-type)
-                                                             :auto-list)]]
-                                (do
-                                  (when (and effective-type
-                                             base-type
-                                             (not= effective-type base-type)
-                                             (nil? coercion-strategy))
-                                    (log/warn (u/format-color 'red
-                                                              (str
-                                                               "WARNING: Field `%s`: effective type `%s` provided but no coercion strategy provided."
-                                                               " Using base-type: `%s`")
-                                                              field-name
-                                                              effective-type
-                                                              base-type)))
-                                  {:table_id                   (u/the-id table)
-                                   :name                       field-name
-                                   :display_name               (humanization/name->human-readable-name field-name)
-                                   :database_type              (or database-type "NULL") ; placeholder for Fields w/ no type info (e.g. Mongo) & all NULL
-                                   :base_type                  base-type
+    (let [active-subset? (driver/should-sync-active-subset? (driver.u/database->driver database))]
+      (t2/insert-returning-pks! :model/Field
+                                (for [{:keys [base-type coercion-strategy database-is-auto-increment database-partitioned database-position
+                                              database-is-generated database-is-nullable database-default pk?
+                                              database-required database-type effective-type field-comment json-unfolding nfc-path visibility-type]
+                                       field-name :name :as field} (sort-by :database-position new-field-metadatas)
+                                      :let [semantic-type (common/semantic-type field)
+                                            has-field-values (when (sync-util/can-be-list? base-type semantic-type)
+                                                               :auto-list)]]
+                                  (do
+                                    (when (and effective-type
+                                               base-type
+                                               (not= effective-type base-type)
+                                               (nil? coercion-strategy))
+                                      (log/warn (u/format-color 'red
+                                                                (str
+                                                                 "WARNING: Field `%s`: effective type `%s` provided but no coercion strategy provided."
+                                                                 " Using base-type: `%s`")
+                                                                field-name
+                                                                effective-type
+                                                                base-type)))
+                                    (merge
+                                     {:table_id                   (u/the-id table)
+                                      :name                       field-name
+                                      :display_name               (humanization/name->human-readable-name field-name)
+                                      :database_type              (or database-type "NULL") ; placeholder for Fields w/ no type info (e.g. Mongo) & all NULL
+                                      :base_type                  base-type
            ;; todo test this?
-                                   :effective_type             (if (and effective-type coercion-strategy) effective-type base-type)
-                                   :coercion_strategy          (when effective-type coercion-strategy)
-                                   :semantic_type              semantic-type
-                                   :parent_id                  parent-id
-                                   :nfc_path                   nfc-path
-                                   :description                field-comment
-                                   :position                   database-position
-                                   :database_position          database-position
-                                   :json_unfolding             (or json-unfolding false)
-                                   :database_is_auto_increment (or database-is-auto-increment false)
-                                   :database_is_generated      database-is-generated
-                                   :database_is_nullable       database-is-nullable
-                                   :database_is_pk             pk?
-                                   :database_default           database-default
-                                   :database_required          (or database-required false)
-                                   :database_partitioned       database-partitioned ;; nullable for database that doesn't support partitioned fields
-                                   :has_field_values           has-field-values
-                                   :visibility_type            (or visibility-type :normal)})))))
+                                      :effective_type             (if (and effective-type coercion-strategy) effective-type base-type)
+                                      :coercion_strategy          (when effective-type coercion-strategy)
+                                      :semantic_type              semantic-type
+                                      :parent_id                  parent-id
+                                      :nfc_path                   nfc-path
+                                      :description                field-comment
+                                      :position                   database-position
+                                      :database_position          database-position
+                                      :json_unfolding             (or json-unfolding false)
+                                      :database_is_auto_increment (or database-is-auto-increment false)
+                                      :database_is_generated      database-is-generated
+                                      :database_is_nullable       database-is-nullable
+                                      :database_is_pk             pk?
+                                      :database_default           database-default
+                                      :database_required          (or database-required false)
+                                      :database_partitioned       database-partitioned ;; nullable for database that doesn't support partitioned fields
+                                      :has_field_values           has-field-values
+                                      :visibility_type            (or visibility-type :normal)}
+                                     (when active-subset?
+                                       {:active_subset true}))))))))
 
 (mu/defn- create-or-reactivate-fields! :- [:maybe [:sequential i/FieldInstance]]
   "Create (or reactivate) Metabase Field object(s) for any Fields in `new-field-metadatas`. Does *NOT* recursively
   handle nested Fields."
-  [table               :- i/TableInstance
+  [database
+   table               :- i/TableInstance
    new-field-metadatas :- [:maybe [:sequential i/TableMetadataField]]
    parent-id           :- common/ParentID]
-  (let [fields-to-reactivate (matching-inactive-fields table new-field-metadatas parent-id)]
+  (let [fields-to-reactivate (matching-inactive-fields database table new-field-metadatas parent-id)]
     ;; if the fields already exist but were just marked inactive then reäctivate them
     (when (seq fields-to-reactivate)
       (t2/update! :model/Field {:id [:in (map u/the-id fields-to-reactivate)]}
-                  {:active true}))
+                  (merge {:active true}
+                         (when (driver/should-sync-active-subset? (driver.u/database->driver database))
+                           {:active_subset true}))))
     (let [reactivated?  (comp (set (map common/canonical-name fields-to-reactivate))
                               common/canonical-name)
           ;; If we reactivated the fields, no need to insert them; insert new rows for any that weren't reactivated
-          new-field-ids (insert-new-fields! table (remove reactivated? new-field-metadatas) parent-id)]
+          new-field-ids (insert-new-fields! database table (remove reactivated? new-field-metadatas) parent-id)]
       ;; now return the newly created or reactivated Fields
       (when-let [new-and-updated-fields (seq (map u/the-id (concat fields-to-reactivate new-field-ids)))]
         (t2/select :model/Field :id [:in new-and-updated-fields])))))
@@ -124,7 +146,8 @@
   "Sync instances of `Field` in the application database with 'active' Fields in the DB being synced (i.e., ones that
   are returned as part of the `db-metadata`). Creates or reactivates Fields as needed. Returns number of Fields
   synced and updated `our-metadata` including the new Fields and their IDs."
-  [table        :- i/TableInstance
+  [database
+   table        :- i/TableInstance
    db-metadata  :- [:set i/TableMetadataField]
    our-metadata :- [:set common/TableMetadataFieldWithID]
    parent-id    :- common/ParentID]
@@ -133,12 +156,13 @@
     {:num-updates
      ;; Field sync logic below is broken out into chunks of 1000 fields for huge star schemas or other situations
      ;; where we don't want to be updating way too many rows at once
-     (sync-util/sum-for [db-field-chunk (partition-all 1000 db-metadata)]
+     (sync-util/sum-for
+      [db-field-chunk (partition-all 1000 db-metadata)]
        (sync-util/with-error-handling (format "Error checking if Fields %s need to be created or reactivated"
                                               (pr-str (map :name db-field-chunk)))
          (let [known-field?        (comp known-fields common/canonical-name)
                new-fields          (remove known-field? db-field-chunk)
-               new-field-instances (create-or-reactivate-fields! table new-fields parent-id)]
+               new-field-instances (create-or-reactivate-fields! database table new-fields parent-id)]
            ;; save any updates to `our-metadata`
            (swap! our-metadata into (fields.our-metadata/fields->our-metadata new-field-instances parent-id))
            ;; now return count of rows updated
@@ -183,7 +207,8 @@
 (mu/defn- sync-nested-fields-of-one-field! :- [:maybe ms/IntGreaterThanOrEqualToZero]
   "Recursively sync Field instances (i.e., rows in application DB) for nested Fields of a single Field, one or both
   `field-metadata` (from synced DB) and `metabase-field` (from application DB)."
-  [table          :- i/TableInstance
+  [database
+   table          :- i/TableInstance
    field-metadata :- [:maybe i/TableMetadataField]
    metabase-field :- [:maybe common/TableMetadataFieldWithID]]
   (let [nested-fields-metadata (:nested-fields field-metadata)
@@ -191,6 +216,7 @@
     (when (or (seq nested-fields-metadata)
               (seq metabase-nested-fields))
       (sync-instances!
+       database
        table
        (set nested-fields-metadata)
        (set metabase-nested-fields)
@@ -200,7 +226,8 @@
   "Recursively sync Field instances (i.e., rows in application DB) for *all* the nested Fields of all Fields in
   `db-metadata` and `our-metadata`.
   Not for the flattened nested fields for JSON columns in normal RDBMSes (nested field columns)"
-  [table        :- i/TableInstance
+  [database
+   table        :- i/TableInstance
    db-metadata  :- [:set i/TableMetadataField]
    our-metadata :- [:set common/TableMetadataFieldWithID]]
   (let [name->field-metadata (m/index-by common/canonical-name db-metadata)
@@ -210,17 +237,19 @@
     (sync-util/sum-for [field-name all-field-names
                         :let [field-metadata (get name->field-metadata field-name)
                               metabase-field (get name->metabase-field field-name)]]
-      (sync-nested-fields-of-one-field! table field-metadata metabase-field))))
+      (sync-nested-fields-of-one-field! database table field-metadata metabase-field))))
 
 (mu/defn sync-instances! :- ms/IntGreaterThanOrEqualToZero
   "Sync rows in the Field table with `db-metadata` describing the current schema of the Table currently being synced,
   creating Field objects or marking them active/inactive as needed."
-  ([table        :- i/TableInstance
+  ([database
+    table        :- i/TableInstance
     db-metadata  :- [:set i/TableMetadataField]
     our-metadata :- [:set common/TableMetadataFieldWithID]]
-   (sync-instances! table db-metadata our-metadata nil))
+   (sync-instances! database table db-metadata our-metadata nil))
 
-  ([table        :- i/TableInstance
+  ([database
+    table        :- i/TableInstance
     db-metadata  :- [:set i/TableMetadataField]
     our-metadata :- [:set common/TableMetadataFieldWithID]
     parent-id    :- common/ParentID]
@@ -231,7 +260,11 @@
                (sync-util/name-for-logging table)
                (pr-str (sort (map common/canonical-name db-metadata)))
                (pr-str (sort (map common/canonical-name our-metadata))))
-   (let [{:keys [num-updates our-metadata]} (sync-active-instances! table db-metadata our-metadata parent-id)]
+   (let [{:keys [num-updates our-metadata]} (sync-active-instances! database table db-metadata our-metadata parent-id)]
+     ;; TODO: instead of retire fields, remove from active subset
+     ;; TODO: to handle 5M, need to make `our-metadata` reducible.
      (+ num-updates
-        (retire-fields! table db-metadata our-metadata)
-        (sync-nested-field-instances! table db-metadata our-metadata)))))
+        (if (driver/should-sync-active-subset? (driver.u/database->driver database))
+          (sync-metadata.fields.active-subset/adjust-active-subset! table db-metadata our-metadata)
+          (retire-fields! table db-metadata our-metadata))
+        (sync-nested-field-instances! database table db-metadata our-metadata)))))

src/metabase/sync/sync_metadata/fields/sync_metadata.clj

@@ -4,6 +4,8 @@
   Fields that were not newly created; newly created Fields are given appropriate metadata when first synced."
   (:require
    [clojure.string :as str]
+   [metabase.driver :as driver]
+   [metabase.driver.util :as driver.u]
    [metabase.sync.interface :as i]
    [metabase.sync.sync-metadata.crufty :as crufty]
    [metabase.sync.sync-metadata.fields.common :as common]

test/metabase/sync/sync_metadata/fields/active_subset_test.clj

@@ -0,0 +1,352 @@
+(ns ^:mb/driver-tests metabase.sync.sync-metadata.fields.active-subset-test
+  (:require
+   [clojure.test :refer [deftest is testing]]
+   [metabase.driver :as driver]
+   [metabase.driver.settings :as driver.settings]
+   [metabase.sync.analyze.fingerprint :as sync.fingerprint]
+   [metabase.sync.core :as sync]
+   [metabase.sync.field-values :as sync.field-values]
+   [metabase.test :as mt]
+   [toucan2.core :as t2]))
+
+;; TODO: Move dataset elsewhere!
+
+;; Sample of the following dataset
+;; {
+;;   "users": {
+;;     "otherId": "user001",
+;;     "name": "Alice Johnson",
+;;     "email": "alice.johnson@email.com",
+;;     "createdAt": "2024-01-15T00:00:00.000Z"
+;;   },
+;;   "products": {
+;;     "otherId": "prod001",
+;;     "name": "Laptop Pro 15",
+;;     "price": 1299.99,
+;;     "category": "Electronics",
+;;     "stock": 45
+;;   },
+;;   "orderItems": {
+;;     "otherId": "item001",
+;;     "links": {
+;;       "customerId": "user001",
+;;       "productId": "prod001"
+;;     },
+;;     "data": {
+;;       "orderDate": "2024-06-01T00:00:00.000Z",
+;;       "status": "delivered",
+;;       "quantity": 1,
+;;       "price": 1299.99
+;;     }
+;;   }
+;;  }
+
+(mt/defdataset orderItemsDb
+  "OrderItems dataset"
+  [["users"
+    [{:field-name "otherId" :base-type :type/Text}
+     {:field-name "name" :base-type :type/Text}
+     {:field-name "email" :base-type :type/Text}
+     {:field-name "createdAt" :base-type :type/Date}]
+    [["user001" "Alice Johnson" "alice.johnson@email.com" "2024-01-15"]
+     ["user002" "Bob Smith" "bob.smith@email.com" "2024-02-20"]
+     ["user003" "Carol White" "carol.white@email.com" "2024-03-10"]
+     ["user004" "David Brown" "david.brown@email.com" "2024-04-05"]
+     ["user005" "Emma Davis" "emma.davis@email.com" "2024-05-12"]]]
+   ["products"
+    [{:field-name "otherId" :base-type :type/Text}
+     {:field-name "name" :base-type :type/Text}
+     {:field-name "price" :base-type :type/Decimal}
+     {:field-name "category" :base-type :type/Text}
+     {:field-name "stock" :base-type :type/Integer}]
+    [["prod001" "Laptop Pro 15" 1299.99 "Electronics" 45]
+     ["prod002" "Wireless Mouse" 29.99 "Accessories" 150]
+     ["prod003" "USB-C Hub" 49.99 "Accessories" 80]
+     ["prod004" "Mechanical Keyboard" 149.99 "Accessories" 60]
+     ["prod005" "4K Monitor" 599.99 "Electronics" 25]]]
+   ["orderItems"
+    [{:field-name "otherId" :base-type :type/Text}
+     {:field-name "links" :base-type :type/Dictionary :nested-fields [{:field-name "customerId"
+                                                                       :base-type :type/Text}
+                                                                      {:field-name "productId"
+                                                                       :base-type :type/Text}]}
+     {:field-name "data" :base-type :type/Dictionary :nested-fields [{:field-name "orderDate"
+                                                                      :base-type :type/DateTime}
+                                                                     {:field-name "status"
+                                                                      :base-type :type/Text}
+                                                                     {:field-name "quantity"
+                                                                      :base-type :type/Integer}
+                                                                     {:field-name "price"
+                                                                      :base-type :type/Decimal}]}]
+    [["item001"
+      {"customerId" "user001", "productId" "prod001"}
+      {"orderDate" #t "2024-06-01", "status" "delivered", "quantity" 1, "price" 1299.99}]
+     ["item002"
+      {"customerId" "user001", "productId" "prod003"}
+      {"orderDate" #t "2024-06-01", "status" "delivered", "quantity" 1, "price" 49.99}]
+     ["item003"
+      {"customerId" "user002", "productId" "prod004"}
+      {"orderDate" #t "2024-06-15", "status" "shipped", "quantity" 1, "price" 149.99}]
+     ["item004"
+      {"customerId" "user003", "productId" "prod005"}
+      {"orderDate" #t "2024-07-02", "status" "processing", "quantity" 1, "price" 599.99}]
+     ["item005"
+      {"customerId" "user004", "productId" "prod002"}
+      {"orderDate" #t "2024-07-20", "status" "delivered", "quantity" 1, "price" 29.99}]]]])
+
+(defmulti sync-test-dataset-def
+  {:added "0.57.0" :arglists '([driver])}
+  driver/dispatch-on-initialized-driver
+  :hierarchy #'driver/hierarchy)
+
+(defmethod sync-test-dataset-def :mongo [_driver] orderItemsDb)
+(defmulti sync-test-active-subset-ids
+  {:added "0.57.0" :arglists '([driver])}
+  driver/dispatch-on-initialized-driver
+  :hierarchy #'driver/hierarchy)
+
+(defmethod sync-test-active-subset-ids :mongo [_driver]
+  (let [users-active-subset #{(mt/id :users :_id)
+                              (mt/id :users :email)
+                              (mt/id :users :createdAt)}
+        orderItems-active-subset #{(mt/id :orderItems :_id)
+                                   (mt/id :orderItems :data)
+                                   (mt/id :orderItems :data :orderDate)
+                                   (mt/id :orderItems :data :price)}
+        products-active-subset #{(mt/id :products :_id)
+                                 (mt/id :products :name)
+                                 (mt/id :products :category)}]
+    (into #{} cat [users-active-subset
+                   orderItems-active-subset
+                   products-active-subset])))
+
+(deftest ^:synchronized sync-active-subset-test
+  (mt/test-drivers
+    (mt/normal-drivers)
+    (when (driver/should-sync-active-subset? driver/*driver*)
+      (mt/dataset
+        (sync-test-dataset-def driver/*driver*)
+
+        (testing "After initial sync all fields have active_subset true"
+          (sync/sync-database! (mt/db))
+          (let [all-db-fields (t2/select :model/Field :table_id
+                                         [:in (t2/select-fn-vec :id :model/Table :db_id (mt/id))])]
+            (is (every? (comp true? :active_subset) all-db-fields))))
+
+        (testing "Syncing the db when there is more leaf fields than `driver.settings/sync-leaf-fields-limit`..."
+          (with-redefs [driver.settings/sync-leaf-fields-limit (constantly 3)]
+            (sync/sync-database! (mt/db))
+            (let [active-subset-field-ids (sync-test-active-subset-ids driver/*driver*)
+                  all-db-fields (t2/select :model/Field :table_id
+                                           [:in (t2/select-fn-vec :id :model/Table :db_id (mt/id))])]
+              (testing "... no fields are deactivated"
+                (is (every? (comp true? :active) all-db-fields)))
+              (testing "... expected fields are part of active_subset"
+              ;; Sorting of fields picked up for sync on mongo should be stable, hence we can expect
+              ;; always same set of fields.
+                (is (every? (comp true? :active_subset) (filter (comp active-subset-field-ids
+                                                                      :id)
+                                                                all-db-fields))))
+              (testing "... and all other fields are not part of the active subset"
+                (is (every? (comp false? :active_subset) (filter (comp (complement active-subset-field-ids)
+                                                                       :id)
+                                                                 all-db-fields)))))))
+
+        (testing "Follow-up sync with `driver.settings/sync-leaf-fields-limit` catching all the fields..."
+          (sync/sync-database! (mt/db))
+          (let [all-db-fields (t2/select :model/Field :table_id
+                                         [:in (t2/select-fn-vec :id :model/Table :db_id (mt/id))])]
+            (testing "... should not touch active key of any of the fields"
+              (is (every? (comp true? :active) all-db-fields)))
+            (testing "... should make every field part of the active subset again"
+              (is (every? (comp true? :active_subset) all-db-fields)))))))))
+(defmulti fingerprint-test-dataset-def
+  {:added "0.57.0" :arglists '([driver])}
+  driver/dispatch-on-initialized-driver
+  :hierarchy #'driver/hierarchy)
+
+(defmethod fingerprint-test-dataset-def :mongo [_driver] orderItemsDb)
+(defmulti fingerprint-test-active-subset-ids
+  {:added "0.57.0" :arglists '([driver])}
+  driver/dispatch-on-initialized-driver
+  :hierarchy #'driver/hierarchy)
+
+(defmethod fingerprint-test-active-subset-ids :mongo [_driver]
+  (let [users-active-subset #{#_(mt/id :users :_id)
+                              (mt/id :users :email)
+                              (mt/id :users :createdAt)}
+        orderItems-active-subset #{;; ignored because of "primary key"
+                                   #_(mt/id :orderItems :_id)
+                                   ;; ignored because of structured type
+                                   #_(mt/id :orderItems :data)
+                                   (mt/id :orderItems :data :orderDate)
+                                   ;; ignored because of float type
+                                   #_(mt/id :orderItems :data :price)}
+        products-active-subset #{#_(mt/id :products :_id)
+                                 (mt/id :products :name)
+                                 (mt/id :products :category)}]
+    (into #{} cat [users-active-subset
+                   orderItems-active-subset
+                   products-active-subset])))
+(defmulti fingerprint-test-act-and-gather!
+  {:added "0.57.0" :arglists '([driver thunk])}
+  driver/dispatch-on-initialized-driver
+  :hierarchy #'driver/hierarchy)
+
+(defmethod fingerprint-test-act-and-gather! :mongo
+  [_driver results-atom thunk]
+  (let [original-fields-to-fingerprint @#'sync.fingerprint/fields-to-fingerprint]
+    (with-redefs [driver.settings/sync-leaf-fields-limit (constantly 3)
+                  sync.fingerprint/fields-to-fingerprint
+                  (fn [& args]
+                    (let [result (apply original-fields-to-fingerprint args)]
+                      (swap! results-atom into (map :id) result)
+                      result))]
+      (thunk))))
+
+(deftest fingerprint-active-subset-test
+  (mt/test-drivers
+    (mt/normal-drivers)
+    (when (driver/should-sync-active-subset? driver/*driver*)
+      (mt/dataset
+        (fingerprint-test-dataset-def driver/*driver*)
+        (let [active-subset-field-ids (fingerprint-test-active-subset-ids driver/*driver*)
+              results-atom (atom #{})]
+          (try
+
+            ;; First adjust fingerprint version so fields are considered for fingerprinting
+            (t2/update! :model/Field :id [:in active-subset-field-ids] {:fingerprint_version 0})
+
+            (testing "Check that only active subset fields are picked for fingerprinting"
+
+              (fingerprint-test-act-and-gather!
+               driver/*driver* results-atom
+               #(testing "... for that do full sync catching synthetic exception with considered fields"
+                  (sync/sync-database! (mt/db) {:scan :full})))
+
+              (testing "... and now the actual check"
+                (is (= (count active-subset-field-ids)
+                       (count @results-atom)))
+                (loop [[first-exp-id & rest-exp-ids] active-subset-field-ids
+                       returned-field-ids  @results-atom]
+                  (if (nil? first-exp-id)
+                    (is (empty? returned-field-ids))
+                    (do (is (contains? returned-field-ids first-exp-id))
+                        (recur rest-exp-ids
+                               (disj returned-field-ids first-exp-id)))))))
+
+            (finally
+              (testing "Finally restore fields to the original state"
+              ;; Full sync should adjust fingerprint version of modified fields back to the original
+              ;; and make everything active.
+                (sync/sync-database! (mt/db) {:scan :full})
+                (let [all-db-fields (t2/select :model/Field :table_id
+                                               [:in (t2/select-fn-vec :id :model/Table :db_id (mt/id))])]
+                  (is (every? (comp true? :active) all-db-fields))
+                  (is (every? (comp true? :active_subset) all-db-fields))
+                  (is (every? (comp #{0 5} :fingerprint_version)
+                              all-db-fields)))))))))))
+
+(defmulti field-values-test-dataset-def
+  {:added "0.57.0" :arglists '([driver])}
+  driver/dispatch-on-initialized-driver
+  :hierarchy #'driver/hierarchy)
+
+(defmethod field-values-test-dataset-def :mongo [_driver] orderItemsDb)
+(defmulti field-values-test-active-subset-ids
+  {:added "0.57.0" :arglists '([driver])}
+  driver/dispatch-on-initialized-driver
+  :hierarchy #'driver/hierarchy)
+
+(defmethod field-values-test-active-subset-ids :mongo [_driver]
+  (let [users-active-subset #{(mt/id :users :_id)
+                              (mt/id :users :email)
+                              (mt/id :users :createdAt)}
+        orderItems-active-subset #{(mt/id :orderItems :_id)
+                                   ;; This is part of active subset, but has visibility other than normal
+                                   ;; hence not part of scanned fields
+                                   #_(mt/id :orderItems :data)
+                                   (mt/id :orderItems :data :orderDate)
+                                   (mt/id :orderItems :data :price)}
+        products-active-subset #{(mt/id :products :_id)
+                                 (mt/id :products :name)
+                                 (mt/id :products :category)}]
+    (into #{} cat [users-active-subset
+                   orderItems-active-subset
+                   products-active-subset])))
+
+(defmulti field-values-test-act-and-gather!
+  {:added "0.57.0" :arglists '([driver])}
+  driver/dispatch-on-initialized-driver
+  :hierarchy #'driver/hierarchy)
+
+(defmethod field-values-test-act-and-gather! :mongo
+  [_driver results-atom thunk]
+  (let [original-table->fields-to-scan @#'sync.field-values/table->fields-to-scan]
+    (with-redefs [driver.settings/sync-leaf-fields-limit (constantly 3)
+                  sync.field-values/table->fields-to-scan
+                  (fn [& args]
+                    (let [result (apply original-table->fields-to-scan args)]
+                      (swap! results-atom into (map :id) result)
+                      result))]
+      (thunk))))
+
+(deftest ewqfield-values-active-subset-test
+  (mt/test-drivers
+    (mt/normal-drivers)
+    (when (driver/should-sync-active-subset? driver/*driver*)
+      (mt/dataset
+        (field-values-test-dataset-def driver/*driver*)
+        (let [active-subset-field-ids (field-values-test-active-subset-ids driver/*driver*)
+              results-atom (atom #{})]
+          (try
+            (testing "Check that only active subset fields are picked for field values scan"
+
+              (field-values-test-act-and-gather!
+               driver/*driver* results-atom
+               #(testing "... for that do full sync catching synthetic exception with considered fields"
+                  (sync/sync-database! (mt/db) {:scan :full})))
+
+              (testing "... and now the actual check"
+                (is (= (count active-subset-field-ids)
+                       (count @results-atom)))
+                (loop [[first-exp-id & rest-exp-ids] active-subset-field-ids
+                       returned-field-ids @results-atom]
+                  (if (nil? first-exp-id)
+                    (is (empty? returned-field-ids))
+                    (do (is (contains? returned-field-ids first-exp-id))
+                        (recur rest-exp-ids
+                               (disj returned-field-ids first-exp-id)))))))
+            (finally
+              (testing "Finally restore fields to the original state"
+              ;; Full sync should clean up active_subset of modified fields
+                (sync/sync-database! (mt/db) {:scan :full})
+                (let [all-db-fields (t2/select :model/Field :table_id
+                                               [:in (t2/select-fn-vec :id :model/Table :db_id (mt/id))])]
+                  (is (every? (comp true? :active) all-db-fields))
+                  (is (every? (comp true? :active_subset) all-db-fields)))))))))))
+
+(deftest ^:synchronized null-on-unsupported-drivers-active-subset-test
+  (mt/test-drivers
+    (mt/normal-drivers)
+    (when-not (driver/should-sync-active-subset? driver/*driver*)
+      (mt/dataset
+        test-data
+        (testing "active_subset key is null after sync on drivers that do not support it..."
+          (sync/sync-database! (mt/db))
+          (let [all-db-fields (t2/select :model/Field :table_id
+                                         [:in (t2/select-fn-vec :id :model/Table :db_id (mt/id))])]
+            (is (every? (comp true? :active) all-db-fields))
+            (is (every? (comp nil? :active_subset) all-db-fields)))
+
+          (let [field-id (mt/id :venues :price)]
+            (try
+              (testing "... even after a field was re-activated"
+                (t2/update! :model/Field :id field-id {:active false})
+                (sync/sync-database! (mt/db))
+                (let [all-db-fields (t2/select :model/Field :table_id
+                                               [:in (t2/select-fn-vec :id :model/Table :db_id (mt/id))])]
+                  (is (every? (comp true? :active) all-db-fields))
+                  (is (every? (comp nil? :active_subset) all-db-fields))))
+              (finally
+                (t2/update! :model/Field :id field-id {:active true})))))))))

test/metabase/sync/sync_metadata/fields/sync_instances_test.clj

@@ -1,4 +1,4 @@
-(ns metabase.sync.sync-metadata.fields.sync-instances-test
+(ns ^:mb/driver-tests metabase.sync.sync-metadata.fields.sync-instances-test
   (:require
    [clojure.test :refer :all]
    [metabase.sync.sync-metadata :as sync-metadata]