DO NOT SUBMIT a first draft of the /check_card API

Not properly tested; needs EE token changes for testing. Just posting
this so it isn't blocking; I'll clean it up later today.

.clj-kondo/config/modules/config.edn

@@ -1967,6 +1967,16 @@
            task-history
            util}}
 
+  enterprise/dependencies
+  {:team "Querying"
+   :api  #{metabase-enterprise.dependencies.core}
+   :uses #{api
+           lib
+           lib-be
+           models
+           queries
+           util}}
+
   enterprise/upload-management
   {:team "UX West"
    :api  #{metabase-enterprise.upload-management.api}

enterprise/backend/src/metabase_enterprise/api/routes.clj

@@ -19,6 +19,7 @@
    [metabase-enterprise.content-verification.api.routes]
    [metabase-enterprise.database-replication.api :as database-replication.api]
    [metabase-enterprise.database-routing.api]
+   [metabase-enterprise.dependencies.api]
    [metabase-enterprise.documents.api]
    [metabase-enterprise.email.api]
    [metabase-enterprise.gsheets.api :as gsheets.api]
@@ -49,6 +50,7 @@
    :ai-entity-analysis         (deferred-tru "AI Entity Analysis")
    :collection-cleanup         (deferred-tru "Collection Cleanup")
    :content-translation        (deferred-tru "Content translation")
+   :dependencies               (deferred-tru "Dependency Tracking")
    :documents                  (deferred-tru "Documents")
    :etl-connections            (deferred-tru "ETL Connections")
    :etl-connections-pg         (deferred-tru "ETL Connections PG replication")
@@ -98,6 +100,7 @@
                                        (premium-handler :etl-connections)
                                        (premium-handler :etl-connections-pg))
    "/database-routing"             (premium-handler metabase-enterprise.database-routing.api/routes :database-routing)
+   "/dependencies"                 (premium-handler metabase-enterprise.dependencies.api/routes :dependencies)
    "/document"                     (premium-handler metabase-enterprise.documents.api/routes :documents)
    "/email"                        (premium-handler metabase-enterprise.email.api/routes :cloud-custom-smtp)
    "/gsheets"                      (-> gsheets.api/routes ;; gsheets requires both features.

enterprise/backend/src/metabase_enterprise/dependencies/api.clj

@@ -0,0 +1,51 @@
+(ns metabase-enterprise.dependencies.api
+  (:require
+   [metabase-enterprise.dependencies.core :as dependencies]
+   [metabase.api.macros :as api.macros]
+   [metabase.api.routes.common :refer [+auth]]
+   [metabase.api.util.handlers :as handlers]
+   [metabase.lib-be.metadata.jvm :as lib-be.metadata.jvm]
+   [metabase.lib.metadata :as lib.metadata]
+   [metabase.models.interface :as mi]
+   [metabase.queries.schema :as queries.schema]
+   [metabase.util.log :as log]
+   [metabase.util.malli.registry :as mr]
+   [metabase.util.malli.schema :as ms]
+   [toucan2.core :as t2]))
+
+(mr/def ::card-body
+  [:map
+   [:id            {:optional false} ms/PositiveInt]
+   [:database_id   {:optional false} ms/PositiveInt]
+   [:dataset_query {:optional true}  [:maybe ms/Map]]
+   [:type          {:optional true}  [:maybe ::queries.schema/card-type]]])
+
+(api.macros/defendpoint :post "/check_card"
+  "Check a proposed edit to a card, and return the card IDs for those cards this edit will break."
+  [_route-params
+   _query-params
+   body :- ::card-body]
+  (let [base-provider (lib-be.metadata.jvm/application-database-metadata-provider (:database_id body))
+        original      (lib.metadata/card base-provider (:id body))
+        card          (-> original
+                          (assoc :dataset-query (:dataset_query body)
+                                 :type          (:type body (:type original)))
+                          (dissoc :result-metadata))
+        ;; TODO: This sucks - it's getting all cards for the same database_id, which is slow and over-reaching.
+        all-cards     (t2/select-fn-set :id :model/Card :database_id (:database_id body))
+        breakages     (dependencies/check-cards-have-sound-refs base-provider [card] all-cards)
+        broken-ids    (keys breakages)
+        broken-cards  (when (seq broken-ids)
+                        (t2/select :model/Card :id [:in broken-ids]))]
+    {:success   (empty? breakages)
+     :bad_cards (into [] (filter (fn [card]
+                                   (if (mi/can-read? card)
+                                     card
+                                     (do (log/warnf "Eliding broken card %d - not readable by the user" (:id card))
+                                         nil))))
+                      broken-cards)}))
+
+(def ^{:arglists '([request respond raise])} routes
+  "`/api/ee/dependencies` routes."
+  (handlers/routes
+   (api.macros/ns-handler *ns* +auth)))

enterprise/backend/src/metabase_enterprise/dependencies/core.clj

@@ -0,0 +1,49 @@
+(ns metabase-enterprise.dependencies.core
+  "API namespace for the `metabase-enterprise.dependencies` module."
+  (:require
+   [metabase.lib.core :as lib]
+   [metabase.lib.metadata :as lib.metadata]
+   [metabase.lib.metadata.cached-provider :as lib.metadata.cached-provider]
+   [metabase.lib.metadata.protocols :as lib.metadata.protocols]
+   [metabase.lib.schema.id :as lib.schema.id]
+   [metabase.lib.schema.mbql-clause :as lib.schema.mbql-clause]
+   [metabase.lib.schema.metadata :as lib.schema.metadata]
+   [metabase.util.malli :as mu]))
+
+(defn- provider-with-updated-cards [base-provider cards]
+  (-> base-provider
+      lib.metadata.cached-provider/cached-metadata-provider
+      (doto (lib.metadata.protocols/store-metadatas! (map #(dissoc % :result-metadata) cards)))))
+
+(mu/defn check-cards-have-sound-refs :- [:map-of ::lib.schema.id/card [:sequential ::lib.schema.mbql-clause/clause]]
+  "Given a list `updated-cards` of `:metadata/card`s which may have changes vs. AppDB, and a list `check-cards` of
+  cards to check, return any bad clauses found in the `check-cards`.
+
+  May return false positives: that is, if one of the `check-cards` has a bad ref in it, then it will be returned
+  even if that bad ref has nothing to do with the `updated-cards`.
+
+  Returns a map `{card-id [bad refs...]}`, which will be empty if there are no bad refs detected."
+  [base-provider  :- ::lib.schema.metadata/metadata-provider
+   updated-cards  :- [:sequential ::lib.schema.metadata/card]
+   check-card-ids :- [:sequential ::lib.schema.id/card]]
+  (let [provider (provider-with-updated-cards base-provider updated-cards)]
+    (reduce (fn [errors card-id]
+              (let [card     (lib.metadata/card provider card-id)
+                    query    (lib/query provider (:dataset-query card))
+                    bad-refs (lib/find-bad-refs query)]
+                (cond-> errors
+                  bad-refs (assoc (:id card) bad-refs))))
+            {}
+            check-card-ids)))
+
+(comment
+  ;; This should work on any fresh-ish Metabase instance; these are the built-in example questions.
+  (let [card-ids [1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
+                  21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37]
+        base-mp  #_{:clj-kondo/ignore [:unresolved-namespace]}
+        (metabase.lib-be.metadata.jvm/application-database-metadata-provider 1)
+        card     (lib.metadata/card base-mp 1)
+        card'    (update-in card [:dataset-query :query :expressions]
+                            ;; Replacing the expression Age with Duration - this breaks downstream uses!
+                            update-keys (constantly "Duration"))]
+    (check-cards-have-sound-refs base-mp [card'] card-ids)))

src/metabase/lib/core.cljc

@@ -47,6 +47,7 @@
    [metabase.lib.template-tags :as lib.template-tags]
    [metabase.lib.temporal-bucket :as lib.temporal-bucket]
    [metabase.lib.util :as lib.util]
+   [metabase.lib.validate :as lib.validate]
    [metabase.util.namespaces :as shared.ns]))
 
 (comment lib.aggregation/keep-me
@@ -410,4 +411,6 @@
   previous-stage-number
   query-stage
   source-table-id
-  update-query-stage])
+  update-query-stage]
+ [lib.validate
+  find-bad-refs])

src/metabase/lib/validate.cljc

@@ -0,0 +1,40 @@
+(ns metabase.lib.validate
+  "Checks and validation for queries."
+  (:require
+   [metabase.lib.field.resolution :as lib.field.resolution]
+   [metabase.lib.schema :as lib.schema]
+   [metabase.lib.schema.mbql-clause :as lib.schema.mbql-clause]
+   [metabase.lib.walk :as lib.walk]
+   [metabase.util.malli :as mu]))
+
+(mu/defn find-bad-refs :- [:maybe [:sequential ::lib.schema.mbql-clause/clause]]
+  "Returns a list of bad `:field` refs on this query.
+
+  Returns nil if all refs on the query are sound, that is if they can be resolved to a column from some source."
+  [query :- ::lib.schema/query]
+  (let [bad-fields (volatile! [])]
+    (lib.walk/walk-clauses
+     query (fn [query path-type path clause]
+             (when (and (= path-type :lib.walk/stage)
+                        (vector? clause)
+                        (= (first clause) :field))
+               (let [column (lib.walk/apply-f-for-stage-at-path
+                             lib.field.resolution/resolve-field-ref
+                             query path clause)]
+                 (when (or (not column)
+                           (::lib.field.resolution/fallback-metadata? column))
+                   (vswap! bad-fields conj clause))))
+             nil))
+    (not-empty @bad-fields)))
+
+(comment
+  (require '[metabase.lib.core :as lib])
+  (require '[metabase.lib.test-metadata :as meta])
+  (let [base             (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
+                             (lib/breakout (meta/field-metadata :orders :quantity))
+                             (lib/aggregate (lib/count))
+                             lib/append-stage)
+        [_category cnt] (lib/visible-columns base)
+        query            (lib/filter base (lib/> cnt 100))]
+    {:valid-query (find-bad-refs query)
+     :bad-query   (find-bad-refs (assoc-in query [:stages 1 :filters 0 2 2] "n"))}))