Compare commits

...

6 Commits

Author SHA1 Message Date
William Zimrin
fc9780ed9f make resolve-field public again 2025-10-10 17:35:56 -05:00
William
fe89c9bc8a Merge branch 'master' into fix-bigquery-dependency-error 2025-10-10 16:26:46 -05:00
William Zimrin
09ec2e171d support oracle rownum col 2025-10-10 16:23:49 -05:00
William Zimrin
5730815cdf fix rename 2025-10-07 15:58:04 -05:00
William Zimrin
cefceb1cc5 fix bigquery validation failures 2025-10-07 15:43:55 -05:00
William Zimrin
d5516cf7e6 Fix bigquery dependencies code erroring
Bigquery had a defmethod that wasn't updated to support the new native-query-deps arglists.  Also adds cross-driver
tests for the various native validation/parsing functions.
2025-10-07 14:51:19 -05:00
4 changed files with 73 additions and 10 deletions

View File

@@ -1,4 +1,4 @@
(ns metabase-enterprise.dependencies.native-validation-test
(ns ^:mb/driver-tests metabase-enterprise.dependencies.native-validation-test
(:require
[clojure.test :refer :all]
[metabase-enterprise.dependencies.native-validation :as deps.native-validation]
@@ -6,7 +6,9 @@
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.test-metadata :as meta]))
[metabase.lib.test-metadata :as meta]
[metabase.query-processor.compile :as qp.compile]
[metabase.test :as mt]))
(defn- fake-query
([mp query]
@@ -15,6 +17,23 @@
(-> (lib/native-query mp query)
(lib/with-template-tags template-tags))))
(defn- sample-mbql-query []
(let [mp (mt/metadata-provider)
driver (:engine (lib.metadata/database mp))
base-query (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))
(lib/join (-> (lib/join-clause
(lib.metadata/table mp (mt/id :products))
[(lib/= (lib.metadata/field mp (mt/id :orders :product_id))
(-> (lib.metadata/field mp (mt/id :products :id))
(lib/with-join-alias "Products")))])
(lib/with-join-fields :all)
(lib/with-join-alias "Products"))))
available-fields (lib/fieldable-columns base-query)
query (lib/with-fields base-query (take 2 available-fields))]
{:driver driver
:query query
:mp mp}))
(defn- validates?
[mp driver card-id expected]
(is (=? expected
@@ -61,6 +80,13 @@
(lib-be/normalize-query mp)
(deps.native-validation/native-query-deps driver))))))
(deftest ^:parallel cross-driver-deps-test
(testing "native-query-deps handles valid queries in current driver"
(mt/test-drivers (mt/normal-drivers-with-feature :dependencies/native)
(let [{:keys [driver query mp]} (sample-mbql-query)]
(is (= #{{:table (mt/id :products)} {:table (mt/id :orders)}}
(deps.native-validation/native-query-deps driver mp query)))))))
(deftest ^:parallel validate-bad-queries-test
(testing "validate-native-query handles nonsense queries"
(let [mp (deps.tu/default-metadata-provider)
@@ -87,6 +113,12 @@
driver
(fake-query mp "select bad from products"))))))))
(deftest ^:parallel cross-driver-validation-test
(testing "validate-native-query handles valid queries in current driver"
(mt/test-drivers (mt/normal-drivers-with-feature :dependencies/native)
(let [{:keys [driver query mp]} (sample-mbql-query)]
(is (empty? (deps.native-validation/validate-native-query driver mp query)))))))
(deftest ^:parallel validate-native-query-with-subquery-columns-test
(testing "validate-native-query should detect invalid columns in subqueries"
(let [mp (deps.tu/default-metadata-provider)
@@ -236,3 +268,12 @@
driver mp
"this is not a query"
[])))))
(deftest ^:parallel cross-driver-metadata-test
(testing "native-result-metadata handles valid queries in current driver"
(mt/test-drivers (mt/normal-drivers-with-feature :dependencies/native)
(let [{:keys [driver query mp]} (sample-mbql-query)]
(is (=? (-> (into [(lib.metadata/field mp (mt/id :orders :id))
(lib.metadata/field mp (mt/id :orders :user_id))]
(lib.metadata/fields mp (mt/id :products))))
(deps.native-validation/native-result-metadata driver mp query)))))))

View File

@@ -1024,6 +1024,15 @@
list-datasets
(some #{schema}))))
(defmethod driver.sql/resolve-field [:bigquery-cloud-sdk :all-columns]
[driver metadata-provider col-spec]
(let [parent-method (get-method driver.sql/resolve-field [:sql :all-columns])
old-table-ref (:table col-spec)
[new-schema new-table] (str/split (:table old-table-ref) #"\.")
new-table-ref (assoc old-table-ref :table new-table :schema new-schema)]
(->> (assoc col-spec :table new-table-ref)
(parent-method driver metadata-provider))))
(defmethod driver/table-name-length-limit :bigquery-cloud-sdk
[_driver]
;; https://cloud.google.com/bigquery/docs/tables

View File

@@ -749,3 +749,15 @@
(defmethod sql-jdbc/impl-table-known-to-not-exist? :oracle
[_ ^SQLException e]
(= (.getErrorCode e) 942))
(defmethod driver.sql/resolve-field [:oracle :single-column]
[driver metadata-provider col-spec]
(let [parent-method (get-method driver.sql/resolve-field [:sql :single-column])]
(if (= (:column col-spec) "rownum")
[{:base-type :type/Integer
:name "rownum"
:lib/desired-column-alias (or (:alias col-spec) "rownum")
:display-name "Row Num"
:effective-type :type/Integer
:semantic-type :Semantic/*}]
(parent-method driver metadata-provider col-spec))))

View File

@@ -193,7 +193,7 @@
;;; | Dependencies |
;;; +----------------------------------------------------------------------------------------------------------------+
(defmulti ^:private resolve-field
(defmulti resolve-field
"Resolves a field reference to one or more actual database fields.
This uses a supplied metadata provider instead of hitting the db directly. 'Field reference' refers to the field
@@ -202,10 +202,11 @@
Note: this currently sets :lib/desired-column-alias but no other :lib/* fields, because the callers of this function
don't need the other fields. If we care about other :lib/* fields in the future, we can add them then."
{:added "0.57.0" :arglists '([driver metadata-provider col-spec])}
(fn [_driver _metadata-provider col-spec]
(:type col-spec)))
(fn [driver _metadata-provider col-spec]
[(driver/dispatch-on-initialized-driver driver) (:type col-spec)])
:hierarchy #'driver/hierarchy)
(defmethod resolve-field :all-columns
(defmethod resolve-field [:sql :all-columns]
[driver metadata-provider col-spec]
(or (some->> (:table col-spec)
(find-table-or-transform driver (driver-api/tables metadata-provider) (driver-api/transforms metadata-provider))
@@ -214,7 +215,7 @@
(map #(assoc % :lib/desired-column-alias (:name %))))
[(assoc col-spec ::bad-reference true)]))
(defmethod resolve-field :single-column
(defmethod resolve-field [:sql :single-column]
[driver metadata-provider {:keys [alias] :as col-spec}]
[(if-let [{:keys [name] :as found}
(->> (:source-columns col-spec)
@@ -238,7 +239,7 @@
(->> (get-name m)
(u.humanization/name->human-readable-name :simple)))
(defmethod resolve-field :custom-field
(defmethod resolve-field [:sql :custom-field]
[_driver _metadata-provider col-spec]
[{:base-type :type/*
:name (get-name col-spec)
@@ -247,7 +248,7 @@
:effective-type :type/*
:semantic-type :Semantic/*}])
(defmethod resolve-field :invalid-table-wildcard
(defmethod resolve-field [:sql :invalid-table-wildcard]
[_driver _metadata-provider col-spec]
[(assoc col-spec ::bad-reference true)])
@@ -261,7 +262,7 @@
(apply (partial max-key (comp count ancestors)) common-ancestors)
default-type)))
(defmethod resolve-field :composite-field
(defmethod resolve-field [:sql :composite-field]
[driver metadata-provider col-spec]
(let [member-fields (mapcat (partial resolve-field driver metadata-provider)
(:member-fields col-spec))]