Compare commits

...

11 Commits

Author SHA1 Message Date
Bryan Maass
3620458199 going too far 2025-07-29 16:25:51 -06:00
Bryan Maass
a8ee9ea81e add more tests 2025-07-29 11:39:36 -06:00
Bryan Maass
a51fc71609 remove debug def 2025-07-29 09:26:15 -06:00
Bryan Maass
ceba54bfbe add more comments + fix some test schemas 2025-07-28 15:15:58 -06:00
Bryan Maass
f459a6a74a fix schema oopsie 2025-07-28 14:58:40 -06:00
Bryan Maass
df2d52ac81 fix merge paren balance 2025-07-28 14:53:07 -06:00
Bryan Maass
e302a8cec6 Merge remote-tracking branch 'origin/master' into fixing-malli-schema-caching 2025-07-28 14:51:01 -06:00
Bryan Maass
41807fd4e6 uncomment functions, removing nocommits 2025-07-28 14:42:19 -06:00
Bryan Maass
5c8b8e5557 app starts up 2025-07-28 14:25:24 -06:00
Bryan Maass
b2cde2261d fix another 150 2025-07-28 12:04:58 -06:00
Bryan Maass
993af7ba38 wip 2025-07-25 16:59:44 -06:00
59 changed files with 1349 additions and 749 deletions

View File

@@ -334,7 +334,7 @@
[:type [:enum :number, :string, :date, :datetime, :time, :boolean, :null]]
[:description {:optional true} [:maybe :string]]
[:table-reference {:optional true} :string]]
[:map {:encode/ai-service-request #(set/rename-keys % {:table-reference :table_reference})}]])
[:map {:encode/ai-service-request (mr/with-key #(set/rename-keys % {:table-reference :table_reference}))}]])
(mr/def ::example-generation-payload
[:map
@@ -352,8 +352,8 @@
[:description {:optional true} [:maybe :string]]
[:queryable-dimensions [:sequential ::example-generation-column]]
[:default-time-dimension {:optional true} ::example-generation-column]]
[:map {:encode/ai-service-request #(set/rename-keys % {:default-time-dimension :default_time_dimension
:queryable-dimensions :queryable_dimensions})}]]]]])
[:map {:encode/ai-service-request (mr/with-key #(set/rename-keys % {:default-time-dimension :default_time_dimension
:queryable-dimensions :queryable_dimensions}))}]]]]])
(mu/defn generate-example-questions
"Generate example questions for the given models and metrics."

View File

@@ -13,11 +13,11 @@
(mr/def ::message
[:and
[:map
{:decode/api-response #(update-keys % metabot-v3.u/safe->kebab-case-en)}
{:decode/api-response (mr/with-key #(update-keys % metabot-v3.u/safe->kebab-case-en))}
[:role ::role]
[:content {:optional true} [:maybe :string]]
[:navigate-to {:optional true} [:maybe :string]]]
[:map {:encode/api-request #(update-keys % metabot-v3.u/safe->snake_case_en)}]])
[:map {:encode/api-request (mr/with-key #(update-keys % metabot-v3.u/safe->snake_case_en))}]])
(mr/def ::messages
[:sequential ::message])

View File

@@ -14,10 +14,11 @@
"A MetaBot v3 reaction type keyword e.g. `:metabot.reaction/message`"
[:fn
{:error/message "Reaction type must be a kebab-case keyword starting whose namespace is `metabot.reaction`."}
(fn [x]
(and (qualified-keyword? x)
(= (u/->kebab-case-en (u/qualified-name x)) (u/qualified-name x))
(= (namespace x) "metabot.reaction")))])
(mr/with-key
(fn [x]
(and (qualified-keyword? x)
(= (u/->kebab-case-en (u/qualified-name x)) (u/qualified-name x))
(= (namespace x) "metabot.reaction"))))])
;;; TODO -- need Kondo hook that registers the keyword
(mu/defn defreaction
@@ -36,7 +37,7 @@
;;; this is just a placeholder so LSP can register the place it lives for jump-to-definition functionality. Actual
;;; schema gets created below by [[reaction-schema]] and [[update-reaction-schema!]]
(mr/def ::reaction any?)
(mr/def ::reaction :any)
(defn- reaction-schema
"Build the schema for `::reaction`."

View File

@@ -86,7 +86,7 @@
"is-null" "is-not-null"
"string-is-empty" "string-is-not-empty"
"is-true" "is-false"]]]
[:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])
[:map {:encode/tool-api-request (mr/with-key #(update-keys % metabot-v3.u/safe->kebab-case-en))}]])
(mr/def ::temporal-extraction-filter
[:and
@@ -101,7 +101,7 @@
"minute-equals" "minute-not-equals"
"second-equals" "second-not-equals"]]
[:value :int]]
[:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])
[:map {:encode/tool-api-request (mr/with-key #(update-keys % metabot-v3.u/safe->kebab-case-en))}]])
(mr/def ::disjunctive-temporal-extraction-filter
[:and
@@ -116,7 +116,7 @@
"minute-equals" "minute-not-equals"
"second-equals" "second-not-equals"]]
[:values [:sequential :int]]]
[:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])
[:map {:encode/tool-api-request (mr/with-key #(update-keys % metabot-v3.u/safe->kebab-case-en))}]])
(mr/def ::temporal-filter
[:and
@@ -131,7 +131,7 @@
"date-before" "date-on-or-before"
"date-after" "date-on-or-after"]]
[:value [:or :string :int]]]
[:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])
[:map {:encode/tool-api-request (mr/with-key #(update-keys % metabot-v3.u/safe->kebab-case-en))}]])
(mr/def ::disjunctive-temporal-filter
[:and
@@ -143,7 +143,7 @@
"greater-than" "greater-than-or-equal"
"less-than" "less-than-or-equal"]]
[:values [:sequential [:or :string :int]]]]
[:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])
[:map {:encode/tool-api-request (mr/with-key #(update-keys % metabot-v3.u/safe->kebab-case-en))}]])
(mr/def ::string-filter
[:and
@@ -155,7 +155,7 @@
"string-contains" "string-not-contains"
"string-starts-with" "string-ends-with"]]
[:value :string]]
[:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])
[:map {:encode/tool-api-request (mr/with-key #(update-keys % metabot-v3.u/safe->kebab-case-en))}]])
(mr/def ::disjunctive-string-date-filter
[:and
@@ -166,7 +166,7 @@
"string-contains" "string-not-contains"
"string-starts-with" "string-ends-with"]]
[:values [:sequential :string]]]
[:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])
[:map {:encode/tool-api-request (mr/with-key #(update-keys % metabot-v3.u/safe->kebab-case-en))}]])
(mr/def ::numeric-filter
[:and
@@ -180,7 +180,7 @@
"number-greater-than" "number-greater-than-or-equal"
"number-less-than" "number-less-than-or-equal"]]
[:value [:or :int :double]]]
[:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])
[:map {:encode/tool-api-request (mr/with-key #(update-keys % metabot-v3.u/safe->kebab-case-en))}]])
(mr/def ::disjunctive-numeric-filter
[:and
@@ -190,7 +190,7 @@
"equals" "not-equals"
"number-equals" "number-not-equals"]]
[:values [:sequential [:or :int :double]]]]
[:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])
[:map {:encode/tool-api-request (mr/with-key #(update-keys % metabot-v3.u/safe->kebab-case-en))}]])
(mr/def ::filter
[:or
@@ -207,7 +207,7 @@
[:field_granularity {:optional true}
[:maybe [:enum {:encode/tool-api-request keyword}
"day" "week" "month" "quarter" "year"]]]]
[:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])
[:map {:encode/tool-api-request (mr/with-key #(update-keys % metabot-v3.u/safe->kebab-case-en))}]])
(mr/def ::query-metric-arguments
[:and
@@ -215,7 +215,7 @@
[:metric_id :int]
[:filters {:optional true} [:maybe [:sequential ::filter]]]
[:group_by {:optional true} [:maybe [:sequential ::group-by]]]]
[:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])
[:map {:encode/tool-api-request (mr/with-key #(update-keys % metabot-v3.u/safe->kebab-case-en))}]])
(mr/def ::aggregation
[:and
@@ -225,14 +225,14 @@
[:sort_order {:optional true} [:maybe [:enum {:encode/tool-api-request keyword} "asc" "desc"]]]
[:function [:enum {:encode/tool-api-request keyword}
"avg" "count" "count-distinct" "max" "min" "sum"]]]
[:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])
[:map {:encode/tool-api-request (mr/with-key #(update-keys % metabot-v3.u/safe->kebab-case-en))}]])
(mr/def ::field
[:and
[:map
[:field_id :string]
[:bucket {:optional true} ::bucket]]
[:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])
[:map {:encode/tool-api-request (mr/with-key #(update-keys % metabot-v3.u/safe->kebab-case-en))}]])
(mr/def ::query-model-arguments
[:and
@@ -246,21 +246,17 @@
[:field ::field]
[:direction [:enum {:encode/tool-api-request keyword} "asc" "desc"]]]]]]
[:limit {:optional true} [:maybe :int]]]
[:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])
[:map {:encode/tool-api-request (mr/with-key #(update-keys % metabot-v3.u/safe->kebab-case-en))}]])
(mr/def ::count
[:and
:int
[:fn
{:error/message "Valid count, a natural number"}
#(<= 0 %)]])
[:int {:error/message "Valid count, a natural number" :min 0}])
(mr/def ::proportion
[:and
number?
[:fn
{:error/message "Valid proportion between (inclusive) 0 and 1."}
#(<= 0 % 1)]])
[:or {:error/message "Valid proportion between (inclusive) 0 and 1."}
[:double {:min 0 :max 1}]
[:float {:min 0 :max 1}]
[:= 1]
[:= 0]])
(mr/def ::field-values
[:or
@@ -269,7 +265,7 @@
[:sequential :string]])
(mr/def ::statistics
[:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
[:map {:decode/tool-api-response (mr/with-key #(update-keys % metabot-v3.u/safe->snake_case_en))}
[:distinct_count {:optional true} [:maybe ::count]]
[:percent_null {:optional true} [:maybe ::proportion]]
[:min {:optional true} [:maybe number?]]
@@ -289,9 +285,9 @@
(mr/def ::field-values-result
[:or
[:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
[:map {:decode/tool-api-response (mr/with-key #(update-keys % metabot-v3.u/safe->snake_case_en))}
[:structured_output
[:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
[:map {:decode/tool-api-response (mr/with-key #(update-keys % metabot-v3.u/safe->snake_case_en))}
[:field_id :string]
[:statistics {:optional true} [:maybe ::statistics]]
[:values {:optional true} [:maybe [:sequential :any]]]]]]
@@ -299,17 +295,20 @@
[:output :string]]])
(mr/def ::field-type
[:enum {:decode/tool-api-response #(when % (-> % name u/->snake_case_en))}
[:enum {:decode/tool-api-response (mr/with-key
#(when % (-> % name u/->snake_case_en)))}
"boolean" "date" "datetime" "time" "number" "string"])
(mr/def ::column
[:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
[:map {:decode/tool-api-response (mr/with-key
#(update-keys % metabot-v3.u/safe->snake_case_en))}
[:field_id :string]
[:name :string]
[:type [:maybe ::field-type]]
[:description {:optional true} [:maybe :string]]
[:semantic_type {:optional true
:decode/tool-api-response #(some-> % name u/->snake_case_en)}
:decode/tool-api-response (mr/with-key
#(some-> % name u/->snake_case_en))}
[:maybe :string]]
[:field_values {:optional true} ::field-values]])
@@ -318,9 +317,11 @@
(mr/def ::filtering-result
[:or
[:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
[:map {:decode/tool-api-response (mr/with-key
#(update-keys % metabot-v3.u/safe->snake_case_en))}
[:structured_output
[:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
[:map {:decode/tool-api-response (mr/with-key
#(update-keys % metabot-v3.u/safe->snake_case_en))}
[:type [:= :query]]
[:query_id :string]
[:query mbql.s/Query]
@@ -338,10 +339,11 @@
[:with_metric_default_temporal_breakout {:optional true, :default true} :boolean]
[:with_metric_queryable_dimensions {:optional true, :default true} :boolean]]
[:map {:encode/tool-api-request
#(set/rename-keys % {:with_model_fields :with-fields?
:with_model_metrics :with-metrics?
:with_metric_default_temporal_breakout :with-default-temporal-breakout?
:with_metric_queryable_dimensions :with-queryable-dimensions?})}]])
(mr/with-key
#(set/rename-keys % {:with_model_fields :with-fields?
:with_model_metrics :with-metrics?
:with_metric_default_temporal_breakout :with-default-temporal-breakout?
:with_metric_queryable_dimensions :with-queryable-dimensions?}))}]])
(mr/def ::subscription-schedule
(let [days ["sunday" "monday" "tuesday" "wednesday" "thursday" "friday" "saturday"]]
@@ -364,7 +366,7 @@
(for [fl ["first" "last"]
day days]
(str fl "-" day)))]]]
[:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]]))
[:map {:encode/tool-api-request (mr/with-key #(update-keys % metabot-v3.u/safe->kebab-case-en))}]]))
(mr/def ::create-dashboard-subscription-arguments
[:and
@@ -372,7 +374,7 @@
[:dashboard_id :int]
[:email :string]
[:schedule ::subscription-schedule]]
[:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])
[:map {:encode/tool-api-request (mr/with-key #(update-keys % metabot-v3.u/safe->kebab-case-en))}]])
(mr/def ::field-values-arguments
[:and
@@ -381,7 +383,7 @@
[:entity_id :int]
[:field_id :string]
[:limit {:optional true} [:maybe :int]]]
[:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])
[:map {:encode/tool-api-request (mr/with-key #(update-keys % metabot-v3.u/safe->kebab-case-en))}]])
(mr/def ::filter-records-arguments
[:and
@@ -393,12 +395,12 @@
[:query_id {:optional true} :string]]
[:map [:report_id :int]]
[:map [:table_id :string]]]
[:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]]]
[:map {:encode/tool-api-request (mr/with-key #(update-keys % metabot-v3.u/safe->kebab-case-en))}]]]
[:filters [:sequential ::filter]]]
[:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])
[:map {:encode/tool-api-request (mr/with-key #(update-keys % metabot-v3.u/safe->kebab-case-en))}]])
(mr/def ::basic-metric
[:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
[:map {:decode/tool-api-response (mr/with-key #(update-keys % metabot-v3.u/safe->snake_case_en))}
[:id :int]
[:type [:= :metric]]
[:name :string]
@@ -408,13 +410,13 @@
(mr/def ::full-metric
[:merge
::basic-metric
[:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
[:map {:decode/tool-api-response (mr/with-key #(update-keys % metabot-v3.u/safe->snake_case_en))}
[:queryable_dimensions {:optional true} ::columns]]])
(mr/def ::find-metric-result
[:or
[:map
{:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
{:decode/tool-api-response (mr/with-key #(update-keys % metabot-v3.u/safe->snake_case_en))}
[:structured_output [:map
[:id :int]
[:type [:= :metric]]
@@ -442,12 +444,12 @@
[:map
[:table_id :string]
[:result_field_id :string]]]
[:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]]]]
[:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])
[:map {:encode/tool-api-request (mr/with-key #(update-keys % metabot-v3.u/safe->kebab-case-en))}]]]]
[:map {:encode/tool-api-request (mr/with-key #(update-keys % metabot-v3.u/safe->kebab-case-en))}]])
(mr/def ::find-outliers-result
[:or
[:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
[:map {:decode/tool-api-response (mr/with-key #(update-keys % metabot-v3.u/safe->snake_case_en))}
[:structured_output [:sequential
[:map
[:dimension :any]
@@ -457,7 +459,7 @@
(mr/def ::generate-insights-arguments
[:map
;; query should not be changed to kebab-case
{:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}
{:encode/tool-api-request (mr/with-key #(update-keys % metabot-v3.u/safe->kebab-case-en))}
[:for [:or
[:map [:metric_id :int]]
[:map [:table_id :string]]
@@ -466,8 +468,8 @@
(mr/def ::get-current-user-result
[:or
[:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
[:structured_output [:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
[:map {:decode/tool-api-response (mr/with-key #(update-keys % metabot-v3.u/safe->snake_case_en))}
[:structured_output [:map {:decode/tool-api-response (mr/with-key #(update-keys % metabot-v3.u/safe->snake_case_en))}
[:id :int]
[:type [:= :user]]
[:name :string]
@@ -477,7 +479,7 @@
(mr/def ::get-dashboard-details-result
[:or
[:map
{:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
{:decode/tool-api-response (mr/with-key #(update-keys % metabot-v3.u/safe->snake_case_en))}
[:structured_output [:map
[:id :int]
[:type [:= :dashboard]]
@@ -493,23 +495,25 @@
[:with_field_values {:optional true, :default true} :boolean]
[:with_queryable_dimensions {:optional true, :default true} :boolean]]
[:map {:encode/tool-api-request
#(set/rename-keys % {:metric_id :metric-id
:with_default_temporal_breakout :with-default-temporal-breakout?
:with_field_values :with-field-values?
:with_queryable_dimensions :with-queryable-dimensions?})}]])
(mr/with-key
#(set/rename-keys % {:metric_id :metric-id
:with_default_temporal_breakout :with-default-temporal-breakout?
:with_field_values :with-field-values?
:with_queryable_dimensions :with-queryable-dimensions?}))}]])
(mr/def ::get-metric-details-result
[:or
[:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
[:map {:decode/tool-api-response (mr/with-key
#(update-keys % metabot-v3.u/safe->snake_case_en))}
[:structured_output ::full-metric]]
[:map [:output :string]]])
(mr/def ::get-query-details-result
[:or
[:map
{:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
{:decode/tool-api-response (mr/with-key #(update-keys % metabot-v3.u/safe->snake_case_en))}
[:structured_output [:map
{:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
{:decode/tool-api-response (mr/with-key #(update-keys % metabot-v3.u/safe->snake_case_en))}
[:type [:= :query]]
[:query_id :string]
[:query :map]
@@ -523,16 +527,16 @@
[:with_fields {:optional true, :default true} :boolean]
[:with_field_values {:optional true, :default true} :boolean]]
[:map {:encode/tool-api-request
#(set/rename-keys % {:report_id :report-id
:with_fields :with-fields?
:with_field_values :with-field-values?})}]])
(mr/with-key #(set/rename-keys % {:report_id :report-id
:with_fields :with-fields?
:with_field_values :with-field-values?}))}]])
(mr/def ::get-report-details-result
[:or
[:map
{:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
{:decode/tool-api-response (mr/with-key #(update-keys % metabot-v3.u/safe->snake_case_en))}
[:structured_output [:map
{:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
{:decode/tool-api-response (mr/with-key #(update-keys % metabot-v3.u/safe->snake_case_en))}
[:id :int]
[:type [:= :question]]
[:name :string]
@@ -550,14 +554,15 @@
[:with_metrics {:optional true, :default true} :boolean]
[:with_metric_default_temporal_breakout {:optional true, :default true} :boolean]]
[:fn {:error/message "Exactly one of model_id and table_id required"}
#(= (count (select-keys % [:model_id :table_id])) 1)]
(mr/with-key #(= (count (select-keys % [:model_id :table_id])) 1))]
[:map {:encode/tool-api-request
#(set/rename-keys % {:model_id :model-id
:table_id :table-id
:with_fields :with-fields?
:with_field_values :with-field-values?
:with_metrics :with-metrics?
:with_metric_default_temporal_breakout :with-default-temporal-breakout?})}]])
(mr/with-key
#(set/rename-keys % {:model_id :model-id
:table_id :table-id
:with_fields :with-fields?
:with_field_values :with-field-values?
:with_metrics :with-metrics?
:with_metric_default_temporal_breakout :with-default-temporal-breakout?}))}]])
(mr/def ::basic-table
[:map
@@ -571,19 +576,19 @@
(mr/def ::full-table
[:merge
::basic-table
[:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
[:map {:decode/tool-api-response (mr/with-key #(update-keys % metabot-v3.u/safe->snake_case_en))}
[:queryable_foreign_key_tables {:optional true} [:sequential ::basic-table]]]])
(mr/def ::get-table-details-result
[:or
[:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
[:map {:decode/tool-api-response (mr/with-key #(update-keys % metabot-v3.u/safe->snake_case_en))}
[:structured_output ::full-table]]
[:map [:output :string]]])
(mr/def ::answer-sources-result
[:or
[:map
{:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
{:decode/tool-api-response (mr/with-key #(update-keys % metabot-v3.u/safe->snake_case_en))}
[:structured_output [:map
[:metrics [:sequential ::full-metric]]
[:models [:sequential ::full-table]]]]]

View File

@@ -11,12 +11,15 @@
(mr/def ::metadata.parameter.name
[:and
{:decode/metadata-file (fn [x]
(keyword (u/->kebab-case-en x)))
:encode/api-request (fn [x]
(u/->snake_case_en (name x)))
:decode/api-response (fn [x]
(keyword (u/->kebab-case-en x)))}
{:decode/metadata-file (mr/with-key
(fn [x]
(keyword (u/->kebab-case-en x))))
:encode/api-request (mr/with-key
(fn [x]
(u/->snake_case_en (name x))))
:decode/api-response (mr/with-key
(fn [x]
(keyword (u/->kebab-case-en x))))}
:keyword])
(mr/def ::metadata.parameter
@@ -29,12 +32,14 @@
[:description {:optional true} [:maybe ::lib.schema.common/non-blank-string]]]
[:fn
{:error/message "metadata.parameter must specify either `:type`` or `:anyOf`, but not both."}
(fn [query]
(= (count (select-keys query [:type :anyOf])) 1))]])
(mr/with-key
(fn [query]
(= (count (select-keys query [:type :anyOf])) 1)))]])
(mr/def ::metadata.name
[:and
{:encode/api-request (fn [x] (u/->snake_case_en (name x)))
{:encode/api-request (mr/with-key
(fn [x] (u/->snake_case_en (name x))))
:decode/metadata-file keyword
:decode/api-request keyword
:decode/api-response keyword}

View File

@@ -23,12 +23,13 @@
(mr/def ::qp-results-cased-map
[:fn
{:error/message "Map where all simple keywords are snake_case; namespaced keywords can be any case."}
(fn [m]
(and (map? m)
(every? (fn [k]
(or (qualified-keyword? k)
(not (str/includes? (name k) "-"))))
(keys m))))])
(mr/with-key
(fn [m]
(and (map? m)
(every? (fn [k]
(or (qualified-keyword? k)
(not (str/includes? (name k) "-"))))
(keys m)))))])
(mr/def ::Field
[:and

View File

@@ -192,10 +192,11 @@
(mr/def ::parameter.schema.or
"Not sure what the difference is between `:oneOf` and `:anyOf` but they seem to both mean 'or'."
[:multi
{:dispatch (fn [x]
(if (contains? x :anyOf)
:anyOf
:oneOf))}
{:dispatch (mr/with-key
(fn [x]
(if (contains? x :anyOf)
:anyOf
:oneOf)))}
[:anyOf
[:merge
::parameter.schema.common
@@ -238,17 +239,18 @@
[:and
:map
[:multi
{:dispatch (fn [x]
(cond
(not (map? x)) :invalid
(:type x) :typed
(contains? x :$ref) :ref
(contains? x :oneOf) :or
(contains? x :anyOf) :or
(contains? x :allOf) :and
(contains? x :const) :const
(:enum x) :untyped-enum
:else :empty))}
{:dispatch (mr/with-key
(fn [x]
(cond
(not (map? x)) :invalid
(:type x) :typed
(contains? x :$ref) :ref
(contains? x :oneOf) :or
(contains? x :anyOf) :or
(contains? x :allOf) :and
(contains? x :const) :const
(:enum x) :untyped-enum
:else :empty)))}
[:invalid :map]
[:typed ::parameter.schema.typed]
[:ref ::parameter.schema.ref]

View File

@@ -77,6 +77,8 @@
;; this doesn't use [[schema.core/defn]] because [[schema.core/defn]] doesn't like optional keyword args
{:pre [(#{:h2 :mysql :postgres} db-type)
(instance? javax.sql.DataSource data-source)]}
(def db-type db-type)
(def data-source data-source)
(map->ApplicationDB
{:db-type db-type
:data-source (if create-pool?

View File

@@ -62,10 +62,11 @@
(mr/def ::cache-strategy
(if config/ee-available?
[:multi
{:dispatch (fn [_value]
(if (premium-features/has-feature? :cache-granular-controls)
:ee
:oss))}
{:dispatch (mr/with-key
(fn [_value]
(if (premium-features/has-feature? :cache-granular-controls)
:ee
:oss)))}
[:ee ::cache-strategy.ee]
[:oss ::cache-strategy.oss]]
::cache-strategy.oss))

View File

@@ -90,7 +90,8 @@
[:type (apply ms/enum-keywords-and-strings channel-template-details-type)]
[:subject string?]
[:recipient-type {:optional true} (ms/enum-keywords-and-strings :cc :bcc)]]
[:multi {:dispatch (comp keyword :type)}
[:multi {:dispatch (mr/with-key
(comp keyword :type))}
[:email/handlebars-resource
[:map
[:path string?]]]
@@ -102,7 +103,8 @@
"Channel Template schema."
[:merge
[:map
[:channel_type [:fn #(= "channel" (-> % keyword namespace))]]]
[:channel_type [:fn (mr/with-key
#(= "channel" (-> % keyword namespace)))]]]
[:multi {:dispatch :channel_type}
[:channel/email
[:map

View File

@@ -24,6 +24,7 @@
[metabase.util.i18n :refer [tru]]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
[metabase.util.performance :as perf]
[potemkin :as p])
(:import
@@ -270,7 +271,7 @@
;; responsible for its lifecycle and that means you can't use
;; `with-open` on the Connection you'd get from the DataSource
{:error/message "Cannot be a JDBC spec wrapping a java.sql.Connection"}
(complement :connection)]])
(mr/with-key (complement :connection))]])
(mu/defn do-with-resolved-connection-data-source :- (driver-api/instance-of-class DataSource)
"Part of the default implementation for [[do-with-connection-with-options]]: get an appropriate `java.sql.DataSource`

View File

@@ -11,7 +11,7 @@
(mu/defn event-schema
"Get the Malli schema we should use for events of `topic`. By default, this looks in our registry for a schema
matching the event topic name; if it fails to find one, it falls back to `:map`."
[topic :- keyword?]
[topic :- :keyword]
(or (mr/registered-schema topic)
:map))
@@ -41,15 +41,15 @@
(mr/def :event/collection-read
[:map {:closed true}
[:user-id pos-int?]
[:object [:fn #(t2/instance-of? :model/Collection %)]]])
[:user-id [:int {:min 1}]]
[:object [:fn (mr/with-key #(t2/instance-of? :model/Collection %))]]])
;; dashboard events
(mr/def ::dashboard
[:map {:closed true}
[:user-id pos-int?]
[:object [:fn #(t2/instance-of? :model/Dashboard %)]]])
[:user-id [:int {:min 1}]]
[:object [:fn (mr/with-key #(t2/instance-of? :model/Dashboard %))]]])
(mr/def :event/dashboard-create ::dashboard)
(mr/def :event/dashboard-update ::dashboard)
@@ -57,21 +57,22 @@
(mr/def ::dashboard-with-dashcards
(-> (event-schema ::dashboard)
(mut/assoc :dashcards [:sequential [:map [:id pos-int?]]])))
(mut/assoc :dashcards [:sequential [:map [:id [:int {:min 1}]]]])
mr/with-key))
(mr/def :event/dashboard-remove-cards ::dashboard-with-dashcards)
(mr/def :event/dashboard-add-cards ::dashboard-with-dashcards)
(mr/def :event/dashboard-read
[:map {:closed true}
[:user-id [:maybe pos-int?]]
[:object-id [:maybe pos-int?]]])
[:user-id [:maybe [:int {:min 1}]]]
[:object-id [:maybe [:int {:min 1}]]]])
;; user events
(mr/def ::user
[:map {:closed true}
[:user-id pos-int?]])
[:user-id [:int {:min 1}]]])
(mr/def :event/user-login ::user)
(mr/def :event/user-joined ::user)
@@ -94,8 +95,8 @@
(mr/def ::segment
[:map {:closed true}
[:user-id pos-int?]
[:object [:fn #(t2/instance-of? :model/Segment %)]]])
[:user-id [:int {:min 1}]]
[:object [:fn (mr/with-key #(t2/instance-of? :model/Segment %))]]])
(mr/def :event/segment-create ::segment)
@@ -112,10 +113,10 @@
(mr/def ::database
[:map {:closed true}
[:object [:fn #(t2/instance-of? :model/Database %)]]
[:previous-object {:optional true} [:fn #(t2/instance-of? :model/Database %)]]
[:object [:fn (mr/with-key #(t2/instance-of? :model/Database %))]]
[:previous-object {:optional true} [:fn (mr/with-key #(t2/instance-of? :model/Database %))]]
[:details {:optional true} :map]
[:user-id pos-int?]
[:user-id [:int {:min 1}]]
[:details-changed? {:optional true} [:maybe :boolean]]])
(mr/def :event/database-create ::database)
@@ -126,31 +127,31 @@
(mr/def :event/alert-create
[:map {:closed true}
(-> [:user-id pos-int?]
(-> [:user-id [:int {:min 1}]]
(with-hydrate :user user-hydrate))
[:object [:and
[:fn #(t2/instance-of? :model/Pulse %)]
[:fn (mr/with-key #(t2/instance-of? :model/Pulse %))]
[:map
[:card [:fn #(t2/instance-of? :model/Card %)]]]]]])
[:card [:fn (mr/with-key #(t2/instance-of? :model/Card %))]]]]]])
;; pulse schemas
(mr/def :event/pulse-create
[:map {:closed true}
[:user-id pos-int?]
[:object [:fn #(t2/instance-of? :model/Pulse %)]]])
[:user-id [:int {:min 1}]]
[:object [:fn (mr/with-key #(t2/instance-of? :model/Pulse %))]]])
;; table events
(mr/def :event/table-read
[:map {:closed true}
[:user-id pos-int?]
[:object [:fn #(t2/instance-of? :model/Table %)]]])
[:user-id [:int {:min 1}]]
[:object [:fn (mr/with-key #(t2/instance-of? :model/Table %))]]])
(mr/def ::permission-failure
[:map {:closed true}
[:user-id [:maybe pos-int?]]
[:object [:maybe [:fn #(boolean (t2/model %))]]]
[:user-id [:maybe [:int {:min 1}]]]
[:object [:maybe [:fn (mr/with-key #(boolean (t2/model %)))]]]
[:has-access {:optional true} [:maybe :boolean]]])
(mr/def :event/read-permission-failure ::permission-failure)
@@ -159,5 +160,5 @@
(mr/def :event/create-permission-failure
[:map {:closed true}
[:user-id [:maybe pos-int?]]
[:user-id [:maybe [:int {:min 1}]]]
[:model [:or :keyword :string]]])

View File

@@ -132,14 +132,15 @@
(mr/def ::absolute-datetime
[:multi {:error/message "valid :absolute-datetime clause"
:doc/title [:span [:code ":absolute-datetime"] " clause"]
:dispatch (fn [x]
(cond
(core/not (is-clause? :absolute-datetime x)) :invalid
(mr/validate ::lib.schema.literal/date (second x)) :date
:else :datetime))}
:dispatch (mr/with-key
(fn [x]
(cond
(core/not (is-clause? :absolute-datetime x)) :invalid
(mr/validate ::lib.schema.literal/date (second x)) :date
:else :datetime)))}
[:invalid [:fn
{:error/message "not an :absolute-datetime clause"}
(constantly false)]]
(mr/with-key (constantly false))]]
[:date (helpers/clause
:absolute-datetime
"date" ::lib.schema.literal/date
@@ -252,7 +253,7 @@
(mr/def ::no-binning-options-at-top-level
[:fn
{:error/message ":binning keys like :strategy are not allowed at the top level of :field options."}
(complement :strategy)])
(mr/with-key (complement :strategy))])
(mr/def ::FieldOptions
[:and
@@ -327,10 +328,11 @@
(mr/def ::require-base-type-for-field-name
[:fn
{:error/message ":field clauses using a string field name must specify :base-type."}
(fn [[_ id-or-name {:keys [base-type]}]]
(if (string? id-or-name)
base-type
true))])
(mr/with-key
(fn [[_ id-or-name {:keys [base-type]}]]
(if (string? id-or-name)
base-type
true)))])
(mr/def ::field
[:and
@@ -390,8 +392,8 @@
(mr/def ::Reference
[:schema
;; this is provided for the convenience of Lib which uses this schema for `:field-ref` in results metadata
{:decode/normalize (fn [x]
((#?(:clj requiring-resolve :cljs resolve) 'metabase.legacy-mbql.normalize/normalize-field-ref) x))}
{:decode/normalize (mr/with-key (fn [x]
((#?(:clj requiring-resolve :cljs resolve) 'metabase.legacy-mbql.normalize/normalize-field-ref) x)))}
(one-of aggregation expression field)])
(def Reference
@@ -418,12 +420,13 @@
(mr/def ::StringExpressionArg
[:multi
{:dispatch (fn [x]
(cond
(string? x) :string
(is-clause? string-functions x) :string-expression
(is-clause? :value x) :value
:else :else))}
{:dispatch (mr/with-key
(fn [x]
(cond
(string? x) :string
(is-clause? string-functions x) :string-expression
(is-clause? :value x) :value
:else :else)))}
[:string :string]
[:string-expression StringExpression]
[:value value]
@@ -472,13 +475,14 @@
(mr/def ::NumericExpressionArg
[:multi
{:error/message "numeric expression argument"
:dispatch (fn [x]
(cond
(number? x) :number
(is-clause? numeric-functions x) :numeric-expression
(is-clause? aggregations x) :aggregation
(is-clause? :value x) :value
:else :field))}
:dispatch (mr/with-key
(fn [x]
(cond
(number? x) :number
(is-clause? numeric-functions x) :numeric-expression
(is-clause? aggregations x) :aggregation
(is-clause? :value x) :value
:else :field)))}
[:number number?]
[:numeric-expression NumericExpression]
[:aggregation Aggregation]
@@ -490,12 +494,13 @@
(mr/def ::DateTimeExpressionArg
[:multi
{:error/message "datetime expression argument"
:dispatch (fn [x]
(cond
(is-clause? aggregations x) :aggregation
(is-clause? :value x) :value
(is-clause? datetime-functions x) :datetime-expression
:else :else))}
:dispatch (mr/with-key
(fn [x]
(cond
(is-clause? aggregations x) :aggregation
(is-clause? :value x) :value
(is-clause? datetime-functions x) :datetime-expression
:else :else)))}
[:aggregation Aggregation]
[:value value]
[:datetime-expression DatetimeExpression]
@@ -507,18 +512,19 @@
(mr/def ::ExpressionArg
[:multi
{:error/message "expression argument"
:dispatch (fn [x]
(cond
(number? x) :number
(boolean? x) :boolean
(is-clause? boolean-functions x) :boolean-expression
(is-clause? numeric-functions x) :numeric-expression
(is-clause? datetime-functions x) :datetime-expression
(is-clause? aggregations x) :aggregation
(string? x) :string
(is-clause? string-functions x) :string-expression
(is-clause? :value x) :value
:else :else))}
:dispatch (mr/with-key
(fn [x]
(cond
(number? x) :number
(boolean? x) :boolean
(is-clause? boolean-functions x) :boolean-expression
(is-clause? numeric-functions x) :numeric-expression
(is-clause? datetime-functions x) :datetime-expression
(is-clause? aggregations x) :aggregation
(string? x) :string
(is-clause? string-functions x) :string-expression
(is-clause? :value x) :value
:else :else)))}
[:number number?]
[:boolean :boolean]
[:boolean-expression BooleanExpression]
@@ -546,10 +552,11 @@
(mr/def ::IntGreaterThanZeroOrNumericExpression
[:multi
{:error/message "int greater than zero or numeric expression"
:dispatch (fn [x]
(if (number? x)
:number
:else))}
:dispatch (mr/with-key
(fn [x]
(if (number? x)
:number
:else)))}
[:number PositiveInt]
[:else NumericExpression]])
@@ -792,10 +799,11 @@
(mr/def ::OrderComparable
[:multi
{:error/message "order comparable"
:dispatch (fn [x]
(if (is-clause? :value x)
:value
:else))}
:dispatch (mr/with-key
(fn [x]
(if (is-clause? :value x)
:value
:else)))}
[:value value]
[:else [:or
number?
@@ -954,15 +962,16 @@
(mr/def ::Filter
[:multi
{:error/message "valid filter expression"
:dispatch (fn [x]
(cond
(is-clause? datetime-functions x) :datetime
(is-clause? numeric-functions x) :numeric
(is-clause? string-functions x) :string
(is-clause? boolean-functions x) :boolean
(is-clause? :value x) :value
(is-clause? :segment x) :segment
:else :else))}
:dispatch (mr/with-key
(fn [x]
(cond
(is-clause? datetime-functions x) :datetime
(is-clause? numeric-functions x) :numeric
(is-clause? string-functions x) :string
(is-clause? boolean-functions x) :boolean
(is-clause? :value x) :value
(is-clause? :segment x) :segment
:else :else)))}
[:datetime DatetimeExpression]
[:numeric NumericExpression]
[:string StringExpression]
@@ -1004,17 +1013,18 @@
[:multi
{:error/message ":field or :expression reference or expression"
:doc/title "expression definition"
:dispatch (fn [x]
(cond
(is-clause? numeric-functions x) :numeric
(is-clause? string-functions x) :string
(is-clause? boolean-functions x) :boolean
(is-clause? datetime-functions x) :datetime
(is-clause? :case x) :case
(is-clause? :if x) :if
(is-clause? :offset x) :offset
(is-clause? :value x) :value
:else :else))}
:dispatch (mr/with-key
(fn [x]
(cond
(is-clause? numeric-functions x) :numeric
(is-clause? string-functions x) :string
(is-clause? boolean-functions x) :boolean
(is-clause? datetime-functions x) :datetime
(is-clause? :case x) :case
(is-clause? :if x) :if
(is-clause? :offset x) :offset
(is-clause? :value x) :value
:else :else)))}
[:numeric NumericExpression]
[:string StringExpression]
[:boolean BooleanExpression]
@@ -1084,10 +1094,11 @@
(mr/def ::UnnamedAggregation
[:multi
{:error/message "unnamed aggregation clause or numeric expression"
:dispatch (fn [x]
(if (is-clause? numeric-functions x)
:numeric-expression
:else))}
:dispatch (mr/with-key
(fn [x]
(if (is-clause? numeric-functions x)
:numeric-expression
:else)))}
[:numeric-expression NumericExpression]
[:else (one-of aggregation avg cum-sum distinct distinct-where stddev sum min max metric share count-where
sum-where case case:if median percentile ag:var cum-count count offset)]])
@@ -1111,10 +1122,11 @@
(mr/def ::Aggregation
[:multi
{:error/message "aggregation clause or numeric expression"
:dispatch (fn [x]
(if (is-clause? :aggregation-options x)
:aggregation-options
:unnamed-aggregation))}
:dispatch (mr/with-key
(fn [x]
(if (is-clause? :aggregation-options x)
:aggregation-options
:unnamed-aggregation)))}
[:aggregation-options aggregation-options]
[:unnamed-aggregation UnnamedAggregation]])
@@ -1314,10 +1326,11 @@
;; make sure people don't try to pass in a `:name` that's different from the actual key in the map.
[:fn
{:error/message "keys in template tag map must match the :name of their values"}
(fn [m]
(every? (fn [[tag-name tag-definition]]
(core/= tag-name (:name tag-definition)))
m))]])
(mr/with-key
(fn [m]
(every? (fn [[tag-name tag-definition]]
(core/= tag-name (:name tag-definition)))
m)))]])
(def ^:private NativeQuery:Common
[:map
@@ -1346,10 +1359,11 @@
(mr/def ::SourceQuery
[:multi
{:dispatch (fn [x]
(if ((every-pred map? :native) x)
:native
:mbql))}
{:dispatch (mr/with-key
(fn [x]
(if ((every-pred map? :native) x)
:native
:mbql)))}
;; when using native queries as source queries the schema is exactly the same except use `:native` in place of
;; `:query` for reasons I do not fully remember (perhaps to make it easier to differentiate them from MBQL source
;; queries).
@@ -1391,7 +1405,7 @@
[:visibility_type {:optional true} [:maybe [:ref ::lib.schema.metadata/column.visibility-type]]]]
[:fn
{:error/message "Legacy results metadata should not have :lib/type, use :metabase.lib.schema.metadata/column for Lib metadata"}
(complement :lib/type)]
(mr/with-key (complement :lib/type))]
(letfn [(disallowed-key? [k]
(core/or (core/not (keyword? k))
(let [disallowed-char (if (qualified-keyword? k)
@@ -1400,13 +1414,15 @@
(str/includes? (str k) disallowed-char))))]
[:fn
{:error/message "legacy source query metadata should use snake_case keys (except for namespaced lib keys, which should use kebab-case)"
:error/fn (fn [{m :value} _]
(str "legacy source query metadata should use snake_case keys (except for namespaced lib keys, which should use kebab-case), got: "
(when (map? m)
(into #{} (filter disallowed-key?) (keys m)))))}
(fn [m]
(core/and (map? m)
(every? (complement disallowed-key?) (keys m))))])])
:error/fn (mr/with-key
(fn [{m :value} _]
(str "legacy source query metadata should use snake_case keys (except for namespaced lib keys, which should use kebab-case), got: "
(when (map? m)
(into #{} (filter disallowed-key?) (keys m))))))}
(mr/with-key
(fn [m]
(core/and (map? m)
(every? (complement disallowed-key?) (keys m)))))])])
(def source-table-card-id-regex
"Pattern that matches `card__id` strings that can be used as the `:source-table` of MBQL queries."
@@ -1526,16 +1542,19 @@
;; additional constraints
[:fn
{:error/message "Joins must have either a `source-table` or `source-query`, but not both."}
(every-pred
(some-fn :source-table :source-query)
(complement (every-pred :source-table :source-query)))]
(mr/with-key
(every-pred
(some-fn :source-table :source-query)
(complement (every-pred :source-table :source-query))))]
(let [disallowed-keys #{:filter :breakout :aggreggation :expressions :joins}]
[:fn
{:error/message "Join should not have top-level 'inner' query keys like :fields or :filter -- they should go in :source-query"
:error/fn (fn [{join :value} _]
(when (map? join)
(str "join should not have top-level 'inner' query keys, found " (set/intersection (set (keys join)) disallowed-keys))))}
(complement (apply some-fn disallowed-keys))])])
:error/fn (mr/with-key
(fn [{join :value} _]
(when (map? join)
(str "join should not have top-level 'inner' query keys, found " (set/intersection (set (keys join)) disallowed-keys)))))}
(mr/with-key
(complement (apply some-fn disallowed-keys)))])])
(def Join
"Alias for ::Join. Prefer that going forward."
@@ -1547,7 +1566,8 @@
(helpers/non-empty [:sequential Join])
[:fn
{:error/message "All join aliases must be unique."}
#(helpers/empty-or-distinct? (filter some? (map :alias %)))]])
(mr/with-key
#(helpers/empty-or-distinct? (filter some? (map :alias %))))]])
(mr/def ::Fields
[:schema
@@ -1589,12 +1609,14 @@
;;
[:fn
{:error/message "Query must specify either `:source-table` or `:source-query`, but not both."}
(fn [query]
(core/= 1 (core/count (select-keys query [:source-query :source-table]))))]
(mr/with-key
(fn [query]
(core/= 1 (core/count (select-keys query [:source-query :source-table])))))]
[:fn
{:error/message "Fields specified in `:breakout` should not be specified in `:fields`; this is implied."}
(fn [{:keys [breakout fields]}]
(empty? (set/intersection (set breakout) (set fields))))]])
(mr/with-key
(fn [{:keys [breakout fields]}]
(empty? (set/intersection (set breakout) (set fields)))))]])
;;; ----------------------------------------------------- Params -----------------------------------------------------
@@ -1613,7 +1635,9 @@
(mr/def ::dimension
[:and
{:doc/title [:span [:code ":dimension"] " clause"]}
[:fn {:error/message "must be a `:dimension` clause"} (partial helpers/is-clause? :dimension)]
[:fn {:error/message "must be a `:dimension` clause"}
(mr/with-key
(partial helpers/is-clause? :dimension))]
[:catn
[:tag [:= :dimension]]
[:target [:schema [:or [:ref ::field-or-expression-ref] [:ref ::template-tag]]]]
@@ -1685,10 +1709,11 @@
[:fn
{:error/message "max-results-bare-rows must be less or equal to than max-results"}
(fn [{:keys [max-results max-results-bare-rows]}]
(if-not (core/and max-results max-results-bare-rows)
true
(core/>= max-results max-results-bare-rows)))]])
(mr/with-key
(fn [{:keys [max-results max-results-bare-rows]}]
(if-not (core/and max-results max-results-bare-rows)
true
(core/>= max-results max-results-bare-rows))))]])
(mr/def ::MiddlewareOptions
"Additional options that can be used to toggle middleware on or off."
@@ -1769,14 +1794,16 @@
[:and
[:fn
{:error/message "Query must specify at most one of `:native` or `:query`, but not both."}
(complement (every-pred :native :query))]
(mr/with-key
(complement (every-pred :native :query)))]
[:fn
{:error/message "Native queries must not specify `:query`; MBQL queries must not specify `:native`."}
(fn [{native :native, mbql :query, query-type :type}]
(core/case query-type
:native (core/not mbql)
:query (core/not native)
false))]])
(mr/with-key
(fn [{native :native, mbql :query, query-type :type}]
(core/case query-type
:native (core/not mbql)
:query (core/not native)
false)))]])
(mr/def ::check-query-does-not-have-source-metadata
"`:source-metadata` is added to queries when `card__id` source queries are resolved. It contains info about the
@@ -1790,7 +1817,8 @@
properly."
[:fn
{:error/message "`:source-metadata` should be added in the same level as `:source-query` (i.e., the 'inner' MBQL query.)"}
(complement :source-metadata)])
(mr/with-key
(complement :source-metadata))])
(def Query
"Schema for an [outer] query, e.g. the sort of thing you'd pass to the query processor or save in `Card.dataset_query`."

View File

@@ -53,19 +53,19 @@
:rest [:* (wrap-clause-arg-schema arg-schema)]
(wrap-clause-arg-schema vector-arg-schema)))))
(defn clause
(defmacro clause
"Impl of [[metabase.legacy-mbql.schema.macros/defclause]] macro. Creates a Malli schema."
[tag & arg-schemas]
[:and
{:doc/title [:span [:code (pr-str tag)] " clause"]}
[:fn
{:error/message (str "must be a `" tag "` clause")}
(partial is-clause? tag)]
(into
[:catn
["tag" [:= tag]]]
(for [[arg-name arg-schema] (partition 2 arg-schemas)]
[arg-name (clause-arg-schema arg-schema)]))])
`[:and
{:doc/title [:span [:code (pr-str ~tag)] " clause"]}
[:fn
{:error/message (str "must be a `" ~tag "` clause")}
(mr/with-key (partial is-clause? ~tag))]
~(into
[:catn
["tag" [:= tag]]]
(for [[arg-name arg-schema] (partition 2 arg-schemas)]
[arg-name (clause-arg-schema arg-schema)]))])
(defn- clause-tag [a-clause]
(when (and (vector? a-clause)

View File

@@ -5,12 +5,15 @@
[metabase.lib.drill-thru.zoom :as lib.drill-thru.zoom]
[metabase.lib.schema :as lib.schema]
[metabase.lib.schema.drill-thru :as lib.schema.drill-thru]
[metabase.util.malli :as mu]))
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]))
(mu/defn object-detail-drill :- [:maybe [:or
::lib.schema.drill-thru/drill-thru.pk
::lib.schema.drill-thru/drill-thru.zoom
::lib.schema.drill-thru/drill-thru.fk-details]]
(mr/def ::object-detail-drill-out [:maybe [:or
::lib.schema.drill-thru/drill-thru.pk
::lib.schema.drill-thru/drill-thru.zoom
::lib.schema.drill-thru/drill-thru.fk-details]])
(mu/defn object-detail-drill :- ::object-detail-drill-out
"When clicking a foreign key or primary key value, drill through to the details for that specific object.
Contrast [[metabase.lib.drill-thru.fk-filter/fk-filter-drill]], which filters this query to only those rows with a

View File

@@ -33,9 +33,10 @@
[:lib/source ::lib.schema.metadata/column.source]]
[:fn
{:error/message "map with debugging output"}
(fn [m]
(or (not *debug*)
(::debug.origin m)))]])
(mr/with-key
(fn [m]
(or (not *debug*)
(::debug.origin m))))]])
(defn- merge-metadata
[m & more]

View File

@@ -270,7 +270,7 @@
qualified-keyword?
[:fn
{:error/message ":lib/type should be a :metadata/ keyword"}
#(= (namespace %) "metadata")]]]])
(mr/with-key #(= (namespace %) "metadata"))]]]])
(mu/defn metadata :- ::metadata-map
"Calculate an appropriate `:metadata/*` object for something. What this looks like depends on what we're calculating
@@ -467,13 +467,15 @@
[:fn
;; should be dev-facing only, so don't need to i18n
{:error/message "Column :lib/desired-column-alias values must be distinct for each stage!"
:error/fn (fn [{:keys [value]} _]
(str "Column :lib/desired-column-alias values must be distinct, got: "
(pr-str (mapv :lib/desired-column-alias value))))}
(fn [columns]
(or
(empty? columns)
(apply distinct? (map :lib/desired-column-alias columns))))]])
:error/fn (mr/with-key
(fn [{:keys [value]} _]
(str "Column :lib/desired-column-alias values must be distinct, got: "
(pr-str (mapv :lib/desired-column-alias value)))))}
(mr/with-key
(fn [columns]
(or
(empty? columns)
(apply distinct? (map :lib/desired-column-alias columns)))))]])
(mr/def ::returned-columns.options
"Schema for options passed to [[returned-columns]] and [[returned-columns-method]]."
@@ -482,7 +484,7 @@
[:include-remaps? {:optional true, :default false} :boolean]]
[:fn
{:error/message "unique-name-fn is no longer allowed as an option."}
(complement :unique-name-fn)]])
(mr/with-key (complement :unique-name-fn))]])
(defmulti returned-columns-method
"Impl for [[returned-columns]]."

View File

@@ -44,11 +44,12 @@
::col
[:fn
{:error/message "column with all kebab-cased keys"}
(fn [m]
(every? (fn [k]
(and (keyword? k)
(not (str/includes? k "_"))))
(keys m)))]])
(mr/with-key
(fn [m]
(every? (fn [k]
(and (keyword? k)
(not (str/includes? k "_"))))
(keys m))))]])
(mr/def ::cols
[:maybe [:sequential ::col]])
@@ -394,15 +395,17 @@
[:sequential ::kebab-cased-map]
[:fn
{:error/message "columns should have unique :name(s)"}
(fn [cols]
(or (empty? cols)
(apply distinct? (map :name cols))))]
(mr/with-key
(fn [cols]
(or (empty? cols)
(apply distinct? (map :name cols)))))]
;; QUE-1623
[:fn
{:error/message "columns should have unique :field-ref(s)"}
(fn [cols]
(or (empty? cols)
(apply distinct? (map :field-ref cols))))]]
(mr/with-key
(fn [cols]
(or (empty? cols)
(apply distinct? (map :field-ref cols)))))]]
"Return metadata for columns returned by a pMBQL `query`.
`initial-cols` are (optionally) the initial minimal metadata columns as returned by the driver (usually just column

View File

@@ -43,12 +43,13 @@
(mr/def ::stage.native
[:and
[:map
{:decode/normalize #(->> %
common/normalize-map
;; filter out null :collection keys -- see #59675
(m/filter-kv (fn [k v]
(not (and (= k :collection)
(nil? v))))))}
{:decode/normalize (mr/with-key
#(->> %
common/normalize-map
;; filter out null :collection keys -- see #59675
(m/filter-kv (fn [k v]
(not (and (= k :collection)
(nil? v)))))))}
[:lib/type [:= {:decode/normalize common/normalize-keyword} :mbql.stage/native]]
[:lib/stage-metadata {:optional true} [:maybe [:ref ::lib.schema.metadata/stage]]]
;; the actual native query, depends on the underlying database. Could be a raw SQL string or something like that.
@@ -77,13 +78,13 @@
]
[:fn
{:error/message ":source-table is not allowed in a native query stage."}
#(not (contains? % :source-table))]
(mr/with-key #(not (contains? % :source-table)))]
[:fn
{:error/message ":source-card is not allowed in a native query stage."}
#(not (contains? % :source-card))]
(mr/with-key #(not (contains? % :source-card)))]
[:fn
{:error/message ":query is not allowed in a native query stage, you probably meant to use :native instead."}
(complement :query)]])
(mr/with-key (complement :query))]])
(mr/def ::breakout
[:ref ::ref/ref])
@@ -167,9 +168,10 @@
(mr/def ::stage.valid-refs
[:fn
{:error/message "Valid references for a single query stage"
:error/fn (fn [{:keys [value]} _]
(ref-error-for-stage value))}
(complement ref-error-for-stage)])
:error/fn (mr/with-key
(fn [{:keys [value]} _]
(ref-error-for-stage value)))}
(mr/with-key (complement ref-error-for-stage))])
;;; TODO -- should `::page` have a `:lib/type`, like all the other maps in pMBQL?
(mr/def ::page
@@ -202,20 +204,24 @@
[:page {:optional true} [:ref ::page]]]
[:fn
{:error/message ":source-query is not allowed in pMBQL queries."}
#(not (contains? % :source-query))]
(mr/with-key #(not (contains? % :source-query)))]
[:fn
{:error/message ":native is not allowed in an MBQL stage."}
#(not (contains? % :native))]
(mr/with-key #(not (contains? % :native)))]
[:fn
{:error/message "A query must have exactly one of :source-table or :source-card"}
(complement (comp #(= (count %) 1) #{:source-table :source-card}))]
(mr/with-key (complement (comp #(= (count %) 1) #{:source-table :source-card})))]
[:ref ::stage.valid-refs]
(into [:and]
(map (fn [k]
[:fn
{:error/message (str k " is deprecated and should not be used")}
(complement k)]))
[:aggregation-idents :breakout-idents :expression-idents])])
[:and
[:fn
{:error/message ":aggregation-idents is deprecated and should not be used"}
(mr/with-key (complement :aggregation-idents))]
[:fn
{:error/message ":breakout-idents is deprecated and should not be used"}
(mr/with-key (complement :breakout-idents))]
[:fn
{:error/message ":expression-idents is deprecated and should not be used"}
(mr/with-key (complement :expression-idents))]]])
;;; the schemas are constructed this way instead of using `:or` because they give better error messages
(mr/def ::stage.type
@@ -233,12 +239,12 @@
[:and
{:default {}
:decode/normalize common/normalize-map
:encode/serialize #(dissoc %
;; this stuff is all added at runtime by QP middleware.
:params
:parameters
:lib/stage-metadata
:middleware)}
:encode/serialize (mr/with-key #(dissoc %
;; this stuff is all added at runtime by QP middleware.
:params
:parameters
:lib/stage-metadata
:middleware))}
[:map
[:lib/type [:ref ::stage.type]]]
[:multi {:dispatch lib-type
@@ -247,7 +253,7 @@
[:mbql.stage/mbql [:ref ::stage.mbql]]]
[:fn
{:error/message "A query stage should not have :source-metadata, the prior stage should have :lib/stage-metadata instead"}
(complement :source-metadata)]])
(mr/with-key (complement :source-metadata))]])
(mr/def ::stage.initial
[:multi {:dispatch lib-type
@@ -260,10 +266,10 @@
:error/message "Invalid stage :lib/type: expected :mbql.stage/native or :mbql.stage/mbql"}
[:mbql.stage/native [:fn
{:error/message "Native stages are only allowed as the first stage of a query or join."}
(constantly false)]]
(mr/with-key (constantly false))]]
[:mbql.stage/mbql [:fn
{:error/message "Only the initial stage of a query can have a :source-table or :source-card"}
(complement (some-fn :source-table :source-card))]]])
(mr/with-key (complement (some-fn :source-table :source-card)))]]])
(defn- visible-join-alias?-fn
"Apparently you're allowed to use a join alias for a join that appeared in any previous stage or the current stage, or
@@ -316,9 +322,10 @@
(mr/def ::stages.valid-refs
[:fn
{:error/message "Valid references for all query stages"
:error/fn (fn [{stages :value} _]
(ref-error-for-stages stages))}
(complement ref-error-for-stages)])
:error/fn (mr/with-key
(fn [{stages :value} _]
(ref-error-for-stages stages)))}
(mr/with-key (complement ref-error-for-stages))])
(mr/def ::stages
[:and
@@ -358,47 +365,49 @@
(= (namespace k) "lib"))))
query)))
(mr/def ::query
[:and
[:map
{:decode/normalize common/normalize-map
:encode/serialize serialize-query}
[:lib/type [:=
{:decode/normalize common/normalize-keyword, :default :mbql/query}
:mbql/query]]
;; TODO (Cam 6/12/25) -- why in the HECC is `:lib/metadata` not a required key here? It's virtually REQUIRED for
;; anything to work correctly outside of the low-level conversion code. We should make it required and then fix
;; whatever breaks.
[:lib/metadata {:optional true} ::lib.schema.metadata/metadata-provider]
[:database {:optional true} [:multi {:dispatch (partial = id/saved-questions-virtual-database-id)}
[true ::id/saved-questions-virtual-database]
[false ::id/database]]]
[:stages [:ref ::stages]]
[:parameters {:optional true} [:maybe [:ref ::parameter/parameters]]]
;;
;; OPTIONS
;;
;; These keys are used to tweak behavior of the Query Processor.
;;
[:settings {:optional true} [:maybe [:ref ::settings]]]
[:constraints {:optional true} [:maybe [:ref ::constraints]]]
[:middleware {:optional true} [:maybe [:ref ::middleware-options]]]
;; TODO -- `:viz-settings` ?
;;
;; INFO
;;
;; Used when recording info about this run in the QueryExecution log; things like context query was ran in and
;; User who ran it
[:info {:optional true} [:maybe [:ref ::info/info]]]
;;
;; ACTIONS
;;
;; This stuff is only used for Actions.
[:create-row {:optional true} [:maybe [:ref ::actions/row]]]
[:update-row {:optional true} [:maybe [:ref ::actions/row]]]]
;;
;; CONSTRAINTS
[:ref ::lib.schema.util/unique-uuids]
[:fn
{:error/message ":expressions is not allowed in the top level of a query -- it is only allowed in MBQL stages"}
#(not (contains? % :expressions))]])
(let [db-dispatcher (partial = id/saved-questions-virtual-database-id)
expressionless #(not (contains? % :expressions))]
(mr/def ::query
[:and
[:map
{:decode/normalize common/normalize-map
:encode/serialize serialize-query}
[:lib/type [:=
{:decode/normalize common/normalize-keyword, :default :mbql/query}
:mbql/query]]
;; TODO (Cam 6/12/25) -- why in the HECC is `:lib/metadata` not a required key here? It's virtually REQUIRED for
;; anything to work correctly outside of the low-level conversion code. We should make it required and then fix
;; whatever breaks.
[:lib/metadata {:optional true} ::lib.schema.metadata/metadata-provider]
[:database {:optional true} [:multi {:dispatch db-dispatcher}
[true ::id/saved-questions-virtual-database]
[false ::id/database]]]
[:stages [:ref ::stages]]
[:parameters {:optional true} [:maybe [:ref ::parameter/parameters]]]
;;
;; OPTIONS
;;
;; These keys are used to tweak behavior of the Query Processor.
;;
[:settings {:optional true} [:maybe [:ref ::settings]]]
[:constraints {:optional true} [:maybe [:ref ::constraints]]]
[:middleware {:optional true} [:maybe [:ref ::middleware-options]]]
;; TODO -- `:viz-settings` ?
;;
;; INFO
;;
;; Used when recording info about this run in the QueryExecution log; things like context query was ran in and
;; User who ran it
[:info {:optional true} [:maybe [:ref ::info/info]]]
;;
;; ACTIONS
;;
;; This stuff is only used for Actions.
[:create-row {:optional true} [:maybe [:ref ::actions/row]]]
[:update-row {:optional true} [:maybe [:ref ::actions/row]]]]
;;
;; CONSTRAINTS
[:ref ::lib.schema.util/unique-uuids]
[:fn
{:error/message ":expressions is not allowed in the top level of a query -- it is only allowed in MBQL stages"}
expressionless]]))

View File

@@ -63,7 +63,7 @@
[:ref ::expression/number]
[:fn
{:error/message "percentile must be between zero and one"}
#(<= 0 % 1)]])
(mr/with-key #(<= 0 % 1))]])
(mbql-clause/define-tuple-mbql-clause :percentile
#_expr [:ref ::expression/number]

View File

@@ -29,10 +29,11 @@
[:map
{:decode/normalize lib.schema.common/normalize-map}
[:strategy [:ref ::strategy]]]
[:multi {:dispatch (fn [x]
(keyword (some #(get x %) [:strategy "strategy"])))
:error/fn (fn [{:keys [value]} _]
(str "Invalid binning strategy" (pr-str value)))}
[:multi (mr/with-key
{:dispatch (fn [x]
(keyword (some #(get x %) [:strategy "strategy"])))
:error/fn (fn [{:keys [value]} _]
(str "Invalid binning strategy" (pr-str value)))})
[:default [:map
[:strategy [:= :default]]]]
[:bin-width [:map

View File

@@ -92,7 +92,7 @@
[:string {:min 1}]
[:fn
{:error/message "non-blank string"}
(complement str/blank?)]])
(mr/with-key (complement str/blank?))]])
(mr/def ::int-greater-than-or-equal-to-zero
"Schema representing an integer than must also be greater than or equal to zero."
@@ -103,7 +103,7 @@
(mr/def ::positive-number
[:fn
{:error/message "positive number"}
(every-pred number? pos?)])
(mr/with-key (every-pred number? pos?))])
(mr/def ::uuid
[:string
@@ -114,55 +114,50 @@
:min 36
:max 36}])
(defn- semantic-type? [x]
(isa? x :Semantic/*))
(mr/def ::semantic-type
[:and
[:keyword
{:decode/normalize normalize-keyword}]
[:fn
{:error/message "valid semantic type"
:error/fn (fn [{:keys [value]} _]
(str "Not a valid semantic type: " (pr-str value)))}
semantic-type?]])
(defn- relation-type? [x]
(isa? x :Relation/*))
[:fn (mr/with-key
{:error/message "valid semantic type"
:error/fn (mr/with-key
(fn [{:keys [value]} _]
(str "Not a valid semantic type: " (pr-str value))))})
(mr/with-key #(isa? % :Semantic/*))]])
(mr/def ::relation-type
[:and
[:keyword
{:decode/normalize normalize-keyword}]
[:fn
{:error/message "valid relation type"
:error/fn (fn [{:keys [value]} _]
(str "Not a valid relation type: " (pr-str value)))}
relation-type?]])
[:fn (mr/with-key
{:error/message "valid relation type"
:error/fn (fn [{:keys [value]} _]
(str "Not a valid relation type: " (pr-str value)))})
(mr/with-key #(isa? % :Relation/*))]])
(mr/def ::semantic-or-relation-type
[:and
{:description "valid semantic or relation type"}
[:keyword
{:decode/normalize normalize-keyword}]
[:fn
{:error/message "valid semantic or relation type"
:error/fn (fn [{:keys [value]} _]
(str "Not a valid semantic or relation type: " (pr-str value)))}
(some-fn semantic-type? relation-type?)]])
(defn- base-type? [x]
(isa? x :type/*))
[:fn (mr/with-key
{:error/message "valid semantic or relation type"
:error/fn (fn [{:keys [value]} _]
(str "Not a valid semantic or relation type: " (pr-str value)))})
(mr/with-key
(some-fn #(isa? % :Semantic/*)
#(isa? % :Relation/*)))]])
(mr/def ::base-type
[:and
[:keyword
{:decode/normalize normalize-keyword}]
[:fn
{:error/message "valid base type"
:error/fn (fn [{:keys [value]} _]
(str "Not a valid base type: " (pr-str value)))}
base-type?]])
(mr/with-key
[:fn
{:error/message "valid base type"
:error/fn (fn [{:keys [value]} _]
(str "Not a valid base type: " (pr-str value)))}
(mr/with-key #(isa? % :type/*))])])
(defn- normalize-options-map [m]
(let [m (normalize-map m)]
@@ -188,7 +183,7 @@
[:display-name {:optional true} [:maybe ::non-blank-string]]]
[:fn
{:error/message ":ident is deprecated and should not be included in options maps"}
(complement :ident)]])
(mr/with-key (complement :ident))]])
(mr/def ::external-op
[:map

View File

@@ -24,9 +24,10 @@
(mr/def ::drill-thru.type
[:fn
{:error/message "valid drill-thru :type keyword"}
(fn [k]
(and (qualified-keyword? k)
(= (namespace k) "drill-thru")))])
(mr/with-key
(fn [k]
(and (qualified-keyword? k)
(= (namespace k) "drill-thru"))))])
(mr/def ::drill-thru.common
[:map
@@ -53,7 +54,7 @@
;; we should ignore NULL values for PKs and FKs -- do not add filters on them.
[:value [:and
:some
[:fn {:error/message "Non-NULL value"} #(not= % :null)]]]])
[:fn {:error/message "Non-NULL value"} (mr/with-key #(not= % :null))]]]])
(mr/def ::drill-thru.object-details.dimensions
[:sequential {:min 1} [:ref ::drill-thru.object-details.dimension]])
@@ -215,7 +216,7 @@
[:map
[:semantic-type [:fn
{:error/message "Latitude semantic type"}
#(isa? % :type/Latitude)]]]])
(mr/with-key #(isa? % :type/Latitude))]]]])
(mr/def ::drill-thru.zoom-in.geographic.column.longitude
[:merge
@@ -223,7 +224,7 @@
[:map
[:semantic-type [:fn
{:error/message "Longitude semantic type"}
#(isa? % :type/Longitude)]]]])
(mr/with-key #(isa? % :type/Longitude))]]]])
(mr/def ::drill-thru.zoom-in.geographic.column.county-state-city
[:merge
@@ -231,9 +232,9 @@
[:map
[:semantic-type [:fn
{:error/message "Country/State/City semantic type"}
#(some (fn [semantic-type]
(isa? % semantic-type))
[:type/Country :type/State :type/City])]]]])
(mr/with-key #(some (fn [semantic-type]
(isa? % semantic-type))
[:type/Country :type/State :type/City]))]]]])
(mr/def ::drill-thru.zoom-in.geographic.country-state-city->binned-lat-lon
[:merge
@@ -275,8 +276,9 @@
[:type [:= :drill-thru/zoom-in.geographic]]
[:subtype keyword?]]]
[:multi {:dispatch :subtype
:error/fn (fn [{:keys [value]} _]
(str "Invalid zoom-in.geographic drill thru subtype" (pr-str value)))}
:error/fn (mr/with-key
(fn [{:keys [value]} _]
(str "Invalid zoom-in.geographic drill thru subtype" (pr-str value))))}
[:drill-thru.zoom-in.geographic/country-state-city->binned-lat-lon
::drill-thru.zoom-in.geographic.country-state-city->binned-lat-lon]
[:drill-thru.zoom-in.geographic/binned-lat-lon->binned-lat-lon
@@ -291,12 +293,13 @@
[:max-value number?]
[:new-binning ::lib.schema.binning/binning]]])
(defn invalid-drill-error-fn [{:keys [value]} _]
(str "Invalid drill thru (unknown :type): " (pr-str value)))
(mr/def ::drill-thru
[:and
::drill-thru.common
[:multi {:dispatch :type
:error/fn (fn [{:keys [value]} _]
(str "Invalid drill thru (unknown :type): " (pr-str value)))}
[:multi {:dispatch :type :error/fn invalid-drill-error-fn}
[:drill-thru/pk ::drill-thru.pk]
[:drill-thru/fk-details ::drill-thru.fk-details]
[:drill-thru/zoom ::drill-thru.zoom]
@@ -351,7 +354,7 @@
[:column-ref [:ref ::lib.schema.ref/ref]]
[:value [:fn
{:error/message ":null should not be used in context row values, only for top-level context value"}
#(not= % :null)]]])
(mr/with-key #(not= % :null))]]])
;;; Sequence of maps with keys `:column`, `:column-ref`, and `:value`
;;;

View File

@@ -31,7 +31,7 @@
:hierarchy lib.hierarchy/hierarchy)
(mr/def ::base-type
[:multi {:dispatch (partial = ::type.unknown)}
[:multi {:dispatch (mr/with-key (partial = ::type.unknown))}
[true [:= ::type.unknown]]
[false [:ref ::common/base-type]]])
@@ -93,7 +93,7 @@
(boolean (and (vector? expr)
(#{:asc :desc} (first expr)))))
(defn- expression-schema
(defmacro ^:private expression-schema
"Schema that matches the following rules:
1a. expression is *not* an MBQL clause, OR
@@ -103,17 +103,18 @@
2. expression's [[type-of]] isa? `base-type`"
[base-type description]
[:and
;; vector = MBQL clause, anything else = not an MBQL clause
[:multi
{:dispatch vector?}
[true [:ref :metabase.lib.schema.mbql-clause/clause]]
[false [:ref :metabase.lib.schema.literal/literal]]]
[:fn
{:error/message description}
#(and (not (non-expression-clause? %))
(or *suppress-expression-type-check?*
(type-of? % base-type)))]])
`[:and
;; vector = MBQL clause, anything else = not an MBQL clause
[:multi
{:dispatch vector?}
[true [:ref :metabase.lib.schema.mbql-clause/clause]]
[false [:ref :metabase.lib.schema.literal/literal]]]
[:fn
{:error/message ~description}
(mr/with-key
#(and (not (non-expression-clause? %))
(or *suppress-expression-type-check?*
(type-of? % ~base-type))))]])
(mr/def ::boolean
(expression-schema :type/Boolean "expression returning a boolean"))
@@ -201,38 +202,41 @@
#_args [:* :any]]
[:fn
{:error/message "non-aggregation expression"}
#(letfn [(agg-tag? [tag]
(lib.hierarchy/isa? tag :metabase.lib.schema.aggregation/aggregation-clause-tag))
(agg-expr? [expr]
(and (vector? expr)
(or (agg-tag? (first expr))
(some agg-expr? (nnext expr)))))]
(not (agg-expr? %)))]])
(mr/with-key
#(letfn [(agg-tag? [tag]
(lib.hierarchy/isa? tag :metabase.lib.schema.aggregation/aggregation-clause-tag))
(agg-expr? [expr]
(and (vector? expr)
(or (agg-tag? (first expr))
(some agg-expr? (nnext expr)))))]
(not (agg-expr? %))))]])
(mr/def ::expressions
"The `:expressions` definition map as found as a top-level key in an MBQL stage."
[:and
[:sequential {:min 1} [:ref ::expression.definition]]
[:fn
{:error/message "expressions must have unique names"}
(fn [expressions]
(or (empty? expressions)
(apply distinct? (map #(:lib/expression-name (lib.options/options %)) expressions))))]])
(mr/with-key
[:fn
{:error/message "expressions must have unique names"}
(fn [expressions]
(or (empty? expressions)
(apply distinct? (map #(:lib/expression-name (lib.options/options %)) expressions))))])])
(mr/def ::positive-integer-or-numeric-expression
[:and
[:ref ::integer]
[:fn
{:error/message "positive integer literal or numeric expression"}
#(cond
(vector? %) ;; non-literal (checked above)
true
(mr/with-key
#(cond
(vector? %) ;; non-literal (checked above)
true
(not (int? %))
false
(not (int? %))
false
(not (pos? %))
false
(not (pos? %))
false
:else
true)]])
:else
true))]])

View File

@@ -54,9 +54,10 @@
[:schema [:ref ::expression/temporal]]
[:repeat [:schema [:ref :mbql.clause/interval]]]]
[:fn
{:error/fn (fn [{:keys [value]} _]
(str "Invalid :+ or :- clause: " (validate-plus-minus-temporal-arithmetic-expression value)))}
(complement validate-plus-minus-temporal-arithmetic-expression)]])
{:error/fn (mr/with-key (fn [{:keys [value]} _]
(str "Invalid :+ or :- clause: " (validate-plus-minus-temporal-arithmetic-expression value))))}
(mr/with-key
(complement validate-plus-minus-temporal-arithmetic-expression))]])
(mr/def ::plus-minus-numeric-schema
[:cat

View File

@@ -179,8 +179,9 @@
[:or
(into [:enum
{:error/message "valid timezone ID"
:error/fn (fn [{:keys [value]} _]
(str "invalid timezone ID: " (pr-str value)))}]
:error/fn (mr/with-key
(fn [{:keys [value]} _]
(str "invalid timezone ID: " (pr-str value))))}]
(sort
#?(;; 600 timezones on java 17
:clj (ZoneId/getAvailableZoneIds)
@@ -205,10 +206,11 @@
[:ref ::common/base-type]
[:fn
{:error/message ":absolute-datetime base-type must derive from :type/Date or :type/DateTime"}
(fn [base-type]
(some #(isa? base-type %)
[:type/Date
:type/DateTime]))]])
(mr/with-key
(fn [base-type]
(some #(isa? base-type %)
[:type/Date
:type/DateTime])))]])
(mr/def ::absolute-datetime.options
[:merge
@@ -269,7 +271,7 @@
value-type))))
(mr/def ::relative-datetime.amount
[:multi {:dispatch (some-fn keyword? string?)}
[:multi {:dispatch (mr/with-key (some-fn keyword? string?))}
[true [:= {:decode/normalize common/normalize-keyword-lower} :current]]
[false :int]])

View File

@@ -5,11 +5,9 @@
[metabase.util.malli.registry :as mr]))
(mr/def ::offset.n
[:and
[:and {:error/message "offset cannot be zero"}
:int
[:fn
{:error/message "offset cannot be zero"}
(some-fn pos-int? neg-int?)]])
[:not [:= 0]]])
;;; added 0.50.0
(mbql-clause/define-tuple-mbql-clause :offset

View File

@@ -6,7 +6,7 @@
;;; them, e.g. when we start working on the generative stuff
(mr/def ::database
[:schema {:doc/title "Valid Database ID"} pos-int?])
[:schema {:doc/title "Valid Database ID"} [:int {:min 1}]])
(def saved-questions-virtual-database-id
"The ID used to signify that a database is 'virtual' rather than physical.
@@ -32,34 +32,34 @@
saved-questions-virtual-database-id])
(mr/def ::table
[:schema {:doc/title "Valid Table ID"} pos-int?])
[:schema {:doc/title "Valid Table ID"} [:int {:min 1}]])
(mr/def ::field
[:schema {:doc/title "Valid Field ID"} pos-int?])
[:schema {:doc/title "Valid Field ID"} [:int {:min 1}]])
(mr/def ::card
[:schema {:doc/title "Valid Card ID"} pos-int?])
[:schema {:doc/title "Valid Card ID"} [:int {:min 1}]])
(mr/def ::segment
[:schema {:doc/title "Valid legacy Segment ID"} pos-int?])
[:schema {:doc/title "Valid legacy Segment ID"} [:int {:min 1}]])
(mr/def ::snippet
[:schema {:doc/title "Valid Snippet ID"} pos-int?])
[:schema {:doc/title "Valid Snippet ID"} [:int {:min 1}]])
(mr/def ::dimension
[:schema {:doc/title "Valid Dimension ID"} pos-int?])
[:schema {:doc/title "Valid Dimension ID"} [:int {:min 1}]])
(mr/def ::action
[:schema {:doc/title "Valid Action ID"} pos-int?])
[:schema {:doc/title "Valid Action ID"} [:int {:min 1}]])
(mr/def ::dashboard
[:schema {:doc/title "Valid Dashboard ID"} pos-int?])
[:schema {:doc/title "Valid Dashboard ID"} [:int {:min 1}]])
(mr/def ::dashcard
[:schema {:doc/title "Valid DashboardCard ID"} pos-int?])
[:schema {:doc/title "Valid DashboardCard ID"} [:int {:min 1}]])
(mr/def ::user
[:schema {:doc/title "Valid User ID"} pos-int?])
[:schema {:doc/title "Valid User ID"} [:int {:min 1}]])
(mr/def ::pulse
[:schema {:doc/title "Valid Pulse ID"} pos-int?])
[:schema {:doc/title "Valid Pulse ID"} [:int {:min 1}]])

View File

@@ -23,7 +23,7 @@
;;; with appropriate aliases.
(mr/def ::fields
[:multi
{:dispatch (some-fn keyword? string?)}
{:dispatch (mr/with-key (some-fn keyword? string?))}
[true [:enum {:decode/normalize common/normalize-keyword} :all :none]]
;; TODO -- `:fields` is supposed to be distinct (ignoring UUID), e.g. you can't have `[:field {} 1]` in there
;; twice. (#32489)
@@ -36,7 +36,8 @@
;;; Driver implementations: This is guaranteed to be present after pre-processing.
(mr/def ::alias
[:schema
{:gen/fmap #(str % "-" (random-uuid))}
(mr/with-key
{:gen/fmap #(str % "-" (random-uuid))})
::common/non-blank-string])
(mr/def ::condition
@@ -97,18 +98,21 @@
[:strategy {:optional true} ::strategy]]
[:fn
{:error/message "join should not have metadata attached directly to them; attach metadata to their last stage instead"}
(complement (some-fn :lib/stage-metadata :source-metadata))]])
(mr/with-key
(complement (some-fn :lib/stage-metadata :source-metadata)))]])
(mr/def ::joins
[:and
[:sequential {:min 1} [:ref ::join]]
[:fn
{:error/fn (fn [& _]
(i18n/tru "Join aliases must be unique at a given stage of a query"))}
(fn ensure-unique-join-aliases [joins]
(if-let [aliases (not-empty (filter some? (map :alias joins)))]
(apply distinct? aliases)
true))]])
(mr/with-key
{:error/fn (fn [& _]
(i18n/tru "Join aliases must be unique at a given stage of a query"))})
(mr/with-key
(fn ensure-unique-join-aliases [joins]
(if-let [aliases (not-empty (filter some? (map :alias joins)))]
(apply distinct? aliases)
true)))]])
(mr/def ::strategy.option
[:map

View File

@@ -111,9 +111,10 @@
[:time/offset-date-time
{:error/message "instance of java.time.OffsetDateTime"
:encode/serialize str}]
[:time/zoned-date-time
{:error/message "instance of java.time.ZonedDateTime"
:encode/serialize #(str (t/offset-date-time %))}]]
(mr/with-key
[:time/zoned-date-time
{:error/message "instance of java.time.ZonedDateTime"
:encode/serialize #(str (t/offset-date-time %))}])]
:cljs ::string.datetime))
(mr/def ::temporal

View File

@@ -9,7 +9,7 @@
(comment metabase.types.core/keep-me)
(defonce ^:private ^{:doc "Set of all registered MBQL clause tags e.g. #{:starts-with}"} tag-registry
(defonce ^{:doc "Set of all registered MBQL clause tags e.g. #{:starts-with}"} tag-registry
(atom #{}))
(defn tag->registered-schema-name
@@ -19,7 +19,7 @@
(keyword "mbql.clause" (name tag)))
(def ^:private invalid-clause-schema
[:fn {:error/message "not a known MBQL clause"} (constantly false)])
[:fn {:error/message "not a known MBQL clause"} (mr/with-key (constantly false))])
(defn- clause-schema
"Build the schema for `::clause`, a `:multi` schema that maps MBQL clause tag -> the schema
@@ -27,10 +27,11 @@
[]
(into [:multi
{:dispatch common/mbql-clause-tag
:error/fn (fn [{:keys [value]} _]
(if-let [tag (common/mbql-clause-tag value)]
(str "Invalid " tag " clause: " (pr-str value))
"not an MBQL clause"))}
:error/fn (mr/with-key
(fn [{:keys [value]} _]
(if-let [tag (common/mbql-clause-tag value)]
(str "Invalid " tag " clause: " (pr-str value))
"not an MBQL clause")))}
[::mc/default invalid-clause-schema]]
(map (fn [tag]
[tag [:ref (tag->registered-schema-name tag)]]))

View File

@@ -23,10 +23,11 @@
(mr/def ::kebab-cased-map
[:fn
{:error/message "map with all kebab-cased keys"
:error/fn (fn [{:keys [value]} _]
(if-not (map? value)
"map with all kebab-cased keys"
(str "map with all kebab-cased keys, got: " (pr-str (remove kebab-cased-key? (keys value))))))}
:error/fn (mr/with-key
(fn [{:keys [value]} _]
(if-not (map? value)
"map with all kebab-cased keys"
(str "map with all kebab-cased keys, got: " (pr-str (remove kebab-cased-key? (keys value)))))))}
kebab-cased-map?])
;;; Column vs Field?
@@ -60,13 +61,14 @@
source card, or a join, and should get field name refs instead of field ID refs --
see [[metabase.lib.field.util/inherited-column?]])."
[:enum
{:decode/normalize (fn [k]
(when-let [k (lib.schema.common/normalize-keyword k)]
;; TODO (Cam 7/1/25) -- if we wanted, we could use `:source/breakouts` to populate
;; `:lib/breakout?` -- but I think that isn't really necessary since we can
;; recalculate that information anyway.
(when-not (#{:source/fields :source/breakouts} k)
k)))}
{:decode/normalize (mr/with-key
(fn [k]
(when-let [k (lib.schema.common/normalize-keyword k)]
;; TODO (Cam 7/1/25) -- if we wanted, we could use `:source/breakouts` to populate
;; `:lib/breakout?` -- but I think that isn't really necessary since we can
;; recalculate that information anyway.
(when-not (#{:source/fields :source/breakouts} k)
k))))}
;; these are for things from some sort of source other than the current stage;
;; they must be referenced with string names rather than Field IDs
:source/card
@@ -199,10 +201,11 @@
means it's getting incorrectly propagated from a previous stage (QUE-1342)."
[:fn
{:error/message ":lib/expression-name should only be set for expressions in the current stage (:lib/source = :source/expressions)"}
(fn [m]
(if (:lib/expression-name m)
(= (:lib/source m) :source/expressions)
true))])
(mr/with-key
(fn [m]
(if (:lib/expression-name m)
(= (:lib/source m) :source/expressions)
true)))])
(mr/def ::column.validate-native-column
"Certain keys cannot possibly be set when a column comes from directly from native query results, for example
@@ -211,26 +214,29 @@
:lib/expression-name]] ; if it comes from a native query then it can't come from an expression.
[:fn
{:error/message "Invalid column metadata for a column with :lib/source :source/native"
:error/fn (fn [{m :value} _]
(some (fn [k]
(when (k m)
(str "Column has :lib/source :source/native but also " k "; this is impossible")))
disallowed-keys))}
(fn [m]
(if (= (:lib/source m) :source/native)
(every? (fn [k]
(not (k m)))
disallowed-keys)
true))]))
:error/fn (mr/with-key
(fn [{m :value} _]
(some (fn [k]
(when (k m)
(str "Column has :lib/source :source/native but also " k "; this is impossible")))
disallowed-keys)))}
(mr/with-key
(fn [m]
(if (= (:lib/source m) :source/native)
(every? (fn [k]
(not (k m)))
disallowed-keys)
true)))]))
(mr/def ::column.validate-table-defaults-column
"A column with :lib/source :source/table-defaults cannot possibly have a join alias."
[:fn
{:error/message "A column with :lib/source :source/table-defaults cannot possibly have a join alias"}
(fn [m]
(if (= (:lib/source m) :source/table-defaults)
(not (:metabase.lib.join/join-alias m))
true))])
(mr/with-key
(fn [m]
(if (= (:lib/source m) :source/table-defaults)
(not (:metabase.lib.join/join-alias m))
true)))])
;;; TODO (Cam 7/1/25) -- disabled for now because of bugs like QUE-1496; once that's fixed we should re-enable this.
#_(mr/def ::column.validate-join-alias
@@ -482,14 +488,15 @@
`:metabase.lib.schema.metadata/result-metadata` (i.e., kebab-cased) maps, or map snake_cased as returned by QP
metadata, but they should NOT be a mixture of both -- if we mixed them somehow there is a bug in our code."
[:multi
{:dispatch (fn [col]
;; if this has `:lib/type` we know FOR SURE that it's lib-style metadata; but we should also be able
;; to infer this fact automatically if it's using `kebab-case` keys. `:base-type` is required for both
;; styles so look at that.
(let [col (lib.schema.common/normalize-map-no-kebab-case col)]
(if ((some-fn :lib/type :base-type) col)
:lib
:legacy)))}
{:dispatch (mr/with-key
(fn [col]
;; if this has `:lib/type` we know FOR SURE that it's lib-style metadata; but we should also be able
;; to infer this fact automatically if it's using `kebab-case` keys. `:base-type` is required for both
;; styles so look at that.
(let [col (lib.schema.common/normalize-map-no-kebab-case col)]
(if ((some-fn :lib/type :base-type) col)
:lib
:legacy))))}
[:lib
[:merge
[:ref ::column]

View File

@@ -9,7 +9,7 @@
number?
[:fn
{:error/message "Valid percentage between (inclusive) 0 and 1."}
#(<= 0 % 1)]])
(mr/with-key #(<= 0 % 1))]])
(mr/def ::fingerprint.global
"Fingerprint values that Fields of all types should have."
@@ -59,8 +59,9 @@
[:type/DateTime {:optional true} [:ref ::fingerprint.temporal]]]
[:fn
{:error/message "Type-specific fingerprint with exactly one key"}
(fn [m]
(= 1 (count (keys m))))]])
(mr/with-key
(fn [m]
(= 1 (count (keys m)))))]])
(mr/def ::fingerprint
"Schema for a Field 'fingerprint' generated as part of the analysis stage. Used to power the 'classification'

View File

@@ -192,9 +192,11 @@
(mr/def ::dimension.target
[:multi {:dispatch lib.schema.common/mbql-clause-tag
:error/fn (fn [{:keys [value]} _]
(str "Invalid :dimension target: must be either a :field or a :template-tag, got: "
(pr-str value)))}
:error/fn
(mr/with-key
(fn [{:keys [value]} _]
(str "Invalid :dimension target: must be either a :field or a :template-tag, got: "
(pr-str value))))}
[:field [:ref ::legacy-field-ref]]
[:expression [:ref ::legacy-expression-ref]]
[:template-tag [:ref ::template-tag]]])
@@ -226,9 +228,11 @@
(mr/def ::target
[:multi {:dispatch lib.schema.common/mbql-clause-tag
:error/fn (fn [{:keys [value]} _]
(str "Invalid parameter :target, must be either :field, :dimension, or :variable; got: "
(pr-str value)))}
:error/fn
(mr/with-key
(fn [{:keys [value]} _]
(str "Invalid parameter :target, must be either :field, :dimension, or :variable; got: "
(pr-str value))))}
[:field [:ref ::legacy-field-ref]]
[:dimension [:ref ::dimension]]
[:variable [:ref ::variable]]])

View File

@@ -19,11 +19,12 @@
(mr/def ::field.options
[:merge
{:encode/serialize (fn [opts]
(m/filter-keys (fn [k]
(or (simple-keyword? k)
(= (namespace k) "lib")))
opts))}
{:encode/serialize (mr/with-key
(fn [opts]
(m/filter-keys (fn [k]
(or (simple-keyword? k)
(= (namespace k) "lib")))
opts)))}
::common/options
[:map
[:join-alias {:optional true} [:ref ::lib.schema.join/alias]]
@@ -137,8 +138,10 @@
[:and
::mbql-clause/clause
[:fn
{:error/fn (fn [_ _]
(str "Valid reference, must be one of these clauses: "
(str/join ", " (sort (descendants @lib.hierarchy/hierarchy ::ref)))))}
(fn [[tag :as _clause]]
(lib.hierarchy/isa? tag ::ref))]])
{:error/fn (mr/with-key
(fn [_ _]
(str "Valid reference, must be one of these clauses: "
(str/join ", " (sort (descendants @lib.hierarchy/hierarchy ::ref))))))}
(mr/with-key
(fn [[tag :as _clause]]
(lib.hierarchy/isa? tag ::ref)))]])

View File

@@ -89,9 +89,10 @@
(mr/def ::disallow-dimension
[:fn
{:decode/normalize #(dissoc % :dimension)
{:decode/normalize (mr/with-key #(dissoc % :dimension))
:error/message ":dimension is only allowed for :type :dimension template tags"}
#(not (contains? % :dimension))])
(mr/with-key
#(not (contains? % :dimension)))])
;; Example:
;;
@@ -160,7 +161,8 @@
{:decode/normalize common/normalize-map}
[:map
[:type [:ref ::type]]]
[:multi {:dispatch #(keyword (:type %))}
[:multi {:dispatch
(mr/with-key #(keyword (:type %)))}
[:temporal-unit [:ref ::temporal-unit]]
[:dimension [:ref ::field-filter]]
[:snippet [:ref ::snippet]]
@@ -174,7 +176,8 @@
;; make sure people don't try to pass in a `:name` that's different from the actual key in the map.
[:fn
{:error/message "keys in template tag map must match the :name of their values"}
(fn [m]
(every? (fn [[tag-name tag-definition]]
(= tag-name (:name tag-definition)))
m))]])
(mr/with-key
(fn [m]
(every? (fn [[tag-name tag-definition]]
(= tag-name (:name tag-definition)))
m)))]])

View File

@@ -50,8 +50,9 @@
(mr/def ::unique-uuids
[:fn
{:error/message "all :lib/uuids must be unique"
:error/fn (fn [{:keys [value]} _]
(str "Duplicate :lib/uuid " (pr-str (find-duplicate-uuid value))))}
:error/fn (mr/with-key
(fn [{:keys [value]} _]
(str "Duplicate :lib/uuid " (pr-str (find-duplicate-uuid value)))))}
#'unique-uuids?])
(defn ref-distinct-key
@@ -113,9 +114,11 @@
(mr/def ::distinct-ignoring-uuids
[:fn
{:error/message "values must be distinct ignoring uuids"
:error/fn (fn [{:keys [value]} _]
(str "Duplicate values ignoring uuids in: " (pr-str (remove-lib-uuids value))))}
(comp u/empty-or-distinct? remove-lib-uuids)])
:error/fn (mr/with-key
(fn [{:keys [value]} _]
(str "Duplicate values ignoring uuids in: " (pr-str (remove-lib-uuids value)))))}
(mr/with-key
(comp u/empty-or-distinct? remove-lib-uuids))])
(defn distinct-ignoring-uuids
"Add an additional constraint to `schema` that requires all elements to be distinct after removing uuids."

View File

@@ -103,15 +103,15 @@
[:merge
[:map
[:payload_type (ms/enum-decode-keyword notification-types)]]
[:multi {:dispatch (comp keyword :payload_type)}
[:multi {:dispatch (mr/with-key (comp keyword :payload_type))}
[:notification/system-event
[:map
[:payload_id {:optional true} nil?]]]
[:notification/card
[:map
;; optional during creation
[:payload_id {:optional true} int?]
[:creator_id {:optional true} int?]]]
[:payload_id {:optional true} :int]
[:creator_id {:optional true} :int]]]
[:notification/testing :any]]])
(defn- validate-notification
@@ -183,7 +183,8 @@
"Schema for :model/NotificationSubscription."
[:merge [:map
[:type (ms/enum-decode-keyword subscription-types)]]
[:multi {:dispatch (comp keyword :type)}
[:multi {:dispatch (mr/with-key
(comp keyword :type))}
[:notification-subscription/system-event
[:map
[:event_name [:or :keyword :string]]
@@ -291,7 +292,8 @@
[:map
;; optional during insertion
[:notification_id {:optional true} ms/PositiveInt]
[:channel_type {:decode/json keyword} [:fn #(= "channel" (-> % keyword namespace))]]
[:channel_type {:decode/json keyword} [:fn (mr/with-key
#(= "channel" (-> % keyword namespace)))]]
[:channel_id {:optional true} [:maybe ms/PositiveInt]]
[:template_id {:optional true} [:maybe ms/PositiveInt]]
[:active {:optional true} [:maybe :boolean]]])
@@ -332,30 +334,30 @@
[:merge [:map
[:type (ms/enum-decode-keyword notification-recipient-types)]
[:notification_handler_id {:optional true} ms/PositiveInt]]
[:multi {:dispatch (comp keyword :type)}
[:multi {:dispatch (mr/with-key (comp keyword :type))}
[:notification-recipient/user
[:map
[:user_id ms/PositiveInt]
[:permissions_group_id {:optional true} [:fn nil?]]
[:details {:optional true} [:fn empty?]]]]
[:permissions_group_id {:optional true} :nil]
[:details {:optional true} [:fn (mr/with-key empty?)]]]]
[:notification-recipient/group
[:map
[:permissions_group_id ms/PositiveInt]
[:user_id {:optional true} [:fn nil?]]
[:details {:optional true} [:fn empty?]]]]
[:user_id {:optional true} :nil]
[:details {:optional true} [:fn (mr/with-key empty?)]]]]
[:notification-recipient/raw-value
[:map
[:details [:map {:closed true}
[:value :any]]]
[:user_id {:optional true} [:fn nil?]]
[:permissions_group_id {:optional true} [:fn nil?]]]]
[:user_id {:optional true} :nil]
[:permissions_group_id {:optional true} :nil]]]
[:notification-recipient/template
[:map
[:details [:map {:closed true}
[:pattern :string]
[:is_optional {:optional true} :boolean]]]
[:user_id {:optional true} [:fn nil?]]
[:permissions_group_id {:optional true} [:fn nil?]]]]]])
[:user_id {:optional true} :nil]
[:permissions_group_id {:optional true} :nil]]]]])
(defn- check-valid-recipient
[recipient]
@@ -502,7 +504,7 @@
[:template {:optional true} [:maybe ::models.channel/ChannelTemplate]]
[:channel {:optional true} [:maybe ::models.channel/Channel]]
[:recipients {:optional true} [:sequential ::NotificationRecipient]]]]]]]
[:multi {:dispatch (comp keyword :payload_type)}
[:multi {:dispatch (mr/with-key (comp keyword :payload_type))}
[:notification/card [:map
[:payload ::NotificationCard]]]
[::mc/default :map]]])

View File

@@ -41,7 +41,7 @@
[:payload
[:map {:closed true}
;; TODO: event-info schema for each event type
[:event_topic [:fn #(= "event" (-> % keyword namespace))]]
[:event_topic [:fn (mr/with-key #(= "event" (-> % keyword namespace)))]]
[:event_info [:maybe :map]]]]]]
[:notification/card
[:map
@@ -79,7 +79,7 @@
;; override the payload with extra context
[:payload
[:map {:closed true}
[:event_topic [:fn #(= "event" (-> % keyword namespace))]]
[:event_topic [:fn (mr/with-key #(= "event" (-> % keyword namespace)))]]
[:event_info [:maybe :map]]
[:custom {:optional true} [:maybe :map]]]]]]
[:notification/dashboard

View File

@@ -14,9 +14,10 @@
(mr/def ::legacy-field-or-expression-reference
"Schema for a valid legacy `:field` or `:expression` reference (possibly not yet normalized)."
[:fn
(fn [k]
((comp (mr/validator mbql.s/Field)
mbql.normalize/normalize-tokens) k))])
(mr/with-key
(fn [k]
((comp (mr/validator mbql.s/Field)
mbql.normalize/normalize-tokens) k)))])
(mr/def ::values-source-config
"Schema for valid source_options within a Parameter"

View File

@@ -399,3 +399,7 @@
;; The following arity is provided soley for convenience for tests/REPL usage
([ks :- [:vector :any] new-value]
(update-data-perms-graph! (assoc-in (api-graph) (cons :groups ks) new-value))))
(mu/defn my-f :- [:fn {:pos :output} (constantly true)]
[a :- [:fn {:arg :a} (constantly true)]
b :- [:fn {:arg :b} (constantly true)]])

View File

@@ -6,8 +6,8 @@
(mr/def ::card
[:map {:closed true}
[:user-id [:maybe pos-int?]]
[:object [:fn #(t2/instance-of? :model/Card %)]]])
[:user-id [:maybe [:int {:min 1}]]]
[:object [:fn (mr/with-key #(t2/instance-of? :model/Card %))]]])
(mr/def :event/card-create ::card)
(mr/def :event/card-update ::card)
@@ -17,11 +17,11 @@
[:map {:closed true}
;; context is deliberately coupled to view-log's context
[:context view-log/context]
[:user-id [:maybe pos-int?]]
[:object-id [:maybe pos-int?]]])
[:user-id [:maybe [:int {:min 1}]]]
[:object-id [:maybe [:int {:min 1}]]]])
(mr/def :event/card-query
[:map {:closed true}
[:card-id pos-int?]
[:user-id [:maybe pos-int?]]
[:card-id [:int {:min 1}]]
[:user-id [:maybe [:int {:min 1}]]]
[:context {:optional true} :any]])

View File

@@ -58,7 +58,7 @@
:map
[:fn
{:error/message "options map without namespaced keys and base-type/effective-type"}
(complement (some-fn :base-type :effective-type :ident :lib/uuid))]]
(mr/with-key (complement (some-fn :base-type :effective-type :ident :lib/uuid)))]]
[:or
::lib.schema.id/field
:string]])

View File

@@ -48,10 +48,11 @@
::col
[:fn
{:error/message "map with QP results casing rules for keys"}
(fn [m]
(every? (fn [k]
(= k (key->qp-results-key k)))
(keys m)))]])
(mr/with-key
(fn [m]
(every? (fn [k]
(= k (key->qp-results-key k)))
(keys m))))]])
(mr/def ::cols
[:maybe [:sequential ::col]])

View File

@@ -89,7 +89,8 @@
qualified-keyword?
[:fn
{:error/message "do-not-bucket-reason keyword"}
#(= (namespace %) "do-not-bucket-reason")]])
(mr/with-key
#(= (namespace %) "do-not-bucket-reason"))]])
;;; This returns a keyword corresponding to why we're not autobucketing for debugging/testing purposes
(mu/defn- should-not-be-autobucketed? :- [:maybe ::do-not-bucket-reason]

View File

@@ -84,8 +84,8 @@
[:sequential ::breakout-combination]
[:fn
{:error/message "Distinct combinations"}
#(or (empty? %)
(apply distinct? %))]])
(mr/with-key #(or (empty? %)
(apply distinct? %)))]])
(mu/defn- breakout-combinations :- ::breakout-combinations
"Return a sequence of all breakout combinations (by index) we should generate queries for.

View File

@@ -11,7 +11,7 @@
:map
[:fn
{:error/message "Query with a :type or :lib/type key"}
(some-fn :type :lib/type)]])
(mr/with-key (some-fn :type :lib/type))]])
;; TODO -- fill this out a bit.
(mr/def ::metadata :any)

View File

@@ -20,6 +20,7 @@
[metabase.util.json :as json]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
[methodical.core :as methodical]
[toucan2.core :as t2])
(:import
@@ -99,8 +100,9 @@
(def ^:private Type
[:fn
{:error/message "Valid Setting :type"}
(fn [a-type]
(contains? (set (keys (methods get-value-of-type))) a-type))])
(mr/with-key
(fn [a-type]
(contains? (set (keys (methods get-value-of-type))) a-type)))])
(def ^:private Visibility
[:enum :public :authenticated :settings-manager :admin :internal])
@@ -172,11 +174,11 @@
[:type Type]
;; different getters/setters take care of parsing/unparsing
[:getter ifn?]
[:setter ifn?]
[:getter 'ifn?]
[:setter 'ifn?]
;; an init function can be used to seed initial values
[:init [:maybe ifn?]]
[:init [:maybe 'ifn?]]
;; type annotation, e.g. ^String, to be applied. Defaults to tag based on :type
[:tag :symbol]
@@ -211,7 +213,7 @@
;; called whenever setting value changes, whether from update-setting! or a cache refresh. used to handle cases
;; where a change to the cache necessitates a change to some value outside the cache, like when a change the
;; `:site-locale` setting requires a call to `java.util.Locale/setDefault`
[:on-change [:maybe ifn?]]
[:on-change [:maybe 'ifn?]]
;; If non-nil, determines the Enterprise feature flag required to use this setting. If the feature is not enabled,
;; the setting will behave the same as if `enabled?` returns `false` (see below).
@@ -219,7 +221,7 @@
;; Function which returns true if the setting should be enabled. If it returns false, the setting will throw an
;; exception when it is attempted to be set, and will return its default value when read. Defaults to always enabled.
[:enabled? [:maybe ifn?]]
[:enabled? [:maybe 'ifn?]]
;; Keyword that determines what kind of audit log entry should be created when this setting is written. Options are
;; `:never`, `:no-value`, `:raw-value`, and `:getter`. User- and database-local settings are never audited. `:getter`

View File

@@ -24,7 +24,7 @@
(set! *warn-on-reflection* true)
(def ^:private ^:deprcated SetupToken
(def ^:private ^:deprecated SetupToken
"Schema for a string that matches the instance setup token."
(mu/with-api-error-message
[:and

View File

@@ -1,6 +1,7 @@
(ns metabase.sync.interface
"Schemas and constants used by the sync code."
(:require
[malli.core :as mc]
[malli.util :as mut]
[metabase.lib.schema.common :as lib.schema.common]
[metabase.util.malli.registry :as mr]
@@ -83,7 +84,8 @@
(mr/def ::FieldMetadataEntry
(-> (mr/schema ::TableMetadataField)
(mut/assoc :table-schema [:maybe ::lib.schema.common/non-blank-string])
(mut/assoc :table-name ::lib.schema.common/non-blank-string)))
(mut/assoc :table-name ::lib.schema.common/non-blank-string)
mc/form))
(def FieldMetadataEntry
"Schema for an item in the expected output of [[metabase.driver/describe-fields]]."

View File

@@ -31,12 +31,12 @@
"A namespace for the key-value pair"
[:and
{;; api request comes in, turn `foo` into `:namespace/foo`
:decode/api-request (partial keyword "namespace")
:decode/api-request (mr/with-key (partial keyword "namespace"))
:encode/api-request name
;; writing to the DB, turn `:namespace/foo` into `foo`
:encode/database name
;; reading from the DB, turn `foo` into `:namespace/foo`
:decode/database (partial keyword "namespace")}
:decode/database (mr/with-key (partial keyword "namespace"))}
:keyword])
(mr/def ::expires-at
@@ -66,7 +66,7 @@
[:expires-at [:maybe ::expires-at]]
[:namespace ::namespace]
[:value {:encode/database json/encode
:decode/database #(json/decode % keyword)}
:decode/database (mr/with-key #(json/decode % keyword))}
:any]]
(into [:multi
{:dispatch :namespace}]

View File

@@ -14,22 +14,45 @@
(set! *warn-on-reflection* true)
(mr/def ::CronScheduleString
(mu/with-api-error-message
[:and
ms/NonBlankString
[:fn
{:error/fn (fn [{:keys [value]} _]
(try
(CronExpression/validateExpression value)
(catch Throwable e
(str "Invalid cron schedule string: " (.getMessage e)))))}
(fn [^String s]
(try
(CronExpression/validateExpression s)
true
(catch Throwable _
false)))]]
(i18n/deferred-tru "value must be a valid Quartz cron schedule string.")))
(mr/with-key
(mu/with-api-error-message
[:and
ms/NonBlankString
[:fn
{:error/fn (mr/with-key
(fn [{:keys [value]} _]
(try
(CronExpression/validateExpression value)
(catch Throwable e
(str "Invalid cron schedule string: " (.getMessage e))))))}
(mr/with-key
(fn [^String s]
(try
(CronExpression/validateExpression s)
true
(catch Throwable _
false))))]]
(i18n/deferred-tru "value must be a valid Quartz cron schedule string."))))
(meta (mr/with-key
(mu/with-api-error-message
[:and
ms/NonBlankString
[:fn
{:error/fn (mr/with-key
(fn [{:keys [value]} _]
(try
(CronExpression/validateExpression value)
(catch Throwable e
(str "Invalid cron schedule string: " (.getMessage e))))))}
(mr/with-key
(fn [^String s]
(try
(CronExpression/validateExpression s)
true
(catch Throwable _
false))))]]
(i18n/deferred-tru "value must be a valid Quartz cron schedule string."))))
(def CronScheduleString
"Malli Schema for a valid cron schedule string."

View File

@@ -33,13 +33,12 @@
(me/humanize {:wrap humanize-include-value})))
(def ^:private Schema
[:and any?
[:fn {:description "a malli schema"} mc/schema]])
[:fn {:description "a malli schema"} (mr/with-key mc/schema)])
(def localized-string-schema
"Schema for localized string."
#?(:clj [:fn {:error/message "must be a localized string"}
i18n/localized-string?]
(mr/with-key i18n/localized-string?)]
;; TODO Is there a way to check if a string is being localized in CLJS, by the `ttag`?
;; The compiler seems to just inline the translated strings with no annotation or wrapping.
:cljs :string))
@@ -50,8 +49,7 @@
[mschema props]
(mut/update-properties (mc/schema mschema) merge props))
;; Kondo gets confused by :refer [defn] on this, so it's referenced fully qualified.
(metabase.util.malli/defn with-api-error-message
(defmacro with-api-error-message
"Update a malli schema to have a :description (used by umd/describe, which is used by api docs),
and a :error/fn (used by me/humanize, which is used by defendpoint).
They don't have to be the same, but usually are.
@@ -60,16 +58,15 @@
[:string {:min 1}]
(deferred-tru \"Must be a string with at least 1 character representing a User ID.\"))"
{:style/indent [:form]}
([mschema :- Schema error-message :- localized-string-schema]
(with-api-error-message mschema error-message error-message))
([mschema :- :any
description-message :- localized-string-schema
specific-error-message :- localized-string-schema]
(mut/update-properties (mc/schema mschema) assoc
;; override generic description in api docs and :errors key in API's response
:description description-message
;; override generic description in :specific-errors key in API's response
:error/fn (constantly specific-error-message))))
([mschema error-message]
`(with-api-error-message ~mschema ~error-message ~error-message))
([mschema description-message specific-error-message]
`(mr/with-key
(mut/update-properties (mc/schema ~mschema) assoc
;; override generic description in api docs and :errors key in API's response
:description ~description-message
;; override generic description in :specific-errors key in API's response
:error/fn (mr/with-key (constantly ~specific-error-message))))))
#?(:clj
(defmacro disable-enforcement

View File

@@ -257,7 +257,7 @@
'more [:maybe [:* :any]]
'kvs [:* :any]
:any))
`(validate-input ~error-context ~schema ~arg-name)))
`(validate-input ~error-context (mr/with-key ~schema) ~arg-name)))
arg-names
schemas)
(filter some?))))
@@ -292,11 +292,12 @@
arglist (input-schema->arglist input-schema)
input-validation-forms (input-schema->validation-forms error-context input-schema)
result-form (input-schema->application-form input-schema)
result-form (if (and output-schema
(not= output-schema :any))
`(->> ~result-form
(validate-output ~error-context ~output-schema))
result-form (if (and output-schema (not= output-schema :any))
`(validate-output ~error-context
(mr/with-key ~output-schema)
~result-form)
result-form)]
`(~arglist
(try
~@input-validation-forms
@@ -314,6 +315,52 @@
(for [schema schemas]
(instrumented-arity error-context schema)))))
#_parsed
;; => #malli.core.Tags{:values
;; {:name my-multi-f,
;; :return #malli.core.Tags{:values {:- :-, :schema (fn my-odd [x] (odd? x))}},
;; :doc nil,
;; :meta nil,
;; :arities
;; #malli.core.Tag{:key :multiple,
;; :value
;; #malli.core.Tags{:values
;; {:arities
;; [#malli.core.Tags{:values
;; {:args [a :- [:fn (fn my-even [x] (even? x))]],
;; :prepost nil,
;; :body []}}
;; #malli.core.Tags{:values
;; {:args
;; [a
;; :-
;; [:fn (fn my-even [x] (even? x))]
;; b
;; :-
;; :int],
;; :prepost nil,
;; :body []}}],
;; :meta nil}}}}}
;; => #malli.core.Tags{:values
;; {:name my-f,
;; :return #malli.core.Tags{:values {:- :-, :schema (fn my-odd [x] (odd? x))}},
;; :doc nil,
;; :meta nil,
;; :arities
;; #malli.core.Tag{:key :single,
;; :value
;; #malli.core.Tags{:values
;; {:args [a :- [:fn (fn my-even [x] (even? x))]],
;; :prepost nil,
;; :body []}}}}}
'(let [input-validator (mc/validator [:cat [:fn (fn my-even [x] (even? x))] :int])])
#_(def input-schema '[:cat [:fn (fn my-even [x] (even? x))]])
#_(instrumented-fn-form {} parsed 'my-f)
(defn instrumented-fn-form
"Nota Bene: not safe for expansion into Clojurescript!
Given a `fn-tail` like
@@ -327,6 +374,8 @@
(mc/-instrument {:schema [:=> [:cat :int :any] :any]}
(fn [x y] (+ 1 2)))"
[error-context parsed & [fn-name]]
(def parsed parsed)
;; (prn ["parsed" parsed])
`(let [~'&f ~(deparameterized-fn-form parsed fn-name)]
(core/fn ~@(instrumented-fn-tail error-context (fn-schema parsed)))))

View File

@@ -2,43 +2,125 @@
(:refer-clojure :exclude [declare def])
(:require
#?@(:clj ([malli.experimental.time :as malli.time]
[net.cgrand.macrovich :as macros]))
[net.cgrand.macrovich :as macros])) ;; TODO: consider using a ttl cache incase any bad schemas start slipping through? [clojure.core.cache :as cache]
[clojure.string :as str]
[clojure.walk :as walk]
[malli.core :as mc]
[malli.registry]
[malli.util :as mut])
[malli.util :as mut]
[metabase.config.core :as config]
[metabase.util.log :as log])
#?(:cljs (:require-macros [metabase.util.malli.registry])))
(defonce ^:private cache (atom {}))
(defonce
^{:private true
:doc "Has the form:
{[k cache-key] function}
Example:
{[:validator ::my/int] \"#function[clojure.core/int]\"}"}
cache (atom {}))
(defn- schema-cache-key
"Make schemas that aren't `=` to identical ones e.g.
[:re #\"\\d{4}\"]
[:or [:re #\"\\d{4}\"] :int]
[:fn (constantly true)]
[:fn even?]
work correctly as cache keys instead of creating new entries every time the code is evaluated."
[x]
(if (and (vector? x)
(= (first x) :re))
(into (empty x)
(map (fn [child]
(cond-> child
(instance? #?(:clj java.util.regex.Pattern :cljs js/RegExp) child) str)))
x)
x))
work correctly as cache keys instead of creating new entries every time the code is evaluated.
(defn cached
"Get a cached value for `k` + `schema`. Cache is cleared whenever a schema is (re)defined
Also handles functions to prevent cache key instability from composed/anonymous functions.
;; both are anti-patterns and will throw:
(let [my-fn (constantly 3)]
(schema-cache-key [:fn my-fn]))
(let [my-fn (constantly 3)]
(schema-cache-key (with-key my-fn)))"
[cache-keyable]
(or
(some-> cache-keyable meta ::key)
(walk/postwalk
(fn [x]
(if #?(:clj (instance? java.util.regex.Pattern x)
:cljs (instance? js/RegExp x)) (str x)
(let [meta-key (some-> x meta ::key)
_ (when (and (not config/is-prod?)
meta-key
(not (#{"(" "["} (str (first meta-key)))))
(log/info
(str/join ["Schema cache key forms (not their values per se) must be wrapped with [[mu/with-key]] to ensure stability."
"If you are using a derived schema, either wrap the form in the generation function, "
"or a macro with [[mu/with-key]] to wrap the schema form itself."
(pr-str {::error "iono" :meta-key meta-key :schema x})])))]
(if meta-key meta-key
(if (fn? x)
(do
(log/info (str "function schemas must be wrapped with [[mu/with-key]], see the docstring for more info.\n"
(pr-str {::error "function without key" :schema x})))
x)
x)))))
cache-keyable)))
(defmacro with-key
"Adds `::mr/key` metadata, which is a pr-str'd string of body, to body. Be careful not to call this
on functions taking parameters, since it uses the shape of the literal body passed to it as a key.
e.g.:
(defn my-schema [] :int)
(mr/with-key (my-schema))
If you change `my-schema` to return `:keyword` here, the cache will not invalidate properly.
All functions that are used with mr/validate, mu/defn, and defendpoint need their function schema _forms_ wrapped with
this macro."
[body]
`(try
(with-meta ~body (merge (meta ~body) {::key ~(pr-str body)}))
(catch Exception _# ~body)))
(defmacro stable-key?
"Evaluates the schema form twice, and if the results are not equal, it is not usable as a cache key."
[schema]
`(let [computed-schema# (#'schema-cache-key ~schema)]
(if (= computed-schema#
(#'schema-cache-key ~schema))
computed-schema#
false)))
(def ^{:dynamic true
:doc "A hook that is called whenever the cache is updated, for side effects.
This is used in tests or to log cache updates."}
*cache-miss-hook* (fn [_k _schema _value-thunk] nil))
(binding [*cache-miss-hook* (fn [k schema value-thunk]
(log/info (str "Cache miss for " (pr-str k) " with schema " (pr-str schema)
". Calculating value with " (pr-str value-thunk) ".")))]
(defn cached
"Get a cached value for `k` + `schema`. Cache is cleared whenever a schema is (re)defined
with [[metabase.util.malli.registry/def]]. If value doesn't exist, `value-thunk` is used to calculate (and cache)
it.
You generally shouldn't use this outside of this namespace unless you have a really good reason to do so! Make sure
you used namespaced keys if you are using it elsewhere."
[k schema value-thunk]
(let [schema-key (schema-cache-key schema)]
(or (get (get @cache k) schema-key) ; get-in is terribly inefficient
(let [v (value-thunk)]
(swap! cache assoc-in [k schema-key] v)
v))))
[k schema value-thunk]
(let [schema-key (schema-cache-key schema)]
(or (get @cache [k schema-key])
(do (*cache-miss-hook* k schema value-thunk) false)
(let [v (value-thunk)]
(swap! cache assoc [k schema-key] v)
v)))))
(defn cache-size-info
"Return information about the current cache size and structure for debugging memory issues."
[cache-val]
(let [total-entries (count cache-val)
by-type (->> (group-by (fn [[k _]] (first k)) cache-val)
(mapv (fn [[k v]]
[k (count v)]))
(into {}))]
{:total-cache-entries total-entries
:entries-by-type by-type}))
(defn validator
"Fetch a cached [[mc/validator]] for `schema`, creating one if needed. The cache is flushed whenever the registry
@@ -97,7 +179,8 @@
"Register a spec with our Malli spec registry."
[schema definition]
(swap! registry* assoc schema definition)
(reset! cache {})
;; TODO: Fixme cache fix prereq
;; (reset! cache {})
nil)
(defn registered-schema
@@ -110,8 +193,6 @@
[type]
(malli.registry/schema registry type))
;;; TODO -- we should change `:doc/message` to `:description` so it's inline
;;; with [[metabase.util.malli.describe/describe]] and [[malli.experimental.describe/describe]]
(defn -with-doc
"Add a `:description` option to a `schema`. Tries to merge it in existing vector schemas to avoid unnecessary
indirection."
@@ -133,7 +214,7 @@
(defmacro def
"Like [[clojure.spec.alpha/def]]; add a Malli schema to our registry."
([type schema]
`(register! ~type ~schema))
`(register! ~type (with-key ~schema)))
([type docstring schema]
(assert (string? docstring))
`(metabase.util.malli.registry/def ~type
@@ -149,7 +230,7 @@
(-> schema
(mc/-set-properties (merge (mc/properties schema) properties))))
(deref* [schema]
(let [dereffed (-> schema mc/deref deref-all-preserving-properties)
(let [dereffed (-> schema mc/deref deref-all-preserving-properties)
properties (mc/properties schema)]
(cond-> dereffed
(seq properties) (with-properties properties))))]
@@ -179,3 +260,11 @@
;; not sure this option is really needed, but [[mc/deref-recursive]] sets it... turning it off doesn't
;; seem to make any of our tests fail so maybe I'm not capturing something
{::mc/walk-schema-refs true})))
(comment
(count @cache)
;; inspect cache keys:
;; (map (comp schema-cache-key second) (keys @cache))
)

View File

@@ -9,6 +9,7 @@
[metabase.util.i18n :as i18n :refer [deferred-tru]]
[metabase.util.json :as json]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
[metabase.util.password :as u.password]
[toucan2.core :as t2]))
@@ -70,6 +71,8 @@
;;; -------------------------------------------------- Schemas --------------------------------------------------
;; TODO: these should be moved over to `mr/def`.
(def NonBlankString
"Schema for a string that cannot be blank."
(mu/with-api-error-message
@@ -83,7 +86,7 @@
[:string {:min 1}]
[:fn
{:error/message "non-blank string"}
(complement str/blank?)]]
(mr/with-key (complement str/blank?))]]
(deferred-tru "value must be a non-blank string.")))
(def IntGreaterThanOrEqualToZero
@@ -92,8 +95,7 @@
[:int
{:min 0
:description (str message)
:error/fn (fn [_ _]
(str message))
:error/fn (mr/with-key (fn [_ _] (str message)))
:api/regex #"\d+"}]))
(def Int
@@ -101,8 +103,7 @@
(let [message (deferred-tru "value must be an integer.")]
[:int
{:description (str message)
:error/fn (fn [_ _]
(str message))
:error/fn (mr/with-key (fn [_ _] (str message)))
:api/regex #"-?\d+"}]))
(def PositiveInt
@@ -223,20 +224,20 @@
(one of `\"true\"` or `true` or `\"false\"` or `false`.).
Used by [[metabase.api.common/defendpoint]] to coerce the value for this schema to a boolean.
Garanteed to evaluate to `true` or `false` when passed through a json decoder."
(-> [:enum {:decode/json (fn [b] (contains? #{"true" true} b))
:json-schema {:type "boolean"}}
"true" "false" true false]
(mu/with-api-error-message
(deferred-tru "value must be a valid boolean string (''true'' or ''false'')."))))
(mu/with-api-error-message
[:enum {:decode/json (fn [b] (contains? #{"true" true} b))
:json-schema {:type "boolean"}}
"true" "false" true false]
(deferred-tru "value must be a valid boolean string (''true'' or ''false'').")))
(def MaybeBooleanValue
"Same as above, but allows distinguishing between `nil` (the user did not specify a value)
and `false` (the user specified `false`)."
(-> [:enum {:decode/json (fn [b] (some->> b (contains? #{"true" true})))
:json-schema {:type "boolean" :optional true}}
"true" "false" true false nil]
(mu/with-api-error-message
(deferred-tru "value must be a valid boolean string (''true'' or ''false'')."))))
(mu/with-api-error-message
[:enum {:decode/json (fn [b] (some->> b (contains? #{"true" true})))
:json-schema {:type "boolean" :optional true}}
"true" "false" true false nil]
(deferred-tru "value must be a valid boolean string (''true'' or ''false'').")))
(def RemappedFieldValue
"Has two components:
@@ -297,19 +298,20 @@
(defn MapWithNoKebabKeys
"Helper for creating a schema to check if a map doesn't contain kebab case keys."
[]
[:fn
{:error/message "Map should not contain any kebab-case keys"}
(fn [m]
;; reduce-kv is more efficient that iterating over (keys m). But we have to extract the underlying map from
;; Toucan2 Instance because it doesn't implement IKVReduce (yet).
(let [m (if (instance? toucan2.instance.Instance m)
(.m ^toucan2.instance.Instance m)
m)]
(reduce-kv (fn [_ k _]
(if (str/includes? k "-")
(reduced false)
true))
true m)))])
(mr/with-key
[:fn
{:error/message "Map should not contain any kebab-case keys"}
(fn [m]
;; reduce-kv is more efficient that iterating over (keys m). But we have to extract the underlying map from
;; Toucan2 Instance because it doesn't implement IKVReduce (yet).
(let [m (if (instance? toucan2.instance.Instance m)
(.m ^toucan2.instance.Instance m)
m)]
(reduce-kv (fn [_ k _]
(if (str/includes? k "-")
(reduced false)
true))
true m)))]))
(def File
"Schema for a file coming in HTTP request from multipart/form-data"

View File

@@ -1,6 +1,7 @@
(ns metabase.util.retry
"Support for in-memory, thread-blocking retrying."
(:require
[malli.core :as mc]
[malli.util :as mut]
[metabase.config.core :as config]
[metabase.settings.core :refer [defsetting]]
@@ -25,7 +26,8 @@
[:retry-on-result-pred {:optional true} [:=> [:cat :any] :boolean]]])
(mr/def ::retry-overrides
(mut/optional-keys [:ref ::retry-config]))
(mc/form
(mut/optional-keys [:ref ::retry-config])))
;;; these kondo warnings are ignored for now because I'm planning on moving this namespace out of `util` to eliminate
;;; the dependency of `util` of `settings` -- will fix them after this namespace gets moved. -- Cam

View File

@@ -179,21 +179,23 @@
(mr/def ::map
[:map
{:gen/schema map-pairs
:gen/fmap (fn [kvs]
(into {} kvs))}])
:gen/fmap (mr/with-key
(fn [kvs]
(into {} kvs)))}])
(mr/def ::exprs
[:cat
{:gen/fmap (fn [exprs]
;; sequence of exprs can't start with a keyword, otherwise it would be an MBQL clause. Add an extra
;; value at the beginning if the generated value starts with a keyword.
(let [exprs (if (keyword? (first exprs))
(cons {:a 1} exprs)
exprs)]
;; randomly return either a vector or sequence.
(mg/generate [:or
[:= {} (apply list exprs)]
[:= {} (vec exprs)]])))}
{:gen/fmap (mr/with-key
(fn [exprs]
;; sequence of exprs can't start with a keyword, otherwise it would be an MBQL clause. Add an extra
;; value at the beginning if the generated value starts with a keyword.
(let [exprs (if (keyword? (first exprs))
(cons {:a 1} exprs)
exprs)]
;; randomly return either a vector or sequence.
(mg/generate [:or
[:= {} (apply list exprs)]
[:= {} (vec exprs)]]))))}
[:repeat
{:max 5}
[:ref ::expr]]])

View File

@@ -1,25 +1,124 @@
(ns metabase.util.malli.registry-test
#?(:cljs (:require-macros
[metabase.util.malli.registry-test :refer [stable-key?]]))
(:require
#?@(:clj
([metabase.util.i18n :as i18n]))
[clojure.test :refer [are deftest is testing]]
#?@(:clj ([metabase.util.i18n :as i18n :refer [deferred-tru]]))
[clojure.string :as str]
[clojure.test :refer [are deftest is testing use-fixtures]]
[clojure.walk :as walk]
[malli.core :as mc]
[malli.error :as me]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]))
(defn- clear-cache []
(reset! @#'mr/cache {}))
(use-fixtures :each (fn [t] (clear-cache) (t)))
(let [cache-misses (atom [])]
(binding [mr/*cache-miss-hook* (fn [k schema _value-thunk]
(swap! cache-misses conj [k schema]))]
(mr/validate [:fn (fn [x] (or (int? x) (string? x)))] "x")
(mr/validate [:fn (fn [x] (or (int? x) (string? x)))] "x"))
@cache-misses)
(def to-wrap? #{'partial
'comp
'complement
'constantly
'juxt
'every-pred
'some-fn
'fnil
'completing
'comparator
'fn
'fn*})
;; (defmacro auto-wrap-schema [schema-form]
;; `'~(walk/postwalk
;; (fn [x]
;; (if (and (list? x)
;; (to-wrap? (first x)))
;; (list 'mr/with-key x)
;; x))
;; schema-form))
;; (auto-wrap-schema [:map [:x [:fn (constantly true)]]])
(defmacro with-returning-cache-misses [& body]
`(let [cache-misses# (atom [])]
(binding [mr/*cache-miss-hook* (fn [k# schema# _#]
(swap! cache-misses# conj [k# schema#]))]
~@body)
@cache-misses#))
(deftest mu-defn-with-cachable-schemas-does-not-grow-cache
(is (= []
(with-returning-cache-misses
(mu/defn my-good-fn :- [:fn (mr/with-key (fn [_] true))]
"A good function that should be stable in the cache."
[_a :- [:fn (mr/with-key (fn [_]))]]
true)))))
(defn- excise-fns [x]
(walk/postwalk (fn [x] (if (fn? x) :function x)) x))
(deftest mu-defn-with-cachable-schemas-and-calling-it-does-not-grow-cache
(is (= [[:explainer [:fn {:marker 1} :function]]]
(excise-fns
(with-returning-cache-misses
(mu/defn my-good-fn :- (mr/with-key [:fn {:marker 1} (fn [_] true)])
"A good function that should be stable in the cache."
[_a :- (mr/with-key [:fn {:marker 1} (fn [_] true)])]
true)
;; calling it once should not grow the cache:
(my-good-fn 42)
;; redefining the function should not grow the cache:
(mu/defn my-good-fn :- (mr/with-key [:fn {:marker 1} (fn [_] true)])
"A good function that should be stable in the cache."
[_a :- (mr/with-key [:fn {:marker 1} (fn [_] true)])]
true)
;; calling it again should not grow the cache:
(my-good-fn 42))))))
(deftest mu-defn-with-uncachable-schema-and-calling-it-creates-explainer
(is (= [:explainer]
(mapv first
(with-returning-cache-misses
(mu/defn my-good-fn :- [:fn (fn [_] true)]
"A good function that should be stable in the cache."
[_a :- [:fn (fn [_] true)]]
true)
(my-good-fn 42))))))
(deftest ^:parallel cache-handle-regexes-test
(testing (str "For things that aren't ever equal when you re-evaluate them (like Regex literals) maybe sure we do"
" something smart to avoid creating infinite cache entries")
(mr/validate [:re #"\d{4}"] "1234")
(:validator @@#'mr/cache)
(let [before-count (count (:validator @@#'mr/cache))]
(mr/validate [:re #"\d{4}"] "1234")
(mr/validate [:re #"\d{4}"] "1234")
(is (= (count (:validator @@#'mr/cache))
before-count)))))
(is (mr/stable-key? [:re #"\d{4}"]))
(is (mr/stable-key? [:or [:re #"\d{4}"] [:re #"\d{3}"]]))
(is (mr/stable-key? [:map [:id [:re #"\d{4}"]] [:name :string]]))))
(mr/def ::int
:int)
(deftest ^:parallel cache-handle-fn-test
(testing "For things that are functions, we should be able to cache them"
(is (mr/stable-key? [:fn {:error/message "positive number"} pos?]))
(is (mr/stable-key? [:fn {:error/message "non-blank string"} (complement str/blank?)]))
(is (mr/stable-key? [:fn {:error/message "the vacuous schema"} (constantly true)]))))
(deftest ^:parallel cache-does-not-handle-inline-anonymous-fn-test
(testing "Inline anonymous functions are not cachable!"
(is (not (mr/stable-key? [:fn (fn [s] (if (str/blank? s) false true))])))
(is (not (mr/stable-key? [:fn {:error/fn (fn [_ _] "no!")} number?])))
(is (not (mr/stable-key? [:map [:x [:fn (fn [_] false)]]])))))
(deftest ^:parallel cache-handles-inline-anonymous-fn-with-key-test
(testing "Inline anonymous functions are not cachable!"
(is (mr/stable-key? [:fn (mr/with-key (fn [s] (if (str/blank? s) false true)))]))
(is (mr/stable-key? [:fn {:error/fn (mr/with-key (fn [_ _] "no!"))} number?]))
(is (mr/stable-key? [:map [:x [:fn (mr/with-key (fn [_] false))]]]))))
(mr/def ::int :int)
(deftest ^:parallel explainer-test
(is (= ["should be an integer"]
@@ -89,11 +188,237 @@
(deftest ^:parallel deref-circular-refs-test
(testing "Don't resolve circular refs"
(are [schema] (= [:map
[:parent {:optional true} [:ref ::location]]
[:name :string]
[:id [:int {:min 1}]]
[:id-2 [:int {:description "another ID", :min 1}]]]
(mc/form (mr/resolve-schema schema)))
::location
[:ref ::location])))
(is (= [:map
[:parent {:optional true} [:ref ::location]]
[:name :string]
[:id [:int {:min 1}]]
[:id-2 [:int {:description "another ID", :min 1}]]]
(mc/form (mr/resolve-schema ::location))))
(is (= [:map
[:parent {:optional true} [:ref ::location]]
[:name :string]
[:id [:int {:min 1}]]
[:id-2 [:int {:description "another ID", :min 1}]]]
(mc/form (mr/resolve-schema [:ref ::location]))))))
(deftest ^:parallel cache-function-objects-stability-test
(testing "Enhanced schema-cache-key function handles function objects to prevent memory leaks"
(testing "Basic function references should be stable"
(let [schema [:fn {:description "number check"} number?]]
(is (= [:validator] (map first (with-returning-cache-misses (mr/validate schema 42)))))
(is (= []
(with-returning-cache-misses
(mr/validate [:fn {:description "number check"} number?] 42)
(mr/validate [:fn {:description "number check"} number?] 42)
(mr/validate [:fn {:description "number check"} number?] 42)))
"Function schema reuse should not create multiple cache entries")
(is (= [:validator]
(map first (with-returning-cache-misses
(mr/validate [:fn number?] 42))))
"New schemas create new a new cache entry")))
(testing "Function composition should be stable (every-pred)"
(let [schema [:fn {:error/message "positive number"} (every-pred number? pos?)]]
(is (= [:validator] (map first (with-returning-cache-misses (mr/validate schema 42)))))
(is (= []
(with-returning-cache-misses
(mr/validate [:fn {:error/message "positive number"} (every-pred number? pos?)] 42)
(mr/validate [:fn {:error/message "positive number"} (every-pred number? pos?)] 42)
(mr/validate [:fn {:error/message "positive number"} (every-pred number? pos?)] 42)))
"Function schema reuse should not create multiple cache entries")
(is (= [:validator]
(map first (with-returning-cache-misses
(mr/validate [:fn (every-pred number? pos?)] 42))))
"New schemas create new a new cache entry")))
(testing "Complement functions should be stable"
(let [schema [:fn {:error/message "non-blank string"} (complement str/blank?)]]
(is (= [:validator] (map first (with-returning-cache-misses (mr/validate schema "hello")))))
(is (= []
(with-returning-cache-misses
(mr/validate [:fn {:error/message "positive number"} (every-pred number? pos?)] "hello")
(mr/validate [:fn {:error/message "positive number"} (every-pred number? pos?)] "hello")
(mr/validate [:fn {:error/message "positive number"} (every-pred number? pos?)] "hello")))
"Function schema reuse should not create multiple cache entries")
(is (= [:validator]
(map first (with-returning-cache-misses
(mr/validate [:fn {:i-am "a different schema"} (every-pred number? pos?)] "hello"))))
"New schemas create new a new cache entry")))
(testing "Constant functions should be stable"
(let [schema [:fn (constantly false)]]
(is (= [:validator] (map first (with-returning-cache-misses (mr/validate schema :anything)))))
(is (= []
(with-returning-cache-misses
(mr/validate [:fn (constantly false)] :anything)
(mr/validate [:fn (constantly false)] :anything)
(mr/validate [:fn (constantly false)] :anything)))
"Function schema reuse should not create multiple cache entries")
(is (= [:validator]
(map first (with-returning-cache-misses
(mr/validate [:fn {:i-am "a different schema"} (constantly false)] :anything))))
"New schemas create new a new cache entry")))
(testing "Anonymous functions with same reference should be stable"
(let [pred-fn (fn [x] (and (number? x) (pos? x)))
schema [:fn {:desc "positive number"} pred-fn]]
(mr/validate schema 42)
(let [before-count (count (:validator @@#'mr/cache))]
;; Same function reference should reuse cache
(mr/validate [:fn {:desc "positive number"} pred-fn] 42)
(mr/validate [:fn {:desc "positive number"} pred-fn] 42)
(is (= (count (:validator @@#'mr/cache))
before-count)
"Same anonymous function reference should not create multiple cache entries"))))))
;; What if the stable key is made with constantly? how is that getting caught?
;; I am catching it in the schema key mechanism. It's illegal to use
;; `(constantly true)`, `(every-pred even?)`, `(complement str/blank?)` or similar things without
;; wrapping them in `mr/with-key`!! That's the only way we can ensure that the cache key is stable, AND
;; that the functions getting cached are the ones we expect.
;;
;; If we don't do it that way, then this will happen:
;; (mr/validate [:fn (constantly true)] 42) ;=> true
;; ^ caches the function `(constantly true)` with a cache key of [:fn #function[clojure.core/constantly_1234]]
;; now, when we calculate the cache key for a fn schema created with constantly again, the KEY WILL BE THE SAME:
;; (mr/validate [:fn (constantly false)] 42) ;=> true
;; ^ This schema should fail, but it willreturn true because the cache key is the same as the previous one, so
;; it will reuse the cached function `(constantly true)`!!!!
(defn rewrite-schema-with-key
"Rewrites a schema with a stable key, ensuring that the cache key is consistent."
[schema]
(walk/postwalk
(fn [x]
(if (and (fn? x) (not= x (constantly true)))
(mr/with-key x)
x))
schema))
(deftest partial-schemas-are-cached-correctly
(is (#'mr/schema-cache-key [:fn (mr/with-key (every-pred even?))]))
(is (true? (mr/validate [:fn (every-pred even?)] 42)))
(is (false? (mr/validate [:fn (every-pred odd?)] 42))))
(deftest ^:parallel cache-memory-leak-prevention-test
(testing "Memory leak prevention through stable cache keys"
(testing "Multiple identical schemas with function composition don't grow cache unboundedly"
;; Add multiple identical schemas that would have created different cache keys before the fix
(is (mr/stable-key? [:fn (every-pred number? pos?)]))
(is (= 1 (count (with-returning-cache-misses
(dotimes [i 20]
(mr/validate [:fn {:error/message "positive"} (every-pred number? pos?)] (inc i)))))))
;; Should only have 1 validator in cache (plus any others from previous tests)
(let [validator-cache (:validator @@#'mr/cache)
our-validators (filter (fn [[k _]]
(and (vector? k)
(= (first k) :fn)
(= (get-in k [1 :error/message]) "positive")))
validator-cache)]
(is (<= (count our-validators) 2)
"Should have at most 1-2 cache entries for identical schemas, not 20")))))
(deftest ^:parallel schema-cache-key-backward-compatibility-test
(testing "Enhanced schema-cache-key maintains backward compatibility"
(testing "Regex patterns still work correctly (existing functionality)"
(let [schema [:re #"\d{4}"]]
(is (mr/stable-key? schema))
(mr/validate schema "1234")
(let [before-count (count (:validator @@#'mr/cache))]
;; Multiple evaluations should reuse cache (this was already working)
(mr/validate [:re #"\d{4}"] "1234")
(mr/validate [:re #"\d{4}"] "1234")
(is (= (count (:validator @@#'mr/cache))
before-count)
"Regex patterns should continue to work as before"))))
(testing "Non-function, non-regex schemas still work"
(let [schema [:and :int [:> 0]]]
(mr/validate schema 42)
(let [before-count (count (:validator @@#'mr/cache))]
(mr/validate [:and :int [:> 0]] 42)
(mr/validate [:and :int [:> 0]] 42)
(is (= (count (:validator @@#'mr/cache))
before-count)
"Non-function schemas should continue to work"))))))
(deftest ^:parallel function-object-patterns-test
(testing "Specific problematic patterns found in Metabase codebase are now stable"
(testing "MBQL clause validation pattern"
(let [schema [:fn {:error/message "not a known MBQL clause"} (constantly false)]]
(mr/explain schema "anything")
(let [before-count (count (:explainer @@#'mr/cache))]
(mr/explain [:fn {:error/message "not a known MBQL clause"} (constantly false)] "anything")
(is (= (count (:explainer @@#'mr/cache))
before-count)
"MBQL clause pattern should be stable"))))
(testing "Error function in schema metadata"
(let [error-fn (fn [{:keys [value]} _] (str "Invalid value: " value))
schema [:fn {:error/fn error-fn} number?]]
(mr/explain schema "not-a-number")
(let [before-count (count (:explainer @@#'mr/cache))]
(mr/explain [:fn {:error/fn error-fn} number?] "not-a-number")
(is (= (count (:explainer @@#'mr/cache))
before-count)
"Error function pattern should be stable"))))
#?(:clj
(testing "Localized string validation (common in Metabase)"
(testing "Function references in i18n context should be stable"
;; This simulates patterns like [:fn {:error/message "..."} i18n/localized-string?]
(let [schema [:fn {:error/message "must be a string"} string?]]
(mr/validate schema "hello")
(let [before-count (count (:validator @@#'mr/cache))]
(mr/validate [:fn {:error/message "must be a string"} string?] "hello")
(is (= (count (:validator @@#'mr/cache))
before-count)
"i18n-style validation should be stable"))))))))
(deftest ^:parallel performance-characteristics-test
(testing "Enhanced schema-cache-key performance is acceptable"
(let [schemas [[:fn {:desc "num"} number?]
[:fn {:desc "str"} string?]
[:fn {:desc "pos"} (every-pred number? pos?)]
[:fn {:desc "blank"} (complement str/blank?)]
[:re {:desc "raw re"} #"\d+"]
[:or {:desc "deep re"} [:re #"\d+"]]
[:and number? pos?]]
iterations 100
start-time (System/nanoTime)]
;; Generate cache keys many times
(dotimes [_ iterations]
(doseq [schema schemas]
(mr/validate schema (case (first schema)
:fn 42
:re "123"
:or "123"
:and 42))))
(let [elapsed-ms (/ (- (System/nanoTime) start-time) 1000000.0)]
;; Should complete reasonably quickly, I see 3ms locally
(is (< elapsed-ms 10)
"Cache key generation should be fast")))))
(deftest ^:parallel evil-schemas-test
(testing "Evil schemas reproduce themselves in the cache"
(let [evil-schema-gen (fn [] [:int {:evil (rand)}])
iterations 100]
(is (= 100 (count
(with-returning-cache-misses
(dotimes [_ iterations]
(let [evil-schema (evil-schema-gen)]
(mr/validate evil-schema 42))))))))))
(deftest ^:parallel with-api-error-message-key-stability-test
(is (mr/stable-key? (mu/with-api-error-message
[:and
{:error/message "non-blank string"
:json-schema {:type "string" :minLength 1}}
[:string {:min 1}]
[:fn
{:error/message "non-blank string"}
(mr/with-key (complement str/blank?))]]
(deferred-tru "value must be a non-blank string.")))))