Compare commits

...

1 Commits

Author SHA1 Message Date
dan sutton
26a421f0bb Lowering memory usage of malli cache
```
registry=> (update-vals @cache mm/measure)
{:validator "16.4 MiB",
 :metabase.api.macros/decoder "98.6 KiB",
 :metabase.lib.normalize/coercer "144.3 MiB",
 :explainer "10.2 MiB"}
```

Explanation of each:

binding `[filter? (mr/validator ::mbql.s/Filter)]`
This doesn't actually lower ram usage, just dramatically drops the cache
usage. This filter is hit ~170,000 times when loading the examples
dashboard. It walks over each query and for each piece in the query it
hits the cache. Very wasteful but not exactly slow.

```diff
-   #'metadata-provider?])
+   metadata-provider?])
```
I found that changing this from a var to the var's value here can drop
the validator usage from ~125mb to ~10mb. The weird thing is that it
would happen on the second time, not the first. Leaving it here as it
seems innocuous.

```
@@ -53,12 +52,9 @@
   default-error-fn)

 (defn- coercer [schema]
-  (mr/cached ::coercer
-             schema
-             (fn []
-               (let [respond identity
-                     raise   #'*error-fn*] ; capture var rather than the bound value at the time this is eval'ed
-                 (mc/coercer schema (mtx/transformer mtx/default-value-transformer {:name :normalize}) respond raise)))))
+  (let [respond identity
+        raise   #'*error-fn*] ; capture var rather than the bound value at the time this is eval'ed
+    (mc/coercer schema (mtx/transformer mtx/default-value-transformer {:name :normalize}) respond raise)))
```

This is a tradeoff. It is slower without this.

```
normalize=> (time
              (do
                (dotimes [_ 50]
                  (normalize
                    :metabase.lib.schema/stage stage))
                :done))
"Elapsed time: 526.081834 msecs"
:done
;; redefine coercer to use cache
normalize=> (time
              (do
                (dotimes [_ 50]
                  (normalize
                    :metabase.lib.schema/stage stage))
                :done))
"Elapsed time: 37.066083 msecs"
:done
```

I'm hopeful that while it is slower, it's not meaningfully slower. Doing
this 50 times is half a second. Hopefully this is academic.

```diff
@@ -400,7 +400,7 @@
     ;; TODO (Cam 6/12/25) -- why in the HECC is `:lib/metadata` not a required key here? It's virtually REQUIRED for
     ;; anything to work correctly outside of the low-level conversion code. We should make it required and then fix
     ;; whatever breaks.
-    [:lib/metadata {:optional true} ::lib.schema.metadata/metadata-provider]
+    #_[:lib/metadata {:optional true} ::lib.schema.metadata/metadata-provider]
```

This one i found just completely transformed memory usage.
The validator and explainers dropped from ~130mb to ~15 or 10mb.
```clojure
registry=> (update-vals @cache mm/measure)
{:validator "16.4 MiB",
 :metabase.api.macros/decoder "98.6 KiB",
 :metabase.lib.normalize/coercer "144.3 MiB",
 :explainer "10.2 MiB"}
```

```diff
@@ -429,7 +429,7 @@
     [:update-row {:optional true} [:maybe [:ref ::actions/row]]]]
    ;;
    ;; CONSTRAINTS
-   [:ref ::lib.schema.util/unique-uuids]
+   #_[:ref ::lib.schema.util/unique-uuids]
```
This one could undo the changes above. I don't know why
2025-09-10 11:05:23 -05:00
4 changed files with 14 additions and 16 deletions

View File

@@ -42,6 +42,7 @@
[metabase.util.i18n :as i18n]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
[metabase.util.performance :as perf]
[metabase.util.time :as u.time]))
@@ -1054,11 +1055,12 @@
"Replaces legacy filter clauses with modern alternatives."
[query]
(try
(lib.util.match/replace query
(filter-clause :guard mbql.preds/Filter?)
(-> filter-clause
replace-relative-date-filters
replace-exclude-date-filters))
(let [filter? (mr/validator ::mbql.s/Filter)]
(lib.util.match/replace query
(filter-clause :guard filter? #_ mbql.preds/Filter?)
(-> filter-clause
replace-relative-date-filters
replace-exclude-date-filters)))
(catch #?(:clj Throwable :cljs :default) e
(throw (ex-info (i18n/tru "Error replacing legacy filters")
{:query query}

View File

@@ -77,7 +77,7 @@
"Schema for something that satisfies the [[metabase.lib.metadata.protocols/MetadataProvider]] protocol."
[:fn
{:error/message "Valid MetadataProvider"}
#'metadata-provider?])
metadata-provider?])
(defn metadata-providerable?
"Whether `x` is a [[metadata-provider?]], or has one attached at `:lib/metadata` (i.e., a query)."

View File

@@ -8,8 +8,7 @@
[metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.util :as u]
[metabase.util.i18n :as i18n]
[metabase.util.log :as log]
[metabase.util.malli.registry :as mr]))
[metabase.util.log :as log]))
(defn- lib-type [x]
(when (map? x)
@@ -53,12 +52,9 @@
default-error-fn)
(defn- coercer [schema]
(mr/cached ::coercer
schema
(fn []
(let [respond identity
raise #'*error-fn*] ; capture var rather than the bound value at the time this is eval'ed
(mc/coercer schema (mtx/transformer mtx/default-value-transformer {:name :normalize}) respond raise)))))
(let [respond identity
raise #'*error-fn*] ; capture var rather than the bound value at the time this is eval'ed
(mc/coercer schema (mtx/transformer mtx/default-value-transformer {:name :normalize}) respond raise)))
(defn normalize
"Ensure some part of an MBQL query `x`, e.g. a clause or map, is in the right shape after coming in from JavaScript or

View File

@@ -400,7 +400,7 @@
;; TODO (Cam 6/12/25) -- why in the HECC is `:lib/metadata` not a required key here? It's virtually REQUIRED for
;; anything to work correctly outside of the low-level conversion code. We should make it required and then fix
;; whatever breaks.
[:lib/metadata {:optional true} ::lib.schema.metadata/metadata-provider]
#_[:lib/metadata {:optional true} ::lib.schema.metadata/metadata-provider]
[:database {:optional true} [:multi {:dispatch (partial = id/saved-questions-virtual-database-id)}
[true ::id/saved-questions-virtual-database]
[false ::id/database]]]
@@ -429,7 +429,7 @@
[:update-row {:optional true} [:maybe [:ref ::actions/row]]]]
;;
;; CONSTRAINTS
[:ref ::lib.schema.util/unique-uuids]
#_[:ref ::lib.schema.util/unique-uuids]
(common/disallowed-keys
{:filter ":filter is not allowed in MBQL 5, and it's not allowed in the top-level of a stage in any MBQL version"
:source-query ":source-query is not allowed in MBQL 5, and it's not allowed in the top-level of a stage in any MBQL version"