Spike: sqlc -> Clojure query codegen with Malli schemas

Generate Clojure query fns with Malli result schemas from .sql files, using
sqlc for SQL parsing and type inference and sqlc-gen-json to expose its
analysis as JSON.

Verified end to end: migrations against Postgres 16 (1427 changesets, 190
tables), pg_dump fed to sqlc unmodified, real queries ported from the
glossary/sso HugSQL spike branches, and the generated file compiles with
schemas that accept valid rows and reject bad ones.

See dev/src/dev/sqlc/README.md for open issues (no H2 engine, weak param
type inference, no dynamic SQL).

.clj-kondo/config.edn

@@ -181,7 +181,7 @@
      clojure.core.async/take!
      clojure.core.async/to-chan!
      clojure.core.async/to-chan!!
-     ;; only writes .clj-kondo/ratchets.edn, from a :once fixture that finishes before the parallel tests
+     ;; writes the ratchet files from a :once fixture that finishes before the parallel tests
      dev.kondo-ratchet/fix!
      metabase.ai-tracing.core/record!
      metabase.api.macros.defendpoint.tools-manifest/assert-optional-fields-nullable!
@@ -844,7 +844,7 @@
    :name    test-namespaces}
   {:pattern "^(?!.*-test(?:\\..*)?$)(?!.*test-util)(?!^metabase\\.test.*).*$"
    :name    source-namespaces}
-  {:pattern "(?:(?:^metabase\\.cmd.*)|(?:^build.*)|(?:^metabuild-common.*)|(?:^release.*)|(?:^i18n.*)|(?:^lint-migrations-file$))|(?:^mage.*)|(?:^load-namespaces.*)"
+  {:pattern "(?:(?:^metabase\\.cmd.*)|(?:^build.*)|(?:^metabuild-common.*)|(?:^release.*)|(?:^i18n.*)|(?:^lint-migrations-file$))|(?:^mage.*)|(?:^load-namespaces.*)|(?:^dev\\..*)"
    :name    printable-namespaces}
   {:pattern "^metabase\\.(?:(?:lib)|(?:legacy-mbql))\\..*"
    :name    metabase-lib}

.clj-kondo/config/modules/config.edn

@@ -1399,7 +1399,6 @@
               metabase.metabot.settings
               metabase.metabot.tools.entity-details}
    :friends #{agent-api
-              enterprise/metabot
               enterprise/metabot-analytics
               slackbot}
    :uses    #{activity-feed
@@ -2388,7 +2387,6 @@
               metabase.sso.core
               metabase.sso.init
               metabase.sso.settings}
-   :friends #{enterprise/sso}
    :uses    #{api
               auth-identity
               config
@@ -2597,8 +2595,7 @@
            tracing
            transforms-base
            util}
-   :friends #{models
-              enterprise/transforms-python}
+   :friends #{enterprise/transforms-python}
    :model-exports #{:model/Transform
                     :model/TransformDagRun
                     :model/TransformJob
@@ -3273,15 +3270,28 @@
    :uses #{premium-features}}
 
   enterprise/data-apps
-  {:team "Gadget"
+  {:team "Embedding"
    :api  #{metabase-enterprise.data-apps.api
            metabase-enterprise.data-apps.init
            metabase-enterprise.data-apps.sync}
    :uses #{api
+           collections
+           lib
+           lib-be
            models
+           permissions
            premium-features
+           request
            settings
-           util}}
+           util}
+   :model-imports #{:model/Card
+                    :model/Collection
+                    :model/DataPermissions
+                    :model/Database
+                    :model/Permissions
+                    :model/PermissionsGroup
+                    :model/Table
+                    :model/User}}
 
   enterprise/data-complexity-score
   {:team "Metabot"
@@ -3568,9 +3578,6 @@
    :api     #{metabase-enterprise.metabot.api
               metabase-enterprise.metabot.api.routes
               metabase-enterprise.metabot.init}
-   :friends #{slackbot
-              mcp
-              metabot}
    :uses    #{api
               lib
               lib-be

.clj-kondo/config/modules/ratchets.edn

@@ -0,0 +1,6 @@
+;; Budgets for escape hatches in config.edn. Counts may fall but not exceed these values.
+;; Leave reductions to the post-merge shrink workflow; explain increases in the PR.
+{:api-any              1
+ :friend-edges         22
+ :model-imports-bypass 2
+ :uses-any             4}

.clj-kondo/ratchets.edn

@@ -5,7 +5,7 @@
 ;; Any ignore outside :comment-exempt needs an explanatory comment directly above or trailing on its line.
 ;; `./bin/mage kondo-ratchets-shrink` lowers budgets unless `--seed` explicitly adds or raises one.
 ;; The workflow runs it on master, so feature branches do not need to record reductions.
-;; Raising or adding a budget (`--seed` for inline ignores, a manual edit for config) or widening the
+;; Raising or adding a budget (`--seed` for inline ignores, a manual edit otherwise) or widening the
 ;; exemptions must be explained in the PR.
 ;; :all is the vector-less ignore form, which suppresses every linter on the next form.
 {:ignore-counts  {:aliased-namespace-symbol                                            3
@@ -21,7 +21,7 @@
                   :duplicate-require                                                   1
                   :equals-true                                                         1
                   :main-without-gen-class                                              1
-                  :metabase/defsetting-namespace                                       2
+                  :metabase/defsetting-namespace                                       1
                   :metabase/disallow-hardcoded-driver-names-in-tests                   34
                   :metabase/discourage-millis-duration                                 1
                   :metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests 13

.clj-kondo/src/hooks/clojure/core/ns.clj

@@ -92,7 +92,7 @@
 (defn- lint-modules [ns-form-node config]
   (let [ns-symb (ns-form-node->ns-symb ns-form-node)]
     (when-not (modules/ignored-namespace? config ns-symb)
-      (when-let [current-module (modules/module ns-symb)]
+      (when-let [current-module (modules/module config ns-symb)]
         (let [required-namespace-symb-nodes (-> ns-form-node
                                                 ns-form-node->require-node
                                                 require-node->namespace-symb-nodes)]

.clj-kondo/src/hooks/clojure/core/require.clj

@@ -11,10 +11,10 @@
          (every? simple-symbol? required-namespaces)]}
   (when-not (modules/ignored-namespace? config current-ns)
     ;; ignore namespaces outside of the module system
-    (when-let [current-module (modules/module current-ns)]
+    (when-let [current-module (modules/module config current-ns)]
       (doseq [required-namespace required-namespaces]
         ;; ignore namespaces outside of the module system.
-        (when (modules/module required-namespace)
+        (when (modules/module config required-namespace)
           (when-let [error (modules/usage-error config current-module required-namespace)]
             (hooks/reg-finding! (assoc (meta node)
                                        :message error
@@ -22,7 +22,7 @@
 
 (defn- lint-dynamic-require* [node current-ns config]
   (when (and (not (modules/ignored-namespace? config current-ns))
-             (modules/module current-ns))
+             (modules/module config current-ns))
     (hooks/reg-finding! (assoc (meta node)
                                :message "Module dependency cannot be statically determined from a dynamic require"
                                :type    :metabase/modules))))

.clj-kondo/src/hooks/clojure/test.clj

@@ -194,7 +194,7 @@
                (not (contains? @warned-namespaces ns-symb)))
       (swap! warned-namespaces conj ns-symb)
       (let [known-modules  (set (keys (:metabase/modules (modules/config input))))
-            current-module (modules/module ns-symb)]
+            current-module (modules/module (modules/config input) ns-symb)]
         (when (or (not current-module)
                   (not (contains? known-modules current-module)))
           (hooks/reg-finding! (assoc (meta node)

.clj-kondo/src/hooks/common/modules.clj

@@ -1,4 +1,8 @@
 (ns hooks.common.modules
+  "Resolves namespaces to modules and enforces module boundaries.
+
+  The Kondo linter, config generator, and CI tooling share this namespace so
+  they apply the same rules."
   (:require
    [clojure.string :as str]))
 
@@ -12,25 +16,130 @@
   (merge (get-in config [:linters :metabase/modules])
          (select-keys config [:metabase/modules])))
 
-(defn module
+;;;; Module tree. These functions take the `:metabase/modules` map, or the prefix map built from it.
+;;;;
+;;;; Dots express nesting: `lib.schema` is a child of `lib`. A declared
+;;;; `enterprise/X` module is also a child of OSS module `X`. A namespace
+;;;; belongs to the module with the most specific matching `:ns-prefix`.
+
+(defn default-ns-prefix
+  "Namespace prefix a module owns unless it sets `:ns-prefix`: `metabase.lib.schema` for `lib.schema`,
+  `metabase-enterprise.foo` for `enterprise/foo`."
+  [module]
+  (str (if (= "enterprise" (namespace module)) "metabase-enterprise." "metabase.") (name module)))
+
+(defn module-ns-prefix
+  "Namespace prefix `module` owns: its `:ns-prefix`, else [[default-ns-prefix]]."
+  [modules module]
+  (or (get-in modules [module :ns-prefix])
+      (default-ns-prefix module)))
+
+(defn build-prefix->module
+  "Map each declared namespace prefix to its module, for [[declared-module]]."
+  [modules]
+  (into {}
+        (map (fn [module] [(module-ns-prefix modules module) module]))
+        (keys modules)))
+
+(defn declared-module
+  "Resolve `ns-symb` to its declared module, or `nil`.
+
+  Chooses the most specific dotted prefix and ignores a trailing `-test`."
+  [prefix->module ns-symb]
+  (loop [candidate (str/replace (str ns-symb) #"-test$" "")]
+    (or (get prefix->module candidate)
+        (when-let [dot (str/last-index-of candidate ".")]
+          (recur (subs candidate 0 dot))))))
+
+(defn resolve-module
   "E.g.
 
-    (module 'metabase.qp.middleware.wow) => 'qp
-    (module 'metabase-enterprise.whatever.core) => enterprise/whatever"
-  [ns-symb]
+    (resolve-module prefix->module 'metabase.qp.middleware.wow) => 'qp
+    (resolve-module prefix->module 'metabase-enterprise.whatever.core) => enterprise/whatever
+
+  An unmatched Metabase namespace resolves to its first segment so the linter can report an undeclared module."
+  [prefix->module ns-symb]
   {:pre [(simple-symbol? ns-symb)]}
   ;; treat something like `metabase.driver-test` (for a module that hasn't fully been updated to use `.core`
   ;; namespaces) as being in the `driver` module
   (let [ns-symb (if (str/ends-with? (name ns-symb) "-test")
                   (symbol (str/replace (name ns-symb) #"-test$" ""))
                   ns-symb)]
-    (or (some->> (re-find #"^metabase-enterprise\.([^.]+)" (str ns-symb))
+    (or (declared-module prefix->module ns-symb)
+        (some->> (re-find #"^metabase-enterprise\.([^.]+)" (str ns-symb))
                  second
                  (symbol "enterprise"))
         (some-> (re-find #"^metabase\.([^.]+)" (str ns-symb))
                 second
                 symbol))))
 
+(defn parent-module
+  "The direct parent of `module`, or `nil` when it is top-level."
+  [modules module]
+  (let [module-name (name module)]
+    (if-let [dot (str/last-index-of module-name ".")]
+      (symbol (namespace module) (subs module-name 0 dot))
+      (when (= "enterprise" (namespace module))
+        (let [oss (symbol module-name)]
+          (when (contains? modules oss)
+            oss))))))
+
+(defn descendant-of?
+  "Whether `module` is `ancestor` or one of its descendants."
+  [modules module ancestor]
+  (boolean (some #{ancestor} (take-while some? (iterate #(parent-module modules %) module)))))
+
+(defn- exports-child?
+  [modules parent child]
+  (or (contains? (set (get-in modules [parent :module-exports])) child)
+      ;; OSS module `X` implicitly exports its `enterprise/X` companion.
+      (and (not= (namespace parent) (namespace child))
+           (contains? modules child))))
+
+(defn- blocking-export
+  "The first missing export from `module` toward the root.
+
+  Returns `{:ancestor parent, :child child}`, or `nil` when every link is
+  exported."
+  [modules module]
+  (loop [child module]
+    (when-let [ancestor (parent-module modules child)]
+      (if (exports-child? modules ancestor child)
+        (recur ancestor)
+        {:ancestor ancestor, :child child}))))
+
+(defn namability-error
+  "Explain why `caller` may not refer to `target` in `:uses`, or return `nil` if it may.
+
+  A nested module is private to the nearest ancestor with a missing export.
+  Exporting every link makes it public."
+  [modules caller target]
+  (when-let [{:keys [ancestor child]} (blocking-export modules target)]
+    (when-not (descendant-of? modules caller ancestor)
+      (format (str "Module %s is nested and not exported by its ancestors; %s may not use it. Add %s to %s's "
+                   ":module-exports, or move the caller into the %s subtree. [:metabase/modules %s :module-exports]")
+              target caller child ancestor ancestor ancestor))))
+
+;;;; Lint rules. These functions take the full linter config.
+
+(defonce ^:private prefix->module-cache
+  (atom nil))
+
+(defn- prefix->module
+  "[[build-prefix->module]] for `config`, cached because Kondo hands every hook call the same `:metabase/modules` map."
+  [config]
+  (let [modules (:metabase/modules config)
+        cached  @prefix->module-cache]
+    (if (identical? modules (:modules cached))
+      (:prefix->module cached)
+      (:prefix->module (reset! prefix->module-cache {:modules        modules
+                                                     :prefix->module (build-prefix->module modules)})))))
+
+(defn module
+  "Resolve `ns-symb` to its module under `config`; see [[resolve-module]]."
+  [config ns-symb]
+  (resolve-module (prefix->module config) ns-symb))
+
 (defn- module-api-namespaces
   "Set of API namespace symbols for a given module. `:any` means you can use anything, there are no API namespaces for
   this module (yet). If unspecified, the default is just the `<module>.core` namespace."
@@ -44,9 +153,7 @@
       module-config
 
       :else
-      (let [ns-prefix (if (= (namespace module) "enterprise")
-                        (str "metabase-enterprise." (name module))
-                        (name module))]
+      (let [ns-prefix (module-ns-prefix (:metabase/modules config) module)]
         #{(symbol (str ns-prefix ".api"))
           (symbol (str ns-prefix ".core"))
           (symbol (str ns-prefix ".init"))}))))
@@ -68,15 +175,17 @@
         (contains? (set allowed-modules) required-module))))
 
 (defn- allowed-module-namespace? [config current-module ns-symb]
-  (let [module                (module ns-symb)
+  (let [module                (module config ns-symb)
         module-api-namespaces (module-api-namespaces config module)
         module-friends        (module-friends config module)]
     (or (nil? module-api-namespaces)
         (contains? module-api-namespaces ns-symb)
-        (contains? module-friends current-module))))
+        (contains? module-friends current-module)
+        ;; a child may use its ancestors' internals; a parent still goes through its child's `:api`
+        (descendant-of? (:metabase/modules config) current-module module))))
 
 (defn- rest-module? [module]
-  (str/ends-with? module "-rest"))
+  (re-find #"[.-]rest$" (str module)))
 
 (defn- routes-module? [module]
   (str/ends-with? module "-routes"))
@@ -89,7 +198,7 @@
   error type if there is one, otherwise `nil` if there are no errors."
   [config current-module required-namespace]
   ;; ignore stuff not in a module i.e. non-Metabase stuff.
-  (when-let [required-module (module required-namespace)]
+  (when-let [required-module (module config required-namespace)]
     (when-not (= current-module required-module)
       (cond
         (not (allowed-module? config current-module required-module))
@@ -113,4 +222,8 @@
                 required-module
                 current-module
                 required-module
-                (symbol (str/replace required-module #"-rest$" "")))))))
+                (symbol (str/replace required-module #"[.-]rest$" "")))
+
+        ;; Config tests check explicit `:uses`; wildcard callers are checked here.
+        (= :any (allowed-modules config current-module))
+        (namability-error (:metabase/modules config) current-module required-module)))))

.clj-kondo/src/hooks/metabase/settings/models/setting.clj

@@ -4,9 +4,9 @@
    [hooks.common :as common]))
 
 (defn lint-defsetting-namespace [node context]
-  (when-not (re-matches #"^metabase(?:-enterprise)?\.[^\.]+\.settings$" (name (:ns context)))
+  (when-not (re-matches #"^metabase(?:-enterprise)?\.([^\.]+\.)+settings$" (name (:ns context)))
     (hooks/reg-finding! (assoc (meta node)
-                               :message "All defsettings should live in metabase[-enterprise].<module>.settings namespaces"
+                               :message "All defsettings should live in a namespace ending in .settings under metabase[-enterprise]"
                                :type :metabase/defsetting-namespace))))
 
 ;;; TODO -- move this into a Kondo config file in `.clj-kondo/config/`

.clj-kondo/src/hooks/metabase/toucan/db_ns.clj

@@ -8,10 +8,20 @@
   Registered as an `:analyze-call` hook on each of those functions in `.clj-kondo/config.edn`. The hook
   returns its input unchanged so Kondo's normal analysis of the call (arity, var usage) still runs."
   (:require
-   [clj-kondo.hooks-api :as hooks]))
+   [clj-kondo.hooks-api :as hooks]
+   [hooks.common.modules :as modules]))
 
-(defn- db-namespace? [ns-sym]
-  (boolean (re-matches #"^metabase(?:-enterprise)?\.(?:driver\.)?[^.]+\.db$" (name ns-sym))))
+(def ^:private driver-db-namespace
+  "Matches `metabase.driver.<driver>.db`; individual drivers are not modules."
+  #"^metabase\.driver\.[^.]+\.db$")
+
+(defn- db-namespace? [config ns-sym]
+  (let [ns-str (name ns-sym)]
+    (boolean
+     ;; Resolve ownership first: a nested `.db` name alone does not make a module.
+     (or (when-let [module (modules/module config ns-sym)]
+           (= ns-str (str (modules/module-ns-prefix (:metabase/modules config) module) ".db")))
+         (re-matches driver-db-namespace ns-str)))))
 
 (defn- test-file?
   "Whether `filename` is in a test source tree. Test namespaces are exempt through the `test-namespaces` group in
@@ -24,7 +34,7 @@
   namespace."
   [{:keys [node ns filename] :as input}]
   (when (and ns
-             (not (db-namespace? ns))
+             (not (db-namespace? (modules/config input) ns))
              (not (test-file? filename)))
     (let [fn-node (first (:children node))]
       (hooks/reg-finding!

.clj-kondo/test/hooks/common/modules_test.clj

@@ -5,6 +5,6 @@
 
 (deftest ^:parallel module-test
   (are [symb expected] (= expected
-                          (hooks.common.modules/module symb))
+                          (hooks.common.modules/module {} symb))
     'metabase.qp.middleware.wow        'qp
     'metabase-enterprise.whatever.core 'enterprise/whatever))

.clj-kondo/test/hooks/metabase/toucan/db_ns_test.clj

@@ -5,14 +5,15 @@
    [clojure.test :refer :all]
    [hooks.metabase.toucan.db-ns :as toucan.db-ns]))
 
-(defn- lint-query-call [form ns-sym & [filename]]
+(defn- lint-query-call [form ns-sym & [filename modules]]
   (binding [clj-kondo.impl.utils/*ctx* {:config     {:linters {:metabase/t2-query-namespace {:level :warning}}}
                                         :ignores    (atom nil)
                                         :findings   (atom [])
                                         :namespaces (atom {})}]
     (let [input  {:node     (hooks/parse-string (pr-str form))
                   :ns       ns-sym
-                  :filename (or filename "src/metabase/foo/bar.clj")}
+                  :filename (or filename "src/metabase/foo/bar.clj")
+                  :config   {:metabase/modules modules}}
           output (toucan.db-ns/lint-query-call input)]
       (is (identical? (:node input) (:node output))
           "the hook must return the node unchanged so Kondo's normal analysis still runs")
@@ -48,3 +49,25 @@
     (is (=? [{:type    :metabase/t2-query-namespace
               :message #".*`toucan2.core/insert!`.*"}]
             (lint-query-call '(toucan2.core/insert! :model/Card {}) 'metabase.queries.models.card)))))
+
+(deftest ^:parallel t2-query-namespace-follows-the-module-tree-test
+  (let [modules '{queries            {}
+                  metabot            {}
+                  metabot.llm        {}
+                  lib.be             {:ns-prefix "metabase.lib-be"}
+                  enterprise/sandbox {}}]
+    (testing "a nested module owns its own db namespace"
+      (is (empty? (lint-query-call '(t2/select :model/Card) 'metabase.metabot.llm.db nil modules))))
+    (testing "the parent's db namespace stays its own"
+      (is (empty? (lint-query-call '(t2/select :model/Card) 'metabase.metabot.db nil modules))))
+    (testing "a module with an explicit :ns-prefix is resolved through it, not its dotted name"
+      (is (empty? (lint-query-call '(t2/select :model/Card) 'metabase.lib-be.db nil modules)))
+      (is (=? [{:type :metabase/t2-query-namespace}]
+              (lint-query-call '(t2/select :model/Card) 'metabase.lib.be.db nil modules))))
+    (testing "a db namespace under a directory naming no module is still a finding"
+      (is (=? [{:type :metabase/t2-query-namespace}]
+              (lint-query-call '(t2/query {:select [:*]}) 'metabase.queries.models.db nil modules)))
+      (is (=? [{:type :metabase/t2-query-namespace}]
+              (lint-query-call '(t2/query {:select [:*]}) 'metabase.metabot.llm.models.db nil modules))))
+    (testing "enterprise modules resolve through the metabase-enterprise root"
+      (is (empty? (lint-query-call '(t2/select :model/Card) 'metabase-enterprise.sandbox.db nil modules))))))

.github/actionlint.yaml

@@ -14,3 +14,6 @@ paths:
   .github/workflows/lint.yml:
     ignore:
       - 'step must run script with "run" section or run action with "uses" section'
+  .github/workflows/pr.checks.yml:
+    ignore:
+      - 'step must run script with "run" section or run action with "uses" section'

.github/actions/test-driver/action.yml

@@ -20,6 +20,30 @@ inputs:
     required: false
     default: "ee"
     description: Edition to test (oss or ee)
+  runner-index:
+    required: false
+    description: >-
+      The index of this runner among those sharing a partitioned suite. See
+      `be-tests-bigquery-cloud-sdk-ee` in `.github/workflows/drivers.yml` for an example.
+  runner-count:
+    required: false
+    description: >-
+      How many runners share the suite. Recommended to set to `strategy.job-total` rather
+      than a literal so that it doesn't need updating if the `runner-index` matrix changes.
+  jvms-per-runner:
+    required: false
+    default: "1"
+    description: >-
+      Concurrent JVMs per runner. Recommended to raise above 1 for drivers whose tests
+      spend a lot of time waiting on a remote warehouse.
+  heap-per-jvm:
+    required: false
+    default: "4g"
+    description: >-
+      `-Xmx` per JVM, applied only when `jvms-per-runner` is more than 1 (otherwise the
+      the `:ci` alias's `-Xmx12g` is used). To size it: reserve ~1G for the OS and ~2G
+      of non-heap per JVM, then divide what's left. On a 16G runner that's ~4G each for
+      two JVMs.
   granular-rerun:
     required: false
     default: "true"
@@ -98,6 +122,7 @@ runs:
       # Otherwise run exactly as before. Quote the file so the whole `[ns/var ...]`
       # vector reaches clojure -X as a single argument.
       run: |
+        ALIASES="dev:ci:${{ inputs.edition }}:${{ inputs.edition }}-dev:drivers:drivers-dev:test"
         BASE="clojure -X:dev:ci:${{ inputs.edition }}:${{ inputs.edition }}-dev:drivers:drivers-dev:test"
         # `:profile-tests` overrides `:test`'s exec-fn, so it has to come last. See dev.profile-tests.
         # The `profiled` output is what the collection steps below key off, so that the decision is
@@ -115,12 +140,19 @@ runs:
           $BASE :only "$ONLY"
         elif [ "${{ inputs.granular-rerun }}" = "true" ] && [ "${{ github.run_attempt }}" != "1" ] && [ -s target/failed-tests ]; then
           echo "Granular rerun of previously-failed tests: $(cat target/failed-tests)"
-          $BASE :only "$(cat target/failed-tests)"
+          clojure -X:"$ALIASES" :only "$(cat target/failed-tests)"
+        elif [ -n "$TEST_RUNNER_INDEX" ]; then
+          CLOJURE_ALIASES="$ALIASES" ./.github/scripts/run-backend-test-jvms.sh ${{ inputs.test-args }}
         else
-          $BASE ${{ inputs.test-args }}
+          clojure -X:"$ALIASES" ${{ inputs.test-args }}
         fi
       shell: bash
       id: run-test
+      env:
+        TEST_RUNNER_INDEX: ${{ inputs.runner-index }}
+        TEST_RUNNER_COUNT: ${{ inputs.runner-count }}
+        TEST_JVMS_PER_RUNNER: ${{ inputs.jvms-per-runner }}
+        TEST_HEAP_PER_JVM: ${{ inputs.heap-per-jvm }}
       # Keep this step continue-on-error because the quarantine gate below controls whether it
       # ultimately fails.
       continue-on-error: true
@@ -184,7 +216,9 @@ runs:
       uses: actions/upload-artifact@v7
       with:
         name: ${{ format('test-logs-{0}{1}', github.job, inputs.artifact-suffix && format('-{0}', inputs.artifact-suffix) || '') }}
-        path: logs/test-log.json
+        # Glob: a single-JVM run writes `logs/test-log.json`; a multi-JVM run merges one
+        # `logs/test-log-p<index>.json` per JVM.
+        path: logs/test-log*.json
         retention-days: 1
       # Failing to upload the artifact should never affect the outcome of the whole test run.
       continue-on-error: true

.github/file-paths.yaml

@@ -115,10 +115,10 @@ backend_sources: &backend_sources
   - "src/**"
   - "modules/drivers/{*,*/{*,!(test)/**}}"
   - "resources/**"
-  # The ratchets file drives project_ratchet_checks alone, so the automation PR that lowers the budgets
-  # runs the Project tests job by itself. Patterns are OR-ed, so a separate "!..." entry would match
+  # Ratchet files drive project_ratchet_checks alone, so automation PRs that lower budgets
+  # run the Project tests job by themselves. Patterns are OR-ed, so a separate "!..." entry would match
   # every other path instead of excluding this one.
-  - ".clj-kondo/{!(ratchets.edn),*/**}"
+  - ".clj-kondo/**/!(ratchets.edn)"
 
 backend_specs: &backend_specs
   - *shared_specs
@@ -169,6 +169,7 @@ project_backend_checks:
 # Only the ratchet check reads the budgets, so a diff that touches nothing else runs that suite alone.
 project_ratchet_checks:
   - ".clj-kondo/ratchets.edn"
+  - ".clj-kondo/config/modules/ratchets.edn"
 
 project_migration_checks:
   - *project_tests_infra

.github/scripts/file-paths.unit.spec.ts

@@ -29,7 +29,7 @@ describe("file-paths.yaml", () => {
   // dorny/paths-filter ORs the patterns in a filter, so a "!..." entry does not subtract from the
   // other entries - it matches every path they don't, turning the filter permanently on. Excluding
   // a path means writing the exclusion inside one pattern, the way backend_sources keeps
-  // ratchets.edn out with ".clj-kondo/{!(ratchets.edn),*/**}".
+  // ratchet files out with ".clj-kondo/**/!(ratchets.edn)".
   it("never excludes a path with a standalone negation", () => {
     const negated = Object.entries(filters).flatMap(([name, filter]) =>
       patterns(filter)
@@ -44,6 +44,9 @@ describe("file-paths.yaml", () => {
     "%s skips a ratchets-only change and still follows the rest of .clj-kondo",
     (name) => {
       expect(matches(name, ".clj-kondo/ratchets.edn")).toBe(false);
+      expect(
+        matches(name, ".clj-kondo/config/modules/ratchets.edn"),
+      ).toBe(false);
       expect(matches(name, ".clj-kondo/config.edn")).toBe(true);
       expect(matches(name, ".clj-kondo/config/modules/config.edn")).toBe(true);
     },
@@ -53,5 +56,11 @@ describe("file-paths.yaml", () => {
     expect(matches("project_ratchet_checks", ".clj-kondo/ratchets.edn")).toBe(
       true,
     );
+    expect(
+      matches(
+        "project_ratchet_checks",
+        ".clj-kondo/config/modules/ratchets.edn",
+      ),
+    ).toBe(true);
   });
 });

.github/scripts/run-backend-test-jvms.sh

@@ -0,0 +1,160 @@
+#!/usr/bin/env bash
+#
+# Run this runner's share of a partitioned backend test suite as one or more concurrent
+# JVMs. The caller says how many runners share the suite and how many JVMs each gets.
+#
+# WHY MULTIPLE JVMS PER RUNNER
+# ----------------------------
+# Driver tests spend most of their wall time waiting on the remote warehouse, and they
+# can't use the idle CPU by using more threads because some tests mutate global state.
+# A second JVM gets that isolation for free.
+#
+# Two paths are NOT safe to share:
+#   * `target/junit` — hawk deletes the whole directory at `:begin-test-run`, so a JVM
+#     starting late would wipe an earlier one's results.
+#   * `logs/test-log.json` — the log4j2 File appender truncates it on open.
+# So each JVM runs in its own hardlinked copy of the checkout (inodes, not disk) and
+# outputs are merged back into the workspace afterwards.
+#
+# INPUTS (environment)
+# --------------------
+#   CLOJURE_ALIASES        alias string for `clojure -X`, e.g. dev:ci:ee:ee-dev:drivers:drivers-dev:test
+#   TEST_RUNNER_INDEX      index of this runner among those sharing the suite
+#   TEST_RUNNER_COUNT      how many runners share the suite
+#   TEST_JVMS_PER_RUNNER   concurrent JVMs per runner (default 1)
+#   TEST_HEAP_PER_JVM      -Xmx per JVM, applied only when running more than one
+#
+# Exits non-zero if any JVM fails.
+
+set -uo pipefail
+
+: "${CLOJURE_ALIASES:?CLOJURE_ALIASES is required}"
+: "${TEST_RUNNER_INDEX:?TEST_RUNNER_INDEX is required}"
+: "${TEST_RUNNER_COUNT:?TEST_RUNNER_COUNT is required}"
+TEST_ARGS=("$@")
+TEST_JVMS_PER_RUNNER="${TEST_JVMS_PER_RUNNER:-1}"
+TEST_HEAP_PER_JVM="${TEST_HEAP_PER_JVM:-4g}"
+
+die() {
+  echo "::error::run-backend-test-jvms.sh: $1"
+  exit 1
+}
+
+[[ "$TEST_RUNNER_INDEX" =~ ^[0-9]+$ ]] || die "TEST_RUNNER_INDEX must be a non-negative integer, got '$TEST_RUNNER_INDEX'"
+[[ "$TEST_RUNNER_COUNT" =~ ^[0-9]+$ ]] || die "TEST_RUNNER_COUNT must be a positive integer, got '$TEST_RUNNER_COUNT'"
+[[ "$TEST_JVMS_PER_RUNNER" =~ ^[0-9]+$ ]] || die "TEST_JVMS_PER_RUNNER must be a positive integer, got '$TEST_JVMS_PER_RUNNER'"
+[ "$TEST_RUNNER_COUNT" -ge 1 ] || die "TEST_RUNNER_COUNT must be at least 1"
+[ "$TEST_JVMS_PER_RUNNER" -ge 1 ] || die "TEST_JVMS_PER_RUNNER must be at least 1"
+[ "$TEST_RUNNER_INDEX" -lt "$TEST_RUNNER_COUNT" ] ||
+  die "runner index $TEST_RUNNER_INDEX is out of range for $TEST_RUNNER_COUNT runner(s) — a matrix entry was added without raising runner-count"
+
+# Partition indexes are split across runners rather than handed out in contiguous
+# blocks. E.g. with 2 runners and 2 JVMs the first runner gets partition 0 and 2, and
+# the second runner gets partitions 1 and 3. Hawk partitions in namespace order, so
+# neighbouring partitions have similar runtimes — contiguous blocks would group the slow
+# ones together onto one runner.
+PARTITION_TOTAL=$((TEST_RUNNER_COUNT * TEST_JVMS_PER_RUNNER))
+INDEXES=()
+for ((k = 0; k < TEST_JVMS_PER_RUNNER; k++)); do
+  INDEXES+=($((TEST_RUNNER_INDEX + k * TEST_RUNNER_COUNT)))
+done
+
+# One JVM: run directly in the workspace with no heap override, exactly as a
+# partitioned run behaved before this script existed.
+if [ "$TEST_JVMS_PER_RUNNER" -eq 1 ]; then
+  echo "Running partition ${INDEXES[0]} of $PARTITION_TOTAL"
+  exec clojure -X:"$CLOJURE_ALIASES" "${TEST_ARGS[@]}" \
+    :partition/total "$PARTITION_TOTAL" \
+    :partition/index "${INDEXES[0]}"
+fi
+
+
+WORKSPACE="$PWD"
+# Sibling of the checkout: same filesystem (so `cp -al` can hardlink) but outside the
+# repo, so one JVM's directory never gets copied into another's.
+JVM_DIRS_ROOT="$(dirname "$WORKSPACE")/backend-test-jvms"
+STATUS_DIR="$JVM_DIRS_ROOT/status"
+
+# Merge every JVM's results into the workspace the rest of the job reads from.
+#
+# Each partition's junit XML goes into its own subdirectory: `mb_hawk_var_less_errors.xml`
+# has a fixed name, so a flat copy would silently overwrite it. Nesting is safe because
+# both consumers (dorny/test-reporter and ci-conductor's backend adapter) walk
+# `target/junit` recursively and match on filename suffix.
+#
+# Runs from an EXIT trap so a cancelled or timed-out job still publishes whatever
+# finished.
+# shellcheck disable=SC2329  # invoked by the EXIT trap below, which shellcheck can't see.
+merge_outputs() {
+  mkdir -p "$WORKSPACE/target/junit" "$WORKSPACE/logs"
+  local idx jvm_dir
+  for idx in "${INDEXES[@]}"; do
+    jvm_dir="$JVM_DIRS_ROOT/partition-$idx"
+    if [ -d "$jvm_dir/target/junit" ]; then
+      mkdir -p "$WORKSPACE/target/junit/p$idx"
+      cp -a "$jvm_dir/target/junit/." "$WORKSPACE/target/junit/p$idx/" || true
+    fi
+    # `logs/test-log*.json` is what the action uploads on failure.
+    if [ -f "$jvm_dir/logs/test-log.json" ]; then
+      cp -a "$jvm_dir/logs/test-log.json" "$WORKSPACE/logs/test-log-p$idx.json" || true
+    fi
+  done
+}
+trap merge_outputs EXIT
+trap 'exit 143' TERM
+trap 'exit 130' INT
+
+# Hardlinked copy of the checkout. Skips the paths a run writes to (`target`, `logs`, `.cpcache`)
+# and the two most expensive trees nothing on the test classpath reads (`.git`, `node_modules`).
+prepare_jvm_dir() {
+  local jvm_dir="$1" entry
+  rm -rf "$jvm_dir"
+  mkdir -p "$jvm_dir"
+  for entry in "$WORKSPACE"/* "$WORKSPACE"/.[!.]*; do
+    [ -e "$entry" ] || continue
+    case "$(basename "$entry")" in
+      .git | node_modules | target | logs | .cpcache) continue ;;
+    esac
+    # Fall back to a real copy if the filesystem refuses the hardlink.
+    cp -al "$entry" "$jvm_dir/" 2>/dev/null || cp -a "$entry" "$jvm_dir/"
+  done
+}
+
+# Prefix output with the partition index so interleaved JVM logs stay readable.
+run_partition() {
+  local idx="$1" jvm_dir="$2" rc line
+  (
+    cd "$jvm_dir" || exit 1
+    # `-J` options land after the aliases' `:jvm-opts`, overriding the `:ci` alias's
+    # `-Xms12g -Xmx12g`. Xms stays small so JVMs sharing a runner grow into memory on
+    # demand rather than each committing its ceiling up front.
+    exec clojure -J-Xms512m -J-Xmx"$TEST_HEAP_PER_JVM" -X:"$CLOJURE_ALIASES" \
+      "${TEST_ARGS[@]}" \
+      :partition/total "$PARTITION_TOTAL" \
+      :partition/index "$idx"
+  ) 2>&1 | while IFS= read -r line || [ -n "$line" ]; do printf '[p%s] %s\n' "$idx" "$line"; done
+  rc="${PIPESTATUS[0]}"
+  echo "$rc" > "$STATUS_DIR/p$idx"
+}
+
+rm -rf "$JVM_DIRS_ROOT"
+mkdir -p "$STATUS_DIR"
+
+echo "Runner $TEST_RUNNER_INDEX of $TEST_RUNNER_COUNT: running $TEST_JVMS_PER_RUNNER concurrent JVMs for partition(s) ${INDEXES[*]} of $PARTITION_TOTAL (-Xmx$TEST_HEAP_PER_JVM each)"
+
+for idx in "${INDEXES[@]}"; do
+  prepare_jvm_dir "$JVM_DIRS_ROOT/partition-$idx"
+done
+
+for idx in "${INDEXES[@]}"; do
+  run_partition "$idx" "$JVM_DIRS_ROOT/partition-$idx" &
+done
+wait
+
+overall=0
+for idx in "${INDEXES[@]}"; do
+  rc="$(cat "$STATUS_DIR/p$idx" 2>/dev/null || echo 1)"
+  echo "partition $idx exited $rc"
+  [ "$rc" -eq 0 ] || overall=1
+done
+exit "$overall"

.github/workflows/block-master-source-branch-prs.yml

@@ -1,15 +0,0 @@
-name: Block Master Branch PRs
-on:
-  pull_request:
-    types: [opened, synchronize]
-
-jobs:
-  check-source-branch:
-    if: github.head_ref == 'master'
-    runs-on: ${{ vars.SLIM_RUNNER_KEY }}
-    timeout-minutes: 5
-    steps:
-      - name: Block master branch PRs
-        run: |
-          echo "PRs from \`master\` branches are not allowed, please use a new branch name for your PR."
-          exit 1

.github/workflows/check-backport-script.yml

@@ -1,21 +0,0 @@
-name: Prevent backport.sh from being merged
-
-on:
-  pull_request:
-    types: [opened, synchronize, reopened]
-
-jobs:
-  check-backport-script:
-    runs-on: ${{ vars.SLIM_RUNNER_KEY }}
-    timeout-minutes: 5
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v6
-      - name: Check for backport.sh file
-        run: |
-          if [ -f "./backport.sh" ]; then
-            echo "backport.sh file exists. Please remove it before merging."
-            exit 1
-          else
-            echo "All good."
-          fi

.github/workflows/docs-generate-base.yml

@@ -77,6 +77,9 @@ jobs:
 
       - name: Generate AI provider documentation
         id: generate_ai_provider_docs
+        # The generator only exists on branches that ship the multi-provider registry (master / 64+), but
+        # this reusable workflow always runs from master, so skip the step on release branches without it.
+        if: hashFiles('src/metabase/cmd/ai_provider_dox.clj') != ''
         run: |
           set -e  # Exit on any error
           echo "Generating AI provider documentation..."

.github/workflows/drivers.yml

@@ -168,29 +168,11 @@ jobs:
     needs: determine-driver-skips
     if: ${{ needs.determine-driver-skips.outputs.bigquery-should-run == 'true' }}
     runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
-    timeout-minutes: 70
+    timeout-minutes: 40
     strategy:
       fail-fast: false
       matrix:
-        job:
-          - name: Driver Tests Part 1
-            test-args: >-
-              :only-tags [:mb/driver-tests]
-              :exclude-tags [:mb/transforms-python-test]
-              :fail-fast? true
-              :partition/total 2
-              :partition/index 0
-          - name: Driver Tests Part 2
-            test-args: >-
-              :only-tags [:mb/driver-tests]
-              :exclude-tags [:mb/transforms-python-test]
-              :fail-fast? true
-              :partition/total 2
-              :partition/index 1
-          - name: Transforms Python Tests
-            test-args: >-
-              :only-tags [:mb/transforms-python-test]
-            needs-python-runner: true
+        runner-index: [0, 1]
     env:
       CI: 'true'
       DRIVERS: bigquery-cloud-sdk
@@ -201,11 +183,43 @@ jobs:
       MB_BIGQUERY_TEST_REFRESH_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_REFRESH_TOKEN }}
       MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON }}
       MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING }}
-    name: BigQuery ${{ matrix.job.name }}
+    name: BigQuery Driver Tests ${{ matrix.runner-index }}
+    steps:
+    - uses: actions/checkout@v6
+    - name: Test BigQuery Cloud SDK driver
+      id: test-driver
+      uses: ./.github/actions/test-driver
+      with:
+        junit-name: 'be-tests-bigquery-cloud-sdk-ee'
+        test-args: >-
+          :only-tags [:mb/driver-tests]
+          :exclude-tags [:mb/transforms-python-test]
+          :fail-fast? true
+        runner-index: ${{ matrix.runner-index }}
+        runner-count: ${{ strategy.job-total }}
+        jvms-per-runner: '2'
+        heap-per-jvm: '4g'
+        artifact-suffix: runner-${{ matrix.runner-index }}
+
+  be-tests-bigquery-transforms-python-ee:
+    needs: determine-driver-skips
+    if: ${{ needs.determine-driver-skips.outputs.bigquery-should-run == 'true' }}
+    runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
+    timeout-minutes: 30
+    env:
+      CI: 'true'
+      DRIVERS: bigquery-cloud-sdk
+      MB_BIGQUERY_TEST_PROJECT_ID: ${{ secrets.BIGQUERY_TEST_PROJECT_ID }}
+      MB_BIGQUERY_TEST_CLIENT_ID: ${{ secrets.MB_BIGQUERY_TEST_CLIENT_ID }}
+      MB_BIGQUERY_TEST_CLIENT_SECRET: ${{ secrets.MB_BIGQUERY_TEST_CLIENT_SECRET }}
+      MB_BIGQUERY_TEST_ACCESS_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_ACCESS_TOKEN }}
+      MB_BIGQUERY_TEST_REFRESH_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_REFRESH_TOKEN }}
+      MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON }}
+      MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING }}
+    name: BigQuery Transforms Python Tests
     steps:
     - uses: actions/checkout@v6
     - name: Setup python-runner
-      if: ${{ matrix.job.needs-python-runner == true }}
       uses: ./.github/actions/setup-python-runner
       with:
         metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
@@ -213,15 +227,17 @@ jobs:
       id: test-driver
       uses: ./.github/actions/test-driver
       with:
+        # This is the same as the cloud-sdk job above to preserve quarantine history in ci-conductor.
         junit-name: 'be-tests-bigquery-cloud-sdk-ee'
-        test-args: ${{ matrix.job.test-args }}
-        artifact-suffix: ${{ matrix.job.name }}
+        test-args: >-
+          :only-tags [:mb/transforms-python-test]
+        artifact-suffix: transforms-python
     - name: Cleanup python-runner
-      if: ${{ always() && matrix.job.needs-python-runner == true }}
+      if: ${{ always() }}
       uses: ./.github/actions/cleanup-python-runner
       with:
         upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
-        logs-artifact-suffix: ${{ matrix.job.name }}
+        logs-artifact-suffix: transforms-python
 
   be-tests-clickhouse-ee:
     needs: determine-driver-skips
@@ -1257,6 +1273,7 @@ jobs:
       # if you add a driver job, you must add it to this list for it to be a required check
       - be-tests-athena-ee
       - be-tests-bigquery-cloud-sdk-ee
+      - be-tests-bigquery-transforms-python-ee
       - be-tests-clickhouse-ee
       - be-tests-databricks-ee
       - be-tests-druid-jdbc-ee

.github/workflows/embedding-sdk-host-sample-apps-tests.yml

@@ -424,15 +424,38 @@ jobs:
         if: steps.prebuilt-cljs.outcome != 'success'
         run: bun run build-pure:cljs
 
+      # The data-app suite synchronizes real actions, and Metabase only permits
+      # those against a writable warehouse, which comes from the `qa-db`
+      # snapshot below. That snapshot creator builds every QA database in one
+      # pass — Mongo and MySQL included — so all three have to be up even though
+      # only `postgres-writable` is restored. The other host apps read the sample
+      # database, so they skip the pull entirely.
+      - name: Prepare Docker containers
+        if: ${{ matrix.test_suite == 'vite-6-data-app-host-app-e2e' }}
+        uses: ./.github/actions/e2e-prepare-containers
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+          postgres: true
+          mysql: true
+          mongo: true
+
       - name: Run Metabase
         run: node e2e/runner/run_cypress_ci.js start
 
       - uses: ./.github/actions/resolve-job-id
 
+      # Without `--spec` the snapshot config picks up `qa-db.cy.snap.js` too,
+      # which is what creates the `postgres-writable` snapshot the data-app
+      # action tests restore.
       - name: Generate database snapshots
         run: |
-          node e2e/runner/run_cypress_ci.js snapshot \
-            --spec "e2e/snapshot-creators/default.cy.snap.js"
+          if [ "${{ matrix.test_suite }}" = "vite-6-data-app-host-app-e2e" ]; then
+            node e2e/runner/run_cypress_ci.js snapshot
+          else
+            node e2e/runner/run_cypress_ci.js snapshot \
+              --spec "e2e/snapshot-creators/default.cy.snap.js"
+          fi
 
       - name: Prepare and launch Host App
         run: npx tsx ./e2e/runner/embedding-sdk/host-apps/start-ci.ts ${{ matrix.test_suite }}

.github/workflows/embedding-sdk-package-build.yml

@@ -24,6 +24,8 @@ jobs:
           # unrelated to embedding (EMB-765)
           echo "::remove-matcher owner=tsc::"
           bun run build-embedding-sdk-package
+      - name: Smoke test the packed SDK CLI
+        run: node bin/embedding-sdk/test-sdk-package.mjs
       - name: Prepare Embedding SDK artifact
         uses: actions/upload-artifact@v7
         with:

.github/workflows/kondo-ratchets-shrink.yml

@@ -1,9 +1,9 @@
 name: Shrink ratchets
 
-# Lowers the budgets in .clj-kondo/ratchets.edn to match master and delivers the result through one
+# Lowers the ratchet budgets to match master and delivers the result through one
 # automation PR, because the rulesets require a pull request for every change to master. The PR is
 # approved by github-actions[bot] and merges itself once the ratchet check passes. A merge of that PR
-# touches only ratchets.edn, which matches `.clj-kondo/**` below, so it triggers one more run that finds
+# touches only ratchet files, which match `.clj-kondo/**` below, so it triggers one more run that finds
 # nothing to lower and stops: a single no-op, not a loop.
 
 on:
@@ -32,7 +32,8 @@ concurrency:
 
 env:
   AUTOMATION_BRANCH: automation/shrink-ratchets
-  RATCHETS_FILE: .clj-kondo/ratchets.edn
+  KONDO_RATCHETS_FILE: .clj-kondo/ratchets.edn
+  MODULE_RATCHETS_FILE: .clj-kondo/config/modules/ratchets.edn
 
 jobs:
   shrink:
@@ -56,9 +57,10 @@ jobs:
         with:
           distribution: temurin
           java-version: 25
-      - name: Shrink ignore limits
+      - name: Shrink ratchets
         run: |
-          cp "$RATCHETS_FILE" "$RUNNER_TEMP/ratchets-before.edn"
+          cp "$KONDO_RATCHETS_FILE" "$RUNNER_TEMP/kondo-ratchets-before.edn"
+          cp "$MODULE_RATCHETS_FILE" "$RUNNER_TEMP/module-ratchets-before.edn"
           ./bin/mage kondo-ratchets-shrink
       # Two independent outputs: `push` says the branch needs a new commit, `pr` names the open automation
       # PR if there is one. Approval and auto-merge key on `pr` alone, so a run that failed after opening
@@ -77,23 +79,23 @@ jobs:
             --jq '.[0].number // empty')
           echo "pr=$open_pr" >> "$GITHUB_OUTPUT"
 
-          if git diff --quiet -- "$RATCHETS_FILE"; then
+          if git diff --quiet -- "$KONDO_RATCHETS_FILE" "$MODULE_RATCHETS_FILE"; then
             echo "Ratchet budgets already match master $(git rev-parse --short HEAD)"
             [ -z "$open_pr" ] || echo "The open automation PR #$open_pr is re-approved below"
             echo "push=false" >> "$GITHUB_OUTPUT"
             exit 0
           fi
 
-          # The ratchet file is the whole payload, so an open PR that already carries this result is left
+          # The ratchet files are the whole payload, so an open PR that already carries this result is left
           # alone. Rebasing it onto every master push would restart its CI and churn the diff without
           # changing what it delivers. Compare the open PR with the working tree, which contains the
-          # shrunk ratchets file produced above, to determine whether the PR would deliver the same file.
+          # shrunk ratchet files produced above, to determine whether the PR would deliver the same result.
           #
           # A conflicted PR is the exception: only a fresh commit clears it. Mergeability is computed
           # lazily, so a pending UNKNOWN reads as not conflicted and the next run settles it.
           if [ -n "$open_pr" ] \
              && git fetch --quiet --depth=1 origin "refs/heads/$AUTOMATION_BRANCH" 2>/dev/null \
-             && git diff --quiet FETCH_HEAD -- "$RATCHETS_FILE" \
+             && git diff --quiet FETCH_HEAD -- "$KONDO_RATCHETS_FILE" "$MODULE_RATCHETS_FILE" \
              && [ "$(gh pr view "$open_pr" --repo "$GITHUB_REPOSITORY" \
                        --json mergeable --jq .mergeable)" != CONFLICTING ]; then
             echo "The automation PR #$open_pr already carries this result"
@@ -102,7 +104,7 @@ jobs:
             exit 0
           fi
 
-          git diff --stat -- "$RATCHETS_FILE"
+          git diff --stat -- "$KONDO_RATCHETS_FILE" "$MODULE_RATCHETS_FILE"
           echo "push=true" >> "$GITHUB_OUTPUT"
       - name: Update the automation PR
         id: deliver
@@ -116,14 +118,16 @@ jobs:
           OPEN_PR: ${{ steps.compare.outputs.pr }}
         run: |
           ./bin/mage -kondo-ratchets-pr-body \
-            "$RUNNER_TEMP/ratchets-before.edn" \
-            "$RATCHETS_FILE" \
+            "$RUNNER_TEMP/kondo-ratchets-before.edn" \
+            "$KONDO_RATCHETS_FILE" \
+            "$RUNNER_TEMP/module-ratchets-before.edn" \
+            "$MODULE_RATCHETS_FILE" \
             "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
             > "$RUNNER_TEMP/ratchets-pr-body.md"
 
           git config user.name github-actions
           git config user.email github-actions@github.com
-          git add -- "$RATCHETS_FILE"
+          git add -- "$KONDO_RATCHETS_FILE" "$MODULE_RATCHETS_FILE"
           git commit --quiet -m "Tighten ratchets"
           git push --force --quiet \
             "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" \

.github/workflows/linear-pr-links.yml

@@ -1,25 +0,0 @@
-name: Add Issue References to PR
-
-on:
-  pull_request:
-    types: [opened, edited, synchronize]
-
-jobs:
-  add-issue-references:
-    runs-on: ${{ vars.SLIM_RUNNER_KEY }}
-    timeout-minutes: 5
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v6
-      - name: Run the link script
-        uses: actions/github-script@v9
-        env:
-           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-           LINEAR_API_KEY: ${{ secrets.LINEAR_API_KEY }}
-           PR_NUMBER: ${{ github.event.pull_request.number }}
-           REPO_OWNER: ${{ github.repository_owner }}
-           REPO_NAME: ${{ github.event.repository.name }}
-        with:
-          script: |
-            const { link_issues } = require('${{ github.workspace }}/.github/scripts/update-pr-with-issues.js');
-            await link_issues(github)

.github/workflows/pr.checks.yml

@@ -0,0 +1,52 @@
+# Helpful (and quick) checks that run on PRs.
+
+name: PR Checks
+
+on: pull_request
+
+jobs:
+  checks:
+    runs-on: ${{ vars.SLIM_RUNNER_KEY }}
+    timeout-minutes: 10
+    permissions:
+      pull-requests: write
+      issues: read
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          sparse-checkout: |
+            /.github/scripts/update-pr-with-issues.js
+            /backport.sh
+          sparse-checkout-cone-mode: false
+          # download repo structure without file contents, fetched on demand since we only need a few files
+          # NOTE: this gets slow if we start accessing more files, remove this filter in that case.
+          filter: blob:none
+
+      - parallel:
+          - name: Block master branch PRs (from forks)
+            if: github.head_ref == 'master'
+            run: |
+              echo "PRs from \`master\` branches are not allowed, please use a new branch name for your PR."
+              exit 1
+
+          - name: Link PR and issues
+            uses: actions/github-script@v9
+            env:
+              GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              LINEAR_API_KEY: ${{ secrets.LINEAR_API_KEY }}
+              PR_NUMBER: ${{ github.event.pull_request.number }}
+              REPO_OWNER: ${{ github.repository_owner }}
+              REPO_NAME: ${{ github.event.repository.name }}
+            with:
+              script: |
+                const { link_issues } = require('${{ github.workspace }}/.github/scripts/update-pr-with-issues.js');
+                await link_issues(github)
+
+          - name: Prevent backport.sh from being merged
+            run: |
+              if [ -f "./backport.sh" ]; then
+                echo "backport.sh file exists. Please remove it before merging."
+                exit 1
+              else
+                echo "All good."
+              fi

.github/workflows/pull-request-reminders.yml

@@ -1,45 +0,0 @@
-name: Pull Request Reminders
-
-on:
-  pull_request:
-    types: [opened]
-    branches:
-      - "master"
-
-jobs:
-  external-reminder:
-    name: External PR Helper
-    if: ${{ github.event.pull_request.head.repo.full_name != github.repository }}
-    runs-on: ${{ vars.SLIM_RUNNER_KEY }}
-    timeout-minutes: 5
-    permissions:
-      pull-requests: write
-      issues: read
-    steps:
-      - uses: actions/github-script@v9
-        with:
-          script: | # js
-            const owner = context.repo.owner;
-            const repo = context.repo.repo;
-
-            const pullNumber = '${{ github.event.pull_request.number }}';
-            const author = '${{ github.event.pull_request.user.login }}';
-
-            await github.rest.issues.addLabels({
-              owner,
-              repo,
-              issue_number: pullNumber,
-              labels: ["External"],
-            });
-
-            const guideLink = "https://www.notion.so/metabase/Third-Party-PRs-External-contributions-5954061854424c1ea6391d98746646e0";
-            const message = `@${author} Thanks for your contribution! ❤️  We'll try to review it as soon as we can.\n\n
-              Reviewers, make sure to check out our [External PR Review Guide](${guideLink}) when reviewing.
-            `;
-
-            await github.rest.issues.createComment({
-              owner,
-              repo,
-              issue_number: pullNumber,
-              body: message,
-            });

.github/workflows/release-embedding-sdk.yml

@@ -382,6 +382,9 @@ jobs:
       - name: Generate SDK package.json in the build directory
         run: bun run embedding-sdk:generate-package
 
+      - name: Smoke test the packed SDK CLI
+        run: node bin/embedding-sdk/test-sdk-package.mjs
+
       - name: Upload built SDK package as artifact
         uses: actions/upload-artifact@v7
         with:

.github/workflows/schedule-patch.yml

@@ -21,7 +21,7 @@ jobs:
             // Nightly patches cover the current supported major and the one
             // before it. release-auto.yml releases one major per dispatch.
             const currentRelease = Number('${{ vars.CURRENT_VERSION }}');
-            const majors = [currentRelease - 1, currentRelease];
+            const majors = [currentRelease];
 
             await Promise.all(majors.map(version =>
               github.rest.actions.createWorkflowDispatch({