Upgrade node version

- Add actions/setup-node@v4 with node-version 20 to pin the Node runtime
- Pass github.repository and PR number through env vars instead of
  interpolating directly into shell (avoids expression injection pattern)
- Pin tsx@4.21.0 and js-yaml@4.1.1 for reproducible, faster CI runs

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

.clj-kondo/config.edn

@@ -596,9 +596,6 @@
    metabase.model-persistence.api-test/with-persistence-setup!                                                               hooks.common/with-one-top-level-binding
    metabase.model-persistence.api-test/with-setup!                                                                           hooks.common/with-one-top-level-binding
    metabase.model-persistence.test-util/with-persistence-enabled!                                                            hooks.metabase.model-persistence.test-util/with-persistence-enabled!
-   metabase.mq.core/with-queue                                                                                               hooks.common/let-second
-   metabase.mq.core/with-topic                                                                                               hooks.common/let-second
-   metabase.mq.core/def-listener!                                                                                            hooks.metabase.mq.impl/def-listener
    metabase.models.interface/define-batched-hydration-method                                                                 hooks.metabase.models.interface/define-hydration-method
    metabase.models.interface/define-simple-hydration-method                                                                  hooks.metabase.models.interface/define-hydration-method
    metabase.permissions.core/allow-changing-all-users-group-members                                                          hooks.common/do*

.clj-kondo/config/modules/config.edn

@@ -814,7 +814,6 @@
            legacy-mbql
            lib
            lib-be
-           mq
            premium-features
            query-processor
            settings
@@ -1312,17 +1311,6 @@
                     :model/Table
                     :model/User}}
 
-  mq
-  {:team "UX West"
-   :api  #{metabase.mq.core
-           metabase.mq.init}
-   :uses #{analytics
-           app-db
-           models
-           settings
-           startup
-           util}}
-
   native-query-snippets
   {:team "Graphy"
    :api  #{metabase.native-query-snippets.api
@@ -1842,7 +1830,6 @@
            lib
            lib-be
            models
-           mq
            permissions
            premium-features
            queries
@@ -1944,7 +1931,6 @@
            config
            events
            models
-           mq
            premium-features
            util
            enterprise/advanced-permissions}
@@ -2584,7 +2570,6 @@
            events
            lib
            models
-           mq
            query-processor
            util
            warehouse-schema}

.clj-kondo/src/hooks/metabase/mq/impl.clj

@@ -1,24 +0,0 @@
-(ns hooks.metabase.mq.impl
-  (:require
-   [clj-kondo.hooks-api :as hooks]))
-
-(defn def-listener
-  "Kondo hook for `mq/def-listener`. Rewrites to a `fn` form so kondo can analyze bindings.
-
-   Forms:
-     (mq/def-listener! :topic/foo [msg] body...)
-     (mq/def-listener! :queue/bar {:config map} [msgs] body...)"
-  [{:keys [node]}]
-  (let [[_ _channel-name & args] (:children node)
-        ;; skip optional config map
-        args (if (hooks/map-node? (first args))
-               (rest args)
-               args)
-        [bindings & body] args
-        ;; rewrite as: (fn [bindings...] body...)
-        new-node (hooks/list-node
-                  (list*
-                   (hooks/token-node 'fn)
-                   bindings
-                   body))]
-    {:node new-node}))

.cljfmt.edn

@@ -303,6 +303,7 @@
   metabase.util.malli/with                                                                    [[:default]]
   metabase.util.malli/with-api-error-message                                                  [[:default]]
   metabase.util.namespaces/import-fns                                                         [[:default]]
+  metabase.util.queue-test/await-test-while                                                   [[:block 1]]
   metabase.util/auto-retry                                                                    [[:block 1]]
   metabase.util/case-enum                                                                     [[:block 1]]
   metabase.util/for-map                                                                       [[:block 1]]

.github/workflows/check-migration-version.yml

@@ -0,0 +1,41 @@
+name: Check Migration Versions
+
+on:
+  pull_request:
+    paths:
+      - 'resources/migrations/**'
+
+concurrency:
+  group: check-migration-version-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  check-migration-version:
+    runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          filter: blob:none
+          fetch-depth: 0
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
+      - name: Get changed migration files
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          gh api "repos/${GH_REPO}/pulls/${PR_NUMBER}/files" \
+            --paginate -q '.[] | select(.filename | startswith("resources/migrations/")) | select(.filename | endswith(".yaml")) | "\(.status)\t\(.filename)"' \
+            > /tmp/changed-migrations.tsv
+
+      - name: Check migration versions
+        env:
+          BASE_REF: ${{ github.base_ref }}
+          CURRENT_VERSION: ${{ vars.CURRENT_VERSION }}
+          CHANGED_FILES: /tmp/changed-migrations.tsv
+        run: npx --package=tsx@4.21.0 --package=js-yaml@4.1.1 tsx bin/check-migration-versions.ts

bin/check-migration-versions.ts

@@ -0,0 +1,231 @@
+#!/usr/bin/env tsx
+//
+// Check that new migration files target the correct version.
+//
+// Usage:
+//   bin/check-migration-versions <BASE_REF> [CURRENT_VERSION]
+//
+// In CI, BASE_REF and CURRENT_VERSION are set via env vars. Locally they
+// can be passed as arguments.
+//
+// For PRs targeting a release branch, the expected version is derived from
+// the branch name. For PRs targeting master, CURRENT_VERSION must be set
+// and the expected version is current+1.
+//
+// Checks:
+//   1. New migration files are in the correct version directory (e.g. 061/)
+//   2. Changeset IDs inside those files use the correct version prefix (e.g. v61.)
+//
+// In CI, set CHANGED_FILES to a TSV file of "status\tpath" (from the GitHub
+// API) to avoid needing full git history. Falls back to git diff locally.
+
+/* eslint-disable no-console */
+
+import { execFileSync } from "child_process";
+import { existsSync, readFileSync } from "fs";
+
+import { load } from "js-yaml";
+
+function git(...args: string[]): string {
+  return execFileSync("git", args, { encoding: "utf-8" }).trim();
+}
+
+// -- Resolve base ref --------------------------------------------------------
+
+const baseRef = process.argv[2] ?? process.env.BASE_REF;
+
+if (!baseRef) {
+  console.error("ERROR: No base ref. Pass as argument or set BASE_REF.");
+  process.exit(1);
+}
+
+if (!/^[\w.\-/]+$/.test(baseRef)) {
+  console.error(`ERROR: Invalid base ref: ${baseRef}`);
+  process.exit(1);
+}
+
+console.log(`BASE_REF: ${baseRef}`);
+
+// -- Determine the expected migration version for this target branch ---------
+
+let expectedVersion: number;
+
+const releaseBranchMatch = baseRef.match(/^release-x\.(\d+)\.x$/);
+if (releaseBranchMatch) {
+  expectedVersion = Number(releaseBranchMatch[1]);
+  if (!Number.isInteger(expectedVersion) || expectedVersion <= 0) {
+    console.error(
+      `ERROR: Could not derive a valid version from branch name: '${baseRef}'`,
+    );
+    process.exit(1);
+  }
+} else if (baseRef === "master") {
+  const currentVersionStr = process.argv[3] ?? process.env.CURRENT_VERSION;
+  if (!currentVersionStr) {
+    console.error(
+      "ERROR: CURRENT_VERSION is not set. Export it before running this script.",
+    );
+    process.exit(1);
+  }
+  console.log(`CURRENT_VERSION: ${currentVersionStr}`);
+  const currentVersion = Number(currentVersionStr);
+  if (!Number.isInteger(currentVersion) || currentVersion <= 0) {
+    console.error(
+      `ERROR: CURRENT_VERSION must be a positive integer, got: '${currentVersionStr}'`,
+    );
+    process.exit(1);
+  }
+  expectedVersion = currentVersion + 1;
+} else {
+  console.log(
+    `Target branch '${baseRef}' is not master or a release branch — skipping check.`,
+  );
+  process.exit(0);
+}
+
+const expectedDir = String(expectedVersion).padStart(3, "0");
+console.log(
+  `Expected migration version: v${expectedVersion} (directory ${expectedDir}/)\n`,
+);
+
+// -- Find new and modified migration files ------------------------------------
+
+let newFiles: string[] = [];
+let modifiedFiles: string[] = [];
+
+// -- Ensure base ref is available (needed for git diff and git show) ----------
+
+function ensureBaseRef(): void {
+  try {
+    git("rev-parse", `origin/${baseRef}`);
+  } catch {
+    try {
+      git("fetch", "origin", baseRef, "--depth=1");
+    } catch {
+      console.error(`ERROR: Could not fetch origin/${baseRef}`);
+      process.exit(1);
+    }
+  }
+}
+
+const changedFilesPath = process.env.CHANGED_FILES;
+if (changedFilesPath && existsSync(changedFilesPath)) {
+  // CI path: read pre-computed file list from GitHub API
+  console.log(`Reading changed files from ${changedFilesPath}`);
+  for (const line of readFileSync(changedFilesPath, "utf-8").split("\n")) {
+    if (!line) {
+      continue;
+    }
+    const [status, file] = line.split("\t");
+    if (status === "added") {
+      newFiles.push(file);
+    } else if (status === "modified") {
+      modifiedFiles.push(file);
+    }
+  }
+} else {
+  // Local path: use git diff
+  console.log("Using git diff to find changed files");
+  ensureBaseRef();
+
+  function changedFiles(filter: string): string[] {
+    return git(
+      "diff",
+      "--name-only",
+      `--diff-filter=${filter}`,
+      `origin/${baseRef}..HEAD`,
+      "--",
+      "resources/migrations/**/*.yaml",
+    )
+      .split("\n")
+      .filter(Boolean);
+  }
+
+  newFiles = changedFiles("A");
+  modifiedFiles = changedFiles("M");
+}
+
+// In CI, we still need the base ref to git show old file contents
+if (modifiedFiles.length > 0) {
+  ensureBaseRef();
+}
+
+if (newFiles.length === 0 && modifiedFiles.length === 0) {
+  console.log("No new or modified migration files.");
+  process.exit(0);
+}
+
+// -- Helpers ------------------------------------------------------------------
+
+type ChangeLog = {
+  databaseChangeLog?: Array<{ changeSet?: { id?: string } }>;
+} | null;
+
+function getChangeSetIds(doc: ChangeLog): Set<string> {
+  const ids = new Set<string>();
+  for (const entry of doc?.databaseChangeLog ?? []) {
+    const id = entry.changeSet?.id;
+    if (id) {
+      ids.add(id);
+    }
+  }
+  return ids;
+}
+
+function checkVersion(file: string, id: string): boolean {
+  const m = id.match(/^v(\d+)\./);
+  if (m && Number(m[1]) !== expectedVersion) {
+    console.error(
+      `ERROR: ${file}: changeset id "${id}" uses v${m[1]} but expected v${expectedVersion}`,
+    );
+    return false;
+  }
+  return true;
+}
+
+// -- Validate -----------------------------------------------------------------
+
+let errors = 0;
+
+// New files: check directory and all changeset IDs
+for (const file of newFiles) {
+  const dirMatch = file.match(/^resources\/migrations\/(\d+)\//);
+  if (dirMatch && dirMatch[1] !== expectedDir) {
+    console.error(
+      `ERROR: ${file}: in directory ${dirMatch[1]}/ but expected ${expectedDir}/`,
+    );
+    errors++;
+  }
+
+  const doc = load(readFileSync(file, "utf-8")) as ChangeLog;
+  for (const id of getChangeSetIds(doc)) {
+    if (!checkVersion(file, id)) {
+      errors++;
+    }
+  }
+}
+
+// Modified files: only check changeset IDs that were added
+for (const file of modifiedFiles) {
+  const oldContent = git("show", `origin/${baseRef}:${file}`);
+  const oldIds = getChangeSetIds(load(oldContent) as ChangeLog);
+  const newIds = getChangeSetIds(
+    load(readFileSync(file, "utf-8")) as ChangeLog,
+  );
+
+  for (const id of newIds) {
+    if (!oldIds.has(id) && !checkVersion(file, id)) {
+      errors++;
+    }
+  }
+}
+
+if (errors > 0) {
+  console.error(`\n${errors} error(s) found.\n`);
+  console.error(
+    `PRs targeting ${baseRef} should use version v${expectedVersion} (directory ${expectedDir}/).`,
+  );
+  process.exit(1);
+}
+
+console.log(`All migration versions are correct (v${expectedVersion}).`);

dev/src/dev/mq_perf.clj

@@ -1,532 +0,0 @@
-(ns dev.mq-perf
-  "Performance benchmarks for the appdb-backed MQ system (queues and topics).
-
-  Measures publishing throughput, end-to-end latency, and sustained throughput
-  under different load levels. Benchmarks exercise the full realistic pipeline:
-  publish buffer → DB → poll thread → worker pool → listener.
-
-  Self-contained — starts the MQ subsystem without the full Metabase server.
-  Supports multi-node testing: run multiple JVM instances against the same DB.
-
-  Example usage:
-    ;; Single-node: start MQ and run all benchmarks
-    (start-mq!)
-    (run-all-benchmarks!)
-
-    ;; Multi-node coordination: start multiple JVMs, each calls:
-    (start-mq!)
-    (run-all-benchmarks-coordinated!)
-    ;; Then from any node (or a separate REPL), publish the go signal:
-    (signal-go!)
-
-    ;; Individual benchmarks
-    (bench-publish-throughput! {:n 1000 :batch-sizes [1 10 100]})
-    (bench-e2e-latency! {:n 50 :rate :medium :transport :queue})
-    (bench-sustained-throughput! {:duration-sec 15 :rate :medium :transport :queue})
-    (bench-batch-listener! {:n 200 :batch-sizes [1 10 50]})
-    (bench-multi-queue! {:queue-counts [1 5 10 20] :msgs-per-queue 50})"
-  (:require
-   [clojure.string :as str]
-   [metabase.mq.core :as mq]
-   [metabase.mq.listener :as listener]
-   [metabase.mq.publish-buffer :as publish-buffer]
-   [metabase.mq.queue.backend :as q.backend]
-   [metabase.mq.topic.backend :as topic.backend]
-   [toucan2.core :as t2])
-  (:import
-   (java.util.concurrent CountDownLatch TimeUnit)))
-
-(set! *warn-on-reflection* true)
-
-;;; ------------------------------------------------ Startup ------------------------------------------------
-
-(defonce ^:private started? (atom false))
-
-(defn start-mq!
-  "Minimal startup: connects to app DB and starts the MQ subsystem.
-  No Jetty, no scheduler, no sample data. Idempotent."
-  []
-  (let [setup-db! (requiring-resolve 'metabase.app-db.core/setup-db!)]
-    (setup-db! :create-sample-content? false))
-  (let [setup-prometheus! (requiring-resolve 'metabase.analytics.prometheus/setup!)]
-    (setup-prometheus!))
-  (require 'metabase.mq.init)
-  (let [startup! (requiring-resolve 'metabase.startup.core/def-startup-logic!)]
-    (startup! :metabase.mq.init/MqInit))
-  (reset! started? true)
-  (println "MQ system started."))
-
-(defn- ensure-started!
-  "Starts the MQ system if not yet started, and restarts any dead poll threads."
-  []
-  (when-not @started?
-    (start-mq!))
-  ;; Restart poll threads if they died (e.g., from InterruptedException during a previous benchmark).
-  ;; start! is idempotent and start-polling! detects dead futures.
-  (let [start-transports! (requiring-resolve 'metabase.mq.impl/start-transports)]
-    (start-transports!)))
-
-;;; ------------------------------------------------ Utilities ------------------------------------------------
-
-(def ^:private shared-queues
-  "Fixed set of 10 shared queue names used across all nodes for cross-node contention testing."
-  (mapv #(keyword "queue" (str "bench-shared-" %)) (range 10)))
-
-(def ^:private shared-topic :topic/bench-shared)
-
-(defn- run-id
-  "Generates an 8-character unique identifier for a benchmark run."
-  []
-  (subs (str (random-uuid)) 0 8))
-
-(defn- make-channel
-  "Creates a unique channel keyword for benchmarking."
-  [transport id suffix]
-  (keyword (name transport) (str "bench-" id "-" suffix)))
-
-(defn- random-shared-queue
-  "Returns a random shared queue name."
-  []
-  (nth shared-queues (rand-int (count shared-queues))))
-
-(defn- percentile
-  "Returns the p-th percentile from a sorted vector (p in [0,1])."
-  [sorted-vec p]
-  (if (empty? sorted-vec)
-    0.0
-    (let [idx (min (dec (count sorted-vec))
-                   (int (* p (count sorted-vec))))]
-      (double (nth sorted-vec idx)))))
-
-(defn- collect-stats
-  "Computes summary statistics from a collection of numeric values.
-  Returns {:count :min :max :avg :p50 :p95 :p99}."
-  [values]
-  (if (empty? values)
-    {:count 0 :min 0.0 :max 0.0 :avg 0.0 :p50 0.0 :p95 0.0 :p99 0.0}
-    (let [sorted (vec (sort values))
-          n      (count sorted)]
-      {:count n
-       :min   (double (first sorted))
-       :max   (double (last sorted))
-       :avg   (/ (reduce + 0.0 sorted) n)
-       :p50   (percentile sorted 0.50)
-       :p95   (percentile sorted 0.95)
-       :p99   (percentile sorted 0.99)})))
-
-(defn- fmt
-  "Formats a number for display."
-  [x]
-  (if (integer? x)
-    (str x)
-    (format "%.2f" (double x))))
-
-(defn- print-table!
-  "Prints a formatted ASCII table."
-  [title headers rows]
-  (let [col-widths (mapv (fn [i]
-                           (apply max
-                                  (count (nth headers i))
-                                  (map #(count (str (nth % i))) rows)))
-                         (range (count headers)))
-        separator  (str "+-" (str/join "-+-" (map #(apply str (repeat % \-)) col-widths)) "-+")
-        fmt-row    (fn [vals]
-                     (str "| "
-                          (str/join " | "
-                                    (map-indexed (fn [i v]
-                                                   (format (str "%-" (nth col-widths i) "s") (str v)))
-                                                 vals))
-                          " |"))]
-    (println)
-    (println title)
-    (println separator)
-    (println (fmt-row headers))
-    (println separator)
-    (doseq [row rows]
-      (println (fmt-row row)))
-    (println separator)
-    (println)))
-
-(defn- cleanup!
-  "Removes all benchmark data for the given run-id."
-  [id]
-  (t2/delete! :queue_message_batch :queue_name [:like (str "bench-" id "%")])
-  (t2/delete! :topic_message_batch :topic_name [:like (str "bench-" id "%")])
-  (doseq [[ch _] @listener/*listeners*]
-    (when (str/includes? (name ch) (str "bench-" id))
-      (mq/unlisten! ch))))
-
-(defn- publish-at-rate!
-  "Publishes `n` messages at the given rate (msgs/sec). `publish-fn` is called with each seq num.
-  If rate is nil, publishes as fast as possible (burst mode)."
-  [n rate publish-fn]
-  (if (nil? rate)
-    (dotimes [i n]
-      (publish-fn i))
-    (let [interval-ms (long (/ 1000 rate))]
-      (dotimes [i n]
-        (let [start (System/currentTimeMillis)]
-          (publish-fn i)
-          (let [elapsed (- (System/currentTimeMillis) start)
-                sleep-ms (- interval-ms elapsed)]
-            (when (pos? sleep-ms)
-              (Thread/sleep (long sleep-ms)))))))))
-
-(def ^:private rate-map
-  {:low    10
-   :medium 100
-   :high   1000
-   :burst  nil})
-
-(defn- rate->msgs-sec [rate]
-  (get rate-map rate rate))
-
-;;; ---------------------------------------- Benchmark 1: Publishing Throughput ----------------------------------------
-
-(defn bench-publish-throughput!
-  "Measures raw publishing speed for different batch sizes.
-  This benchmark bypasses the publish buffer to measure direct DB insert speed.
-  Options:
-    :n           - total messages to publish per batch-size test (default 1000)
-    :batch-sizes - vector of batch sizes to test (default [1 10 100])
-    :transport   - :queue or :topic (default :queue)"
-  ([] (bench-publish-throughput! {}))
-  ([{:keys [n batch-sizes transport]
-     :or   {n 1000 batch-sizes [1 10 100] transport :queue}}]
-   (ensure-started!)
-   (let [id      (run-id)
-         results (atom [])]
-     (try
-       (doseq [bs batch-sizes]
-         (let [channel    (make-channel transport id (str "pub-" bs))
-               iterations (/ n bs)
-               start      (System/nanoTime)]
-           (binding [publish-buffer/*publish-buffer-ms* 0]
-             (dotimes [i iterations]
-               (let [msgs (vec (for [j (range bs)]
-                                 {:seq (+ (* i bs) j) :ts (System/currentTimeMillis)}))]
-                 (if (= transport :queue)
-                   (q.backend/publish! :queue.backend/appdb channel msgs)
-                   (topic.backend/publish! :topic.backend/appdb channel msgs)))))
-           (let [elapsed-ms (/ (- (System/nanoTime) start) 1e6)]
-             (swap! results conj
-                    [(str bs) (fmt (/ n (/ elapsed-ms 1000.0))) (fmt elapsed-ms)]))))
-       (print-table!
-        (str "Publishing Throughput (" (name transport) ", n=" n ")")
-        ["Batch Size" "Msgs/sec" "Total ms"]
-        @results)
-       (finally
-         (cleanup! id))))))
-
-;;; ---------------------------------------- Benchmark 2: End-to-End Latency ----------------------------------------
-
-(defn bench-e2e-latency!
-  "Measures end-to-end latency from publish to listener receipt via the full async pipeline.
-  Uses the real publish API including the publish buffer.
-  Publishes randomly to the 10 shared queues (or the shared topic) so multiple nodes
-  compete for the same messages.
-  Options:
-    :n         - number of messages to publish (default 50)
-    :rate      - :low, :medium, :high, or :burst (default :medium)
-    :transport - :queue or :topic (default :queue)"
-  ([] (bench-e2e-latency! {}))
-  ([{:keys [n rate transport]
-     :or   {n 50 rate :medium transport :queue}}]
-   (ensure-started!)
-   (let [latencies (atom [])
-         received  (atom 0)
-         channels  (if (= transport :queue) shared-queues [shared-topic])]
-     (try
-       ;; Register listeners on all shared channels
-       (doseq [ch channels]
-         (when-not (listener/get-listener ch)
-           (mq/listen! ch {}
-                       (fn [msg]
-                         (let [receive-ns  (System/nanoTime)
-                               publish-ns  (get msg "publish-ns")]
-                           (when publish-ns
-                             (swap! latencies conj (/ (- receive-ns (double publish-ns)) 1e6))))
-                         (swap! received inc)))))
-       (let [msgs-sec (rate->msgs-sec rate)
-             start    (System/nanoTime)]
-         ;; Publish to random shared channels
-         (publish-at-rate!
-          n msgs-sec
-          (fn [i]
-            (let [ch (if (= transport :queue) (random-shared-queue) shared-topic)]
-              (if (= transport :queue)
-                (mq/with-queue ch [q]
-                  (mq/put q {:seq i :publish-ns (System/nanoTime)}))
-                (mq/with-topic ch [t]
-                  (mq/put t {:seq i :publish-ns (System/nanoTime)}))))))
-         ;; Force flush the publish buffer then wait for delivery
-         (publish-buffer/flush-publish-buffer!)
-         ;; Wait until we've received at least n messages or timeout
-         (let [deadline (+ (System/currentTimeMillis)
-                           (long (* 1000 (+ 30 (/ n (max 1 (or msgs-sec 1000)))))))]
-           (while (and (< @received n)
-                       (< (System/currentTimeMillis) deadline))
-             (Thread/sleep (long 100))))
-         (let [elapsed-ms (/ (- (System/nanoTime) start) 1e6)
-               stats      (collect-stats @latencies)]
-           (print-table!
-            (str "End-to-End Latency (" (name transport) ", rate=" (name rate) ", n=" n ")")
-            ["Metric" "Value"]
-            [["Published" (str n)]
-             ["Received" (str @received)]
-             ["Other nodes" (str (max 0 (- n @received)))]
-             ["From others" (str (max 0 (- @received n)))]
-             ["Total ms" (fmt elapsed-ms)]
-             ["P50 ms" (fmt (:p50 stats))]
-             ["P95 ms" (fmt (:p95 stats))]
-             ["P99 ms" (fmt (:p99 stats))]
-             ["Min ms" (fmt (:min stats))]
-             ["Max ms" (fmt (:max stats))]
-             ["Avg ms" (fmt (:avg stats))]])))
-       (finally
-         (doseq [ch channels]
-           (when (listener/get-listener ch)
-             (mq/unlisten! ch))))))))
-
-;;; ---------------------------------------- Benchmark 3: Sustained Throughput ----------------------------------------
-
-(defn bench-sustained-throughput!
-  "Publishes continuously while consuming for a given duration via the full async pipeline.
-  Publishes randomly to the 10 shared queues (or shared topic).
-  Options:
-    :duration-sec - how long to publish (default 15)
-    :rate         - :low, :medium, :high, or :burst (default :medium)
-    :transport    - :queue or :topic (default :queue)"
-  ([] (bench-sustained-throughput! {}))
-  ([{:keys [duration-sec rate transport]
-     :or   {duration-sec 15 rate :medium transport :queue}}]
-   (ensure-started!)
-   (let [channels  (if (= transport :queue) shared-queues [shared-topic])
-         received  (atom 0)
-         latencies (atom [])
-         published (atom 0)
-         stop?     (atom false)]
-     (try
-       ;; Register listeners on all shared channels
-       (doseq [ch channels]
-         (when-not (listener/get-listener ch)
-           (mq/listen! ch {}
-                       (fn [msg]
-                         (let [receive-ns  (System/nanoTime)
-                               publish-ns  (get msg "publish-ns")]
-                           (when publish-ns
-                             (swap! latencies conj (/ (- receive-ns (double publish-ns)) 1e6)))
-                           (swap! received inc))))))
-       (let [msgs-sec   (rate->msgs-sec rate)
-             start      (System/nanoTime)
-             pub-thread (Thread.
-                         (fn []
-                           (loop []
-                             (when-not @stop?
-                               (try
-                                 (let [ch (if (= transport :queue) (random-shared-queue) shared-topic)]
-                                   (if (= transport :queue)
-                                     (mq/with-queue ch [q]
-                                       (mq/put q {:seq @published :publish-ns (System/nanoTime)}))
-                                     (mq/with-topic ch [t]
-                                       (mq/put t {:seq @published :publish-ns (System/nanoTime)}))))
-                                 (swap! published inc)
-                                 (catch Exception _))
-                               (when msgs-sec
-                                 (Thread/sleep (long (/ 1000 msgs-sec))))
-                               (recur)))))]
-         (.start pub-thread)
-         (Thread/sleep (long (* duration-sec 1000)))
-         (reset! stop? true)
-         (.join pub-thread 5000)
-         ;; Flush publish buffer and wait for remaining messages to drain
-         (publish-buffer/flush-publish-buffer!)
-         (let [drain-start (System/currentTimeMillis)]
-           (while (and (< @received @published)
-                       (< (- (System/currentTimeMillis) drain-start) 30000))
-             (publish-buffer/flush-publish-buffer!)
-             (Thread/sleep (long 500))))
-         (let [elapsed-ms  (/ (- (System/nanoTime) start) 1e6)
-               stats       (collect-stats @latencies)]
-           (print-table!
-            (str "Sustained Throughput (" (name transport) ", rate=" (name rate)
-                 ", duration=" duration-sec "s)")
-            ["Metric" "Value"]
-            [["Published" (str @published)]
-             ["Received" (str @received)]
-             ["Other nodes" (str (max 0 (- @published @received)))]
-             ["From others" (str (max 0 (- @received @published)))]
-             ["Duration ms" (fmt elapsed-ms)]
-             ["Actual pub/sec" (fmt (/ @published (/ elapsed-ms 1000.0)))]
-             ["Actual recv/sec" (fmt (/ @received (/ elapsed-ms 1000.0)))]
-             ["P50 latency ms" (fmt (:p50 stats))]
-             ["P95 latency ms" (fmt (:p95 stats))]
-             ["P99 latency ms" (fmt (:p99 stats))]
-             ["Max latency ms" (fmt (:max stats))]])))
-       (finally
-         (doseq [ch channels]
-           (when (listener/get-listener ch)
-             (mq/unlisten! ch))))))))
-
-;;; ---------------------------------------- Benchmark 4: Batch Listener ----------------------------------------
-
-(defn bench-batch-listener!
-  "Compares single-message vs batch listener processing rates.
-  Uses the shared queues so multiple nodes compete.
-  Options:
-    :n           - total messages to publish (default 200)
-    :batch-sizes - vector of max-batch-messages values to test (default [1 10 50])"
-  ([] (bench-batch-listener! {}))
-  ([{:keys [n batch-sizes]
-     :or   {n 200 batch-sizes [1 10 50]}}]
-   (ensure-started!)
-   (let [results (atom [])]
-     (doseq [bs batch-sizes]
-       (let [received (atom 0)]
-         (try
-           ;; Register listeners on all shared queues
-           (doseq [ch shared-queues]
-             (when-not (listener/get-listener ch)
-               (if (= bs 1)
-                 (mq/listen! ch {}
-                             (fn [_msg]
-                               (swap! received inc)))
-                 (mq/batch-listen! ch
-                                   (fn [msgs]
-                                     (swap! received + (count msgs)))
-                                   {:max-batch-messages bs}))))
-           ;; Publish all messages randomly to shared queues
-           (dotimes [i n]
-             (mq/with-queue (random-shared-queue) [q]
-               (mq/put q {:seq i})))
-           (publish-buffer/flush-publish-buffer!)
-           ;; Wait for messages to be consumed
-           (let [start   (System/nanoTime)
-                 deadline (+ (System/currentTimeMillis) 30000)]
-             (while (and (< @received n)
-                         (< (System/currentTimeMillis) deadline))
-               (Thread/sleep (long 100)))
-             (let [elapsed-ms (/ (- (System/nanoTime) start) 1e6)]
-               (swap! results conj
-                      [(str bs) (str @received) (fmt elapsed-ms)
-                       (fmt (/ @received (/ elapsed-ms 1000.0)))])))
-           (finally
-             (doseq [ch shared-queues]
-               (when (listener/get-listener ch)
-                 (mq/unlisten! ch)))))))
-     (print-table!
-      (str "Batch Listener Comparison (n=" n ")")
-      ["Max Batch" "Received" "Process ms" "Msgs/sec"]
-      @results))))
-
-;;; ---------------------------------------- Benchmark 5: Multiple Queues ----------------------------------------
-
-(defn bench-multi-queue!
-  "Measures performance with varying numbers of the shared queues.
-  Tests with subsets of the 10 shared queues to show scaling.
-  Options:
-    :queue-counts   - vector of queue counts to test (default [1 5 10])
-    :msgs-per-queue - messages per queue (default 50)"
-  ([] (bench-multi-queue! {}))
-  ([{:keys [queue-counts msgs-per-queue]
-     :or   {queue-counts [1 5 10] msgs-per-queue 50}}]
-   (ensure-started!)
-   (let [results (atom [])]
-     (doseq [qc queue-counts]
-       (let [channels (subvec shared-queues 0 (min qc (count shared-queues)))
-             total    (* (count channels) msgs-per-queue)
-             received (atom 0)]
-         (try
-           ;; Register listeners
-           (doseq [ch channels]
-             (when-not (listener/get-listener ch)
-               (mq/listen! ch {}
-                           (fn [_msg]
-                             (swap! received inc)))))
-           ;; Publish to the queues
-           (let [start (System/nanoTime)]
-             (doseq [ch channels]
-               (dotimes [i msgs-per-queue]
-                 (mq/with-queue ch [q]
-                   (mq/put q {:seq i}))))
-             (publish-buffer/flush-publish-buffer!)
-             ;; Wait for messages
-             (let [deadline (+ (System/currentTimeMillis) 60000)]
-               (while (and (< @received total)
-                           (< (System/currentTimeMillis) deadline))
-                 (Thread/sleep (long 100))))
-             (let [elapsed-ms (/ (- (System/nanoTime) start) 1e6)]
-               (swap! results conj
-                      [(str (count channels)) (str total) (str @received) (fmt elapsed-ms)
-                       (fmt (/ @received (/ elapsed-ms 1000.0)))])))
-           (finally
-             (doseq [ch channels]
-               (when (listener/get-listener ch)
-                 (mq/unlisten! ch)))))))
-     (print-table!
-      (str "Multiple Queues (msgs-per-queue=" msgs-per-queue ")")
-      ["Queues" "Total Msgs" "Received" "Total ms" "Msgs/sec"]
-      @results))))
-
-;;; ---------------------------------------- Run All ----------------------------------------
-
-(defn run-all-benchmarks!
-  "Runs all benchmark scenarios with default parameters."
-  []
-  (println "=== MQ Performance Benchmarks ===")
-  (println)
-
-  (println "--- 1/5: Publishing Throughput (Queue) ---")
-  (bench-publish-throughput! {:n 1000 :transport :queue})
-
-  (println "--- 1b/5: Publishing Throughput (Topic) ---")
-  (bench-publish-throughput! {:n 1000 :transport :topic})
-
-  (println "--- 2/5: End-to-End Latency ---")
-  (doseq [rate [:low :medium :high :burst]]
-    (bench-e2e-latency! {:n 50 :rate rate :transport :queue}))
-  (doseq [rate [:low :medium :burst]]
-    (bench-e2e-latency! {:n 50 :rate rate :transport :topic}))
-
-  (println "--- 3/5: Sustained Throughput ---")
-  (bench-sustained-throughput! {:duration-sec 15 :rate :medium :transport :queue})
-  (bench-sustained-throughput! {:duration-sec 15 :rate :medium :transport :topic})
-
-  (println "--- 4/5: Batch Listener Comparison ---")
-  (bench-batch-listener! {:n 200})
-
-  (println "--- 5/5: Multiple Queues ---")
-  (bench-multi-queue! {:queue-counts [1 5 10 20] :msgs-per-queue 50})
-
-  (println "=== All benchmarks complete ==="))
-
-;;; ---------------------------------------- Multi-Node Coordination ----------------------------------------
-
-(defn run-all-benchmarks-coordinated!
-  "Waits for a 'go' signal on :topic/bench-coordination, then runs all benchmarks.
-  Use with multiple JVM instances to start benchmarks simultaneously.
-
-  Usage:
-    ;; On each node:
-    (start-mq!)
-    (run-all-benchmarks-coordinated!)
-
-    ;; Then from any node or separate REPL:
-    (signal-go!)"
-  []
-  (let [latch (CountDownLatch. 1)]
-    (mq/listen! :topic/bench-coordination {}
-                (fn [_msg]
-                  (.countDown latch)))
-    (println "Waiting for go signal... (call (signal-go!) from any node)")
-    (.await latch 300 TimeUnit/SECONDS)
-    (mq/unlisten! :topic/bench-coordination)
-    (println "Go signal received! Starting benchmarks...")
-    (run-all-benchmarks!)))
-
-(defn signal-go!
-  "Sends the go signal to all waiting nodes."
-  []
-  (mq/with-topic :topic/bench-coordination [t]
-    (mq/put t {:go true :time (System/currentTimeMillis)}))
-  (println "Go signal sent."))

dev/src/dev/mq_perf_multi.sh

@@ -1,104 +0,0 @@
-#!/usr/bin/env bash
-# Runs multiple MQ benchmark nodes against the same database.
-#
-# Usage:
-#   ./dev/src/dev/mq_perf_multi.sh start       # Start 2 nodes (default)
-#   ./dev/src/dev/mq_perf_multi.sh start 4     # Start 4 nodes
-#   ./dev/src/dev/mq_perf_multi.sh go          # Send go signal to all waiting nodes
-#   ./dev/src/dev/mq_perf_multi.sh stop        # Stop all running nodes
-#   ./dev/src/dev/mq_perf_multi.sh logs        # Tail all node logs
-#
-# Each node starts a lightweight JVM with just DB + MQ (no server),
-# waits for a coordinated go signal, then runs all benchmarks.
-
-set -euo pipefail
-cd "$(git rev-parse --show-toplevel)"
-
-LOG_DIR="target/mq-perf-logs"
-PID_DIR="target/mq-perf-pids"
-
-cmd_start() {
-  local num_nodes="${1:-2}"
-  mkdir -p "$LOG_DIR" "$PID_DIR"
-
-  # Clean up stale pids
-  rm -f "$PID_DIR"/*.pid
-
-  echo "Starting $num_nodes MQ benchmark nodes..."
-  echo "Logs in $LOG_DIR/"
-  echo
-
-  for i in $(seq 1 "$num_nodes"); do
-    local log_file="$LOG_DIR/node-$i.log"
-    clojure -M:dev:ee:ee-dev -e "
-      (require 'dev.mq-perf)
-      (dev.mq-perf/start-mq!)
-      (dev.mq-perf/run-all-benchmarks-coordinated!)
-    " > "$log_file" 2>&1 &
-    local pid=$!
-    echo "$pid" > "$PID_DIR/node-$i.pid"
-    echo "  Node $i started (PID $pid, log: $log_file)"
-  done
-
-  echo
-  echo "Nodes are initializing. Once ready, run:"
-  echo "  $0 go"
-}
-
-cmd_go() {
-  echo "Sending go signal..."
-  clojure -M:dev:ee:ee-dev -e "
-    (require 'dev.mq-perf)
-    (dev.mq-perf/start-mq!)
-    (dev.mq-perf/signal-go!)
-  "
-  echo
-  echo "Benchmarks started. Watch progress with:"
-  echo "  $0 logs"
-}
-
-cmd_stop() {
-  if [ ! -d "$PID_DIR" ]; then
-    echo "No nodes running."
-    return
-  fi
-
-  for pid_file in "$PID_DIR"/*.pid; do
-    [ -f "$pid_file" ] || continue
-    local pid
-    pid=$(cat "$pid_file")
-    local name
-    name=$(basename "$pid_file" .pid)
-    if kill -0 "$pid" 2>/dev/null; then
-      kill "$pid"
-      echo "Stopped $name (PID $pid)"
-    else
-      echo "$name (PID $pid) already exited"
-    fi
-    rm -f "$pid_file"
-  done
-}
-
-cmd_logs() {
-  if [ ! -d "$LOG_DIR" ]; then
-    echo "No logs found."
-    return
-  fi
-  tail -f "$LOG_DIR"/node-*.log
-}
-
-case "${1:-}" in
-  start) cmd_start "${2:-2}" ;;
-  go)    cmd_go ;;
-  stop)  cmd_stop ;;
-  logs)  cmd_logs ;;
-  *)
-    echo "Usage: $0 {start [N]|go|stop|logs}"
-    echo
-    echo "  start [N]  Start N benchmark nodes (default 2)"
-    echo "  go         Send go signal to start benchmarks"
-    echo "  stop       Kill all running nodes"
-    echo "  logs       Tail all node logs"
-    exit 1
-    ;;
-esac

dev/src/dev/search_perf.clj

@@ -29,6 +29,7 @@
    [metabase.search.config :as search.config]
    [metabase.search.engine :as search.engine]
    [metabase.search.impl :as search.impl]
+   [metabase.search.ingestion :as search.ingestion]
    [metabase.util :as u]
    [metabase.util.random :as u.random]
    [toucan2.core :as t2]))
@@ -445,7 +446,9 @@
                                              :num-users  num-users
                                              :prefix     prefix})
          user-id (:user-id test-env)
-         _ (Thread/sleep 1000) ;; Wait for search index
+         ;; Wait for search index to update if using async indexing
+         _ (binding [search.ingestion/*force-sync* true]
+             (Thread/sleep 1000))
          queries [;; Exact prefix match
                   (str prefix "_table_00001")
                   ;; Partial prefix match

enterprise/backend/src/metabase_enterprise/action_v2/data_editing.clj

@@ -8,26 +8,35 @@
    [metabase.api.common :as api]
    [metabase.lib.core :as lib]
    [metabase.lib.metadata :as lib.metadata]
-   [metabase.mq.core :as mq]
    [metabase.query-processor.core :as qp]
    ;; legacy usage -- don't do things like this going forward
    ^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.query-processor.store :as qp.store]
    [metabase.util :as u]
+   [metabase.util.queue :as queue]
    [metabase.warehouse-schema.models.field-values :as field-values]
-   [toucan2.core :as t2]))
+   [toucan2.core :as t2])
+  (:import
+   (java.util.concurrent ArrayBlockingQueue)))
 
 (set! *warn-on-reflection* true)
 
+(def ^:private ^ArrayBlockingQueue global-field-value-invalidate-queue
+  "Queue used to recalculate the field values for updated columns in the background."
+  (ArrayBlockingQueue. 1000))
+
+(def ^:dynamic *field-value-invalidate-queue*
+  "A layer of indirection on the actual [[field-value-invalidation-queue]], for testing."
+  nil)
+
 (defn- batch-invalidate-field-values!
   "Recalculate the field values for the given fields."
   [field-batches]
   (->> (t2/select :model/Field :id [:in (into #{} cat field-batches)])
        (run! field-values/create-or-update-full-field-values!)))
 
-(mq/def-listener! :queue/field-value-invalidation
-  {:max-batch-messages 10}
-  [messages]
-  (batch-invalidate-field-values! (into [] cat messages)))
+(defmethod queue/init-listener! ::FieldValueInvalidation [_]
+  (queue/listen! "field-value-invalidate" global-field-value-invalidate-queue batch-invalidate-field-values!
+                 {:max-batch-messages 10, :max-next-ms 10}))
 
 (defn select-table-pk-fields
   "Given a table-id, return the :model/Field instances corresponding to its PK columns. Do not assume any ordering."
@@ -139,8 +148,8 @@
                           (apply concat))]
     ;; Note that for now we only rescan field values when values are *added* and not when they are *removed*.
     (when (seq stale-fields)
-      (mq/with-queue :queue/field-value-invalidation [q]
-        (mq/put q [stale-fields])))))
+      (let [^ArrayBlockingQueue queue (or *field-value-invalidate-queue* global-field-value-invalidate-queue)]
+        (.offer queue stale-fields)))))
 
 ;; TODO this is fairly dirty, would be cleaner to map from db values to de-coerced values via middleware
 ;;      invalidation could perhaps be done in response to effect, or in middleware (to dedupe for chained actions)

enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj

@@ -245,4 +245,4 @@
       ;;       this means that the corresponding entries would have been missing or stale when we indexed them.
       ;;       ideally, we would delay the indexing somehow, or only reindex what we've loaded.
       ;;       while we're figuring that out, here's a crude stopgap.
-        (search/async-reindex!)))))
+        (search/reindex!)))))

enterprise/backend/test/metabase_enterprise/action_v2/api_test.clj

@@ -7,17 +7,19 @@
    [flatland.ordered.map :refer [ordered-map]]
    [metabase-enterprise.action-v2.api]
    [metabase-enterprise.action-v2.coerce :as coerce]
+   [metabase-enterprise.action-v2.data-editing :as data-editing]
    [metabase-enterprise.action-v2.test-util :as action-v2.tu]
    [metabase.actions.test-util :as actions.tu]
    [metabase.driver :as driver]
    [metabase.driver.sql-jdbc :as sql-jdbc]
    [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
-   [metabase.mq.test-util :as mq.tu]
    [metabase.query-processor.test :as qp]
    [metabase.test :as mt]
    [metabase.test.data.sql :as sql.tx]
    [metabase.warehouse-schema.models.field-values :as field-values]
-   [toucan2.core :as t2]))
+   [toucan2.core :as t2])
+  (:import
+   (java.util.concurrent ArrayBlockingQueue)))
 
 (set! *warn-on-reflection* true)
 
@@ -530,26 +532,40 @@
   (mt/with-premium-features #{actions-feature-flag}
     (mt/test-drivers (mt/normal-drivers-with-feature :actions/data-editing)
       (action-v2.tu/with-test-tables! [table-id [{:id 'auto-inc-type, :n [:text]} {:primary-key [:id]}]]
-        (let [field-id     (t2/select-one-fn :id :model/Field :table_id table-id :name "n")
-              _            (t2/update! :model/Field {:id field-id} {:semantic_type "type/Category"})
-              field-values #(vec (:values (field-values/get-latest-full-field-values field-id)))
-              create!      #(action-v2.tu/create-rows! table-id %)
-              update!      #(action-v2.tu/update-rows! table-id %)]
-          (mq.tu/with-sync-mq
+        (let [field-id         (t2/select-one-fn :id :model/Field :table_id table-id :name "n")
+              _                (t2/update! :model/Field {:id field-id} {:semantic_type "type/Category"})
+              field-values     #(vec (:values (field-values/get-latest-full-field-values field-id)))
+              test-queue       (ArrayBlockingQueue. 100)
+              create!          #(action-v2.tu/create-rows! table-id %)
+              update!          #(action-v2.tu/update-rows! table-id %)
+              process-queue!   (fn []
+                                 (when-let [field-ids (.poll test-queue)]
+                                   (#'data-editing/batch-invalidate-field-values! [field-ids])
+                                   (recur)))]
+          (binding [data-editing/*field-value-invalidate-queue* test-queue]
             (is (= [] (field-values)))
 
             (create! [{:n "a"}])
+            (is (pos? (.size test-queue)))
+            (process-queue!)
             (is (= ["a"] (field-values)))
 
             (create! [{:n "b"} {:n "c"}])
+            (is (pos? (.size test-queue)))
+            (process-queue!)
             (is (= ["a" "b" "c"] (field-values)))
 
             (update! [{:id 2, :n "d"}])
+            (is (pos? (.size test-queue)))
+            (process-queue!)
             (is (= ["a" "c" "d"] (field-values)))
 
-            ;; Creating a duplicate value should not trigger invalidation
             (create! [{:n "a"}])
+            (is (zero? (.size test-queue)))
+            (process-queue!)
             (update! [{:id 1, :n "e"}])
+            (is (pos? (.size test-queue)))
+            (process-queue!)
             (is (= ["a" "c" "d" "e"] (field-values)))))))))
 
 (deftest execute-form-built-in-table-action-test

enterprise/backend/test/metabase_enterprise/agent_api/api_test.clj

@@ -6,17 +6,10 @@
    [metabase-enterprise.sso.test-setup :as sso.test-setup]
    [metabase.lib.core :as lib]
    [metabase.lib.metadata :as lib.metadata]
-   [metabase.lib.normalize :as lib.normalize]
-   [metabase.metabot.settings] ; for setting definitions
-   [metabase.metabot.tools.util :as metabot.tools.u]
-   [metabase.search.test-util :as search.tu]
    [metabase.test :as mt]
    [metabase.test.fixtures :as fixtures]
    [metabase.test.http-client :as client]
    [metabase.test.util :as tu]
-   [metabase.util :as u]
-   [metabase.util.json :as json]
-   [metabase.warehouse-schema.models.field-values :as field-values]
    [toucan2.core :as t2]))
 
 (set! *warn-on-reflection* true)
@@ -139,192 +132,6 @@
             (is (= "You don't have permissions to do that."
                    (agent-client :rasta :get 403 (str "agent/v1/metric/" (:id metric)))))))))))
 
-(deftest get-table-details-test
-  (with-agent-api-setup!
-    (testing "Returns table details for valid table ID"
-      (let [table-id (mt/id :orders)]
-        (is (=? {:type           "table"
-                 :id             table-id
-                 :name           "ORDERS"
-                 :display_name   "Orders"
-                 :database_id    (mt/id)
-                 :fields         sequential?
-                 :related_tables sequential?}
-                (agent-client :rasta :get 200 (str "agent/v1/table/" table-id))))))
-
-    (testing "Returns 404 for non-existent table"
-      (is (= "Not found."
-             (agent-client :rasta :get 404 "agent/v1/table/999999"))))
-
-    (testing "Respects query parameters"
-      (let [table-id (mt/id :orders)]
-        (is (=? {:type   "table"
-                 :id     table-id
-                 :fields empty?}
-                (agent-client :rasta :get 200
-                              (str "agent/v1/table/" table-id "?with-fields=false&with-related-tables=false"))))))
-
-    (testing "Field values are excluded by default"
-      (let [table-id (mt/id :orders)
-            table    (agent-client :rasta :get 200 (str "agent/v1/table/" table-id))]
-        (is (every? #(nil? (:field_values %)) (:fields table)))))
-
-    (testing "Field values are included when explicitly requested"
-      (let [table-id (mt/id :orders)
-            table    (agent-client :rasta :get 200 (str "agent/v1/table/" table-id "?with-field-values=true"))]
-        (is (some #(seq (:field_values %)) (:fields table)))))))
-
-(deftest get-metric-details-test
-  (with-agent-api-setup!
-    (mt/with-temp [:model/Card metric {:name          "Test Metric"
-                                       :type          :metric
-                                       :database_id   (mt/id)
-                                       :dataset_query (orders-count-query)}]
-      (testing "Returns metric details for valid metric ID"
-        (is (=? {:type                 "metric"
-                 :id                   (:id metric)
-                 :name                 "Test Metric"
-                 :queryable_dimensions sequential?}
-                (agent-client :rasta :get 200 (str "agent/v1/metric/" (:id metric))))))
-
-      (testing "Respects query parameters"
-        (is (=? {:type "metric"
-                 :id   (:id metric)}
-                (agent-client :rasta :get 200
-                              (str "agent/v1/metric/" (:id metric)
-                                   "?with-queryable-dimensions=false&with-field-values=false")))))
-
-      (testing "Returns 404 for non-existent metric"
-        (is (= "Not found."
-               (agent-client :rasta :get 404 "agent/v1/metric/999999")))))))
-
-(defn- ensure-fresh-field-values!
-  "Ensure field values exist for a field by deleting any existing ones and recreating them."
-  [field-id]
-  (t2/delete! :model/FieldValues :field_id field-id :type :full)
-  (field-values/get-or-create-full-field-values! (t2/select-one :model/Field :id field-id)))
-
-(defn- visible-field-id
-  "Find the field-id string for a field by display name within a table's visible columns."
-  [table-id field-display-name]
-  (let [mp            (mt/metadata-provider)
-        query         (lib/query mp (lib.metadata/table mp table-id))
-        field-prefix  (metabot.tools.u/table-field-id-prefix table-id)
-        visible-cols  (lib/visible-columns query)]
-    (->> (keep-indexed (fn [i col]
-                         (when (= (lib/display-name query col) field-display-name)
-                           (str field-prefix i)))
-                       visible-cols)
-         first)))
-
-(deftest get-table-field-values-test
-  (with-agent-api-setup!
-    ;; Ensure field values exist for the field we'll test
-    (ensure-fresh-field-values! (mt/id :people :state))
-
-    (testing "Returns field statistics and values with default limit of 30"
-      (let [table-id (mt/id :people)
-            field-id (visible-field-id table-id "State")]
-        (is (some? field-id) "Should find the State field")
-        (let [result (agent-client :crowberto :get 200
-                                   (format "agent/v1/table/%d/field/%s/values" table-id field-id))]
-          (is (=? {:value_metadata {:statistics   {:distinct-count 49}
-                                    :field_values sequential?}}
-                  result))
-          (is (<= (count (:values result)) 30) "Should apply default limit of 30"))))
-
-    (testing "Respects explicit limit parameter"
-      (let [table-id (mt/id :people)
-            field-id (visible-field-id table-id "State")]
-        (is (=? {:value_metadata {:field_values #(= 5 (count %))}}
-                (agent-client :crowberto :get 200
-                              (format "agent/v1/table/%d/field/%s/values?limit=5" table-id field-id))))))
-
-    (testing "Returns 404 for non-existent table"
-      (is (= "Not found."
-             (agent-client :crowberto :get 404 "agent/v1/table/999999/field/t999999-0/values"))))
-
-    (testing "Returns 400 for invalid field-id format"
-      (let [table-id (mt/id :people)]
-        (is (= "Invalid field_id format: not-a-valid-id"
-               (agent-client :crowberto :get 400 (format "agent/v1/table/%d/field/not-a-valid-id/values" table-id))))))))
-
-(deftest search-test
-  (with-agent-api-setup!
-    (search.tu/with-new-search-if-available-otherwise-legacy
-      (mt/with-temp [:model/Table _ {:name "AgentSearchTestTable"}]
-        (testing "Returns search results for term queries"
-          (is (=? {:data        [{:type "table" :name "AgentSearchTestTable"}]
-                   :total_count 1}
-                  (agent-client :rasta :post 200 "agent/v1/search"
-                                {:term_queries ["AgentSearchTestTable"]}))))))))
-
-(defn- decode-query
-  "Decode a base64-encoded query response to a Clojure map, then normalize it so lib functions work."
-  [response]
-  (-> response :query u/decode-base64 json/decode+kw lib.normalize/normalize))
-
-(deftest construct-query-test
-  (with-agent-api-setup!
-    (testing "Constructs a simple query from a table"
-      (let [table-id (mt/id :orders)
-            response (agent-client :rasta :post 200 "agent/v1/construct-query"
-                                   {:table_id table-id})]
-        (is (string? (:query response)) "Response should contain a query string")
-        (let [decoded (decode-query response)]
-          (is (= :mbql/query (lib/normalized-query-type decoded)))
-          (is (= (mt/id) (lib/database-id decoded)))
-          (is (= (mt/id :orders) (lib/primary-source-table-id decoded))))))
-
-    (testing "Applies default limit of 200 when no limit is specified"
-      (let [table-id (mt/id :orders)
-            response (agent-client :rasta :post 200 "agent/v1/construct-query"
-                                   {:table_id table-id})
-            decoded  (decode-query response)]
-        (is (= 200 (lib/current-limit decoded)))))
-
-    (testing "Respects explicit limit"
-      (let [table-id (mt/id :orders)
-            response (agent-client :rasta :post 200 "agent/v1/construct-query"
-                                   {:table_id table-id
-                                    :limit    10})
-            decoded  (decode-query response)]
-        (is (= 10 (lib/current-limit decoded)))))
-
-    (testing "Returns 404 for non-existent table"
-      (is (= "No table found with table_id 999999"
-             (agent-client :rasta :post 404 "agent/v1/construct-query"
-                           {:table_id 999999}))))))
-
-(deftest execute-query-test
-  (with-agent-api-setup!
-    (testing "Executes a query and returns results with column metadata"
-      (let [table-id       (mt/id :orders)
-            construct-resp (agent-client :rasta :post 200 "agent/v1/construct-query"
-                                         {:table_id table-id
-                                          :limit    5})
-            ;; Streaming response returns 202 (accepted) since it starts streaming before completion
-            execute-resp   (agent-client :rasta :post 202 "agent/v1/execute"
-                                         {:query (:query construct-resp)})]
-        (is (=? {:status    "completed"
-                 :row_count 5
-                 :data      {:cols (fn [cols]
-                                     (and (seq cols)
-                                          (every? :name cols)
-                                          (every? :base_type cols)))
-                             :rows (fn [rows] (= 5 (count rows)))}}
-                execute-resp))))
-
-    (testing "Enforces agent query row limit even when query specifies a higher limit"
-      (let [table-id       (mt/id :orders)
-            construct-resp (agent-client :rasta :post 200 "agent/v1/construct-query"
-                                         {:table_id table-id
-                                          :limit    300})
-            execute-resp   (agent-client :rasta :post 202 "agent/v1/execute"
-                                         {:query (:query construct-resp)})]
-        (is (=? {:status "completed" :row_count 200}
-                execute-resp))))))
-
 (deftest execute-query-records-query-execution-test
   (with-agent-api-setup!
     (testing "Executed queries are recorded with :agent context"

enterprise/backend/test/metabase_enterprise/cache/strategies_test.clj

@@ -85,7 +85,8 @@
                             (-> (qp/process-query query) (dissoc :data))))))))))))))
 
 (deftest e2e-advanced-caching
-  (binding [search.ingestion/*disable-updates* true]
+  (binding [search.ingestion/*force-sync* true
+            search.ingestion/*disable-updates* true]
     (mt/with-empty-h2-app-db!
       (mt/with-premium-features #{:cache-granular-controls}
         (mt/dataset (mt/dataset-definition "caching1"

enterprise/backend/test/metabase_enterprise/data_studio/search_test.clj

@@ -6,158 +6,155 @@
    [metabase.collections.models.collection :as collection]
    [metabase.permissions.core :as perms]
    [metabase.permissions.models.data-permissions :as data-perms]
-   [metabase.search.impl :as search.impl]
-   [metabase.search.test-util :as search.tu]
+   [metabase.search.core :as search]
    [metabase.test :as mt]
    [metabase.util.random :refer [random-name]]))
 
 (deftest published-table-collection-filter-test
-  (search.tu/with-temp-index-table-for-http
-    (mt/with-premium-features #{:library}
-      (testing "Published tables are included in collection-filtered search"
-        (mt/with-temp
-          [:model/Collection {parent-coll :id} {:name "Parent Collection" :location "/"}
-           :model/Collection {child-coll :id}  {:name "Child Collection"
-                                                :location (collection/location-path parent-coll)}
-           :model/Card       {card-1 :id}      {:collection_id parent-coll :name "Parent Card"}
-           :model/Table      {table-1 :id}     {:name "Published Table" :is_published true :collection_id parent-coll}
-           :model/Table      {table-2 :id}     {:name "Child Published Table" :is_published true :collection_id child-coll}
-           :model/Table      {table-3 :id}     {:name "Unpublished Table" :is_published false}
-           :model/Table      {table-4 :id}     {:name "Root Published Table" :is_published true :collection_id nil}]
+  (mt/with-premium-features #{:library}
+    (testing "Published tables are included in collection-filtered search"
+      (mt/with-temp
+        [:model/Collection {parent-coll :id} {:name "Parent Collection" :location "/"}
+         :model/Collection {child-coll :id}  {:name "Child Collection"
+                                              :location (collection/location-path parent-coll)}
+         :model/Card       {card-1 :id}      {:collection_id parent-coll :name "Parent Card"}
+         :model/Table      {table-1 :id}     {:name "Published Table" :is_published true :collection_id parent-coll}
+         :model/Table      {table-2 :id}     {:name "Child Published Table" :is_published true :collection_id child-coll}
+         :model/Table      {table-3 :id}     {:name "Unpublished Table" :is_published false}
+         :model/Table      {table-4 :id}     {:name "Root Published Table" :is_published true :collection_id nil}]
         ;; Initialize search index for appdb engine
-          (search.impl/sync-reindex! {:in-place? true})
-          (doseq [engine ["in-place" "appdb"]]
-            (testing (str "with engine " engine)
-              (testing "Global search"
-                (let [results (mt/user-http-request :crowberto :get 200 "search" :search_engine engine)]
-                  (is (=? [{:collection {:authority_level nil, :id parent-coll, :name "Parent Collection", :type nil}
-                            :database_id (mt/id)
-                            :id table-1
-                            :model "table"
-                            :name "Published Table"
-                            :table_id table-1
-                            :table_name "Published Table"}
-                           {:collection {:authority_level nil, :id child-coll, :name "Child Collection", :type nil}
-                            :database_id (mt/id)
-                            :id table-2
-                            :model "table"
-                            :name "Child Published Table"
-                            :table_id table-2
-                            :table_name "Child Published Table"}
-                           {:database_id (mt/id)
-                            :id table-3
-                            :model "table"
-                            :name "Unpublished Table"
-                            :table_id table-3
-                            :table_name "Unpublished Table"}
-                           {:collection {:authority_level nil, :id "root", :name "Our analytics", :type nil}
-                            :database_id (mt/id)
-                            :id table-4
-                            :model "table"
-                            :name "Root Published Table"
-                            :table_id table-4
-                            :table_name "Root Published Table"}]
-                          (->> (:data results)
-                               (filter (comp #{table-1 table-2 table-3 table-4} :id))
-                               (sort-by :id))))))
-              (testing "Filter by parent collection returns published tables in that collection and descendants"
-                (let [results (mt/user-http-request :crowberto :get 200 "search" :collection parent-coll :search_engine engine)]
-                  (is (=? {:collection {:authority_level nil, :id parent-coll, :name "Parent Collection", :type nil}
-                           :database_id (mt/id)
-                           :id table-1
-                           :model "table"
-                           :name "Published Table"
-                           :table_id table-1
-                           :table_name "Published Table"}
-                          (m/find-first (comp #{table-1} :id) (:data results))))
-                  (is (contains? (set (map :id (:data results))) table-1)
-                      "Published table in parent collection should be included")
-                  (is (contains? (set (map :id (:data results))) table-2)
-                      "Published table in child collection should be included")
-                  (is (contains? (set (map :id (:data results))) card-1)
-                      "Card in parent collection should be included")))
-              (testing "Unpublished tables are not included in collection filter results"
-                (let [results (mt/user-http-request :crowberto :get 200 "search" :collection parent-coll :search_engine engine)]
-                  (is (not (contains? (set (map :id (:data results))) table-3))
-                      "Unpublished table should not be in results")))))))
-      (testing "Published tables in root collection (collection_id=nil) appear in unfiltered search"
-        (mt/with-temp
-          [:model/Table {root-table :id} {:name "Root Published Searchable" :is_published true :collection_id nil}]
-          (search.impl/sync-reindex! {:in-place? true})
-          (doseq [engine ["in-place" "appdb"]]
-            (testing (str "with engine " engine)
-              (let [results (mt/user-http-request :crowberto :get 200 "search" :q "Root Published Searchable" :search_engine engine)]
-                (is (some #(= root-table (:id %)) (:data results))
-                    "Root published table should appear in search results"))))))
-      (testing "Published tables appear as collection items, unpublished as database items"
-        (mt/with-temp
-          [:model/Database   {db-id :id}       {:name "Context Test DB"}
-           :model/Collection {coll-id :id}     {:name "Context Test Collection" :location "/"}
-           :model/Table      {pub-table :id}   {:name "ContextTestTablePub"
-                                                :db_id db-id
-                                                :is_published true
-                                                :collection_id coll-id}
-           :model/Table      {unpub-table :id} {:name "ContextTestTableUnpub"
-                                                :db_id db-id
-                                                :is_published false}]
-          (search.impl/sync-reindex! {:in-place? true})
-          (doseq [engine ["in-place" "appdb"]]
-            (testing (str "with engine " engine)
-              (let [results (mt/user-http-request :crowberto :get 200 "search"
-                                                  :q "Context" :models "table" :search_engine engine)
-                    our-result (->> (filter (comp #{pub-table unpub-table} :id) (:data results))
-                                    (sort-by :id))]
-                (testing "each table appears once"
-                  (is (=? [{:id pub-table
-                            :model "table"
-                            :name "ContextTestTablePub"
-                            :collection {:id coll-id}
-                            :database_name "Context Test DB"}
-                           {:id unpub-table
-                            :model "table"
-                            :name "ContextTestTableUnpub"
-                            :collection {:id nil}
-                            :database_name "Context Test DB"}]
-                          our-result))))))))
-      (testing "Users without query permissions cannot discover tables in search"
-        (let [search-term (random-name)
-              name-1 (str search-term " 1")
-              name-2 (str search-term " 2")]
-          (mt/with-no-data-perms-for-all-users!
-            (mt/with-temp [:model/Table {unpub-table :id} {:collection_id nil :is_published false :name name-1}
-                           :model/Table {pub-table :id} {:collection_id nil :is_published true :name name-2}]
+        (search/reindex! {:async? false :in-place? true})
+        (doseq [engine ["in-place" "appdb"]]
+          (testing (str "with engine " engine)
+            (testing "Global search"
+              (let [results (mt/user-http-request :crowberto :get 200 "search" :search_engine engine)]
+                (is (=? [{:collection {:authority_level nil, :id parent-coll, :name "Parent Collection", :type nil}
+                          :database_id (mt/id)
+                          :id table-1
+                          :model "table"
+                          :name "Published Table"
+                          :table_id table-1
+                          :table_name "Published Table"}
+                         {:collection {:authority_level nil, :id child-coll, :name "Child Collection", :type nil}
+                          :database_id (mt/id)
+                          :id table-2
+                          :model "table"
+                          :name "Child Published Table"
+                          :table_id table-2
+                          :table_name "Child Published Table"}
+                         {:database_id (mt/id)
+                          :id table-3
+                          :model "table"
+                          :name "Unpublished Table"
+                          :table_id table-3
+                          :table_name "Unpublished Table"}
+                         {:collection {:authority_level nil, :id "root", :name "Our analytics", :type nil}
+                          :database_id (mt/id)
+                          :id table-4
+                          :model "table"
+                          :name "Root Published Table"
+                          :table_id table-4
+                          :table_name "Root Published Table"}]
+                        (->> (:data results)
+                             (filter (comp #{table-1 table-2 table-3 table-4} :id))
+                             (sort-by :id))))))
+            (testing "Filter by parent collection returns published tables in that collection and descendants"
+              (let [results (mt/user-http-request :crowberto :get 200 "search" :collection parent-coll :search_engine engine)]
+                (is (=? {:collection {:authority_level nil, :id parent-coll, :name "Parent Collection", :type nil}
+                         :database_id (mt/id)
+                         :id table-1
+                         :model "table"
+                         :name "Published Table"
+                         :table_id table-1
+                         :table_name "Published Table"}
+                        (m/find-first (comp #{table-1} :id) (:data results))))
+                (is (contains? (set (map :id (:data results))) table-1)
+                    "Published table in parent collection should be included")
+                (is (contains? (set (map :id (:data results))) table-2)
+                    "Published table in child collection should be included")
+                (is (contains? (set (map :id (:data results))) card-1)
+                    "Card in parent collection should be included")))
+            (testing "Unpublished tables are not included in collection filter results"
+              (let [results (mt/user-http-request :crowberto :get 200 "search" :collection parent-coll :search_engine engine)]
+                (is (not (contains? (set (map :id (:data results))) table-3))
+                    "Unpublished table should not be in results")))))))
+    (testing "Published tables in root collection (collection_id=nil) appear in unfiltered search"
+      (mt/with-temp
+        [:model/Table {root-table :id} {:name "Root Published Searchable" :is_published true :collection_id nil}]
+        (search/reindex! {:async? false :in-place? true})
+        (doseq [engine ["in-place" "appdb"]]
+          (testing (str "with engine " engine)
+            (let [results (mt/user-http-request :crowberto :get 200 "search" :q "Root Published Searchable" :search_engine engine)]
+              (is (some #(= root-table (:id %)) (:data results))
+                  "Root published table should appear in search results"))))))
+    (testing "Published tables appear as collection items, unpublished as database items"
+      (mt/with-temp
+        [:model/Database   {db-id :id}       {:name "Context Test DB"}
+         :model/Collection {coll-id :id}     {:name "Context Test Collection" :location "/"}
+         :model/Table      {pub-table :id}   {:name "ContextTestTablePub"
+                                              :db_id db-id
+                                              :is_published true
+                                              :collection_id coll-id}
+         :model/Table      {unpub-table :id} {:name "ContextTestTableUnpub"
+                                              :db_id db-id
+                                              :is_published false}]
+        (search/reindex! {:async? false :in-place? true})
+        (doseq [engine ["in-place" "appdb"]]
+          (testing (str "with engine " engine)
+            (let [results (mt/user-http-request :crowberto :get 200 "search"
+                                                :q "Context" :models "table" :search_engine engine)
+                  our-result (->> (filter (comp #{pub-table unpub-table} :id) (:data results))
+                                  (sort-by :id))]
+              (testing "each table appears once"
+                (is (=? [{:id pub-table
+                          :model "table"
+                          :name "ContextTestTablePub"
+                          :collection {:id coll-id}
+                          :database_name "Context Test DB"}
+                         {:id unpub-table
+                          :model "table"
+                          :name "ContextTestTableUnpub"
+                          :collection {:id nil}
+                          :database_name "Context Test DB"}]
+                        our-result))))))))
+    (testing "Users without query permissions cannot discover tables in search"
+      (let [search-term (random-name)
+            name-1 (str search-term " 1")
+            name-2 (str search-term " 2")]
+        (mt/with-no-data-perms-for-all-users!
+          (mt/with-temp [:model/Table {unpub-table :id} {:collection_id nil :is_published false :name name-1}
+                         :model/Table {pub-table :id} {:collection_id nil :is_published true :name name-2}]
             ;; Explicitly deny both view and query perms so this assertion doesn't depend on global defaults.
-              (data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/view-data :blocked)
-              (data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/create-queries :no)
-              (doseq [engine ["in-place" "appdb"]]
-                (search.impl/sync-reindex! {:in-place? true})
-                (mt/with-non-admin-groups-no-root-collection-perms
-                  (let [results (mt/user-http-request :rasta :get 200 "search"
-                                                      :q search-term :models "table" :search_engine engine)
-                        our-result (->> (filter (comp #{pub-table unpub-table} :id) (:data results))
-                                        (sort-by :id)
-                                        (map #(select-keys % [:id])))]
-                    (testing "no tables appear"
-                      (is (= []
-                             our-result)))))))))))))
+            (data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/view-data :blocked)
+            (data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/create-queries :no)
+            (doseq [engine ["in-place" "appdb"]]
+              (search/reindex! {:async? false :in-place? true})
+              (mt/with-non-admin-groups-no-root-collection-perms
+                (let [results (mt/user-http-request :rasta :get 200 "search"
+                                                    :q search-term :models "table" :search_engine engine)
+                      our-result (->> (filter (comp #{pub-table unpub-table} :id) (:data results))
+                                      (sort-by :id)
+                                      (map #(select-keys % [:id])))]
+                  (testing "no tables appear"
+                    (is (= []
+                           our-result))))))))))))
 
 (deftest unpublished-table-visible-with-data-perms-test
-  (search.tu/with-temp-index-table-for-http
-    (testing "Unpublished tables are discoverable when the user has data/query permissions"
-      (let [search-term (random-name)
-            table-name  (str search-term " unpublished")]
-        (mt/with-no-data-perms-for-all-users!
-          (mt/with-temp [:model/Table {unpub-table :id} {:collection_id nil :is_published false :name table-name}]
+  (testing "Unpublished tables are discoverable when the user has data/query permissions"
+    (let [search-term (random-name)
+          table-name  (str search-term " unpublished")]
+      (mt/with-no-data-perms-for-all-users!
+        (mt/with-temp [:model/Table {unpub-table :id} {:collection_id nil :is_published false :name table-name}]
           ;; Explicitly grant data/query perms so this does not rely on global test defaults.
-            (data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/view-data :unrestricted)
-            (data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/create-queries :query-builder)
-            (doseq [engine ["in-place" "appdb"]]
-              (search.impl/sync-reindex! {:in-place? true})
-              (mt/with-non-admin-groups-no-root-collection-perms
-                (let [result-ids (->> (mt/user-http-request :rasta :get 200 "search"
-                                                            :q search-term :models "table" :search_engine engine)
-                                      :data
-                                      (map :id)
-                                      (into #{}))]
-                  (is (contains? result-ids unpub-table)
-                      (format "Unpublished table should be visible with data permissions (engine=%s)" engine)))))))))))
+          (data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/view-data :unrestricted)
+          (data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/create-queries :query-builder)
+          (doseq [engine ["in-place" "appdb"]]
+            (search/reindex! {:async? false :in-place? true})
+            (mt/with-non-admin-groups-no-root-collection-perms
+              (let [result-ids (->> (mt/user-http-request :rasta :get 200 "search"
+                                                          :q search-term :models "table" :search_engine engine)
+                                    :data
+                                    (map :id)
+                                    (into #{}))]
+                (is (contains? result-ids unpub-table)
+                    (format "Unpublished table should be visible with data permissions (engine=%s)" engine))))))))))

enterprise/backend/test/metabase_enterprise/metabot/tools/semantic_search_test.clj

@@ -19,9 +19,9 @@
   `(mt/with-additional-premium-features #{:semantic-search}
      (when (search.engine/supported-engine? :search.engine/semantic)
        (semantic.tu/with-mock-embeddings ~mock-embeddings
-         (binding [search.ingestion/*disable-updates* false]
-           (search.tu/with-sync-search-indexing
-             ~@body))))))
+         (binding [search.ingestion/*disable-updates* false
+                   search.ingestion/*force-sync* true]
+           ~@body)))))
 
 (defmacro with-and-without-semantic-search! [mock-embeddings & body]
   `(do ~@body (with-semantic-search-if-available! ~mock-embeddings ~@body)))

enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj

@@ -22,7 +22,7 @@
 ;; `reindex!` below is ok in a parallel test since it's not actually executing anything
 #_{:clj-kondo/ignore [:metabase/validate-deftest]}
 (use-fixtures :each (fn [f]
-                      (mt/with-dynamic-fn-redefs [search/async-reindex! (constantly nil)]
+                      (mt/with-dynamic-fn-redefs [search/reindex! (constantly nil)]
                         (test-helpers/clean-remote-sync-state f))))
 
 ;; import! tests

enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj

@@ -34,7 +34,7 @@
 
 #_{:clj-kondo/ignore [:metabase/validate-deftest]}
 (use-fixtures :each (fn [f]
-                      (mt/with-dynamic-fn-redefs [search/async-reindex! (constantly nil)]
+                      (mt/with-dynamic-fn-redefs [search/reindex! (constantly nil)]
                         (clean-remote-sync-state f))))
 
 ;;; ------------------------------------------- Event Handler Tests -------------------------------------------

enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj

@@ -23,7 +23,7 @@
 
 #_{:clj-kondo/ignore [:metabase/validate-deftest]}
 (use-fixtures :each (fn [f]
-                      (mt/with-dynamic-fn-redefs [search/async-reindex! (constantly nil)]
+                      (mt/with-dynamic-fn-redefs [search/reindex! (constantly nil)]
                         (test-helpers/clean-remote-sync-state f))))
 
 (deftest transform-event-creates-sync-object-when-setting-enabled-test

enterprise/backend/test/metabase_enterprise/semantic_search/api_test.clj

@@ -8,8 +8,6 @@
    [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
    [metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]
    [metabase-enterprise.semantic-search.test-util :as semantic.tu]
-   [metabase.mq.test-util :as mq.tu]
-   [metabase.search.impl :as search.impl]
    [metabase.search.ingestion :as search.ingestion]
    [metabase.test :as mt]))
 
@@ -63,38 +61,36 @@
   (testing "POST /api/search/re-init with semantic search support"
     (mt/with-premium-features #{:semantic-search}
       (semantic.tu/with-test-db! {:mode :mock-indexed}
-        (mq.tu/with-sync-mq
-          (let [original-index      semantic.tu/mock-index
-                original-table-name (:table-name original-index)
-                new-index           (with-redefs [semantic.index/model-table-suffix (constantly 345)]
-                                      (#'semantic.pgvector-api/fresh-index semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model :force-reset? true))
-                new-table-name      (:table-name new-index)
-                pgvector (semantic.env/get-pgvector-datasource!)]
+        (let [original-index      semantic.tu/mock-index
+              original-table-name (:table-name original-index)
+              new-index           (with-redefs [semantic.index/model-table-suffix (constantly 345)]
+                                    (#'semantic.pgvector-api/fresh-index semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model :force-reset? true))
+              new-table-name      (:table-name new-index)
+              pgvector (semantic.env/get-pgvector-datasource!)]
 
+          (is (semantic.tu/table-exists-in-db? original-table-name))
+          (is (not (semantic.tu/table-exists-in-db? new-table-name)))
+
+          (let [best-index (semantic.index-metadata/find-compatible-index! pgvector semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model)]
+            (is (=? original-index (:index best-index)))
+            (is (:active best-index)))
+
+          (testing "re-init creates the new index"
+            (with-redefs [semantic.index/model-table-suffix (constantly 345)]
+              (let [response (mt/user-http-request :crowberto :post 200 "search/re-init")]
+                (is (contains? response :message))))
+
+            (is (not= original-table-name new-table-name))
             (is (semantic.tu/table-exists-in-db? original-table-name))
-            (is (not (semantic.tu/table-exists-in-db? new-table-name)))
+            (is (semantic.tu/table-exists-in-db? new-table-name))
 
-            (let [best-index (semantic.index-metadata/find-compatible-index! pgvector semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model)]
-              (is (=? original-index (:index best-index)))
-              (is (:active best-index)))
+            (is (zero? (semantic.tu/index-count new-index))))
 
-            (testing "re-init creates the new index"
-              (with-redefs [semantic.index/model-table-suffix (constantly 345)]
-                ;; Call init directly instead of via HTTP — with-redefs is thread-local
-                ;; and not visible to the HTTP handler thread.
-                (search.impl/sync-init-index! {:force-reset? true}))
+          (let [best-index (semantic.index-metadata/find-compatible-index! pgvector semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model)]
+            (is (=? new-index (:index best-index)))
+            (is (:active best-index)))
 
-              (is (not= original-table-name new-table-name))
-              (is (semantic.tu/table-exists-in-db? original-table-name))
-              (is (semantic.tu/table-exists-in-db? new-table-name))
+          (testing "Index can be populated after re-init"
+            (semantic.tu/upsert-index! (semantic.tu/mock-documents) :index new-index)
 
-              (is (zero? (semantic.tu/index-count new-index))))
-
-            (let [best-index (semantic.index-metadata/find-compatible-index! pgvector semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model)]
-              (is (=? new-index (:index best-index)))
-              (is (:active best-index)))
-
-            (testing "Index can be populated after re-init"
-              (semantic.tu/upsert-index! (semantic.tu/mock-documents) :index new-index)
-
-              (is (pos? (semantic.tu/index-count new-index))))))))))
+            (is (pos? (semantic.tu/index-count new-index)))))))))

enterprise/backend/test/metabase_enterprise/serialization/api_test.clj

@@ -9,7 +9,6 @@
    [metabase.analytics.snowplow-test :as snowplow-test]
    [metabase.models.serialization :as serdes]
    [metabase.search.core :as search]
-   [metabase.search.impl :as search.impl]
    [metabase.search.test-util :as search.tu]
    [metabase.test :as mt]
    [metabase.util.compress :as u.compress]
@@ -95,7 +94,6 @@
                                                   :dataset_query {:type     :native
                                                                   :database (t2/select-one-pk :model/Database)
                                                                   :native   {:query "SELECT 1"}}}]
-           (search.impl/sync-reindex! {:in-place? true})
            ~@body)))))
 
 (defn- do-export
@@ -213,8 +211,8 @@
   (testing "Successful export creates valid archive with correct files and Snowplow event"
     (with-serialization-test-data! [coll dash card]
       ;; Clear entities from search index to verify export works independently
-      (search/async-delete! :model/Dashboard [(str (:id dash))])
-      (search/async-delete! :model/Card [(str (:id card))])
+      (search/delete! :model/Dashboard [(str (:id dash))])
+      (search/delete! :model/Card [(str (:id card))])
       (is (= 0 (search-result-count "dashboard" "thraddash")))
       (is (= 0 (search-result-count "dataset" "frobinate")))
 
@@ -257,8 +255,8 @@
   (testing "Import restores deleted/renamed entities and updates search index"
     (with-serialization-test-data! [coll dash card]
       ;; Clear entities from search index
-      (search/async-delete! :model/Dashboard [(str (:id dash))])
-      (search/async-delete! :model/Card [(str (:id card))])
+      (search/delete! :model/Dashboard [(str (:id dash))])
+      (search/delete! :model/Card [(str (:id card))])
 
       ;; Export the data
       (let [ba (do-export (:id coll))]
@@ -270,7 +268,7 @@
         (t2/delete! :model/Card (:id card))
 
         (let [re-indexed? (atom false)
-              res         (mt/with-dynamic-fn-redefs [search/async-reindex! (fn [& _] (reset! re-indexed? true) (future nil))]
+              res         (mt/with-dynamic-fn-redefs [search/reindex! (fn [& _] (reset! re-indexed? true) (future nil))]
                             (mt/user-http-request :crowberto :post 200 "ee/serialization/import?reindex=false"
                                                   {:request-options {:headers {"content-type" "multipart/form-data"}}}
                                                   {:file ba}))]

enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj

@@ -27,7 +27,7 @@
 ;; `reindex!` below is ok in a parallel test since it's not actually executing anything
 #_{:clj-kondo/ignore [:metabase/validate-deftest]}
 (use-fixtures :each (fn [thunk]
-                      (mt/with-dynamic-fn-redefs [search/async-reindex! (constantly nil)
+                      (mt/with-dynamic-fn-redefs [search/reindex! (constantly nil)
                                                   models.database/assert-not-h2! (constantly nil)]
                         (thunk))))
 

enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj

@@ -23,7 +23,7 @@
 ;; `reindex!` below is ok in a parallel test since it's not actually executing anything
 #_{:clj-kondo/ignore [:metabase/validate-deftest]}
 (use-fixtures :each (fn [thunk]
-                      (mt/with-dynamic-fn-redefs [search/async-reindex! (constantly nil)
+                      (mt/with-dynamic-fn-redefs [search/reindex! (constantly nil)
                                                   models.database/assert-not-h2! (constantly nil)]
                         (thunk))))
 

enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj

@@ -98,7 +98,7 @@
 (defn load-extract!
   "Perform a round-trip of loading an existing serialization, then serializing it again."
   [input-dir output-dir]
-  (mt/with-dynamic-fn-redefs [search/async-reindex! (constantly nil)]
+  (mt/with-dynamic-fn-redefs [search/reindex! (constantly nil)]
     (serdes/with-cache
       (load/load-metabase! (ingest/ingest-yaml input-dir))))
   ;; Use a separate cache to make sure there is no cross-contamination.

enterprise/backend/test/metabase_enterprise/tenants/search_test.clj

@@ -2,6 +2,7 @@
   (:require
    [clojure.test :refer :all]
    [metabase.search.appdb.core :as search.engines.appdb]
+   [metabase.search.ingestion :as search.ingestion]
    [metabase.search.test-util :as search.tu]
    [metabase.test :as mt]
    [metabase.test.fixtures :as fixtures]
@@ -12,7 +13,8 @@
   search.engines.appdb/keep-me)
 
 (use-fixtures :once (fixtures/initialize :db))
-(use-fixtures :each (fn [thunk] (search.tu/with-new-search-if-available-otherwise-legacy (thunk))))
+(use-fixtures :each (fn [thunk] (binding [search.ingestion/*force-sync* true]
+                                  (search.tu/with-new-search-if-available-otherwise-legacy (thunk)))))
 
 (deftest dedicated-tenant-collection-dataset-search-test
   (testing "Search returns datasets (models) from dedicated tenant collections"

enterprise/backend/test/metabase_enterprise/transforms/search_test.clj

@@ -36,14 +36,13 @@
 (deftest transform-query-ingestion-test
   (testing "Contents of Python transform sources are extracted and indexed for full-text search"
     (when (= (mdb/db-type) :postgres)
-      (search.tu/with-temp-index-table
-        (mt/with-temp [:model/Transform _ {:target {:database (mt/id)}
-                                           :source {:type "python"
-                                                    :source-database (mt/id)
-                                                    :body "import pandas as pd\n"}
-                                           :name "Test python transform"}]
-          (let [ingested-transform (search-test/ingest-then-fetch! "transform" "Test python transform")
-                vector-value (.getValue ^PGobject (:with_native_query_vector ingested-transform))]
-            (is (string? vector-value))
-            (is (re-find #"import" vector-value))
-            (is (re-find #"panda" vector-value))))))))
+      (mt/with-temp [:model/Transform _ {:target {:database (mt/id)}
+                                         :source {:type "python"
+                                                  :source-database (mt/id)
+                                                  :body "import pandas as pd\n"}
+                                         :name "Test python transform"}]
+        (let [ingested-transform (search-test/ingest-then-fetch! "transform" "Test python transform")
+              vector-value (.getValue ^PGobject (:with_native_query_vector ingested-transform))]
+          (is (string? vector-value))
+          (is (re-find #"import" vector-value))
+          (is (re-find #"panda" vector-value)))))))

frontend/src/embedding-sdk-bundle/hooks/private/use-init-data/use-init-data-internal.ts

@@ -16,11 +16,15 @@ import { useLazySelector } from "embedding-sdk-shared/hooks/use-lazy-selector";
 import { useMetabaseProviderPropsStore } from "embedding-sdk-shared/hooks/use-metabase-provider-props-store";
 import { ensureMetabaseProviderPropsStore } from "embedding-sdk-shared/lib/ensure-metabase-provider-props-store";
 import { getBuildInfo } from "embedding-sdk-shared/lib/get-build-info";
+import registerDashboardVisualizations from "metabase/dashboard/visualizations/register";
 import { EMBEDDING_SDK_CONFIG } from "metabase/embedding-sdk/config";
 import api from "metabase/lib/api";
 import registerVisualizations from "metabase/visualizations/register";
 
 const registerVisualizationsOnce = _.once(registerVisualizations);
+const registerDashboardVisualizationsOnce = _.once(
+  registerDashboardVisualizations,
+);
 
 interface InitDataLoaderParameters {
   reduxStore: SdkStore;
@@ -119,5 +123,6 @@ export const useInitDataInternal = ({
 
   useMount(function registerVisualizations() {
     registerVisualizationsOnce();
+    registerDashboardVisualizationsOnce();
   });
 };

frontend/src/metabase/app.js

@@ -35,6 +35,7 @@ import { syncHistoryWithStore } from "react-router-redux";
 
 import { initializePlugins } from "ee-plugins";
 import { ModifiedBackend } from "metabase/common/components/dnd/ModifiedBackend";
+import registerDashboardVisualizations from "metabase/dashboard/visualizations/register";
 import { createTracker } from "metabase/lib/analytics";
 import api from "metabase/lib/api";
 import { initializeEmbedding } from "metabase/lib/embed";
@@ -104,6 +105,7 @@ function _init(reducers, getRoutes, callback) {
   );
 
   registerVisualizations();
+  registerDashboardVisualizations();
 
   store.dispatch(refreshSiteSettings());
 

frontend/src/metabase/dashboard/actions/utils.ts

@@ -1,8 +1,8 @@
 import { t } from "ttag";
 import _ from "underscore";
 
+import { getIframeDomainName } from "metabase/dashboard/visualizations/IFrameViz/utils";
 import { isActionDashCard } from "metabase/lib/dashboard";
-import { getIframeDomainName } from "metabase/visualizations/visualizations/IFrameViz/utils";
 import type {
   DashCardId,
   Dashboard,

frontend/src/metabase/dashboard/components/DashCard/DashCardActionsPanel/ChartSettingsButton/ChartSettingsButton.tsx

@@ -1,8 +1,8 @@
 import { useDisclosure } from "@mantine/hooks";
 import { t } from "ttag";
 
+import { DashboardChartSettings } from "metabase/dashboard/components/DashboardChartSettings";
 import { Modal } from "metabase/ui";
-import { DashboardChartSettings } from "metabase/visualizations/components/ChartSettings";
 import type {
   DashboardCard,
   Series,

frontend/src/metabase/dashboard/components/DashCard/Dashcard.unit.spec.tsx

@@ -12,6 +12,7 @@ import {
   within,
 } from "__support__/ui";
 import * as dashboardSelectors from "metabase/dashboard/selectors";
+import registerDashboardVisualizations from "metabase/dashboard/visualizations/register";
 import {
   MockDashboardContext,
   type MockDashboardContextProps,
@@ -44,6 +45,7 @@ import type { DashCardProps } from "./DashCard";
 import { DashCard } from "./DashCard";
 
 registerVisualizations();
+registerDashboardVisualizations();
 
 const TEST_DATABASE_ID = 1;
 const TEST_TABLE_ID = 2;

frontend/src/metabase/dashboard/components/DashboardChartSettings/DashboardChartSettings.tsx

@@ -4,15 +4,14 @@ import _ from "underscore";
 import { useDashboardContext } from "metabase/dashboard/context";
 import { Divider, Flex } from "metabase/ui";
 import { getVisualizationRaw } from "metabase/visualizations";
+import { BaseChartSettings } from "metabase/visualizations/components/ChartSettings/BaseChartSettings";
+import { ChartSettingsVisualization } from "metabase/visualizations/components/ChartSettings/ChartSettingsVisualization";
+import { useChartSettingsState } from "metabase/visualizations/components/ChartSettings/hooks";
 import { getClickBehaviorSettings } from "metabase/visualizations/lib/settings";
 import { sanitizeDashcardSettings } from "metabase/visualizations/lib/settings/typed-utils";
 import { getSettingsWidgetsForSeries } from "metabase/visualizations/lib/settings/visualization";
 import type { VisualizationSettings } from "metabase-types/api";
 
-import { BaseChartSettings } from "../BaseChartSettings";
-import { ChartSettingsVisualization } from "../ChartSettingsVisualization";
-import { useChartSettingsState } from "../hooks";
-
 import type { DashboardChartSettingsProps } from "./types";
 
 export const DashboardChartSettings = ({

frontend/src/metabase/dashboard/components/DashboardChartSettings/types.ts

@@ -1,8 +1,7 @@
+import type { CommonChartSettingsProps } from "metabase/visualizations/components/ChartSettings/types";
 import type { Widget } from "metabase/visualizations/types";
 import type { Dashboard, DashboardCard } from "metabase-types/api";
 
-import type { CommonChartSettingsProps } from "../types";
-
 export type DashboardChartSettingsProps = {
   className?: string;
   dashboard?: Dashboard;

frontend/src/metabase/dashboard/visualizations/DashCardPlaceholder.tsx

@@ -9,10 +9,12 @@ import {
 import { replaceCard } from "metabase/dashboard/actions";
 import { useDispatch } from "metabase/lib/redux";
 import { Button, Flex } from "metabase/ui";
+import type {
+  VisualizationDefinition,
+  VisualizationProps,
+} from "metabase/visualizations/types";
 import type { Dashboard, VirtualDashboardCard } from "metabase-types/api";
 
-import type { VisualizationDefinition, VisualizationProps } from "../types";
-
 type Props = VisualizationProps & {
   dashcard: VirtualDashboardCard;
   dashboard: Dashboard;

frontend/src/metabase/dashboard/visualizations/register.js

@@ -0,0 +1,14 @@
+import { registerVisualization } from "metabase/visualizations";
+
+import { DashCardPlaceholder } from "./DashCardPlaceholder";
+import { Heading } from "./Heading";
+import { IFrameViz } from "./IFrameViz";
+import { LinkViz } from "./LinkViz";
+
+// eslint-disable-next-line import/no-default-export -- deprecated usage
+export default function () {
+  registerVisualization(DashCardPlaceholder);
+  registerVisualization(Heading);
+  registerVisualization(LinkViz);
+  registerVisualization(IFrameViz);
+}

frontend/src/metabase/parameters/utils/mapping-options.ts

@@ -3,13 +3,13 @@ import _ from "underscore";
 
 import { tag_names } from "cljs/metabase.parameters.shared";
 import { getColumnIcon } from "metabase/common/utils/columns";
+import { getAllowedIframeAttributes } from "metabase/dashboard/visualizations/IFrameViz/utils";
 import {
   isActionDashCard,
   isQuestionDashCard,
   isVirtualDashCard,
 } from "metabase/lib/dashboard";
 import { getGroupName } from "metabase/querying/filters/utils/groups";
-import { getAllowedIframeAttributes } from "metabase/visualizations/visualizations/IFrameViz/utils";
 import * as Lib from "metabase-lib";
 import { TemplateTagDimension } from "metabase-lib/v1/Dimension";
 import type { DimensionOptionsSection } from "metabase-lib/v1/DimensionOptions/types";

frontend/src/metabase/public/containers/PublicOrEmbeddedDashboard/PublicOrEmbeddedDashboardView-card-filters.stories.tsx

@@ -7,6 +7,7 @@ import { getStore } from "__support__/entities-store";
 import { getNextId } from "__support__/utils";
 import { NumberColumn, StringColumn } from "__support__/visualizations";
 import { Api } from "metabase/api";
+import { Heading } from "metabase/dashboard/visualizations/Heading";
 import { MetabaseReduxProvider } from "metabase/lib/redux/custom-context";
 import {
   MockDashboardContext,
@@ -15,7 +16,6 @@ import {
 import { publicReducers } from "metabase/reducers-public";
 import { registerVisualization } from "metabase/visualizations";
 import { BarChart } from "metabase/visualizations/visualizations/BarChart";
-import { Heading } from "metabase/visualizations/visualizations/Heading";
 import type { Dashboard } from "metabase-types/api";
 import {
   createMockCard,

frontend/src/metabase/visualizations/components/ChartSettings/index.ts

@@ -1,5 +1,4 @@
 export { QuestionChartSettings } from "./QuestionChartSettings";
-export { DashboardChartSettings } from "./DashboardChartSettings";
 export { BaseChartSettings } from "./BaseChartSettings";
 export * from "./hooks";
 export * from "./types";

frontend/src/metabase/visualizations/components/settings/ChartNestedSettingSeriesMultiple.unit.spec.js

@@ -2,8 +2,8 @@
 import userEvent from "@testing-library/user-event";
 
 import { renderWithProviders, screen, within } from "__support__/ui";
+import { DashboardChartSettings } from "metabase/dashboard/components/DashboardChartSettings";
 import { MockDashboardContext } from "metabase/public/containers/PublicOrEmbeddedDashboard/mock-context";
-import { DashboardChartSettings } from "metabase/visualizations/components/ChartSettings";
 import registerVisualizations from "metabase/visualizations/register";
 import { createMockCard } from "metabase-types/api/mocks";
 

frontend/src/metabase/visualizations/register.js

@@ -9,13 +9,9 @@ import { AreaChart } from "./visualizations/AreaChart";
 import { BarChart } from "./visualizations/BarChart";
 import { BoxPlot } from "./visualizations/BoxPlot";
 import { ComboChart } from "./visualizations/ComboChart";
-import { DashCardPlaceholder } from "./visualizations/DashCardPlaceholder";
 import { Funnel } from "./visualizations/Funnel";
 import { Gauge } from "./visualizations/Gauge";
-import { Heading } from "./visualizations/Heading";
-import { IFrameViz } from "./visualizations/IFrameViz";
 import { LineChart } from "./visualizations/LineChart";
-import { LinkViz } from "./visualizations/LinkViz";
 import { Map } from "./visualizations/Map";
 import { ObjectDetail } from "./visualizations/ObjectDetail";
 import { PieChart } from "./visualizations/PieChart";
@@ -53,11 +49,7 @@ export default function () {
   registerVisualization(SankeyChart);
 
   registerVisualization(ActionViz);
-  registerVisualization(DashCardPlaceholder);
-  registerVisualization(Heading);
-  registerVisualization(LinkViz);
   registerVisualization(Text);
-  registerVisualization(IFrameViz);
 
   registerVisualization(ListViz);
 

mage/src/mage/modules.clj

@@ -118,7 +118,6 @@
      llm
      login-history
      metabot
-     mq
      notification
      permissions
      premium-features

resources/migrations/061/20260313_persistent_queue.yaml

@@ -1,140 +0,0 @@
-## Adds support for appdb persistent queue backend
-
-databaseChangeLog:
-  - changeSet:
-      id: v61.asdgew
-      author: nvoxland
-      comment: Create the table backing the message queue
-      changes:
-        - createTable:
-            tableName: queue_message_batch
-            remarks: Lightweight message queue for cross-cluster communication
-            columns:
-              - column:
-                  name: id
-                  type: BIGINT
-                  autoIncrement: true
-                  constraints:
-                    primaryKey: true
-                    nullable: false
-              - column:
-                  name: queue_name
-                  type: varchar(254)
-                  remarks: The name of queue this message is in
-                  constraints:
-                    nullable: false
-              - column:
-                  remarks: The timestamp of when the messages were created
-                  name: created_at
-                  type: ${timestamp_type}
-                  defaultValueComputed: current_timestamp
-                  constraints:
-                    nullable: false
-              - column:
-                  remarks: The status of the messages
-                  name: status
-                  type: varchar(10)
-                  defaultValue: 'pending'
-                  constraints:
-                    nullable: false
-              - column:
-                  remarks: Heartbeat to ensure status is not stuck
-                  name: status_heartbeat
-                  type: ${timestamp_type}
-                  defaultValueComputed: current_timestamp
-                  constraints:
-                    nullable: false
-              - column:
-                  remarks: Number of failures processing these messages
-                  name: failures
-                  type: int
-                  defaultValueNumeric: 0
-                  constraints:
-                    nullable: false
-              - column:
-                  remarks: Current owner of the messages
-                  name: owner
-                  type: varchar(36)
-              - column:
-                  remarks: JSON-encoded array of messages
-                  name: messages
-                  type: ${text.type}
-                  constraints:
-                    nullable: false
-
-  - changeSet:
-      id: v61.gdasfg2e
-      author: nvoxland
-      comment: Index queue_message_batch name and status
-      changes:
-        - createIndex:
-            tableName: queue_message_batch
-            indexName: idx_queue_message_batch_queue_status
-            columns:
-              - column:
-                  name: queue_name
-              - column:
-                  name: status
-
-
-  - changeSet:
-      id: v61.g9dasg4
-      author: nvoxland
-      comment: Create topic_message_batch table for pub/sub messaging
-      changes:
-        - createTable:
-            tableName: topic_message_batch
-            remarks: Append-only message log for pub/sub fan-out messaging
-            columns:
-              - column:
-                  name: id
-                  type: BIGINT
-                  autoIncrement: true
-                  constraints:
-                    primaryKey: true
-                    nullable: false
-              - column:
-                  name: topic_name
-                  type: varchar(254)
-                  remarks: The name of the topic this message was published to
-                  constraints:
-                    nullable: false
-              - column:
-                  remarks: The timestamp of when the message was created
-                  name: created_at
-                  type: ${timestamp_type}
-                  defaultValueComputed: current_timestamp
-                  constraints:
-                    nullable: false
-              - column:
-                  remarks: JSON-encoded array of messages
-                  name: messages
-                  type: ${text.type}
-                  constraints:
-                    nullable: false
-
-  - changeSet:
-      id: v61.igoiead
-      author: nvoxland
-      comment: Index topic_message_batch by topic_name and id for efficient polling
-      changes:
-        - createIndex:
-            tableName: topic_message_batch
-            indexName: idx_topic_message_batch_topic_id
-            columns:
-              - column:
-                  name: topic_name
-              - column:
-                  name: id
-
-  - changeSet:
-      id: v61.8867df
-      author: nvoxland
-      comment: Index topic_message_batch.created_at for efficient cleanup queries
-      changes:
-        - createIndex:
-            tableName: topic_message_batch
-            indexName: idx_topic_message_batch_created_at
-            columns:
-              - column:
-                  name: created_at

src/metabase/analytics/prometheus.clj

@@ -595,54 +595,6 @@
                           ;; 100ms -> 10 minutes
                           :buckets [100 500 1000 5000 10000 30000 60000 120000 300000 600000]})
 
-   ;; messaging metrics
-   (prometheus/counter :metabase-mq/appdb-cleanup-deleted
-                       {:description "Messages/batches deleted by cleanup."
-                        :labels [:transport :channel]})
-   (prometheus/gauge :metabase-mq/appdb-queue-depth
-                     {:description "Batch count per queue by status."
-                      :labels [:channel :status]})
-   (prometheus/counter :metabase-mq/appdb-queue-poll-results
-                       {:description "Queue poll results by outcome."
-                        :labels [:result]})
-   (prometheus/gauge :metabase-mq/appdb-topic-subscriber-lag
-                     {:description "Undelivered messages between subscriber offset and latest."
-                      :labels [:channel]})
-   (prometheus/counter :metabase-mq/batch-stale-recoveries
-                       {:description "Batches recovered from stale processing state."
-                        :labels [:transport :channel]})
-   (prometheus/counter :metabase-mq/batches-handled
-                       {:description "Batches handled by status."
-                        :labels [:transport :channel :status]})
-   (prometheus/counter :metabase-mq/dedup-messages-dropped
-                       {:description "Messages dropped by dedup before publishing."
-                        :labels [:channel]})
-   (prometheus/histogram :metabase-mq/handle-duration-ms
-                         {:description "Duration in milliseconds to process a batch."
-                          :labels [:transport :channel]
-                          :buckets [1 5 10 50 100 500 1000 5000 10000 30000]})
-   (prometheus/counter :metabase-mq/messages-published
-                       {:description "Total messages published."
-                        :labels [:transport :channel]})
-   (prometheus/gauge :metabase-mq/publish-buffer-depth
-                     {:description "Messages sitting in the publish buffer awaiting flush."
-                      :labels [:channel]})
-   (prometheus/counter :metabase-mq/publish-buffer-flush-errors
-                       {:description "Publish buffer flush failures where messages were re-buffered for retry."
-                        :labels [:channel]})
-   (prometheus/counter :metabase-mq/queue-batch-permanent-failures
-                       {:description "Queue batches that exhausted retries."
-                        :labels [:channel]})
-   (prometheus/counter :metabase-mq/queue-batch-retries
-                       {:description "Queue batches retried after transient failure."
-                        :labels [:channel]})
-   (prometheus/counter :metabase-mq/messages-received
-                       {:description "Individual messages delivered to handlers."
-                        :labels [:transport :channel]})
-   (prometheus/counter :metabase-mq/topic-handler-errors
-                       {:description "Errors in topic subscriber handlers."
-                        :labels [:transport :channel]})
-
    ;; release dashboard metrics
    (prometheus/counter :metabase-sync/failures
                        {:description "Number of sync operation failures."

src/metabase/app_db/connection.clj

@@ -134,34 +134,11 @@
 
 (def ^:private ^:dynamic *transaction-depth* 0)
 
-(def ^:dynamic *after-commit*
-  "When non-nil, an atom containing a vector of 0-arity fns to call after the outermost transaction commits.
-  Bound to a fresh atom at the outermost transaction boundary."
-  nil)
-
-(def ^:dynamic *transaction-state*
-  "When non-nil, an atom containing a map for arbitrary per-transaction data.
-  Any subsystem can store namespaced keys here. Bound to a fresh atom at the outermost transaction boundary."
-  nil)
-
 (defn in-transaction?
   "Whether we are currently in a transaction."
   []
   (pos? *transaction-depth*))
 
-(defn after-commit!
-  "Register `f` (a 0-arity fn) to run after the current outermost transaction commits.
-  Returns true if registered, false if not currently in a transaction."
-  [f]
-  (if *after-commit*
-    (do (swap! *after-commit* conj f) true)
-    false))
-
-(defn transaction-state
-  "Returns the current transaction state atom, or nil if not in a transaction."
-  []
-  *transaction-state*)
-
 (defn- do-transaction [^java.sql.Connection connection f]
   (letfn [(thunk []
             (let [savepoint (.setSavepoint connection)]
@@ -169,17 +146,7 @@
                 (let [result (f connection)]
                   (when (= *transaction-depth* 1)
                     ;; top-level transaction, commit
-                    (.commit connection)
-                    ;; drain and execute after-commit callbacks
-                    (loop []
-                      (when-let [callbacks (seq (first (reset-vals! *after-commit* [])))]
-                        (doseq [cb callbacks]
-                          (try
-                            (cb)
-                            (catch Throwable t
-                              (log/error t "Error in after-commit callback"))))
-                        ;; If callbacks registered new callbacks, drain those too
-                        (recur))))
+                    (.commit connection))
                   result)
                 (catch Throwable txn-e
                   (try
@@ -235,11 +202,8 @@
                     {:options options}))
 
     :else
-    (let [outermost? (zero? *transaction-depth*)]
-      (binding [*transaction-depth* (inc *transaction-depth*)
-                *after-commit*      (if outermost? (atom []) *after-commit*)
-                *transaction-state* (if outermost? (atom {}) *transaction-state*)]
-        (do-transaction connection f)))))
+    (binding [*transaction-depth* (inc *transaction-depth*)]
+      (do-transaction connection f))))
 
 (methodical/defmethod t2.pipeline/transduce-query :before :default
   "Make sure application database calls are not done inside core.async dispatch pool threads. This is done relatively

src/metabase/app_db/core.clj

@@ -37,9 +37,7 @@
   db-type
   in-transaction?
   quoting-style
-  unique-identifier
-  after-commit!
-  transaction-state]
+  unique-identifier]
 
  [mdb.connection-pool-setup
   recent-activity?]

src/metabase/cloud_migration/models/cloud_migration.clj

@@ -10,7 +10,7 @@
    [metabase.cmd.dump-to-h2 :as dump-to-h2]
    [metabase.config.core :as config]
    [metabase.models.interface :as mi]
-
+   [metabase.settings.core :as setting]
    [metabase.store-api.core :as store-api]
    [metabase.task.bootstrap :as task.bootstrap]
    [metabase.task.core :as task]
@@ -228,8 +228,7 @@
       (cloud-migration.settings/read-only-mode! true)
       (when (cluster?)
         (log/info "Cluster detected, waiting for read-only mode to propagate")
-        ;; Wait briefly for topic-based cache invalidation to propagate to other nodes
-        (Thread/sleep 5000))
+        (Thread/sleep (int (* 1.5 setting/cache-update-check-interval-ms))))
       (log/info "Stopping scheduler")
       (task/stop-scheduler!)
 

src/metabase/core/core.clj

@@ -20,7 +20,6 @@
    [metabase.events.core :as events]
    [metabase.initialization-status.core :as init-status]
    [metabase.logger.core :as logger]
-   [metabase.mq.core :as mq]
    [metabase.notification.core :as notification]
    [metabase.permissions.core :as perms]
    [metabase.plugins.core :as plugins]
@@ -35,6 +34,7 @@
    [metabase.tracing.core :as tracing]
    [metabase.util :as u]
    [metabase.util.log :as log]
+   [metabase.util.queue :as queue]
    [metabase.util.system-info :as u.system-info]
    [metabase.warehouses.models.database :as database])
   (:import
@@ -103,7 +103,7 @@
   "General application shutdown function which should be called once at application shutdown."
   []
   (log/info "Metabase Shutting Down ...")
-  (mq/shutdown!)
+  (queue/stop-listeners!)
   (task/stop-scheduler!)
   (server/stop-web-server!)
   (tracing/shutdown!)
@@ -228,6 +228,7 @@
   (setting/log-deprecated-env-var-usage!)
   (init-status/set-progress! 0.95)
   (task/start-scheduler!)
+  (queue/start-listeners!)
   (init-status/set-complete!)
   (log/info "Metabase Initialization COMPLETE"))
 

src/metabase/core/init.clj

@@ -40,7 +40,6 @@
    [metabase.metabot.init]
    [metabase.model-persistence.init]
    [metabase.models.init]
-   [metabase.mq.init]
    [metabase.notification.init]
    [metabase.oauth-server.init]
    [metabase.parameters.init]

src/metabase/driver/events/driver_notifications.clj

@@ -8,28 +8,8 @@
    [medley.core :as m]
    [metabase.driver :as driver]
    [metabase.driver-api.core :as driver-api]
-   [metabase.mq.core :as mq]
    [metabase.util.log :as log]
-   [methodical.core :as methodical]
-   ^{:clj-kondo/ignore [:discouraged-namespace]}
-   [toucan2.core :as t2]))
-
-;;; ----------------------------------------- Topic subscription -------------------------------------------
-
-(mq/def-listener! :topic/connection-pool-invalidated
-  {:sync-local-delivery true}
-  [{:keys [database-id all-databases]}]
-  (if all-databases
-    (doseq [{driver :engine, :as database} (t2/select :model/Database)]
-      (try
-        (driver/notify-database-updated driver database)
-        (catch Throwable e
-          (log/error e "Failed to notify database updated" {:id (:id database)}))))
-    (when-let [database (t2/select-one :model/Database :id database-id)]
-      (try
-        (driver/notify-database-updated (:engine database) database)
-        (catch Throwable e
-          (log/error e "Failed to notify database updated" {:id database-id}))))))
+   [methodical.core :as methodical]))
 
 (derive ::event :metabase/event)
 (derive :event/database-update ::event)
@@ -49,7 +29,6 @@
       (when (or details-changed?
                 (not= (remove-irrelevant-data database)
                       (remove-irrelevant-data previous-database)))
-        (mq/with-topic :topic/connection-pool-invalidated [t]
-          (mq/put t {:database-id (:id database)}))))
+        (driver/notify-database-updated (:engine database) database)))
     (catch Throwable e
       (log/warnf e "Failed to process driver notifications event. %s" topic))))

src/metabase/driver/events/report_timezone_updated.clj

@@ -1,15 +1,30 @@
 (ns metabase.driver.events.report-timezone-updated
   (:require
+   [metabase.driver :as driver]
    [metabase.driver-api.core :as driver-api]
-   [metabase.mq.core :as mq]
-   [methodical.core :as methodical]))
+   [metabase.util.log :as log]
+   [methodical.core :as methodical]
+   ^{:clj-kondo/ignore [:discouraged-namespace]}
+   [toucan2.core :as t2]))
 
 (derive ::event :metabase/event)
 (derive :event/report-timezone-updated ::event)
 
+(defn- notify-all-databases-updated
+  "Send notification that all Databases should immediately release cached resources (i.e., connection pools).
+
+  Currently only used below by [[report-timezone]] setter (i.e., only used when report timezone changes). Reusing
+  pooled connections with the old session timezone can have weird effects, especially if report timezone is changed to
+  `nil` (meaning subsequent queries will not attempt to change the session timezone) or something considered invalid
+  by a given Database (meaning subsequent queries will fail to change the session timezone)."
+  []
+  (doseq [{driver :engine, id :id, :as database} (t2/select :model/Database)]
+    (try
+      (driver/notify-database-updated driver database)
+      (catch Throwable e
+        (log/errorf e "Failed to notify %s Database %s updated" driver id)))))
+
 (methodical/defmethod driver-api/publish-event! ::event
-  "When the report-timezone Setting is updated, publish to the connection-pool-invalidated topic
-  so all databases get their pools flushed. Local listeners fire immediately inline."
+  "When the report-timezone Setting is updated, call [[metabase.driver/notify-database-updated]] for all Databases."
   [_topic _event]
-  (mq/with-topic :topic/connection-pool-invalidated [t]
-    (mq/put t {:all-databases true})))
+  (notify-all-databases-updated))

src/metabase/indexed_entities/models/model_index.clj

@@ -139,7 +139,7 @@
                 (t2/delete! :model/ModelIndexValue
                             :model_index_id (:id model-index)
                             :model_pk [:in (->> deletions-part (map first))])
-                (search/async-delete! :model/ModelIndexValue search-model-ids)))
+                (search/delete! :model/ModelIndexValue search-model-ids)))
             (when (seq additions)
               (doseq [additions-part (partition-all 10000 additions)]
                 (t2/insert! :model/ModelIndexValue
@@ -154,7 +154,7 @@
                        :state      (if (> (count values-to-index) max-indexed-values)
                                      "overflow"
                                      "indexed")}))
-        (run! search/async-update! (t2/reducible-select :model/ModelIndexValue :model_index_id (:id model-index)))
+        (run! search/update! (t2/reducible-select :model/ModelIndexValue :model_index_id (:id model-index)))
         (catch Exception e
           (log/errorf e "Error saving model-index values for model-index: %d, model: %d"
                       (:id model-index) (:model_id model-index))

src/metabase/models/transforms/transform.clj

@@ -260,7 +260,7 @@
 (t2/define-before-delete :model/Transform [transform]
   (ws.table/delete-orphaned-provisional-table! (:target_table_id transform) (:id transform))
   (events/publish-event! :event/delete-transform {:id (:id transform)})
-  (search.core/async-delete! :model/Transform [(str (:id transform))])
+  (search.core/delete! :model/Transform [(str (:id transform))])
   transform)
 
 (defn update-transform-tags!

src/metabase/mq/analytics.clj

@@ -1,20 +0,0 @@
-(ns metabase.mq.analytics
-  "Prometheus analytics helpers for the mq subsystem that avoids circular dependencies.
-  Lives in its own namespace to avoid further circular dependencies.")
-
-(set! *warn-on-reflection* true)
-
-(defn inc!
-  "Analytics helper that avoids circular dependencies"
-  [& args]
-  (apply (requiring-resolve 'metabase.analytics.core/inc!) args))
-
-(defn set!
-  "Analytics helper that avoids circular dependencies"
-  [& args]
-  (apply (requiring-resolve 'metabase.analytics.core/set!) args))
-
-(defn observe!
-  "Analytics helper that avoids circular dependencies"
-  [& args]
-  (apply (requiring-resolve 'metabase.analytics.core/observe!) args))

src/metabase/mq/core.clj

@@ -1,87 +0,0 @@
-(ns metabase.mq.core
-  "Unified public API for the Metabase messaging subsystem.
-
-  Two patterns are supported:
-
-  **Queue** — single-consumer, at-least-once delivery.  Each message is processed by exactly
-  one listener and removed after successful processing.  Failed messages are retried up to a
-  configurable limit.
-
-  **Topic** — single-consumer pub/sub.  Each active node receives every published message.
-  There can be up to one listener per node for each topic. Failed messages are never retried.
-
-  Listeners for both types are registered with `def-listener`
-
-      (mq/def-listener! :queue/simple-task [msg]
-        (process msg))
-
-      (mq/def-listener! :topic/my-events [msg]
-        (handle-event msg))
-
-  For queues, a config map enables batching:
-       (mq/def-listener! :queue/my-task
-         {:max-batch-messages 10}
-         [messages]
-         (process messages))
-
-  Publishing is done using `with-queue` or `with-topic` which binds a queue you can put messages on.
-  When the body finishes successfully, the message(s) in the bound queue actually publishes the messages.
-
-      (with-queue :queue/simple-task [q]
-        (put q message))
-
-      (with-topic :topic/my-events [t]
-        (put t payload))"
-  (:require
-   [metabase.mq.impl :as mq.impl]
-   [metabase.mq.listener :as mq.listener]
-   [metabase.mq.publish :as mq.publish]
-   [metabase.mq.queue.appdb :as q.appdb]
-   [metabase.mq.queue.impl :as q.impl]
-   [metabase.mq.queue.memory :as q.memory]
-   [metabase.mq.queue.sync :as q.sync]
-   [metabase.mq.queue.transport-impl :as q.transport-impl]
-   [metabase.mq.topic.appdb :as topic.appdb]
-   [metabase.mq.topic.impl :as topic.impl]
-   [metabase.mq.topic.memory :as topic.memory]
-   [metabase.mq.topic.sync :as topic.sync]
-   [metabase.mq.topic.transport-impl :as topic.transport-impl]
-   [potemkin :as p]))
-
-(set! *warn-on-reflection* true)
-
-(comment
-  mq.impl/keep-me
-  mq.listener/keep-me
-  mq.publish/keep-me
-  q.appdb/keep-me
-  q.impl/keep-me
-  q.memory/keep-me
-  q.sync/keep-me
-  q.transport-impl/keep-me
-  topic.appdb/keep-me
-  topic.transport-impl/keep-me
-  topic.impl/keep-me
-  topic.memory/keep-me
-  topic.sync/keep-me)
-
-(p/import-vars
- [mq.listener
-  batch-listen!
-  def-listener!
-  listen!
-  register-listeners!
-  unlisten!]
-
- [mq.publish
-  put]
-
- [mq.impl
-  last-activity
-  shutdown!]
-
- [q.impl
-  with-queue]
-
- [topic.impl
-  with-topic])

src/metabase/mq/impl.clj

@@ -1,200 +0,0 @@
-(ns metabase.mq.impl
-  "Backend coordination: message delivery, analytics helpers, and lifecycle management."
-  (:require
-   [metabase.mq.analytics :as mq.analytics]
-   [metabase.mq.listener :as listener]
-   [metabase.mq.polling :as mq.polling]
-   [metabase.mq.publish-buffer :as publish-buffer]
-   [metabase.mq.queue.backend :as q.backend]
-   [metabase.util.log :as log])
-  (:import
-   (java.util.concurrent Callable ExecutorService Executors Future)))
-
-(set! *warn-on-reflection* true)
-
-(def ^:private active-handlers
-  "channel keyword → {:future Future, :metadata map}
-   Tracks which channels have an active handler running across all backends."
-  (atom {}))
-
-(def ^:private worker-pool
-  "Shared thread pool for processing messages off backend poll threads."
-  (atom nil))
-
-(def ^:private last-activity*
-  "channel keyword → nanoTime of last publish or handler completion."
-  (atom {}))
-
-(defn last-activity
-  "Returns the nanoTime of the last publish or handler completion for the given channel, or nil."
-  [channel]
-  (get @last-activity* channel))
-
-(defn channel-busy?
-  "Returns true if the given channel has an active handler running."
-  [channel]
-  (contains? @active-handlers channel))
-
-(defn busy-channels
-  "Returns the set of channels that currently have an active handler."
-  []
-  (set (keys @active-handlers)))
-
-(defn active-handler-metadata
-  "Returns the metadata map for the active handler on the given channel, or nil.
-   Backends store backend-specific data here (e.g. queue backends store :bundle-id for heartbeats)."
-  [channel]
-  (get-in @active-handlers [channel :metadata]))
-
-(defn invoke-listener!
-  "Common listener invocation skeleton for both queues and topics.
-  Looks up listener via `listener-fn`, times execution, logs errors,
-  and records metrics. Calls `on-success` / `on-error` for system-specific
-  side effects (e.g. queue ACK/NACK)."
-  [{:keys [channel listener-fn invoke-fn on-success on-error]}]
-  (let [transport (namespace channel)
-        listener    (listener-fn)
-        labels      {:transport transport :channel (name channel)}
-        start       (System/nanoTime)]
-    (try
-      (if-not listener
-        (log/debugf "No listener registered for %s %s, skipping message" transport (name channel))
-        (do
-          (invoke-fn listener)
-          (when on-success (on-success))
-          (mq.analytics/inc! :metabase-mq/batches-handled (assoc labels :status "success"))))
-      (catch Exception e
-        (log/error e (str "Error handling " transport " message") labels)
-        (when on-error (on-error e))
-        (mq.analytics/inc! :metabase-mq/batches-handled (assoc labels :status "error")))
-      (finally
-        (mq.analytics/observe! :metabase-mq/handle-duration-ms labels
-                               (/ (double (- (System/nanoTime) start)) 1e6))))))
-
-(defn handle!
-  "Handles accumulated messages for a channel by invoking the registered listener.
-   Processes each message/batch with error isolation — one failure doesn't block others.
-   For queues, bundles use all-or-nothing ACK semantics: if ANY message in the bundle
-   fails, the ENTIRE bundle is nacked and will be retried. This means successfully
-   processed messages may be re-delivered. Queue listeners MUST be idempotent.
-   The :dedup-fn option on listeners helps mitigate duplicate processing on retry."
-  [channel message-bundles messages]
-  (swap! last-activity* assoc channel (System/nanoTime))
-  (mq.analytics/inc! :metabase-mq/messages-received
-                     {:transport (namespace channel) :channel (name channel)}
-                     (count messages))
-  (let [{:keys [max-batch-messages]} (listener/get-listener channel)]
-    (invoke-listener!
-     {:channel      channel
-      :listener-fn  #(:listener (listener/get-listener channel))
-      :invoke-fn    (fn [h]
-                      (let [error (atom nil)]
-                        (if (and max-batch-messages (> max-batch-messages 1))
-                          (doseq [batch (partition-all max-batch-messages messages)]
-                            (try (h (vec batch))
-                                 (catch Exception e
-                                   (reset! error e)
-                                   (log/errorf e "Error handling %s message for %s, continuing"
-                                               (namespace channel) (name channel)))))
-                          (doseq [msg messages]
-                            (try (h msg)
-                                 (catch Exception e
-                                   (reset! error e)
-                                   (log/errorf e "Error handling %s message for %s, continuing"
-                                               (namespace channel) (name channel))))))
-                        (when-let [e @error]
-                          (throw (ex-info "One or more messages failed" {} e)))))
-      :on-success   #(doseq [[bid backend] message-bundles]
-                       (q.backend/bundle-successful! backend channel bid))
-      :on-error     (fn [_e]
-                      (doseq [[bid backend] message-bundles]
-                        (q.backend/bundle-failed! backend channel bid)))})))
-
-(defn deliver!
-  "Called by backends when messages are ready for delivery.
-   Passes messages directly to handle! for processing.
-   `bundle-id` and `backend` are optional — queues pass them for ACK/NACK, topics pass nil."
-  [channel messages bundle-id backend]
-  (handle! channel (if bundle-id {bundle-id backend} {}) messages))
-
-(defn submit-delivery!
-  "Submits a delivery to the shared worker pool for non-blocking processing.
-   Returns true if submitted, false if the channel already has an active handler.
-   `metadata` is an opaque map stored alongside the future — backends can use it for
-   backend-specific tracking (e.g. queues store {:bundle-id id} for heartbeat updates).
-   Uses bound-fn to convey dynamic bindings to the worker thread.
-   The busy-check and registration are atomic via a single swap!.
-   A generation counter prevents a completed future's cleanup from clobbering a
-   re-submission that won the slot between the finally and the dissoc."
-  [channel messages bundle-id backend metadata]
-  (let [claimed? (atom false)
-        gen      (Object.)]
-    (swap! active-handlers
-           (fn [handlers]
-             (if (contains? handlers channel)
-               handlers
-               (do (reset! claimed? true)
-                   (assoc handlers channel {:future nil :metadata metadata :gen gen})))))
-    (if-not @claimed?
-      false
-      (let [f (.submit ^ExecutorService @worker-pool
-                       ^Callable (bound-fn []
-                                   (try
-                                     (deliver! channel messages bundle-id backend)
-                                     (finally
-                                       ;; Only remove if this generation still owns the slot
-                                       (swap! active-handlers
-                                              (fn [handlers]
-                                                (if (identical? gen (:gen (get handlers channel)))
-                                                  (dissoc handlers channel)
-                                                  handlers)))
-                                       (mq.polling/notify-all!)))))]
-        ;; Only set the future if this generation still owns the slot
-        (swap! active-handlers
-               (fn [handlers]
-                 (if (identical? gen (:gen (get handlers channel)))
-                   (assoc-in handlers [channel :future] f)
-                   handlers)))
-        true))))
-
-(defn start-worker-pool!
-  "Starts the shared worker pool for non-blocking message delivery. Idempotent."
-  []
-  (when-not @worker-pool
-    (let [pool (Executors/newCachedThreadPool)]
-      (when-not (compare-and-set! worker-pool nil pool)
-        ;; Another thread won the race — shut down our pool
-        (.shutdown ^ExecutorService pool)))))
-
-(defn shutdown-worker-pool!
-  "Cancels all active handler futures and shuts down the worker pool."
-  []
-  (doseq [[_ {:keys [^Future future]}] @active-handlers]
-    (when future
-      (.cancel future true)))
-  (reset! active-handlers {})
-  (when-let [^ExecutorService pool @worker-pool]
-    (.shutdownNow pool)
-    (reset! worker-pool nil)))
-
-(defn start-transports
-  "Starts the queue/topic backends."
-  []
-  (let [transport-start! (requiring-resolve 'metabase.mq.transport/start!)]
-    (transport-start! :queue)
-    (transport-start! :topic)))
-
-(defn shutdown-transports
-  "Shuts down all queue/topic backends."
-  []
-  (let [transport-shutdown! (requiring-resolve 'metabase.mq.transport/shutdown!)]
-    (transport-shutdown! :queue)
-    (transport-shutdown! :topic)))
-
-(defn shutdown!
-  "Shuts down all mq infrastructure: publish buffer, worker pool, backends, and listener state."
-  []
-  (publish-buffer/stop-publish-buffer-flush!)
-  (shutdown-transports)
-  (shutdown-worker-pool!)
-  (reset! listener/*listeners* {}))

src/metabase/mq/init.clj

@@ -1,37 +0,0 @@
-(ns metabase.mq.init
-  "Initializes the mq subsystem at startup."
-  (:require
-   [metabase.mq.impl :as mq.impl]
-   [metabase.mq.listener :as listener]
-   [metabase.mq.publish-buffer :as publish-buffer]
-   [metabase.mq.queue.backend :as q.backend]
-   [metabase.mq.settings :as mq.settings]
-   [metabase.mq.topic.backend :as topic.backend]
-   [metabase.startup.core :as startup]
-   [metabase.util.log :as log]))
-
-(def ^:private valid-queue-backends
-  #{:queue.backend/appdb :queue.backend/memory})
-
-(def ^:private valid-topic-backends
-  #{:topic.backend/appdb :topic.backend/memory})
-
-(defmethod startup/def-startup-logic! ::MqInit [_]
-  (let [queue-be (keyword "queue.backend" (mq.settings/queue-backend))
-        topic-be (keyword "topic.backend" (mq.settings/topic-backend))]
-    (when-not (contains? valid-queue-backends queue-be)
-      (throw (ex-info (str "Invalid queue backend: " queue-be
-                           ". Valid backends: " valid-queue-backends)
-                      {:backend queue-be :valid valid-queue-backends})))
-    (when-not (contains? valid-topic-backends topic-be)
-      (throw (ex-info (str "Invalid topic backend: " topic-be
-                           ". Valid backends: " valid-topic-backends)
-                      {:backend topic-be :valid valid-topic-backends})))
-    (alter-var-root #'q.backend/*backend* (constantly queue-be))
-    (alter-var-root #'topic.backend/*backend* (constantly topic-be))
-    (log/infof "Queue backend set to %s" queue-be)
-    (log/infof "Topic backend set to %s" topic-be)
-    (listener/register-listeners!)
-    (publish-buffer/start-publish-buffer-flush!)
-    (mq.impl/start-worker-pool!)
-    (mq.impl/start-transports)))

src/metabase/mq/listener.clj

@@ -1,141 +0,0 @@
-(ns metabase.mq.listener
-  "Listener registry: registration, lookup, instrumentation, and the `def-listener!` macro."
-  (:require
-   [metabase.mq.analytics :as mq.analytics]
-   [metabase.mq.transport :as transport]
-   [metabase.util.log :as log]
-   [metabase.util.malli.registry :as mr]))
-
-(set! *warn-on-reflection* true)
-
-(def ^:dynamic *listeners*
-  "channel → {:listener fn :max-batch-messages int ...} for all channels."
-  (atom {}))
-
-(defn queue-names
-  "Returns the seq of queue channel names currently registered in `*listeners*`."
-  []
-  (filter #(= "queue" (namespace %)) (keys @*listeners*)))
-
-(defn topic-names
-  "Returns the seq of topic channel names currently registered in `*listeners*`."
-  []
-  (filter #(= "topic" (namespace %)) (keys @*listeners*)))
-
-(defn get-listener
-  "Returns the listener config map for `channel`, or nil if not registered."
-  [channel]
-  (get @*listeners* channel))
-
-(defn register-listener!
-  "Atomically registers a listener for the given channel, throwing if one already exists."
-  [channel listener-map]
-  (let [[old _] (swap-vals! *listeners*
-                            (fn [m]
-                              (if (contains? m channel)
-                                m
-                                (assoc m channel listener-map))))]
-    (when (contains? old channel)
-      (throw (ex-info (str "Listener already registered for " (namespace channel) " " (name channel))
-                      {:channel channel})))))
-
-(defn make-instrumented-listener
-  "Wraps a listener with Prometheus metrics instrumentation."
-  [channel listener]
-  (let [labels    {:transport (namespace channel) :channel (name channel)}
-        received  :metabase-mq/messages-received
-        errors    :metabase-mq/topic-handler-errors]
-    (fn [msg]
-      (try
-        (listener msg)
-        (mq.analytics/inc! received labels)
-        (catch Exception e
-          (mq.analytics/inc! errors labels)
-          (throw e))))))
-
-(defn listen!
-  "Registers a listener for a queue or topic.
-   `opts` is an optional map; pass nil or {} for defaults. Queues support `{:exclusive true}`."
-  [channel opts listener]
-  (let [defaults  (transport/on-listen! channel opts)
-        listener' (transport/wrap-listener channel listener)]
-    (register-listener! channel
-                        (merge defaults opts {:listener listener' :max-batch-messages 1}))))
-
-(defn batch-listen!
-  "Registers a batch listener for a queue or topic.
-   The listener will be called with a vec of messages, sized up to :max-batch-messages."
-  [channel listener config]
-  (let [defaults  (transport/on-listen! channel config)
-        listener' (transport/wrap-listener channel listener)]
-    (register-listener! channel
-                        (merge defaults config {:listener listener'}))))
-
-(defn unlisten!
-  "Removes the listener for a channel."
-  [channel]
-  (swap! *listeners* dissoc channel))
-
-(mr/def ::channel
-  [:and :keyword [:fn {:error/message "Channel must be namespaced to 'queue' or 'topic'"}
-                  #(#{"queue" "topic"} (namespace %))]])
-
-(mr/def ::listen-opts
-  [:map [:exclusive {:optional true} :boolean]])
-
-(defmulti def-listener*
-  "Multimethod backing [[def-listener!]]. Each implementation registers its listener
-   by calling [[listen!]] or [[batch-listen!]].
-   Do not call or implement directly — use [[def-listener!]] instead."
-  {:arglists '([channel])}
-  identity)
-
-(defmacro def-listener!
-  "Defines a listener for a queue or topic, detected from the keyword namespace.
-
-   **Topic listener** — receives a single message:
-
-       (mq/def-listener! :topic/settings-cache-invalidated [msg]
-         (restore-cache!))
-
-   **Queue listener** — receives a single message, with optional config:
-
-       (mq/def-listener! :queue/simple-task {:exclusive true} [msg]
-         (process msg))
-
-   **Queue batch listener** — receives a vec of messages (config must include :max-batch-messages):
-
-       (mq/def-listener! :queue/search-reindex
-         {:max-batch-messages 50 :exclusive true}
-         [messages]
-         (process-batch messages))"
-  {:arglists '([channel bindings & body]
-               [channel config bindings & body])}
-  [channel & args]
-  (let [[config & args] (if (map? (first args))
-                          args
-                          (cons nil args))
-        [bindings & body] args
-        batch?          (and config (:max-batch-messages config))]
-    (if batch?
-      `(defmethod def-listener* ~channel [~'_]
-         (batch-listen!
-          ~channel
-          (fn [~@bindings] ~@body)
-          ~(select-keys config [:max-batch-messages :exclusive :dedup-fn])))
-      `(defmethod def-listener* ~channel [~'_]
-         (listen! ~channel ~(or config {}) (fn [~@bindings] ~@body))))))
-
-(defn register-listeners!
-  "Call all [[def-listener!]] implementations to register their listeners.
-   Called at startup and in test setup (from `with-sync-mq`).
-   Throws on the first registration failure so broken listeners are caught early."
-  []
-  (doseq [[k f] (methods def-listener*)]
-    (try
-      (f k)
-      (catch Throwable e
-        (log/errorf e "Failed to register listener %s" k)
-        (throw (ex-info (str "Failed to register listener " k)
-                        {:channel k}
-                        e))))))

src/metabase/mq/memory.clj

@@ -1,89 +0,0 @@
-(ns metabase.mq.memory
-  "Shared in-memory message store for the memory queue and topic backends.
-  Each channel gets a LinkedBlockingQueue. Uses the shared polling infrastructure
-  from mq.polling to drain non-busy channels and submit messages for delivery."
-  (:require
-   [metabase.mq.impl :as mq.impl]
-   [metabase.mq.listener :as listener]
-   [metabase.mq.polling :as mq.polling])
-  (:import
-   (java.util ArrayList)
-   (java.util.concurrent LinkedBlockingQueue)))
-
-(set! *warn-on-reflection* true)
-
-;;; ------------------------------------------- Channel state -------------------------------------------
-
-(def ^:dynamic *channels*
-  "channel-name -> LinkedBlockingQueue"
-  (atom {}))
-
-(def ^:private poll-state (mq.polling/make-poll-state))
-
-(defn- ensure-channel!
-  "Ensures a LinkedBlockingQueue exists for the channel, creating one if needed. Returns the queue."
-  ^LinkedBlockingQueue [channel-name]
-  (or (get @*channels* channel-name)
-      (let [new-q (LinkedBlockingQueue.)]
-        (-> (swap! *channels* (fn [chs]
-                                (if (contains? chs channel-name)
-                                  chs
-                                  (assoc chs channel-name new-q))))
-            (get channel-name)))))
-
-;;; ------------------------------------------- Publish / Drain -------------------------------------------
-
-(defn publish!
-  "Adds messages to the channel's queue and wakes the polling thread."
-  [channel-name messages]
-  (let [^LinkedBlockingQueue q (ensure-channel! channel-name)]
-    (doseq [msg messages]
-      (.offer q msg)))
-  (when-not (mq.impl/channel-busy? channel-name)
-    (mq.polling/notify! poll-state)))
-
-(defn- drain!
-  "Removes and returns all messages currently in the channel's queue, or nil if empty."
-  [channel-name]
-  (when-let [^LinkedBlockingQueue q (get @*channels* channel-name)]
-    (when-not (.isEmpty q)
-      (let [batch (ArrayList.)]
-        (.drainTo q batch)
-        (when-not (.isEmpty batch)
-          (vec batch))))))
-
-;;; ------------------------------------------- Polling -------------------------------------------
-
-(def ^:private bundle-registry
-  (delay (requiring-resolve 'metabase.mq.queue.memory/*bundle-registry*)))
-
-(defn- register-bundle!
-  "Registers a bundle in the queue memory backend's bundle registry for retry tracking."
-  [bundle-id messages]
-  (swap! @@bundle-registry assoc bundle-id {:messages messages :failures 0}))
-
-(defn- poll-once!
-  "Drains all non-busy channels and submits messages for delivery.
-   For queue channels, generates a bundle-id and passes :queue.backend/memory as backend.
-   For topic channels, passes nil bundle-id and nil backend (fire-and-forget)."
-  []
-  (doseq [channel-name (remove mq.impl/channel-busy?
-                               (concat (listener/queue-names) (listener/topic-names)))]
-    (when-let [messages (drain! channel-name)]
-      (if (= "queue" (namespace channel-name))
-        (let [bundle-id (str (random-uuid))]
-          (register-bundle! bundle-id messages)
-          (mq.impl/submit-delivery! channel-name messages bundle-id :queue.backend/memory
-                                    {:bundle-id bundle-id}))
-        (mq.impl/submit-delivery! channel-name messages nil nil nil)))))
-
-(defn start!
-  "Starts the memory polling thread. Idempotent — second call is a no-op."
-  []
-  (mq.polling/start-polling! poll-state "Memory" 50 poll-once!))
-
-(defn shutdown!
-  "Stops the polling thread and clears channel state."
-  []
-  (mq.polling/stop-polling! poll-state "Memory")
-  (reset! *channels* {}))

src/metabase/mq/polling.clj

@@ -1,78 +0,0 @@
-(ns metabase.mq.polling
-  "Shared polling infrastructure for appdb and memory mq backends:
-  time-gated execution and polling thread lifecycle."
-  (:require
-   [metabase.util.log :as log])
-  (:import
-   (java.util.concurrent Future)))
-
-(set! *warn-on-reflection* true)
-
-(defn periodically!
-  "Runs `f` at most once per `interval-ms`. Uses `state-atom` (an atom holding a long timestamp)
-   to track the last run time. Catches and logs exceptions using `label` for context."
-  [state-atom interval-ms label f]
-  (let [now (System/currentTimeMillis)]
-    (when (> (- now @state-atom) interval-ms)
-      (reset! state-atom now)
-      (try
-        (f)
-        (catch Exception e
-          (log/error e (str "Error in " label)))))))
-
-(def ^:private active-poll-states
-  "Set of poll states that currently have a running polling thread."
-  (atom #{}))
-
-(defn make-poll-state
-  "Creates a polling state map with an atom for the background process and a notify object."
-  []
-  {:process (atom nil)
-   :notify  (Object.)})
-
-(defn start-polling!
-  "Starts a polling thread that calls `poll-fn` in a loop, waiting on the notify object
-   for `wait-ms` between iterations. If `poll-fn` returns a truthy value (indicating work
-   was found), the next iteration runs immediately without waiting. Idempotent — second
-   call is a no-op."
-  [{:keys [process notify] :as poll-state} label wait-ms poll-fn]
-  (when-not @process
-    (log/infof "Starting %s polling thread" label)
-    (swap! active-poll-states conj poll-state)
-    (reset! process
-            (future
-              (try
-                (loop []
-                  (when @process
-                    (let [found-work? (try
-                                        (poll-fn)
-                                        (catch InterruptedException e (throw e))
-                                        (catch Exception e
-                                          (log/errorf e "Error in %s polling loop" label)
-                                          false))]
-                      (when-not found-work?
-                        (locking notify (.wait ^Object notify (long wait-ms)))))
-                    (recur)))
-                (catch InterruptedException _
-                  (log/infof "%s polling thread interrupted" label)))))))
-
-(defn stop-polling!
-  "Stops a polling thread."
-  [{:keys [process] :as poll-state} label]
-  (swap! active-poll-states disj poll-state)
-  (when-let [f @process]
-    (reset! process nil)
-    (when (instance? Future f)
-      (.cancel ^Future f true))
-    (log/infof "%s polling thread stopped" label)))
-
-(defn notify!
-  "Wakes the polling thread so it runs immediately."
-  [{:keys [notify]}]
-  (locking notify (.notifyAll ^Object notify)))
-
-(defn notify-all!
-  "Wakes all active polling threads so they run immediately."
-  []
-  (doseq [ps @active-poll-states]
-    (notify! ps)))

src/metabase/mq/publish.clj

@@ -1,103 +0,0 @@
-(ns metabase.mq.publish
-  "Publishing pipeline: buffering, transaction support, and the `with-buffer` macro."
-  (:require
-   [metabase.app-db.core :as mdb]
-   [metabase.mq.analytics :as mq.analytics]
-   [metabase.mq.listener :as listener]
-   [metabase.mq.publish-buffer :as publish-buffer]
-   [metabase.util.log :as log]))
-
-(set! *warn-on-reflection* true)
-
-(def ^:private last-activity*
-  (delay (requiring-resolve 'metabase.mq.impl/last-activity*)))
-
-(def ^:dynamic *defer-in-transaction?*
-  "When true (the default), messages published inside a transaction are deferred
-  until after the transaction commits. Bind to false in sync/test mode where
-  listeners must process inline."
-  true)
-
-(defprotocol MessageBuffer
-  "Protocol for buffering messages before publishing."
-  (put [this msg]
-    "Put a message on the buffer."))
-
-(defn publish!
-  "Publishes messages to a channel. Applies dedup-fn if registered, buffers,
-   publishes to the appropriate backend, and records analytics."
-  [channel messages]
-  (let [{:keys [dedup-fn]} (listener/get-listener channel)
-        before-count (count messages)
-        messages     (if dedup-fn (dedup-fn messages) messages)]
-    (when (and dedup-fn (< (count messages) before-count))
-      (mq.analytics/inc! :metabase-mq/dedup-messages-dropped
-                         {:channel (name channel)}
-                         (- before-count (count messages))))
-    (when (seq messages)
-      (swap! @@last-activity* assoc channel (System/nanoTime))
-      (publish-buffer/buffered-publish! channel messages)
-      (mq.analytics/inc! :metabase-mq/messages-published
-                         {:transport (namespace channel) :channel (name channel)}
-                         (count messages)))))
-
-(defn flush-deferred-messages!
-  "Flushes all deferred messages accumulated during a transaction.
-   Called as an after-commit callback. Per-channel try/catch so one failure doesn't block others."
-  []
-  (when-let [state (mdb/transaction-state)]
-    (let [deferred (::deferred-messages @state)]
-      (doseq [[channel msgs] deferred]
-        (try
-          (publish! channel msgs)
-          (catch Exception e
-            (log/error e "Error flushing deferred messages" {:channel channel})))))))
-
-(defn defer-in-transaction!
-  "Accumulates msgs in *transaction-state* under [::deferred-messages channel].
-   Registers flush-deferred-messages! as an after-commit callback once per transaction.
-   If *defer-in-transaction?* is false (set in tests), immediately publishes"
-  [channel msgs]
-  (if *defer-in-transaction?*
-    (let [state     (mdb/transaction-state)
-          old-state (first (swap-vals! state
-                                       (fn [s]
-                                         (-> s
-                                             (update-in [::deferred-messages channel] (fnil into []) msgs)
-                                             (assoc ::flush-registered? true)))))]
-      (when-not (::flush-registered? old-state)
-        (mdb/after-commit! flush-deferred-messages!)))
-    (publish! channel msgs)))
-
-(defn run-with-buffer
-  "Runs `body-fn` with a MessageBuffer. On success, publishes collected messages —
-   deferred if inside a transaction, immediate via publish! otherwise.
-   Channels with `:sync-local-delivery true` in their listener config always publish
-   immediately, even inside a transaction.
-   On exception, discards buffered messages and rethrows."
-  [channel error-label body-fn]
-  (let [buffer     (atom [])
-        msg-buffer (reify MessageBuffer
-                     (put [_ msg] (swap! buffer conj msg)))]
-    (try
-      (let [result (body-fn msg-buffer)
-            msgs   @buffer]
-        (when (seq msgs)
-          (if (and (mdb/transaction-state)
-                   (not (:sync-local-delivery (listener/get-listener channel))))
-            (defer-in-transaction! channel msgs)
-            (publish! channel msgs)))
-        result)
-      (catch Exception e
-        (log/error e error-label)
-        (throw e)))))
-
-(defmacro with-buffer
-  "Runs the body with a MessageBuffer for the given channel.
-   Messages are buffered and only published if the body completes successfully.
-   Inside a transaction, messages are deferred until after commit."
-  [channel [binding] & body]
-  `(run-with-buffer
-    ~channel
-    (str "Error in " (namespace ~channel) " processing")
-    (fn [~binding] ~@body)))

src/metabase/mq/publish_buffer.clj

@@ -1,94 +0,0 @@
-(ns metabase.mq.publish-buffer
-  "Publish buffering: batches rapid-fire publishes into time-windowed groups
-   and flushes them on a background scheduled thread."
-  (:require
-   [metabase.mq.analytics :as mq.analytics]
-   [metabase.mq.transport :as transport]
-   [metabase.util.log :as log])
-  (:import
-   (java.util.concurrent Executors ScheduledExecutorService ThreadFactory TimeUnit)))
-
-(set! *warn-on-reflection* true)
-
-(def ^:dynamic *publish-buffer-ms*
-  "Time window in ms to buffer rapid-fire publishes. 0 = immediate publish.
-   Resets on every new message (sliding window)."
-  100)
-
-(def ^:dynamic *publish-buffer-max-ms*
-  "Maximum time in ms since the first message before a flush is forced. 0 = no max."
-  5000)
-
-(def ^:dynamic *publish-buffer*
-  "Global buffer for buffering publishes.
-   channel → {:messages [...] :deadline-ms long :created-ms long}"
-  (atom {}))
-
-(defn buffered-publish!
-  "Routes messages through *publish-buffer* or publishes immediately based on *publish-buffer-ms*.
-   The background flush thread handles actual delivery."
-  [channel messages]
-  (if (zero? *publish-buffer-ms*)
-    (transport/publish! channel messages)
-    (swap! *publish-buffer*
-           (fn [buf]
-             (let [now (System/currentTimeMillis)]
-               (update buf channel
-                       (fn [entry]
-                         (-> (or entry {:messages [] :deadline-ms 0 :created-ms now})
-                             (update :messages into messages)
-                             (assoc :deadline-ms (+ now *publish-buffer-ms*))))))))))
-
-(defn flush-publish-buffer!
-  "Drains publish buffer entries past their deadline."
-  []
-  (doseq [[channel entry] @*publish-buffer*]
-    (mq.analytics/set! :metabase-mq/publish-buffer-depth
-                       {:channel (name channel)}
-                       (count (:messages entry))))
-  (doseq [channel (keys @*publish-buffer*)]
-    (let [drained (atom nil)]
-      (swap! *publish-buffer*
-             (fn [buf]
-               (if-let [{:keys [deadline-ms created-ms] :as entry} (get buf channel)]
-                 (let [now (System/currentTimeMillis)]
-                   (if (or (and (pos? deadline-ms) (>= now deadline-ms))
-                           (and (pos? *publish-buffer-max-ms*) (>= now (+ created-ms *publish-buffer-max-ms*))))
-                     (do (reset! drained entry) (dissoc buf channel))
-                     buf))
-                 buf)))
-      (when-let [{:keys [messages] :as entry} @drained]
-        (try (transport/publish! channel messages)
-             (catch Exception e
-               (log/error e "Error flushing publish buffer, re-buffering" {:channel channel})
-               (mq.analytics/inc! :metabase-mq/publish-buffer-flush-errors {:channel (name channel)})
-               ;; Put messages back into the buffer for retry on next flush cycle
-               (swap! *publish-buffer*
-                      (fn [buf]
-                        (update buf channel
-                                (fn [existing]
-                                  (if existing
-                                    (update existing :messages into messages)
-                                    (assoc entry :deadline-ms (+ (System/currentTimeMillis) *publish-buffer-ms*)))))))))))))
-
-(defonce ^:private publish-buffer-executor (atom nil))
-
-(defn start-publish-buffer-flush!
-  "Starts a daemon thread that flushes the publish buffer every 100ms."
-  []
-  (let [exec (Executors/newSingleThreadScheduledExecutor
-              (reify ThreadFactory
-                (newThread [_ r]
-                  (doto (Thread. r "mq-publish-buffer-flush")
-                    (.setDaemon true)))))]
-    (if (compare-and-set! publish-buffer-executor nil exec)
-      (.scheduleAtFixedRate exec ^Runnable flush-publish-buffer! 100 100 TimeUnit/MILLISECONDS)
-      (.shutdown exec))))
-
-(defn stop-publish-buffer-flush!
-  "Stops the publish buffer flush thread and drains remaining entries."
-  []
-  (when-let [^ScheduledExecutorService exec
-             (first (reset-vals! publish-buffer-executor nil))]
-    (.shutdownNow exec))
-  (flush-publish-buffer!))

src/metabase/mq/queue/appdb.clj

@@ -1,198 +0,0 @@
-(ns metabase.mq.queue.appdb
-  "Database-backed implementation of the message queue using the application database."
-  (:require
-   [metabase.app-db.core :as mdb]
-   [metabase.models.interface :as mi]
-   [metabase.mq.analytics :as mq.analytics]
-   [metabase.mq.impl :as mq.impl]
-   [metabase.mq.listener :as listener]
-   [metabase.mq.polling :as mq.polling]
-   [metabase.mq.queue.backend :as q.backend]
-   [metabase.mq.queue.impl :as q.impl]
-   [metabase.util.json :as json]
-   [metabase.util.log :as log]
-   [toucan2.core :as t2])
-  (:import (java.sql Timestamp)
-           (java.time Instant)))
-
-(set! *warn-on-reflection* true)
-
-(defn- for-update-clause
-  "Returns the FOR UPDATE clause for the current app DB.
-  Uses SKIP LOCKED on PostgreSQL and MySQL to avoid blocking on rows locked by other nodes.
-  Falls back to plain FOR UPDATE on H2."
-  []
-  (if (#{:postgres :mysql} (mdb/db-type))
-    [:update :skip-locked]
-    [:update]))
-
-(def ^:private owner-id (str (random-uuid)))
-(def ^:private poll-state (mq.polling/make-poll-state))
-
-;;; ------------------------------------------- Fetch -------------------------------------------
-
-(defn- fetch!
-  "Fetches the oldest pending row for each of the given queue names.
-  Returns a seq of maps with :bundle-id, :queue, and :messages keys, or nil if no messages are available.
-  Selects one row per queue, marks them all as 'processing', and returns them — all within one transaction."
-  [queue-names]
-  (when (seq queue-names)
-    (let [exclusive-names (q.impl/exclusive-queue-names)]
-      (t2/with-transaction [conn]
-        (let [blocked-queues  (when (seq exclusive-names)
-                                (into #{}
-                                      (map :queue_name)
-                                      (t2/query conn {:select   [:queue_name]
-                                                      :from     [:queue_message_batch]
-                                                      :where    [:and
-                                                                 [:in :queue_name (vec exclusive-names)]
-                                                                 [:= :status "processing"]]
-                                                      :group-by [:queue_name]})))
-              fetchable-names (if (seq blocked-queues)
-                                (remove #(contains? blocked-queues (name %)) queue-names)
-                                queue-names)]
-          (when (seq fetchable-names)
-            ;; Fetch the oldest pending row per queue in a single query, locking them for update
-            (let [name-list (mapv name fetchable-names)
-                  rows      (t2/query conn {:select [:*]
-                                            :from   [:queue_message_batch]
-                                            :where  [:in :id {:select   [[[:min :id]]]
-                                                              :from     [:queue_message_batch]
-                                                              :where    [:and
-                                                                         [:in :queue_name name-list]
-                                                                         [:= :status "pending"]]
-                                                              :group-by [:queue_name]}]
-                                            :for    (for-update-clause)})]
-              (when (seq rows)
-                (t2/query conn {:update [:queue_message_batch]
-                                :set    {:status_heartbeat (mi/now)
-                                         :status           "processing"
-                                         :owner            owner-id}
-                                :where  [:in :id (mapv :id rows)]})
-                (mapv (fn [row]
-                        {:bundle-id (:id row)
-                         :queue     (keyword "queue" (:queue_name row))
-                         :messages  (json/decode (:messages row))})
-                      rows)))))))))
-
-;; Stale processing recovery
-(def ^:private stale-processing-timeout-ms
-  "Messages in 'processing' status for longer than this are considered stale and recovered."
-  (* 10 60 1000))
-
-(def ^:private last-stale-check-ms (atom 0))
-
-(defn- recover-stale-processing-batches!
-  "Recovers processing batches whose heartbeat is older than the stale timeout.
-  Returns the number of batches recovered."
-  []
-  (let [max-retries (@@q.impl/queue-max-retries)
-        threshold   (Timestamp/from (.minusMillis (Instant/now) stale-processing-timeout-ms))
-        recovered   (atom 0)]
-    (doseq [row (t2/select :queue_message_batch :status "processing" :status_heartbeat [:< threshold])]
-      (let [new-failures (inc (:failures row))]
-        (if (>= new-failures max-retries)
-          (do
-            (log/warnf "Processing batch %d on queue '%s' has reached max failures (%d), marking as failed"
-                       (:id row) (:queue_name row) max-retries)
-            (mq.analytics/inc! :metabase-mq/queue-batch-permanent-failures {:channel (:queue_name row)})
-            (t2/update! :queue_message_batch (:id row)
-                        {:status "failed" :failures new-failures :status_heartbeat (mi/now) :owner nil}))
-          (do
-            (log/warnf "Recovering processing batch %d on queue '%s' (failures: %d -> %d)"
-                       (:id row) (:queue_name row) (:failures row) new-failures)
-            (mq.analytics/inc! :metabase-mq/batch-stale-recoveries {:transport "queue" :channel (:queue_name row)})
-            (t2/update! :queue_message_batch (:id row)
-                        {:status "pending" :failures new-failures :status_heartbeat (mi/now) :owner nil})))
-        (swap! recovered inc)))
-    @recovered))
-
-;; Failed batch cleanup
-(def ^:private last-cleanup-ms (atom 0))
-
-(defn- cleanup-failed-batches! []
-  (let [threshold (Timestamp/from (.minusMillis (Instant/now) (* 7 24 60 60 1000)))
-        deleted   (t2/delete! :queue_message_batch :status "failed" :status_heartbeat [:< threshold])]
-    (when (pos? deleted)
-      (log/infof "Cleaned up %d failed queue batches" deleted)
-      (mq.analytics/inc! :metabase-mq/appdb-cleanup-deleted {:transport "queue" :channel "all"} deleted))))
-
-;; Queue depth gauge
-(def ^:private last-depth-gauge-ms (atom 0))
-
-(defn- update-depth-gauges! []
-  (doseq [{:keys [queue_name status cnt]}
-          (t2/query {:select   [:queue_name :status [[:count :*] :cnt]]
-                     :from     [:queue_message_batch]
-                     :group-by [:queue_name :status]})]
-    (mq.analytics/set! :metabase-mq/appdb-queue-depth {:channel queue_name :status status} cnt)))
-
-;; Heartbeat
-(def ^:private last-heartbeat-ms (atom 0))
-
-(defn- update-heartbeats! []
-  (doseq [channel (filter #(= "queue" (namespace %)) (mq.impl/busy-channels))]
-    (when-let [{:keys [bundle-id]} (mq.impl/active-handler-metadata channel)]
-      (try
-        (t2/update! :queue_message_batch
-                    {:id bundle-id :owner owner-id :status "processing"}
-                    {:status_heartbeat (mi/now)})
-        (catch Exception e
-          (log/warnf e "Failed to update heartbeat for bundle %d" bundle-id))))))
-
-;;; ------------------------------------------- Polling -------------------------------------------
-
-(defn- poll-iteration!
-  "One iteration of the polling loop: run periodic tasks, then try to process batches.
-  Fetches one pending row per available queue and submits each for delivery.
-  Returns true if any work was found."
-  []
-  (mq.polling/periodically! last-stale-check-ms  (* 60 1000)       "stale batch recovery"  recover-stale-processing-batches!)
-  (mq.polling/periodically! last-heartbeat-ms     (* 2 60 1000)     "heartbeat update"      update-heartbeats!)
-  (mq.polling/periodically! last-cleanup-ms       (* 24 60 60 1000) "failed batch cleanup"  cleanup-failed-batches!)
-  (mq.polling/periodically! last-depth-gauge-ms   (* 30 1000)       "queue depth gauge"     update-depth-gauges!)
-  (boolean
-   (when-let [available-queues (seq (remove mq.impl/channel-busy? (listener/queue-names)))]
-     (when-let [batches (seq (fetch! available-queues))]
-       (doseq [{:keys [bundle-id queue messages]} batches]
-         (mq.impl/submit-delivery! queue messages bundle-id :queue.backend/appdb {:bundle-id bundle-id}))
-       true))))
-
-(defmethod q.backend/start! :queue.backend/appdb [_]
-  (mq.polling/start-polling! poll-state "Queue" 5000 poll-iteration!))
-
-(defmethod q.backend/shutdown! :queue.backend/appdb [_]
-  (mq.polling/stop-polling! poll-state "Queue"))
-
-(defmethod q.backend/publish! :queue.backend/appdb
-  [_ queue messages]
-  (t2/insert! :queue_message_batch
-              {:queue_name (name queue)
-               :messages   (json/encode messages)})
-  (when-not (mq.impl/channel-busy? queue)
-    (mq.polling/notify! poll-state)))
-
-(defmethod q.backend/bundle-successful! :queue.backend/appdb
-  [_ _queue-name bundle-id]
-  (let [deleted (t2/delete! :queue_message_batch :id bundle-id :owner owner-id)]
-    (when (= 0 deleted)
-      (log/warnf "Message %d was already deleted from the queue." bundle-id))))
-
-(defmethod q.backend/bundle-failed! :queue.backend/appdb
-  [_ queue-name bundle-id]
-  (let [row     (t2/select-one :queue_message_batch :id bundle-id :owner owner-id)
-        updated (when row
-                  (if (>= (inc (:failures row)) (@@q.impl/queue-max-retries))
-                    (do
-                      (log/warnf "Message %d has reached max failures (%d), marking as failed" bundle-id (@@q.impl/queue-max-retries))
-                      (mq.analytics/inc! :metabase-mq/queue-batch-permanent-failures {:channel (name queue-name)})
-                      (t2/update! :queue_message_batch
-                                  {:id bundle-id :owner owner-id}
-                                  {:status "failed" :failures [:+ :failures 1] :status_heartbeat (mi/now) :owner nil}))
-                    (do
-                      (mq.analytics/inc! :metabase-mq/queue-batch-retries {:channel (name queue-name)})
-                      (t2/update! :queue_message_batch
-                                  {:id bundle-id :owner owner-id}
-                                  {:status "pending" :failures [:+ :failures 1] :status_heartbeat (mi/now) :owner nil}))))]
-    (when (and row (= 0 updated))
-      (log/warnf "Message %d was not found in the queue. Likely error in concurrency handling" bundle-id))))

src/metabase/mq/queue/backend.clj

@@ -1,47 +0,0 @@
-(ns metabase.mq.queue.backend
-  "Backend abstraction layer for the message queue system.
-  Defines multimethods that different queue implementations must provide."
-  (:require
-   [metabase.util.malli.registry :as mr]))
-
-(set! *warn-on-reflection* true)
-
-(mr/def ::backend
-  [:enum :queue.backend/appdb :queue.backend/memory :queue.backend/sync])
-
-(def ^:dynamic *backend*
-  "Dynamic var specifying which queue backend to use.
-  Default is `:queue.backend/appdb` for production database-backed queues."
-  :queue.backend/appdb)
-
-(defmulti publish!
-  "Publishes messages to the given queue. `messages` is always a vector of messages."
-  {:arglists '([backend queue-name messages])}
-  (fn [backend _queue-name _messages]
-    backend))
-
-(defmulti bundle-successful!
-  "Mark a bundle as successfully processed"
-  {:arglists '([backend queue-name bundle-id])}
-  (fn [backend _queue-name _bundle-id]
-    backend))
-
-(defmulti bundle-failed!
-  "Mark a bundle as failed and increment failure count"
-  {:arglists '([backend queue-name bundle-id])}
-  (fn [backend _queue-name _bundle-id]
-    backend))
-
-(defmulti start!
-  "Starts the backend polling loop. Called once at init time."
-  {:arglists '([backend])}
-  identity)
-
-(defmethod start! :default [_] nil)
-
-(defmulti shutdown!
-  "Shuts down all queue resources for this backend: stops all listeners and releases any background threads."
-  {:arglists '([backend])}
-  identity)
-
-(defmethod shutdown! :default [_] nil)

src/metabase/mq/queue/impl.clj

@@ -1,40 +0,0 @@
-(ns metabase.mq.queue.impl
-  "Internal implementation for the queue messaging system"
-  (:require
-   [metabase.mq.listener :as listener]
-   [metabase.mq.publish :as publish]
-   [metabase.util.malli.registry :as mr]))
-
-(set! *warn-on-reflection* true)
-
-(mr/def :metabase.mq.queue/queue-name
-  [:and :keyword [:fn {:error/message "Queue name must be namespaced to 'queue'"}
-                  #(= "queue" (namespace %))]])
-
-(defn exclusive?
-  "Returns true if the given queue name is registered with `:exclusive true`."
-  [queue-name]
-  (:exclusive (listener/get-listener queue-name)))
-
-(defn exclusive-queue-names
-  "Returns the set of queue names (as strings) that are registered with `:exclusive true`."
-  []
-  (into #{}
-        (comp (filter (fn [[_k v]] (:exclusive v)))
-              (map (fn [[k _v]] (name k))))
-        @listener/*listeners*))
-
-(def queue-max-retries
-  "Lazily-resolved var for `metabase.mq.settings/queue-max-retries`. Use `@@q.impl/queue-max-retries` to read the current value."
-  (delay (requiring-resolve 'metabase.mq.settings/queue-max-retries)))
-
-(defmacro with-queue
-  "Runs the body with the ability to add messages to the given queue.
-  Messages are buffered and only published if the body completes successfully.
-  If an exception occurs, no messages are published and the exception is rethrown.
-
-  When called inside a database transaction, messages are accumulated and published
-  as a single batch after the transaction commits successfully. This prevents consumers
-  from reading uncommitted data."
-  [queue-name [queue-binding] & body]
-  `(publish/with-buffer ~queue-name [~queue-binding] ~@body))

src/metabase/mq/queue/memory.clj

@@ -1,40 +0,0 @@
-(ns metabase.mq.queue.memory
-  "In-memory queue backend. Delegates storage and polling to the shared memory layer.
-  Provides queue-specific bundle tracking for retry semantics."
-  (:require
-   [metabase.mq.analytics :as mq.analytics]
-   [metabase.mq.memory :as memory]
-   [metabase.mq.queue.backend :as q.backend]
-   [metabase.mq.queue.impl :as q.impl]
-   [metabase.util.log :as log]))
-
-(set! *warn-on-reflection* true)
-
-(def ^:dynamic *bundle-registry*
-  "Maps bundle-id -> {:messages [...] :failures n} for retry tracking."
-  (atom {}))
-
-(defmethod q.backend/publish! :queue.backend/memory [_ queue-name messages]
-  (memory/publish! queue-name messages))
-
-(defmethod q.backend/start! :queue.backend/memory [_]
-  (memory/start!))
-
-(defmethod q.backend/shutdown! :queue.backend/memory [_]
-  (memory/shutdown!))
-
-(defmethod q.backend/bundle-successful! :queue.backend/memory [_ _queue-name bundle-id]
-  (swap! *bundle-registry* dissoc bundle-id))
-
-(defmethod q.backend/bundle-failed! :queue.backend/memory [_ queue-name bundle-id]
-  (when-let [{:keys [messages failures]} (get @*bundle-registry* bundle-id)]
-    (swap! *bundle-registry* dissoc bundle-id)
-    (let [new-failures (inc failures)]
-      (if (>= new-failures (@@q.impl/queue-max-retries))
-        (do
-          (log/warnf "Bundle %s has reached max failures (%d), dropping" bundle-id (@@q.impl/queue-max-retries))
-          (mq.analytics/inc! :metabase-mq/queue-batch-permanent-failures {:channel (name queue-name)}))
-        (do
-          (mq.analytics/inc! :metabase-mq/queue-batch-retries {:channel (name queue-name)})
-          ;; Re-queue messages for retry
-          (memory/publish! queue-name messages))))))

src/metabase/mq/queue/sync.clj

@@ -1,27 +0,0 @@
-(ns metabase.mq.queue.sync
-  "Synchronous queue backend that calls the listener inline during `publish!`.
-  Useful in tests to avoid needing `*force-sync*` dynamic vars in each namespace."
-  (:require
-   [metabase.mq.impl :as mq.impl]
-   [metabase.mq.listener :as listener]
-   [metabase.mq.queue.backend :as q.backend]
-   [metabase.util.log :as log]))
-
-(set! *warn-on-reflection* true)
-
-(defmethod q.backend/publish! :queue.backend/sync [_ queue-name messages]
-  (let [bundle-id (str (random-uuid))]
-    (if (listener/get-listener queue-name)
-      ;; Call handle! directly to bypass accumulation — the sync backend must
-      ;; deliver immediately so that tests observe side-effects inline.
-      (mq.impl/handle! queue-name {bundle-id :queue.backend/sync} messages)
-      (log/warnf "No listener registered for queue %s, dropping %d message(s)" queue-name (count messages)))))
-
-(defmethod q.backend/bundle-successful! :queue.backend/sync [_ _queue-name _bundle-id]
-  nil)
-
-(defmethod q.backend/bundle-failed! :queue.backend/sync [_ _queue-name _bundle-id]
-  nil)
-
-(defmethod q.backend/shutdown! :queue.backend/sync [_]
-  nil)

src/metabase/mq/queue/transport_impl.clj

@@ -1,22 +0,0 @@
-(ns metabase.mq.queue.transport-impl
-  "Queue implementations of the transport multimethods."
-  (:require
-   [metabase.mq.queue.backend :as q.backend]
-   [metabase.mq.transport :as transport]))
-
-(set! *warn-on-reflection* true)
-
-(defmethod transport/on-listen! :queue [_ opts]
-  {:exclusive (boolean (:exclusive opts))})
-
-(defmethod transport/publish! :queue [channel messages]
-  (q.backend/publish! q.backend/*backend* channel messages))
-
-(defmethod transport/wrap-listener :queue [_channel listener]
-  listener)
-
-(defmethod transport/start! :queue [_]
-  (q.backend/start! q.backend/*backend*))
-
-(defmethod transport/shutdown! :queue [_]
-  (q.backend/shutdown! q.backend/*backend*))

src/metabase/mq/settings.clj

@@ -1,33 +0,0 @@
-(ns metabase.mq.settings
-  (:require
-   [metabase.app-db.core :as mdb]
-   [metabase.settings.core :as setting :refer [defsetting]]
-   [metabase.util.i18n :refer [deferred-tru]]))
-
-(defsetting queue-max-retries
-  (deferred-tru "Maximum number of times a failed queue message will be retried before being dropped.")
-  :type       :integer
-  :default    5
-  :visibility :internal
-  :encryption :no
-  :export?    false)
-
-(defsetting queue-backend
-  (deferred-tru "Which queue backend to use. Valid values: appdb, memory.")
-  :type       :string
-  :default    "appdb"
-  :visibility :internal
-  :encryption :no
-  :export?    false)
-
-(defsetting topic-backend
-  (deferred-tru "Which topic backend to use. Valid values: appdb, memory.")
-  :type       :string
-  :getter     (fn []
-                (or (setting/get-value-of-type :string :topic-backend)
-                    (if (= (mdb/db-type) :postgres)
-                      "appdb"
-                      "memory")))
-  :visibility :internal
-  :encryption :no
-  :export?    false)

src/metabase/mq/topic/appdb.clj

@@ -1,114 +0,0 @@
-(ns metabase.mq.topic.appdb
-  "Database-backed implementation of the pub/sub system using the application database.
-  Messages are stored in the `topic_message_batch` table. Each subscriber on each node polls
-  independently, tracking its read offset in memory."
-  (:require
-   [metabase.mq.analytics :as mq.analytics]
-   [metabase.mq.impl :as mq.impl]
-   [metabase.mq.listener :as listener]
-   [metabase.mq.polling :as mq.polling]
-   [metabase.mq.topic.backend :as topic.backend]
-   [metabase.util.json :as json]
-   [metabase.util.log :as log]
-   [toucan2.core :as t2])
-  (:import (java.sql Timestamp)
-           (java.time Instant)))
-
-(set! *warn-on-reflection* true)
-
-(def ^:private offsets
-  "Map of topic-name -> long offset."
-  (atom {}))
-
-(def ^:private poll-state (mq.polling/make-poll-state))
-
-(def ^:private batch-size
-  "Maximum number of messages to fetch in a single poll."
-  100)
-
-(defn- current-max-id
-  "Returns the current maximum id in topic_message_batch for the given topic, or 0."
-  [topic-name]
-  (or (t2/select-one-fn :max-id [:topic_message_batch [[:max :id] :max-id]]
-                        :topic_name (name topic-name))
-      0))
-
-(defn- poll-messages!
-  "Fetches rows from topic_message_batch with id > offset."
-  [topic-name offset]
-  (t2/query {:select   [:id :messages]
-             :from     [:topic_message_batch]
-             :where    [:and
-                        [:= :topic_name (name topic-name)]
-                        [:> :id offset]]
-             :order-by [[:id :asc]]
-             :limit    batch-size}))
-
-;;; ------------------------------------------- Periodic tasks (run on polling thread) -------------------------------------------
-
-;; Cleanup old messages
-(def ^:private last-cleanup-ms (atom 0))
-
-(defn- cleanup-old-messages! []
-  (let [threshold (Timestamp/from (.minusMillis (Instant/now) (* 60 60 1000)))
-        deleted   (t2/delete! :topic_message_batch :created_at [:< threshold])]
-    (when (pos? deleted)
-      (log/infof "Cleaned up %d old topic messages" deleted)
-      (mq.analytics/inc! :metabase-mq/appdb-cleanup-deleted {:transport "topic" :channel "all"} deleted))))
-
-(def ^:private last-lag-gauge-ms (atom 0))
-
-(defn- update-lag-gauges! []
-  (doseq [[topic-name offset] @offsets]
-    (let [max-id (current-max-id topic-name)]
-      (mq.analytics/set! :metabase-mq/appdb-topic-subscriber-lag
-                         {:channel (name topic-name)}
-                         (- max-id offset)))))
-
-(defn- poll-iteration!
-  "One iteration of the polling loop: run periodic tasks, then poll all topics.
-  Returns true if any topic had messages delivered."
-  []
-  (mq.polling/periodically! last-cleanup-ms   (* 10 60 1000) "topic cleanup"    cleanup-old-messages!)
-  (mq.polling/periodically! last-lag-gauge-ms (* 30 1000)    "topic lag gauge"  update-lag-gauges!)
-  (let [found-work? (atom false)]
-    (doseq [topic-name (remove mq.impl/channel-busy? (listener/topic-names))]
-      (let [offset (if (contains? @offsets topic-name)
-                     (get @offsets topic-name)
-                     (let [o (current-max-id topic-name)]
-                       (swap! offsets assoc topic-name o)
-                       o))
-            rows   (poll-messages! topic-name offset)]
-        (when (seq rows)
-          (let [all-messages (into [] (mapcat (comp json/decode :messages)) rows)
-                max-id       (:id (last rows))]
-            (when (mq.impl/submit-delivery! topic-name all-messages nil nil nil)
-              (swap! offsets assoc topic-name max-id)
-              (reset! found-work? true))))))
-    @found-work?))
-
-(defmethod topic.backend/start! :topic.backend/appdb [_]
-  (mq.polling/start-polling! poll-state "Topic" 2000 poll-iteration!))
-
-(defmethod topic.backend/shutdown! :topic.backend/appdb [_]
-  (mq.polling/stop-polling! poll-state "Topic")
-  (reset! offsets {}))
-
-(defmethod topic.backend/publish! :topic.backend/appdb
-  [_ topic-name messages]
-  (t2/insert-returning-pk! :topic_message_batch
-                           {:topic_name (name topic-name)
-                            :messages   (json/encode messages)})
-  (when-not (mq.impl/channel-busy? topic-name)
-    (mq.polling/notify! poll-state)))
-
-(defmethod topic.backend/subscribe! :topic.backend/appdb
-  [_ topic-name]
-  (let [offset (current-max-id topic-name)]
-    (swap! offsets assoc topic-name offset)
-    (log/infof "Subscribed to topic %s (starting offset %d)" (name topic-name) offset)))
-
-(defmethod topic.backend/unsubscribe! :topic.backend/appdb
-  [_ topic-name]
-  (swap! offsets dissoc topic-name)
-  (log/infof "Unsubscribed from topic %s" (name topic-name)))

src/metabase/mq/topic/backend.clj

@@ -1,56 +0,0 @@
-(ns metabase.mq.topic.backend
-  "Backend abstraction layer for the pub/sub system.
-  Defines multimethods that different pub/sub implementations must provide.
-
-  Topics are fire-and-forget: there is no retry logic. If a listener throws,
-  the error is logged and the batch is skipped. This matches the semantics of
-  the postgres LISTEN/NOTIFY backend where re-delivery is not possible."
-  (:require
-   [metabase.util.malli.registry :as mr]))
-
-(set! *warn-on-reflection* true)
-
-(mr/def ::backend
-  [:enum :topic.backend/appdb :topic.backend/memory :topic.backend/sync])
-
-(def ^:dynamic *backend*
-  "Dynamic var specifying which pub/sub backend to use.
-  Default is `:topic.backend/appdb` for production database-backed pub/sub.
-  Can be bound to `:topic.backend/memory` for testing."
-  :topic.backend/appdb)
-
-(defmulti publish!
-  "Publishes messages to the given topic. `messages` is a vector of values that will be JSON-encoded
-  as an array and stored in a single row for batch efficiency.
-  Return the batch id of the messages published to avoid re-publishing locally."
-  {:arglists '([backend topic-name messages])}
-  (fn [backend _topic-name _messages]
-    backend))
-
-(defmulti start!
-  "Starts the backend polling loop. Called once at init time.
-  The backend polls `mq.impl/*listeners*` dynamically to discover registered topics."
-  {:arglists '([backend])}
-  identity)
-
-(defmethod start! :default [_] nil)
-
-(defmulti subscribe!
-  "Internal operation: initializes backend state (offsets, channels) for a specific topic.
-  Called by the polling loop when it discovers a new topic in `*listeners*`, not by `listen!`."
-  {:arglists '([backend topic-name])}
-  (fn [backend _topic-name]
-    backend))
-
-(defmulti unsubscribe!
-  "Stops the polling loop for the given topic."
-  {:arglists '([backend topic-name])}
-  (fn [backend _topic-name]
-    backend))
-
-(defmulti shutdown!
-  "Shuts down all topic resources for this backend: unsubscribes all subscribers and releases any background threads."
-  {:arglists '([backend])}
-  identity)
-
-(defmethod shutdown! :default [_] nil)

src/metabase/mq/topic/impl.clj

@@ -1,25 +0,0 @@
-(ns metabase.mq.topic.impl
-  "Topic-specific implementation: on-listen! hook and topic schemas."
-  (:require
-   [metabase.mq.publish :as publish]
-   [metabase.util.malli.registry :as mr]))
-
-(set! *warn-on-reflection* true)
-
-(mr/def :metabase.mq.topic/topic-name
-  [:and :keyword [:fn {:error/message "Topic name must be namespaced to 'topic'"}
-                  #(= "topic" (namespace %))]])
-
-(defmacro with-topic
-  "Runs the body with the ability to add messages to the given topic.
-  Messages are buffered and only published if the body completes successfully.
-  If an exception occurs, no messages are published and the exception is rethrown.
-
-  Outside a transaction, publishing is best-effort and fire-and-forget — if the body
-  succeeds but `publish!` throws, the body's side effects will have already occurred.
-
-  Inside a database transaction, messages are deferred and published as a single batch
-  after the transaction commits successfully. This prevents consumers from seeing
-  uncommitted data. If the transaction rolls back, the messages are discarded."
-  [topic-name [buffer-binding] & body]
-  `(publish/with-buffer ~topic-name [~buffer-binding] ~@body))

src/metabase/mq/topic/memory.clj

@@ -1,23 +0,0 @@
-(ns metabase.mq.topic.memory
-  "In-memory topic backend. Delegates storage and polling to the shared memory layer."
-  (:require
-   [metabase.mq.memory :as memory]
-   [metabase.mq.topic.backend :as topic.backend]
-   [metabase.util.log :as log]))
-
-(set! *warn-on-reflection* true)
-
-(defmethod topic.backend/start! :topic.backend/memory [_]
-  (memory/start!))
-
-(defmethod topic.backend/publish! :topic.backend/memory [_ topic-name messages]
-  (memory/publish! topic-name messages))
-
-(defmethod topic.backend/subscribe! :topic.backend/memory [_ topic-name]
-  (log/infof "Memory subscribed to topic %s" (name topic-name)))
-
-(defmethod topic.backend/unsubscribe! :topic.backend/memory [_ topic-name]
-  (log/infof "Memory unsubscribed from topic %s" (name topic-name)))
-
-(defmethod topic.backend/shutdown! :topic.backend/memory [_]
-  (memory/shutdown!))

src/metabase/mq/topic/sync.clj

@@ -1,18 +0,0 @@
-(ns metabase.mq.topic.sync
-  "Synchronous topic backend that calls listeners inline during `publish!`.
-  Useful in tests to avoid Thread/sleep for polling-based backends."
-  (:require
-   [metabase.mq.impl :as mq.impl]
-   [metabase.mq.topic.backend :as topic.backend]))
-
-(set! *warn-on-reflection* true)
-
-(defmethod topic.backend/publish! :topic.backend/sync
-  [_ topic-name messages]
-  (mq.impl/handle! topic-name {} messages)
-  nil)
-
-(defmethod topic.backend/start! :topic.backend/sync [_] nil)
-(defmethod topic.backend/subscribe! :topic.backend/sync [_ _topic-name] nil)
-(defmethod topic.backend/unsubscribe! :topic.backend/sync [_ _topic-name] nil)
-(defmethod topic.backend/shutdown! :topic.backend/sync [_] nil)

src/metabase/mq/topic/transport_impl.clj

@@ -1,24 +0,0 @@
-(ns metabase.mq.topic.transport-impl
-  "Topic implementations of the transport multimethods."
-  (:require
-   [metabase.mq.listener :as listener]
-   [metabase.mq.topic.backend :as topic.backend]
-   [metabase.mq.transport :as transport]))
-
-(set! *warn-on-reflection* true)
-
-(defmethod transport/on-listen! :topic [channel _opts]
-  (topic.backend/subscribe! topic.backend/*backend* channel)
-  {})
-
-(defmethod transport/publish! :topic [channel messages]
-  (topic.backend/publish! topic.backend/*backend* channel messages))
-
-(defmethod transport/wrap-listener :topic [channel listener]
-  (listener/make-instrumented-listener channel listener))
-
-(defmethod transport/start! :topic [_]
-  (topic.backend/start! topic.backend/*backend*))
-
-(defmethod transport/shutdown! :topic [_]
-  (topic.backend/shutdown! topic.backend/*backend*))

src/metabase/mq/transport.clj

@@ -1,35 +0,0 @@
-(ns metabase.mq.transport
-  "Multimethods dispatched by transport type (:queue or :topic).")
-
-(defn transport-type
-  "Returns :queue or :topic for a namespaced channel keyword."
-  [channel]
-  (keyword (namespace channel)))
-
-(defmulti publish!
-  "Publishes messages to the appropriate backend for the transport type."
-  {:arglists '([channel messages])}
-  (fn [channel _messages] (transport-type channel)))
-
-(defmulti on-listen!
-  "Transport-type-specific hook called during listener registration.
-   Returns a map of default opts for the transport type.
-   Implementations may also perform side effects (e.g. subscribing to a topic backend)."
-  {:arglists '([channel opts])}
-  (fn [channel _opts] (transport-type channel)))
-
-(defmulti wrap-listener
-  "Wraps a listener function with transport-type-specific behavior.
-   Queues return the listener as-is; topics add instrumentation."
-  {:arglists '([channel listener])}
-  (fn [channel _listener] (transport-type channel)))
-
-(defmulti start!
-  "Starts the backend for the given transport type."
-  {:arglists '([transport-type])}
-  identity)
-
-(defmulti shutdown!
-  "Shuts down the backend for the given transport type."
-  {:arglists '([transport-type])}
-  identity)

src/metabase/search/api.clj

@@ -6,11 +6,15 @@
    [metabase.api.common :as api]
    [metabase.api.macros :as api.macros]
    [metabase.api.open-api :as open-api]
+   [metabase.config.core :as config]
    [metabase.permissions.core :as perms]
    [metabase.request.core :as request]
    [metabase.search.config :as search.config]
    [metabase.search.core :as search]
+   [metabase.search.ingestion :as ingestion]
    [metabase.search.settings :as search.settings]
+   [metabase.search.task.search-index :as task.search-index]
+   [metabase.task.core :as task]
    [metabase.util :as u]
    [metabase.util.json :as json]
    [metabase.util.log :as log]
@@ -72,9 +76,7 @@
   []
   (api/check-superuser)
   (if (search/supports-index?)
-    (do
-      (search/async-init! :force-reset? true)
-      {:message "re-init enqueued"})
+    {:message (search/init-index! {:force-reset? true})}
     (throw (ex-info "Search index is not supported for this installation." {:status-code 501}))))
 
 ;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
@@ -86,7 +88,10 @@
   []
   (api/check-superuser)
   (if (search/supports-index?)
-    (do (search/async-reindex!) {:message "reindex triggered"})
+    ;; The job appears to wait on the main thread when run from tests, so, unfortunately, testing this branch is hard.
+    (if (and (task/job-exists? task.search-index/reindex-job-key) (or (not ingestion/*force-sync*) config/is-test?))
+      (do (task/trigger-now! task.search-index/reindex-job-key) {:message "task triggered"})
+      (do (search/reindex!) {:message "reindex triggered"}))
 
     (throw (ex-info "Search index is not supported for this installation." {:status-code 501}))))
 

src/metabase/search/appdb/core.clj

@@ -1,6 +1,7 @@
 (ns metabase.search.appdb.core
   (:require
    [clojure.string :as str]
+   [environ.core :as env]
    [honey.sql.helpers :as sql.helpers]
    [java-time.api :as t]
    [metabase.app-db.core :as mdb]
@@ -12,12 +13,12 @@
    [metabase.search.config :as search.config]
    [metabase.search.engine :as search.engine]
    [metabase.search.filter :as search.filter]
-   [metabase.search.impl :as search.impl]
    [metabase.search.ingestion :as search.ingestion]
    [metabase.search.permissions :as search.permissions]
    [metabase.search.settings :as search.settings]
    [metabase.search.spec :as search.spec]
    [metabase.search.util :as search.util]
+   [metabase.settings.core :as setting]
    [metabase.tracing.core :as tracing]
    [metabase.util :as u]
    [metabase.util.i18n :as i18n]
@@ -26,7 +27,8 @@
    [methodical.core :as methodical]
    [toucan2.core :as t2])
   (:import
-   (java.time OffsetDateTime)))
+   (java.time OffsetDateTime)
+   (java.util Queue)))
 
 ;; Register the multimethods for each specialization
 (comment
@@ -129,7 +131,8 @@
       (when init-now?
         (log/warnf "Triggering a late initialization of the %s search index." search-engine)
         (try
-          (search.impl/async-init! :engine search-engine :force-reset? false)
+          (future
+            (search.engine/init! search-engine {:force-reset? false}))
           (catch Exception e
             (log/error e))))
       ;; Even if the index exists now, return an error so that we don't obscure that there was an issue.
@@ -145,6 +148,15 @@
 
   (tracing/with-span :search "search.appdb.query" {:search/query-length (count search-string)}
     (try
+      (when (setting/string->boolean (:mb-experimental-search-block-on-queue env/env))
+        ;; wait for a bit for the queue to be drained
+        (let [pending-updates #(.size ^Queue @#'search.ingestion/queue)]
+          (when-not (u/poll {:thunk       pending-updates
+                             :done?       zero?
+                             :timeout-ms  2000
+                             :interval-ms 100})
+            (log/warn "Returning search results even though they may be stale. Queue size:" (pending-updates)))))
+
       (let [weights (search.config/weights search-ctx)
             scorers (search.scoring/scorers search-ctx)
             query   (->> (search.index/search-query search-string search-ctx [:legacy_input])
@@ -192,7 +204,7 @@
     (if (and index-created (< 3 (t/time-between (t/instant index-created) (t/instant) :days)))
       (do
         (log/info "Forcing early reindex because existing index is old")
-        (search.impl/async-reindex! :engine :search.engine/appdb))
+        (search.engine/reindex! :search.engine/appdb {}))
 
       (let [created? (search.index/ensure-ready! opts)]
         (when (or created? re-populate?)
@@ -223,4 +235,6 @@
   [_topic event]
   (when (and (= :site-locale (-> event :details :key)) (= :postgres (mdb/db-type)))
     (log/info "Reindexing appdb index because the site locale changed.")
-    (search.impl/async-reindex! :engine :search.engine/appdb)))
+    (if search.ingestion/*force-sync*
+      (search.engine/reindex! :search.engine/appdb {})
+      (future (search.engine/reindex! :search.engine/appdb {})))))

src/metabase/search/appdb/index.clj

@@ -12,6 +12,7 @@
    [metabase.search.appdb.specialization.postgres :as postgres]
    [metabase.search.config :as search.config]
    [metabase.search.engine :as search.engine]
+   [metabase.search.ingestion :as search.ingestion]
    [metabase.search.models.search-index-metadata :as search-index-metadata]
    [metabase.search.spec :as search.spec]
    [metabase.tracing.core :as tracing]
@@ -315,7 +316,7 @@
         (log/warnf "Unable to find table %s and no longer tracking it as pending", table)
         (swap! *indexes* assoc :pending nil))))
 
-  (let [reindexing? (= :search/reindexing context)
+  (let [reindexing? (and (= :search/reindexing context) (not search.ingestion/*force-sync*))
         do-writes   (fn []
                       (let [entries         (map document->entry documents)
                             ;; No need to update the active index if we are doing a full index, as this table will be
@@ -412,10 +413,12 @@
               (swap! *indexes* assoc :pending nil))
             (maybe-create-pending!)
             (activate-table!))]
-    ;; Creates and tracks tables with a unique transaction so the empty tables are available to other threads
-    ;; even while the initial startup and data load may be happening
-    (t2/with-connection [_ (mdb/data-source)]
-      (reset-logic))))
+    (if search.ingestion/*force-sync*
+      (reset-logic)
+      ;; Creates and tracks tables with a unique transaction so the empty tables are available to other threads
+      ;; even while the initial startup and data load may be happening
+      (t2/with-connection [_ (mdb/data-source)]
+        (reset-logic)))))
 
 (defn ensure-ready!
   "Ensure the index is ready to be populated. Return false if it was already ready."
@@ -450,16 +453,3 @@
            (finally
              (#'drop-table! table-name#)
              (t2/delete! :model/SearchIndexMetadata :version version#)))))))
-
-(defn expose-temp-table-to-server-threads!
-  "In tests that use HTTP requests, the server thread can't see the test's `*indexes*` binding.
-  Call this after `with-temp-index-table` to update the global atom so server threads find the temp table.
-  Returns a 0-arity fn that restores the previous state — call it in a `finally` block."
-  []
-  (let [prev-indexes @@#'*indexes*
-        prev-sync-at @@#'next-sync-at]
-    (reset! @#'*indexes* {:active (active-table)})
-    (reset! @#'next-sync-at (+ (long (#'now)) (long (* 24 60 60e9))))
-    (fn []
-      (reset! @#'*indexes* prev-indexes)
-      (reset! @#'next-sync-at prev-sync-at))))

src/metabase/search/core.clj

@@ -2,12 +2,17 @@
   "NOT the API namespace for the search module!! See [[metabase.search]] instead."
   (:require
    [metabase.analytics.core :as analytics]
+   [metabase.analytics.prometheus :as prometheus]
+   [metabase.lib-be.core :as lib-be]
    [metabase.search.config :as search.config]
    [metabase.search.engine :as search.engine]
    [metabase.search.impl :as search.impl]
    [metabase.search.ingestion :as search.ingestion]
    [metabase.search.spec :as search.spec]
    [metabase.search.util :as search.util]
+   [metabase.tracing.core :as tracing]
+   [metabase.util :as u]
+   [metabase.util.log :as log]
    [potemkin :as p]))
 
 (set! *warn-on-reflection* true)
@@ -25,11 +30,7 @@
   model-set]
 
  [search.impl
-  async-delete!
-  async-init!
-  async-reindex!
   search
-  supports-index?
   ;; We could avoid exposing this by wrapping `query-model-set` and `search` with it.
   search-context]
 
@@ -77,6 +78,78 @@
   [_ {:keys [engine]}]
   (if (search.engine/supported-engine? (keyword "search.engine" engine)) 1 0))
 
+(defn supports-index?
+  "Does this instance support a search index, of any sort?"
+  []
+  (seq (search.engine/active-engines)))
+
+(defn init-index!
+  "Ensure there is an index ready to be populated."
+  [& {:as opts}]
+  (when (supports-index?)
+    (log/info "Initializing search indexes")
+    (tracing/with-span :search "search.init-index" {}
+      (lib-be/with-metadata-provider-cache
+        ;; If there are multiple indexes, return the peak inserted for each type. In practice, they should all be the same.
+        (try
+          (let [timer    (u/start-timer)
+                report   (reduce (partial merge-with max)
+                                 nil
+                                 (for [e (search.engine/active-engines)]
+                                   (search.engine/init! e opts)))
+                duration (u/since-ms timer)]
+            (if (seq report)
+              (do
+                (analytics/inc! :metabase-search/index-reindex-ms duration)
+                (prometheus/observe! :metabase-search/index-reindex-duration-ms duration)
+                (doseq [[model cnt] report]
+                  (analytics/inc! :metabase-search/index-reindexes {:model model} cnt))
+                (log/infof "Index initialized in %.0fms %s" duration (sort-by (comp - val) report))
+                report)
+              (log/info "Found existing search index, and using it.")))
+          (catch Exception e
+            (analytics/inc! :metabase-search/index-error)
+            (throw e)))))))
+
+(defn- reindex-logic! [opts]
+  (when (supports-index?)
+    (tracing/with-span :search "search.reindex" {}
+      (lib-be/with-metadata-provider-cache
+        (try
+          (log/info "Reindexing searchable entities")
+          (let [timer    (u/start-timer)
+                report   (reduce (partial merge-with max)
+                                 nil
+                                 (for [e (search.engine/active-engines)]
+                                   (search.engine/reindex! e opts)))
+                duration (u/since-ms timer)]
+            (analytics/inc! :metabase-search/index-reindex-ms duration)
+            (prometheus/observe! :metabase-search/index-reindex-duration-ms duration)
+            (doseq [[model cnt] report]
+              (analytics/inc! :metabase-search/index-reindexes {:model model} cnt))
+            (log/infof "Done reindexing in %.0fms %s" duration (sort-by (comp - val) report))
+            report)
+          (catch Exception e
+            (analytics/inc! :metabase-search/index-error)
+            (throw e)))))))
+
+(defn reindex!
+  "Populate a new index, and make it active. Simultaneously updates the current index.
+  Returns a future that will complete when the reindexing is done.
+  Respects `search.ingestion/*force-sync*` and waits for the future if it's true.
+  Alternately, if `:async?` is false, it will also run synchronously."
+  [& {:keys [async?] :or {async? true} :as opts}]
+  (let [f (fn []
+            (try
+              (reindex-logic! opts)
+              (catch Exception e
+                (log/error e "Reindex failed")
+                (analytics/inc! :metabase-search/index-error)
+                (throw e))))]
+    (if (or search.ingestion/*force-sync* (not async?))
+      (doto (promise) (deliver (f)))
+      (future (f)))))
+
 (defn reset-tracking!
   "Stop tracking the current indexes. Used when resetting the appdb."
   []
@@ -92,7 +165,7 @@
     (doseq [e (search.engine/active-engines)]
       (search.engine/sync-from-restored-db! e))))
 
-(defn async-update!
+(defn update!
   "Given a new or updated instance, put all the corresponding search entries if needed in the queue."
   [instance & [always?]]
   (when (supports-index?)
@@ -100,4 +173,14 @@
                             (remove (comp search.util/impossible-condition? second))
                             seq)]
       ;; We need to delay execution to handle deletes, which alert us *before* updating the database.
-      (search.ingestion/async-updates updates))))
+      (search.ingestion/ingest-maybe-async! updates))))
+
+(defn delete!
+  "Given a model and a list of model's ids, remove corresponding search entries."
+  [model ids]
+  (when (supports-index?)
+    (doseq [e            (search.engine/active-engines)
+            search-model (->> (vals (search.spec/specifications))
+                              (filter (comp #{model} :model))
+                              (map :name))]
+      (search.engine/delete! e search-model ids))))