Cleaning up code

Replace scattered is-test? checks at var definition time with a proper
:mq initialization step in the test startup flow. This mirrors how
production initializes MQ via MqInit in startup/run-startup-logic!.

The :mq step:
- Loads driver.init (so def-listener! namespaces are loaded)
- Calls mq.init/init-for-tests! which sets sync backends, disables
  publish buffering, and registers all listeners

Also:
- Reverts var defaults to production values (appdb backends, 100ms buffer)
- Reverts def-listener! to lazy registration (no eager call)
- Restores with-sync-mq to use fresh *listeners* atom
- Removes all [TZ-DEBUG] diagnostic traces
- :db initialization depends on :mq

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

.clj-kondo/config.edn

@@ -596,6 +596,9 @@
    metabase.model-persistence.api-test/with-persistence-setup!                                                               hooks.common/with-one-top-level-binding
    metabase.model-persistence.api-test/with-setup!                                                                           hooks.common/with-one-top-level-binding
    metabase.model-persistence.test-util/with-persistence-enabled!                                                            hooks.metabase.model-persistence.test-util/with-persistence-enabled!
+   metabase.mq.core/with-queue                                                                                               hooks.common/let-second
+   metabase.mq.core/with-topic                                                                                               hooks.common/let-second
+   metabase.mq.core/def-listener!                                                                                            hooks.metabase.mq.impl/def-listener
    metabase.models.interface/define-batched-hydration-method                                                                 hooks.metabase.models.interface/define-hydration-method
    metabase.models.interface/define-simple-hydration-method                                                                  hooks.metabase.models.interface/define-hydration-method
    metabase.permissions.core/allow-changing-all-users-group-members                                                          hooks.common/do*

.clj-kondo/config/modules/config.edn

@@ -814,6 +814,7 @@
            legacy-mbql
            lib
            lib-be
+           mq
            premium-features
            query-processor
            settings
@@ -1311,6 +1312,17 @@
                     :model/Table
                     :model/User}}
 
+  mq
+  {:team "UX West"
+   :api  #{metabase.mq.core
+           metabase.mq.init}
+   :uses #{analytics
+           app-db
+           models
+           settings
+           startup
+           util}}
+
   native-query-snippets
   {:team "Graphy"
    :api  #{metabase.native-query-snippets.api
@@ -1830,6 +1842,7 @@
            lib
            lib-be
            models
+           mq
            permissions
            premium-features
            queries
@@ -1931,6 +1944,7 @@
            config
            events
            models
+           mq
            premium-features
            util
            enterprise/advanced-permissions}
@@ -2570,6 +2584,7 @@
            events
            lib
            models
+           mq
            query-processor
            util
            warehouse-schema}

.clj-kondo/src/hooks/metabase/mq/impl.clj

@@ -0,0 +1,24 @@
+(ns hooks.metabase.mq.impl
+  (:require
+   [clj-kondo.hooks-api :as hooks]))
+
+(defn def-listener
+  "Kondo hook for `mq/def-listener`. Rewrites to a `fn` form so kondo can analyze bindings.
+
+   Forms:
+     (mq/def-listener! :topic/foo [msg] body...)
+     (mq/def-listener! :queue/bar {:config map} [msgs] body...)"
+  [{:keys [node]}]
+  (let [[_ _channel-name & args] (:children node)
+        ;; skip optional config map
+        args (if (hooks/map-node? (first args))
+               (rest args)
+               args)
+        [bindings & body] args
+        ;; rewrite as: (fn [bindings...] body...)
+        new-node (hooks/list-node
+                  (list*
+                   (hooks/token-node 'fn)
+                   bindings
+                   body))]
+    {:node new-node}))

.cljfmt.edn

@@ -303,7 +303,6 @@
   metabase.util.malli/with                                                                    [[:default]]
   metabase.util.malli/with-api-error-message                                                  [[:default]]
   metabase.util.namespaces/import-fns                                                         [[:default]]
-  metabase.util.queue-test/await-test-while                                                   [[:block 1]]
   metabase.util/auto-retry                                                                    [[:block 1]]
   metabase.util/case-enum                                                                     [[:block 1]]
   metabase.util/for-map                                                                       [[:block 1]]

dev/src/dev/mq_perf.clj

@@ -0,0 +1,532 @@
+(ns dev.mq-perf
+  "Performance benchmarks for the appdb-backed MQ system (queues and topics).
+
+  Measures publishing throughput, end-to-end latency, and sustained throughput
+  under different load levels. Benchmarks exercise the full realistic pipeline:
+  publish buffer → DB → poll thread → worker pool → listener.
+
+  Self-contained — starts the MQ subsystem without the full Metabase server.
+  Supports multi-node testing: run multiple JVM instances against the same DB.
+
+  Example usage:
+    ;; Single-node: start MQ and run all benchmarks
+    (start-mq!)
+    (run-all-benchmarks!)
+
+    ;; Multi-node coordination: start multiple JVMs, each calls:
+    (start-mq!)
+    (run-all-benchmarks-coordinated!)
+    ;; Then from any node (or a separate REPL), publish the go signal:
+    (signal-go!)
+
+    ;; Individual benchmarks
+    (bench-publish-throughput! {:n 1000 :batch-sizes [1 10 100]})
+    (bench-e2e-latency! {:n 50 :rate :medium :transport :queue})
+    (bench-sustained-throughput! {:duration-sec 15 :rate :medium :transport :queue})
+    (bench-batch-listener! {:n 200 :batch-sizes [1 10 50]})
+    (bench-multi-queue! {:queue-counts [1 5 10 20] :msgs-per-queue 50})"
+  (:require
+   [clojure.string :as str]
+   [metabase.mq.core :as mq]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.publish-buffer :as publish-buffer]
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.mq.topic.backend :as topic.backend]
+   [toucan2.core :as t2])
+  (:import
+   (java.util.concurrent CountDownLatch TimeUnit)))
+
+(set! *warn-on-reflection* true)
+
+;;; ------------------------------------------------ Startup ------------------------------------------------
+
+(defonce ^:private started? (atom false))
+
+(defn start-mq!
+  "Minimal startup: connects to app DB and starts the MQ subsystem.
+  No Jetty, no scheduler, no sample data. Idempotent."
+  []
+  (let [setup-db! (requiring-resolve 'metabase.app-db.core/setup-db!)]
+    (setup-db! :create-sample-content? false))
+  (let [setup-prometheus! (requiring-resolve 'metabase.analytics.prometheus/setup!)]
+    (setup-prometheus!))
+  (require 'metabase.mq.init)
+  (let [startup! (requiring-resolve 'metabase.startup.core/def-startup-logic!)]
+    (startup! :metabase.mq.init/MqInit))
+  (reset! started? true)
+  (println "MQ system started."))
+
+(defn- ensure-started!
+  "Starts the MQ system if not yet started, and restarts any dead poll threads."
+  []
+  (when-not @started?
+    (start-mq!))
+  ;; Restart poll threads if they died (e.g., from InterruptedException during a previous benchmark).
+  ;; start! is idempotent and start-polling! detects dead futures.
+  (let [start-transports! (requiring-resolve 'metabase.mq.impl/start-transports)]
+    (start-transports!)))
+
+;;; ------------------------------------------------ Utilities ------------------------------------------------
+
+(def ^:private shared-queues
+  "Fixed set of 10 shared queue names used across all nodes for cross-node contention testing."
+  (mapv #(keyword "queue" (str "bench-shared-" %)) (range 10)))
+
+(def ^:private shared-topic :topic/bench-shared)
+
+(defn- run-id
+  "Generates an 8-character unique identifier for a benchmark run."
+  []
+  (subs (str (random-uuid)) 0 8))
+
+(defn- make-channel
+  "Creates a unique channel keyword for benchmarking."
+  [transport id suffix]
+  (keyword (name transport) (str "bench-" id "-" suffix)))
+
+(defn- random-shared-queue
+  "Returns a random shared queue name."
+  []
+  (nth shared-queues (rand-int (count shared-queues))))
+
+(defn- percentile
+  "Returns the p-th percentile from a sorted vector (p in [0,1])."
+  [sorted-vec p]
+  (if (empty? sorted-vec)
+    0.0
+    (let [idx (min (dec (count sorted-vec))
+                   (int (* p (count sorted-vec))))]
+      (double (nth sorted-vec idx)))))
+
+(defn- collect-stats
+  "Computes summary statistics from a collection of numeric values.
+  Returns {:count :min :max :avg :p50 :p95 :p99}."
+  [values]
+  (if (empty? values)
+    {:count 0 :min 0.0 :max 0.0 :avg 0.0 :p50 0.0 :p95 0.0 :p99 0.0}
+    (let [sorted (vec (sort values))
+          n      (count sorted)]
+      {:count n
+       :min   (double (first sorted))
+       :max   (double (last sorted))
+       :avg   (/ (reduce + 0.0 sorted) n)
+       :p50   (percentile sorted 0.50)
+       :p95   (percentile sorted 0.95)
+       :p99   (percentile sorted 0.99)})))
+
+(defn- fmt
+  "Formats a number for display."
+  [x]
+  (if (integer? x)
+    (str x)
+    (format "%.2f" (double x))))
+
+(defn- print-table!
+  "Prints a formatted ASCII table."
+  [title headers rows]
+  (let [col-widths (mapv (fn [i]
+                           (apply max
+                                  (count (nth headers i))
+                                  (map #(count (str (nth % i))) rows)))
+                         (range (count headers)))
+        separator  (str "+-" (str/join "-+-" (map #(apply str (repeat % \-)) col-widths)) "-+")
+        fmt-row    (fn [vals]
+                     (str "| "
+                          (str/join " | "
+                                    (map-indexed (fn [i v]
+                                                   (format (str "%-" (nth col-widths i) "s") (str v)))
+                                                 vals))
+                          " |"))]
+    (println)
+    (println title)
+    (println separator)
+    (println (fmt-row headers))
+    (println separator)
+    (doseq [row rows]
+      (println (fmt-row row)))
+    (println separator)
+    (println)))
+
+(defn- cleanup!
+  "Removes all benchmark data for the given run-id."
+  [id]
+  (t2/delete! :queue_message_batch :queue_name [:like (str "bench-" id "%")])
+  (t2/delete! :topic_message_batch :topic_name [:like (str "bench-" id "%")])
+  (doseq [[ch _] @listener/*listeners*]
+    (when (str/includes? (name ch) (str "bench-" id))
+      (mq/unlisten! ch))))
+
+(defn- publish-at-rate!
+  "Publishes `n` messages at the given rate (msgs/sec). `publish-fn` is called with each seq num.
+  If rate is nil, publishes as fast as possible (burst mode)."
+  [n rate publish-fn]
+  (if (nil? rate)
+    (dotimes [i n]
+      (publish-fn i))
+    (let [interval-ms (long (/ 1000 rate))]
+      (dotimes [i n]
+        (let [start (System/currentTimeMillis)]
+          (publish-fn i)
+          (let [elapsed (- (System/currentTimeMillis) start)
+                sleep-ms (- interval-ms elapsed)]
+            (when (pos? sleep-ms)
+              (Thread/sleep (long sleep-ms)))))))))
+
+(def ^:private rate-map
+  {:low    10
+   :medium 100
+   :high   1000
+   :burst  nil})
+
+(defn- rate->msgs-sec [rate]
+  (get rate-map rate rate))
+
+;;; ---------------------------------------- Benchmark 1: Publishing Throughput ----------------------------------------
+
+(defn bench-publish-throughput!
+  "Measures raw publishing speed for different batch sizes.
+  This benchmark bypasses the publish buffer to measure direct DB insert speed.
+  Options:
+    :n           - total messages to publish per batch-size test (default 1000)
+    :batch-sizes - vector of batch sizes to test (default [1 10 100])
+    :transport   - :queue or :topic (default :queue)"
+  ([] (bench-publish-throughput! {}))
+  ([{:keys [n batch-sizes transport]
+     :or   {n 1000 batch-sizes [1 10 100] transport :queue}}]
+   (ensure-started!)
+   (let [id      (run-id)
+         results (atom [])]
+     (try
+       (doseq [bs batch-sizes]
+         (let [channel    (make-channel transport id (str "pub-" bs))
+               iterations (/ n bs)
+               start      (System/nanoTime)]
+           (binding [publish-buffer/*publish-buffer-ms* 0]
+             (dotimes [i iterations]
+               (let [msgs (vec (for [j (range bs)]
+                                 {:seq (+ (* i bs) j) :ts (System/currentTimeMillis)}))]
+                 (if (= transport :queue)
+                   (q.backend/publish! :queue.backend/appdb channel msgs)
+                   (topic.backend/publish! :topic.backend/appdb channel msgs)))))
+           (let [elapsed-ms (/ (- (System/nanoTime) start) 1e6)]
+             (swap! results conj
+                    [(str bs) (fmt (/ n (/ elapsed-ms 1000.0))) (fmt elapsed-ms)]))))
+       (print-table!
+        (str "Publishing Throughput (" (name transport) ", n=" n ")")
+        ["Batch Size" "Msgs/sec" "Total ms"]
+        @results)
+       (finally
+         (cleanup! id))))))
+
+;;; ---------------------------------------- Benchmark 2: End-to-End Latency ----------------------------------------
+
+(defn bench-e2e-latency!
+  "Measures end-to-end latency from publish to listener receipt via the full async pipeline.
+  Uses the real publish API including the publish buffer.
+  Publishes randomly to the 10 shared queues (or the shared topic) so multiple nodes
+  compete for the same messages.
+  Options:
+    :n         - number of messages to publish (default 50)
+    :rate      - :low, :medium, :high, or :burst (default :medium)
+    :transport - :queue or :topic (default :queue)"
+  ([] (bench-e2e-latency! {}))
+  ([{:keys [n rate transport]
+     :or   {n 50 rate :medium transport :queue}}]
+   (ensure-started!)
+   (let [latencies (atom [])
+         received  (atom 0)
+         channels  (if (= transport :queue) shared-queues [shared-topic])]
+     (try
+       ;; Register listeners on all shared channels
+       (doseq [ch channels]
+         (when-not (listener/get-listener ch)
+           (mq/listen! ch {}
+                       (fn [msg]
+                         (let [receive-ns  (System/nanoTime)
+                               publish-ns  (get msg "publish-ns")]
+                           (when publish-ns
+                             (swap! latencies conj (/ (- receive-ns (double publish-ns)) 1e6))))
+                         (swap! received inc)))))
+       (let [msgs-sec (rate->msgs-sec rate)
+             start    (System/nanoTime)]
+         ;; Publish to random shared channels
+         (publish-at-rate!
+          n msgs-sec
+          (fn [i]
+            (let [ch (if (= transport :queue) (random-shared-queue) shared-topic)]
+              (if (= transport :queue)
+                (mq/with-queue ch [q]
+                  (mq/put q {:seq i :publish-ns (System/nanoTime)}))
+                (mq/with-topic ch [t]
+                  (mq/put t {:seq i :publish-ns (System/nanoTime)}))))))
+         ;; Force flush the publish buffer then wait for delivery
+         (publish-buffer/flush-publish-buffer!)
+         ;; Wait until we've received at least n messages or timeout
+         (let [deadline (+ (System/currentTimeMillis)
+                           (long (* 1000 (+ 30 (/ n (max 1 (or msgs-sec 1000)))))))]
+           (while (and (< @received n)
+                       (< (System/currentTimeMillis) deadline))
+             (Thread/sleep (long 100))))
+         (let [elapsed-ms (/ (- (System/nanoTime) start) 1e6)
+               stats      (collect-stats @latencies)]
+           (print-table!
+            (str "End-to-End Latency (" (name transport) ", rate=" (name rate) ", n=" n ")")
+            ["Metric" "Value"]
+            [["Published" (str n)]
+             ["Received" (str @received)]
+             ["Other nodes" (str (max 0 (- n @received)))]
+             ["From others" (str (max 0 (- @received n)))]
+             ["Total ms" (fmt elapsed-ms)]
+             ["P50 ms" (fmt (:p50 stats))]
+             ["P95 ms" (fmt (:p95 stats))]
+             ["P99 ms" (fmt (:p99 stats))]
+             ["Min ms" (fmt (:min stats))]
+             ["Max ms" (fmt (:max stats))]
+             ["Avg ms" (fmt (:avg stats))]])))
+       (finally
+         (doseq [ch channels]
+           (when (listener/get-listener ch)
+             (mq/unlisten! ch))))))))
+
+;;; ---------------------------------------- Benchmark 3: Sustained Throughput ----------------------------------------
+
+(defn bench-sustained-throughput!
+  "Publishes continuously while consuming for a given duration via the full async pipeline.
+  Publishes randomly to the 10 shared queues (or shared topic).
+  Options:
+    :duration-sec - how long to publish (default 15)
+    :rate         - :low, :medium, :high, or :burst (default :medium)
+    :transport    - :queue or :topic (default :queue)"
+  ([] (bench-sustained-throughput! {}))
+  ([{:keys [duration-sec rate transport]
+     :or   {duration-sec 15 rate :medium transport :queue}}]
+   (ensure-started!)
+   (let [channels  (if (= transport :queue) shared-queues [shared-topic])
+         received  (atom 0)
+         latencies (atom [])
+         published (atom 0)
+         stop?     (atom false)]
+     (try
+       ;; Register listeners on all shared channels
+       (doseq [ch channels]
+         (when-not (listener/get-listener ch)
+           (mq/listen! ch {}
+                       (fn [msg]
+                         (let [receive-ns  (System/nanoTime)
+                               publish-ns  (get msg "publish-ns")]
+                           (when publish-ns
+                             (swap! latencies conj (/ (- receive-ns (double publish-ns)) 1e6)))
+                           (swap! received inc))))))
+       (let [msgs-sec   (rate->msgs-sec rate)
+             start      (System/nanoTime)
+             pub-thread (Thread.
+                         (fn []
+                           (loop []
+                             (when-not @stop?
+                               (try
+                                 (let [ch (if (= transport :queue) (random-shared-queue) shared-topic)]
+                                   (if (= transport :queue)
+                                     (mq/with-queue ch [q]
+                                       (mq/put q {:seq @published :publish-ns (System/nanoTime)}))
+                                     (mq/with-topic ch [t]
+                                       (mq/put t {:seq @published :publish-ns (System/nanoTime)}))))
+                                 (swap! published inc)
+                                 (catch Exception _))
+                               (when msgs-sec
+                                 (Thread/sleep (long (/ 1000 msgs-sec))))
+                               (recur)))))]
+         (.start pub-thread)
+         (Thread/sleep (long (* duration-sec 1000)))
+         (reset! stop? true)
+         (.join pub-thread 5000)
+         ;; Flush publish buffer and wait for remaining messages to drain
+         (publish-buffer/flush-publish-buffer!)
+         (let [drain-start (System/currentTimeMillis)]
+           (while (and (< @received @published)
+                       (< (- (System/currentTimeMillis) drain-start) 30000))
+             (publish-buffer/flush-publish-buffer!)
+             (Thread/sleep (long 500))))
+         (let [elapsed-ms  (/ (- (System/nanoTime) start) 1e6)
+               stats       (collect-stats @latencies)]
+           (print-table!
+            (str "Sustained Throughput (" (name transport) ", rate=" (name rate)
+                 ", duration=" duration-sec "s)")
+            ["Metric" "Value"]
+            [["Published" (str @published)]
+             ["Received" (str @received)]
+             ["Other nodes" (str (max 0 (- @published @received)))]
+             ["From others" (str (max 0 (- @received @published)))]
+             ["Duration ms" (fmt elapsed-ms)]
+             ["Actual pub/sec" (fmt (/ @published (/ elapsed-ms 1000.0)))]
+             ["Actual recv/sec" (fmt (/ @received (/ elapsed-ms 1000.0)))]
+             ["P50 latency ms" (fmt (:p50 stats))]
+             ["P95 latency ms" (fmt (:p95 stats))]
+             ["P99 latency ms" (fmt (:p99 stats))]
+             ["Max latency ms" (fmt (:max stats))]])))
+       (finally
+         (doseq [ch channels]
+           (when (listener/get-listener ch)
+             (mq/unlisten! ch))))))))
+
+;;; ---------------------------------------- Benchmark 4: Batch Listener ----------------------------------------
+
+(defn bench-batch-listener!
+  "Compares single-message vs batch listener processing rates.
+  Uses the shared queues so multiple nodes compete.
+  Options:
+    :n           - total messages to publish (default 200)
+    :batch-sizes - vector of max-batch-messages values to test (default [1 10 50])"
+  ([] (bench-batch-listener! {}))
+  ([{:keys [n batch-sizes]
+     :or   {n 200 batch-sizes [1 10 50]}}]
+   (ensure-started!)
+   (let [results (atom [])]
+     (doseq [bs batch-sizes]
+       (let [received (atom 0)]
+         (try
+           ;; Register listeners on all shared queues
+           (doseq [ch shared-queues]
+             (when-not (listener/get-listener ch)
+               (if (= bs 1)
+                 (mq/listen! ch {}
+                             (fn [_msg]
+                               (swap! received inc)))
+                 (mq/batch-listen! ch
+                                   (fn [msgs]
+                                     (swap! received + (count msgs)))
+                                   {:max-batch-messages bs}))))
+           ;; Publish all messages randomly to shared queues
+           (dotimes [i n]
+             (mq/with-queue (random-shared-queue) [q]
+               (mq/put q {:seq i})))
+           (publish-buffer/flush-publish-buffer!)
+           ;; Wait for messages to be consumed
+           (let [start   (System/nanoTime)
+                 deadline (+ (System/currentTimeMillis) 30000)]
+             (while (and (< @received n)
+                         (< (System/currentTimeMillis) deadline))
+               (Thread/sleep (long 100)))
+             (let [elapsed-ms (/ (- (System/nanoTime) start) 1e6)]
+               (swap! results conj
+                      [(str bs) (str @received) (fmt elapsed-ms)
+                       (fmt (/ @received (/ elapsed-ms 1000.0)))])))
+           (finally
+             (doseq [ch shared-queues]
+               (when (listener/get-listener ch)
+                 (mq/unlisten! ch)))))))
+     (print-table!
+      (str "Batch Listener Comparison (n=" n ")")
+      ["Max Batch" "Received" "Process ms" "Msgs/sec"]
+      @results))))
+
+;;; ---------------------------------------- Benchmark 5: Multiple Queues ----------------------------------------
+
+(defn bench-multi-queue!
+  "Measures performance with varying numbers of the shared queues.
+  Tests with subsets of the 10 shared queues to show scaling.
+  Options:
+    :queue-counts   - vector of queue counts to test (default [1 5 10])
+    :msgs-per-queue - messages per queue (default 50)"
+  ([] (bench-multi-queue! {}))
+  ([{:keys [queue-counts msgs-per-queue]
+     :or   {queue-counts [1 5 10] msgs-per-queue 50}}]
+   (ensure-started!)
+   (let [results (atom [])]
+     (doseq [qc queue-counts]
+       (let [channels (subvec shared-queues 0 (min qc (count shared-queues)))
+             total    (* (count channels) msgs-per-queue)
+             received (atom 0)]
+         (try
+           ;; Register listeners
+           (doseq [ch channels]
+             (when-not (listener/get-listener ch)
+               (mq/listen! ch {}
+                           (fn [_msg]
+                             (swap! received inc)))))
+           ;; Publish to the queues
+           (let [start (System/nanoTime)]
+             (doseq [ch channels]
+               (dotimes [i msgs-per-queue]
+                 (mq/with-queue ch [q]
+                   (mq/put q {:seq i}))))
+             (publish-buffer/flush-publish-buffer!)
+             ;; Wait for messages
+             (let [deadline (+ (System/currentTimeMillis) 60000)]
+               (while (and (< @received total)
+                           (< (System/currentTimeMillis) deadline))
+                 (Thread/sleep (long 100))))
+             (let [elapsed-ms (/ (- (System/nanoTime) start) 1e6)]
+               (swap! results conj
+                      [(str (count channels)) (str total) (str @received) (fmt elapsed-ms)
+                       (fmt (/ @received (/ elapsed-ms 1000.0)))])))
+           (finally
+             (doseq [ch channels]
+               (when (listener/get-listener ch)
+                 (mq/unlisten! ch)))))))
+     (print-table!
+      (str "Multiple Queues (msgs-per-queue=" msgs-per-queue ")")
+      ["Queues" "Total Msgs" "Received" "Total ms" "Msgs/sec"]
+      @results))))
+
+;;; ---------------------------------------- Run All ----------------------------------------
+
+(defn run-all-benchmarks!
+  "Runs all benchmark scenarios with default parameters."
+  []
+  (println "=== MQ Performance Benchmarks ===")
+  (println)
+
+  (println "--- 1/5: Publishing Throughput (Queue) ---")
+  (bench-publish-throughput! {:n 1000 :transport :queue})
+
+  (println "--- 1b/5: Publishing Throughput (Topic) ---")
+  (bench-publish-throughput! {:n 1000 :transport :topic})
+
+  (println "--- 2/5: End-to-End Latency ---")
+  (doseq [rate [:low :medium :high :burst]]
+    (bench-e2e-latency! {:n 50 :rate rate :transport :queue}))
+  (doseq [rate [:low :medium :burst]]
+    (bench-e2e-latency! {:n 50 :rate rate :transport :topic}))
+
+  (println "--- 3/5: Sustained Throughput ---")
+  (bench-sustained-throughput! {:duration-sec 15 :rate :medium :transport :queue})
+  (bench-sustained-throughput! {:duration-sec 15 :rate :medium :transport :topic})
+
+  (println "--- 4/5: Batch Listener Comparison ---")
+  (bench-batch-listener! {:n 200})
+
+  (println "--- 5/5: Multiple Queues ---")
+  (bench-multi-queue! {:queue-counts [1 5 10 20] :msgs-per-queue 50})
+
+  (println "=== All benchmarks complete ==="))
+
+;;; ---------------------------------------- Multi-Node Coordination ----------------------------------------
+
+(defn run-all-benchmarks-coordinated!
+  "Waits for a 'go' signal on :topic/bench-coordination, then runs all benchmarks.
+  Use with multiple JVM instances to start benchmarks simultaneously.
+
+  Usage:
+    ;; On each node:
+    (start-mq!)
+    (run-all-benchmarks-coordinated!)
+
+    ;; Then from any node or separate REPL:
+    (signal-go!)"
+  []
+  (let [latch (CountDownLatch. 1)]
+    (mq/listen! :topic/bench-coordination {}
+                (fn [_msg]
+                  (.countDown latch)))
+    (println "Waiting for go signal... (call (signal-go!) from any node)")
+    (.await latch 300 TimeUnit/SECONDS)
+    (mq/unlisten! :topic/bench-coordination)
+    (println "Go signal received! Starting benchmarks...")
+    (run-all-benchmarks!)))
+
+(defn signal-go!
+  "Sends the go signal to all waiting nodes."
+  []
+  (mq/with-topic :topic/bench-coordination [t]
+    (mq/put t {:go true :time (System/currentTimeMillis)}))
+  (println "Go signal sent."))

dev/src/dev/mq_perf_multi.sh

@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+# Runs multiple MQ benchmark nodes against the same database.
+#
+# Usage:
+#   ./dev/src/dev/mq_perf_multi.sh start       # Start 2 nodes (default)
+#   ./dev/src/dev/mq_perf_multi.sh start 4     # Start 4 nodes
+#   ./dev/src/dev/mq_perf_multi.sh go          # Send go signal to all waiting nodes
+#   ./dev/src/dev/mq_perf_multi.sh stop        # Stop all running nodes
+#   ./dev/src/dev/mq_perf_multi.sh logs        # Tail all node logs
+#
+# Each node starts a lightweight JVM with just DB + MQ (no server),
+# waits for a coordinated go signal, then runs all benchmarks.
+
+set -euo pipefail
+cd "$(git rev-parse --show-toplevel)"
+
+LOG_DIR="target/mq-perf-logs"
+PID_DIR="target/mq-perf-pids"
+
+cmd_start() {
+  local num_nodes="${1:-2}"
+  mkdir -p "$LOG_DIR" "$PID_DIR"
+
+  # Clean up stale pids
+  rm -f "$PID_DIR"/*.pid
+
+  echo "Starting $num_nodes MQ benchmark nodes..."
+  echo "Logs in $LOG_DIR/"
+  echo
+
+  for i in $(seq 1 "$num_nodes"); do
+    local log_file="$LOG_DIR/node-$i.log"
+    clojure -M:dev:ee:ee-dev -e "
+      (require 'dev.mq-perf)
+      (dev.mq-perf/start-mq!)
+      (dev.mq-perf/run-all-benchmarks-coordinated!)
+    " > "$log_file" 2>&1 &
+    local pid=$!
+    echo "$pid" > "$PID_DIR/node-$i.pid"
+    echo "  Node $i started (PID $pid, log: $log_file)"
+  done
+
+  echo
+  echo "Nodes are initializing. Once ready, run:"
+  echo "  $0 go"
+}
+
+cmd_go() {
+  echo "Sending go signal..."
+  clojure -M:dev:ee:ee-dev -e "
+    (require 'dev.mq-perf)
+    (dev.mq-perf/start-mq!)
+    (dev.mq-perf/signal-go!)
+  "
+  echo
+  echo "Benchmarks started. Watch progress with:"
+  echo "  $0 logs"
+}
+
+cmd_stop() {
+  if [ ! -d "$PID_DIR" ]; then
+    echo "No nodes running."
+    return
+  fi
+
+  for pid_file in "$PID_DIR"/*.pid; do
+    [ -f "$pid_file" ] || continue
+    local pid
+    pid=$(cat "$pid_file")
+    local name
+    name=$(basename "$pid_file" .pid)
+    if kill -0 "$pid" 2>/dev/null; then
+      kill "$pid"
+      echo "Stopped $name (PID $pid)"
+    else
+      echo "$name (PID $pid) already exited"
+    fi
+    rm -f "$pid_file"
+  done
+}
+
+cmd_logs() {
+  if [ ! -d "$LOG_DIR" ]; then
+    echo "No logs found."
+    return
+  fi
+  tail -f "$LOG_DIR"/node-*.log
+}
+
+case "${1:-}" in
+  start) cmd_start "${2:-2}" ;;
+  go)    cmd_go ;;
+  stop)  cmd_stop ;;
+  logs)  cmd_logs ;;
+  *)
+    echo "Usage: $0 {start [N]|go|stop|logs}"
+    echo
+    echo "  start [N]  Start N benchmark nodes (default 2)"
+    echo "  go         Send go signal to start benchmarks"
+    echo "  stop       Kill all running nodes"
+    echo "  logs       Tail all node logs"
+    exit 1
+    ;;
+esac

dev/src/dev/search_perf.clj

@@ -29,7 +29,6 @@
    [metabase.search.config :as search.config]
    [metabase.search.engine :as search.engine]
    [metabase.search.impl :as search.impl]
-   [metabase.search.ingestion :as search.ingestion]
    [metabase.util :as u]
    [metabase.util.random :as u.random]
    [toucan2.core :as t2]))
@@ -446,9 +445,7 @@
                                              :num-users  num-users
                                              :prefix     prefix})
          user-id (:user-id test-env)
-         ;; Wait for search index to update if using async indexing
-         _ (binding [search.ingestion/*force-sync* true]
-             (Thread/sleep 1000))
+         _ (Thread/sleep 1000) ;; Wait for search index
          queries [;; Exact prefix match
                   (str prefix "_table_00001")
                   ;; Partial prefix match

enterprise/backend/src/metabase_enterprise/action_v2/data_editing.clj

@@ -8,35 +8,26 @@
    [metabase.api.common :as api]
    [metabase.lib.core :as lib]
    [metabase.lib.metadata :as lib.metadata]
+   [metabase.mq.core :as mq]
    [metabase.query-processor.core :as qp]
    ;; legacy usage -- don't do things like this going forward
    ^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.query-processor.store :as qp.store]
    [metabase.util :as u]
-   [metabase.util.queue :as queue]
    [metabase.warehouse-schema.models.field-values :as field-values]
-   [toucan2.core :as t2])
-  (:import
-   (java.util.concurrent ArrayBlockingQueue)))
+   [toucan2.core :as t2]))
 
 (set! *warn-on-reflection* true)
 
-(def ^:private ^ArrayBlockingQueue global-field-value-invalidate-queue
-  "Queue used to recalculate the field values for updated columns in the background."
-  (ArrayBlockingQueue. 1000))
-
-(def ^:dynamic *field-value-invalidate-queue*
-  "A layer of indirection on the actual [[field-value-invalidation-queue]], for testing."
-  nil)
-
 (defn- batch-invalidate-field-values!
   "Recalculate the field values for the given fields."
   [field-batches]
   (->> (t2/select :model/Field :id [:in (into #{} cat field-batches)])
        (run! field-values/create-or-update-full-field-values!)))
 
-(defmethod queue/init-listener! ::FieldValueInvalidation [_]
-  (queue/listen! "field-value-invalidate" global-field-value-invalidate-queue batch-invalidate-field-values!
-                 {:max-batch-messages 10, :max-next-ms 10}))
+(mq/def-listener! :queue/field-value-invalidation
+  {:max-batch-messages 10}
+  [messages]
+  (batch-invalidate-field-values! (into [] cat messages)))
 
 (defn select-table-pk-fields
   "Given a table-id, return the :model/Field instances corresponding to its PK columns. Do not assume any ordering."
@@ -148,8 +139,8 @@
                           (apply concat))]
     ;; Note that for now we only rescan field values when values are *added* and not when they are *removed*.
     (when (seq stale-fields)
-      (let [^ArrayBlockingQueue queue (or *field-value-invalidate-queue* global-field-value-invalidate-queue)]
-        (.offer queue stale-fields)))))
+      (mq/with-queue :queue/field-value-invalidation [q]
+        (mq/put q [stale-fields])))))
 
 ;; TODO this is fairly dirty, would be cleaner to map from db values to de-coerced values via middleware
 ;;      invalidation could perhaps be done in response to effect, or in middleware (to dedupe for chained actions)

enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj

@@ -245,4 +245,4 @@
       ;;       this means that the corresponding entries would have been missing or stale when we indexed them.
       ;;       ideally, we would delay the indexing somehow, or only reindex what we've loaded.
       ;;       while we're figuring that out, here's a crude stopgap.
-        (search/reindex!)))))
+        (search/async-reindex!)))))

enterprise/backend/test/metabase_enterprise/action_v2/api_test.clj

@@ -7,19 +7,17 @@
    [flatland.ordered.map :refer [ordered-map]]
    [metabase-enterprise.action-v2.api]
    [metabase-enterprise.action-v2.coerce :as coerce]
-   [metabase-enterprise.action-v2.data-editing :as data-editing]
    [metabase-enterprise.action-v2.test-util :as action-v2.tu]
    [metabase.actions.test-util :as actions.tu]
    [metabase.driver :as driver]
    [metabase.driver.sql-jdbc :as sql-jdbc]
    [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
+   [metabase.mq.test-util :as mq.tu]
    [metabase.query-processor.test :as qp]
    [metabase.test :as mt]
    [metabase.test.data.sql :as sql.tx]
    [metabase.warehouse-schema.models.field-values :as field-values]
-   [toucan2.core :as t2])
-  (:import
-   (java.util.concurrent ArrayBlockingQueue)))
+   [toucan2.core :as t2]))
 
 (set! *warn-on-reflection* true)
 
@@ -532,40 +530,26 @@
   (mt/with-premium-features #{actions-feature-flag}
     (mt/test-drivers (mt/normal-drivers-with-feature :actions/data-editing)
       (action-v2.tu/with-test-tables! [table-id [{:id 'auto-inc-type, :n [:text]} {:primary-key [:id]}]]
-        (let [field-id         (t2/select-one-fn :id :model/Field :table_id table-id :name "n")
-              _                (t2/update! :model/Field {:id field-id} {:semantic_type "type/Category"})
-              field-values     #(vec (:values (field-values/get-latest-full-field-values field-id)))
-              test-queue       (ArrayBlockingQueue. 100)
-              create!          #(action-v2.tu/create-rows! table-id %)
-              update!          #(action-v2.tu/update-rows! table-id %)
-              process-queue!   (fn []
-                                 (when-let [field-ids (.poll test-queue)]
-                                   (#'data-editing/batch-invalidate-field-values! [field-ids])
-                                   (recur)))]
-          (binding [data-editing/*field-value-invalidate-queue* test-queue]
+        (let [field-id     (t2/select-one-fn :id :model/Field :table_id table-id :name "n")
+              _            (t2/update! :model/Field {:id field-id} {:semantic_type "type/Category"})
+              field-values #(vec (:values (field-values/get-latest-full-field-values field-id)))
+              create!      #(action-v2.tu/create-rows! table-id %)
+              update!      #(action-v2.tu/update-rows! table-id %)]
+          (mq.tu/with-sync-mq
             (is (= [] (field-values)))
 
             (create! [{:n "a"}])
-            (is (pos? (.size test-queue)))
-            (process-queue!)
             (is (= ["a"] (field-values)))
 
             (create! [{:n "b"} {:n "c"}])
-            (is (pos? (.size test-queue)))
-            (process-queue!)
             (is (= ["a" "b" "c"] (field-values)))
 
             (update! [{:id 2, :n "d"}])
-            (is (pos? (.size test-queue)))
-            (process-queue!)
             (is (= ["a" "c" "d"] (field-values)))
 
+            ;; Creating a duplicate value should not trigger invalidation
             (create! [{:n "a"}])
-            (is (zero? (.size test-queue)))
-            (process-queue!)
             (update! [{:id 1, :n "e"}])
-            (is (pos? (.size test-queue)))
-            (process-queue!)
             (is (= ["a" "c" "d" "e"] (field-values)))))))))
 
 (deftest execute-form-built-in-table-action-test

enterprise/backend/test/metabase_enterprise/agent_api/api_test.clj

@@ -6,10 +6,17 @@
    [metabase-enterprise.sso.test-setup :as sso.test-setup]
    [metabase.lib.core :as lib]
    [metabase.lib.metadata :as lib.metadata]
+   [metabase.lib.normalize :as lib.normalize]
+   [metabase.metabot.settings] ; for setting definitions
+   [metabase.metabot.tools.util :as metabot.tools.u]
+   [metabase.search.test-util :as search.tu]
    [metabase.test :as mt]
    [metabase.test.fixtures :as fixtures]
    [metabase.test.http-client :as client]
    [metabase.test.util :as tu]
+   [metabase.util :as u]
+   [metabase.util.json :as json]
+   [metabase.warehouse-schema.models.field-values :as field-values]
    [toucan2.core :as t2]))
 
 (set! *warn-on-reflection* true)
@@ -132,6 +139,192 @@
             (is (= "You don't have permissions to do that."
                    (agent-client :rasta :get 403 (str "agent/v1/metric/" (:id metric)))))))))))
 
+(deftest get-table-details-test
+  (with-agent-api-setup!
+    (testing "Returns table details for valid table ID"
+      (let [table-id (mt/id :orders)]
+        (is (=? {:type           "table"
+                 :id             table-id
+                 :name           "ORDERS"
+                 :display_name   "Orders"
+                 :database_id    (mt/id)
+                 :fields         sequential?
+                 :related_tables sequential?}
+                (agent-client :rasta :get 200 (str "agent/v1/table/" table-id))))))
+
+    (testing "Returns 404 for non-existent table"
+      (is (= "Not found."
+             (agent-client :rasta :get 404 "agent/v1/table/999999"))))
+
+    (testing "Respects query parameters"
+      (let [table-id (mt/id :orders)]
+        (is (=? {:type   "table"
+                 :id     table-id
+                 :fields empty?}
+                (agent-client :rasta :get 200
+                              (str "agent/v1/table/" table-id "?with-fields=false&with-related-tables=false"))))))
+
+    (testing "Field values are excluded by default"
+      (let [table-id (mt/id :orders)
+            table    (agent-client :rasta :get 200 (str "agent/v1/table/" table-id))]
+        (is (every? #(nil? (:field_values %)) (:fields table)))))
+
+    (testing "Field values are included when explicitly requested"
+      (let [table-id (mt/id :orders)
+            table    (agent-client :rasta :get 200 (str "agent/v1/table/" table-id "?with-field-values=true"))]
+        (is (some #(seq (:field_values %)) (:fields table)))))))
+
+(deftest get-metric-details-test
+  (with-agent-api-setup!
+    (mt/with-temp [:model/Card metric {:name          "Test Metric"
+                                       :type          :metric
+                                       :database_id   (mt/id)
+                                       :dataset_query (orders-count-query)}]
+      (testing "Returns metric details for valid metric ID"
+        (is (=? {:type                 "metric"
+                 :id                   (:id metric)
+                 :name                 "Test Metric"
+                 :queryable_dimensions sequential?}
+                (agent-client :rasta :get 200 (str "agent/v1/metric/" (:id metric))))))
+
+      (testing "Respects query parameters"
+        (is (=? {:type "metric"
+                 :id   (:id metric)}
+                (agent-client :rasta :get 200
+                              (str "agent/v1/metric/" (:id metric)
+                                   "?with-queryable-dimensions=false&with-field-values=false")))))
+
+      (testing "Returns 404 for non-existent metric"
+        (is (= "Not found."
+               (agent-client :rasta :get 404 "agent/v1/metric/999999")))))))
+
+(defn- ensure-fresh-field-values!
+  "Ensure field values exist for a field by deleting any existing ones and recreating them."
+  [field-id]
+  (t2/delete! :model/FieldValues :field_id field-id :type :full)
+  (field-values/get-or-create-full-field-values! (t2/select-one :model/Field :id field-id)))
+
+(defn- visible-field-id
+  "Find the field-id string for a field by display name within a table's visible columns."
+  [table-id field-display-name]
+  (let [mp            (mt/metadata-provider)
+        query         (lib/query mp (lib.metadata/table mp table-id))
+        field-prefix  (metabot.tools.u/table-field-id-prefix table-id)
+        visible-cols  (lib/visible-columns query)]
+    (->> (keep-indexed (fn [i col]
+                         (when (= (lib/display-name query col) field-display-name)
+                           (str field-prefix i)))
+                       visible-cols)
+         first)))
+
+(deftest get-table-field-values-test
+  (with-agent-api-setup!
+    ;; Ensure field values exist for the field we'll test
+    (ensure-fresh-field-values! (mt/id :people :state))
+
+    (testing "Returns field statistics and values with default limit of 30"
+      (let [table-id (mt/id :people)
+            field-id (visible-field-id table-id "State")]
+        (is (some? field-id) "Should find the State field")
+        (let [result (agent-client :crowberto :get 200
+                                   (format "agent/v1/table/%d/field/%s/values" table-id field-id))]
+          (is (=? {:value_metadata {:statistics   {:distinct-count 49}
+                                    :field_values sequential?}}
+                  result))
+          (is (<= (count (:values result)) 30) "Should apply default limit of 30"))))
+
+    (testing "Respects explicit limit parameter"
+      (let [table-id (mt/id :people)
+            field-id (visible-field-id table-id "State")]
+        (is (=? {:value_metadata {:field_values #(= 5 (count %))}}
+                (agent-client :crowberto :get 200
+                              (format "agent/v1/table/%d/field/%s/values?limit=5" table-id field-id))))))
+
+    (testing "Returns 404 for non-existent table"
+      (is (= "Not found."
+             (agent-client :crowberto :get 404 "agent/v1/table/999999/field/t999999-0/values"))))
+
+    (testing "Returns 400 for invalid field-id format"
+      (let [table-id (mt/id :people)]
+        (is (= "Invalid field_id format: not-a-valid-id"
+               (agent-client :crowberto :get 400 (format "agent/v1/table/%d/field/not-a-valid-id/values" table-id))))))))
+
+(deftest search-test
+  (with-agent-api-setup!
+    (search.tu/with-new-search-if-available-otherwise-legacy
+      (mt/with-temp [:model/Table _ {:name "AgentSearchTestTable"}]
+        (testing "Returns search results for term queries"
+          (is (=? {:data        [{:type "table" :name "AgentSearchTestTable"}]
+                   :total_count 1}
+                  (agent-client :rasta :post 200 "agent/v1/search"
+                                {:term_queries ["AgentSearchTestTable"]}))))))))
+
+(defn- decode-query
+  "Decode a base64-encoded query response to a Clojure map, then normalize it so lib functions work."
+  [response]
+  (-> response :query u/decode-base64 json/decode+kw lib.normalize/normalize))
+
+(deftest construct-query-test
+  (with-agent-api-setup!
+    (testing "Constructs a simple query from a table"
+      (let [table-id (mt/id :orders)
+            response (agent-client :rasta :post 200 "agent/v1/construct-query"
+                                   {:table_id table-id})]
+        (is (string? (:query response)) "Response should contain a query string")
+        (let [decoded (decode-query response)]
+          (is (= :mbql/query (lib/normalized-query-type decoded)))
+          (is (= (mt/id) (lib/database-id decoded)))
+          (is (= (mt/id :orders) (lib/primary-source-table-id decoded))))))
+
+    (testing "Applies default limit of 200 when no limit is specified"
+      (let [table-id (mt/id :orders)
+            response (agent-client :rasta :post 200 "agent/v1/construct-query"
+                                   {:table_id table-id})
+            decoded  (decode-query response)]
+        (is (= 200 (lib/current-limit decoded)))))
+
+    (testing "Respects explicit limit"
+      (let [table-id (mt/id :orders)
+            response (agent-client :rasta :post 200 "agent/v1/construct-query"
+                                   {:table_id table-id
+                                    :limit    10})
+            decoded  (decode-query response)]
+        (is (= 10 (lib/current-limit decoded)))))
+
+    (testing "Returns 404 for non-existent table"
+      (is (= "No table found with table_id 999999"
+             (agent-client :rasta :post 404 "agent/v1/construct-query"
+                           {:table_id 999999}))))))
+
+(deftest execute-query-test
+  (with-agent-api-setup!
+    (testing "Executes a query and returns results with column metadata"
+      (let [table-id       (mt/id :orders)
+            construct-resp (agent-client :rasta :post 200 "agent/v1/construct-query"
+                                         {:table_id table-id
+                                          :limit    5})
+            ;; Streaming response returns 202 (accepted) since it starts streaming before completion
+            execute-resp   (agent-client :rasta :post 202 "agent/v1/execute"
+                                         {:query (:query construct-resp)})]
+        (is (=? {:status    "completed"
+                 :row_count 5
+                 :data      {:cols (fn [cols]
+                                     (and (seq cols)
+                                          (every? :name cols)
+                                          (every? :base_type cols)))
+                             :rows (fn [rows] (= 5 (count rows)))}}
+                execute-resp))))
+
+    (testing "Enforces agent query row limit even when query specifies a higher limit"
+      (let [table-id       (mt/id :orders)
+            construct-resp (agent-client :rasta :post 200 "agent/v1/construct-query"
+                                         {:table_id table-id
+                                          :limit    300})
+            execute-resp   (agent-client :rasta :post 202 "agent/v1/execute"
+                                         {:query (:query construct-resp)})]
+        (is (=? {:status "completed" :row_count 200}
+                execute-resp))))))
+
 (deftest execute-query-records-query-execution-test
   (with-agent-api-setup!
     (testing "Executed queries are recorded with :agent context"

enterprise/backend/test/metabase_enterprise/cache/strategies_test.clj

@@ -85,8 +85,7 @@
                             (-> (qp/process-query query) (dissoc :data))))))))))))))
 
 (deftest e2e-advanced-caching
-  (binding [search.ingestion/*force-sync* true
-            search.ingestion/*disable-updates* true]
+  (binding [search.ingestion/*disable-updates* true]
     (mt/with-empty-h2-app-db!
       (mt/with-premium-features #{:cache-granular-controls}
         (mt/dataset (mt/dataset-definition "caching1"

enterprise/backend/test/metabase_enterprise/data_studio/search_test.clj

@@ -6,155 +6,158 @@
    [metabase.collections.models.collection :as collection]
    [metabase.permissions.core :as perms]
    [metabase.permissions.models.data-permissions :as data-perms]
-   [metabase.search.core :as search]
+   [metabase.search.impl :as search.impl]
+   [metabase.search.test-util :as search.tu]
    [metabase.test :as mt]
    [metabase.util.random :refer [random-name]]))
 
 (deftest published-table-collection-filter-test
-  (mt/with-premium-features #{:library}
-    (testing "Published tables are included in collection-filtered search"
-      (mt/with-temp
-        [:model/Collection {parent-coll :id} {:name "Parent Collection" :location "/"}
-         :model/Collection {child-coll :id}  {:name "Child Collection"
-                                              :location (collection/location-path parent-coll)}
-         :model/Card       {card-1 :id}      {:collection_id parent-coll :name "Parent Card"}
-         :model/Table      {table-1 :id}     {:name "Published Table" :is_published true :collection_id parent-coll}
-         :model/Table      {table-2 :id}     {:name "Child Published Table" :is_published true :collection_id child-coll}
-         :model/Table      {table-3 :id}     {:name "Unpublished Table" :is_published false}
-         :model/Table      {table-4 :id}     {:name "Root Published Table" :is_published true :collection_id nil}]
+  (search.tu/with-temp-index-table-for-http
+    (mt/with-premium-features #{:library}
+      (testing "Published tables are included in collection-filtered search"
+        (mt/with-temp
+          [:model/Collection {parent-coll :id} {:name "Parent Collection" :location "/"}
+           :model/Collection {child-coll :id}  {:name "Child Collection"
+                                                :location (collection/location-path parent-coll)}
+           :model/Card       {card-1 :id}      {:collection_id parent-coll :name "Parent Card"}
+           :model/Table      {table-1 :id}     {:name "Published Table" :is_published true :collection_id parent-coll}
+           :model/Table      {table-2 :id}     {:name "Child Published Table" :is_published true :collection_id child-coll}
+           :model/Table      {table-3 :id}     {:name "Unpublished Table" :is_published false}
+           :model/Table      {table-4 :id}     {:name "Root Published Table" :is_published true :collection_id nil}]
         ;; Initialize search index for appdb engine
-        (search/reindex! {:async? false :in-place? true})
-        (doseq [engine ["in-place" "appdb"]]
-          (testing (str "with engine " engine)
-            (testing "Global search"
-              (let [results (mt/user-http-request :crowberto :get 200 "search" :search_engine engine)]
-                (is (=? [{:collection {:authority_level nil, :id parent-coll, :name "Parent Collection", :type nil}
-                          :database_id (mt/id)
-                          :id table-1
-                          :model "table"
-                          :name "Published Table"
-                          :table_id table-1
-                          :table_name "Published Table"}
-                         {:collection {:authority_level nil, :id child-coll, :name "Child Collection", :type nil}
-                          :database_id (mt/id)
-                          :id table-2
-                          :model "table"
-                          :name "Child Published Table"
-                          :table_id table-2
-                          :table_name "Child Published Table"}
-                         {:database_id (mt/id)
-                          :id table-3
-                          :model "table"
-                          :name "Unpublished Table"
-                          :table_id table-3
-                          :table_name "Unpublished Table"}
-                         {:collection {:authority_level nil, :id "root", :name "Our analytics", :type nil}
-                          :database_id (mt/id)
-                          :id table-4
-                          :model "table"
-                          :name "Root Published Table"
-                          :table_id table-4
-                          :table_name "Root Published Table"}]
-                        (->> (:data results)
-                             (filter (comp #{table-1 table-2 table-3 table-4} :id))
-                             (sort-by :id))))))
-            (testing "Filter by parent collection returns published tables in that collection and descendants"
-              (let [results (mt/user-http-request :crowberto :get 200 "search" :collection parent-coll :search_engine engine)]
-                (is (=? {:collection {:authority_level nil, :id parent-coll, :name "Parent Collection", :type nil}
-                         :database_id (mt/id)
-                         :id table-1
-                         :model "table"
-                         :name "Published Table"
-                         :table_id table-1
-                         :table_name "Published Table"}
-                        (m/find-first (comp #{table-1} :id) (:data results))))
-                (is (contains? (set (map :id (:data results))) table-1)
-                    "Published table in parent collection should be included")
-                (is (contains? (set (map :id (:data results))) table-2)
-                    "Published table in child collection should be included")
-                (is (contains? (set (map :id (:data results))) card-1)
-                    "Card in parent collection should be included")))
-            (testing "Unpublished tables are not included in collection filter results"
-              (let [results (mt/user-http-request :crowberto :get 200 "search" :collection parent-coll :search_engine engine)]
-                (is (not (contains? (set (map :id (:data results))) table-3))
-                    "Unpublished table should not be in results")))))))
-    (testing "Published tables in root collection (collection_id=nil) appear in unfiltered search"
-      (mt/with-temp
-        [:model/Table {root-table :id} {:name "Root Published Searchable" :is_published true :collection_id nil}]
-        (search/reindex! {:async? false :in-place? true})
-        (doseq [engine ["in-place" "appdb"]]
-          (testing (str "with engine " engine)
-            (let [results (mt/user-http-request :crowberto :get 200 "search" :q "Root Published Searchable" :search_engine engine)]
-              (is (some #(= root-table (:id %)) (:data results))
-                  "Root published table should appear in search results"))))))
-    (testing "Published tables appear as collection items, unpublished as database items"
-      (mt/with-temp
-        [:model/Database   {db-id :id}       {:name "Context Test DB"}
-         :model/Collection {coll-id :id}     {:name "Context Test Collection" :location "/"}
-         :model/Table      {pub-table :id}   {:name "ContextTestTablePub"
-                                              :db_id db-id
-                                              :is_published true
-                                              :collection_id coll-id}
-         :model/Table      {unpub-table :id} {:name "ContextTestTableUnpub"
-                                              :db_id db-id
-                                              :is_published false}]
-        (search/reindex! {:async? false :in-place? true})
-        (doseq [engine ["in-place" "appdb"]]
-          (testing (str "with engine " engine)
-            (let [results (mt/user-http-request :crowberto :get 200 "search"
-                                                :q "Context" :models "table" :search_engine engine)
-                  our-result (->> (filter (comp #{pub-table unpub-table} :id) (:data results))
-                                  (sort-by :id))]
-              (testing "each table appears once"
-                (is (=? [{:id pub-table
-                          :model "table"
-                          :name "ContextTestTablePub"
-                          :collection {:id coll-id}
-                          :database_name "Context Test DB"}
-                         {:id unpub-table
-                          :model "table"
-                          :name "ContextTestTableUnpub"
-                          :collection {:id nil}
-                          :database_name "Context Test DB"}]
-                        our-result))))))))
-    (testing "Users without query permissions cannot discover tables in search"
-      (let [search-term (random-name)
-            name-1 (str search-term " 1")
-            name-2 (str search-term " 2")]
-        (mt/with-no-data-perms-for-all-users!
-          (mt/with-temp [:model/Table {unpub-table :id} {:collection_id nil :is_published false :name name-1}
-                         :model/Table {pub-table :id} {:collection_id nil :is_published true :name name-2}]
+          (search.impl/sync-reindex! {:in-place? true})
+          (doseq [engine ["in-place" "appdb"]]
+            (testing (str "with engine " engine)
+              (testing "Global search"
+                (let [results (mt/user-http-request :crowberto :get 200 "search" :search_engine engine)]
+                  (is (=? [{:collection {:authority_level nil, :id parent-coll, :name "Parent Collection", :type nil}
+                            :database_id (mt/id)
+                            :id table-1
+                            :model "table"
+                            :name "Published Table"
+                            :table_id table-1
+                            :table_name "Published Table"}
+                           {:collection {:authority_level nil, :id child-coll, :name "Child Collection", :type nil}
+                            :database_id (mt/id)
+                            :id table-2
+                            :model "table"
+                            :name "Child Published Table"
+                            :table_id table-2
+                            :table_name "Child Published Table"}
+                           {:database_id (mt/id)
+                            :id table-3
+                            :model "table"
+                            :name "Unpublished Table"
+                            :table_id table-3
+                            :table_name "Unpublished Table"}
+                           {:collection {:authority_level nil, :id "root", :name "Our analytics", :type nil}
+                            :database_id (mt/id)
+                            :id table-4
+                            :model "table"
+                            :name "Root Published Table"
+                            :table_id table-4
+                            :table_name "Root Published Table"}]
+                          (->> (:data results)
+                               (filter (comp #{table-1 table-2 table-3 table-4} :id))
+                               (sort-by :id))))))
+              (testing "Filter by parent collection returns published tables in that collection and descendants"
+                (let [results (mt/user-http-request :crowberto :get 200 "search" :collection parent-coll :search_engine engine)]
+                  (is (=? {:collection {:authority_level nil, :id parent-coll, :name "Parent Collection", :type nil}
+                           :database_id (mt/id)
+                           :id table-1
+                           :model "table"
+                           :name "Published Table"
+                           :table_id table-1
+                           :table_name "Published Table"}
+                          (m/find-first (comp #{table-1} :id) (:data results))))
+                  (is (contains? (set (map :id (:data results))) table-1)
+                      "Published table in parent collection should be included")
+                  (is (contains? (set (map :id (:data results))) table-2)
+                      "Published table in child collection should be included")
+                  (is (contains? (set (map :id (:data results))) card-1)
+                      "Card in parent collection should be included")))
+              (testing "Unpublished tables are not included in collection filter results"
+                (let [results (mt/user-http-request :crowberto :get 200 "search" :collection parent-coll :search_engine engine)]
+                  (is (not (contains? (set (map :id (:data results))) table-3))
+                      "Unpublished table should not be in results")))))))
+      (testing "Published tables in root collection (collection_id=nil) appear in unfiltered search"
+        (mt/with-temp
+          [:model/Table {root-table :id} {:name "Root Published Searchable" :is_published true :collection_id nil}]
+          (search.impl/sync-reindex! {:in-place? true})
+          (doseq [engine ["in-place" "appdb"]]
+            (testing (str "with engine " engine)
+              (let [results (mt/user-http-request :crowberto :get 200 "search" :q "Root Published Searchable" :search_engine engine)]
+                (is (some #(= root-table (:id %)) (:data results))
+                    "Root published table should appear in search results"))))))
+      (testing "Published tables appear as collection items, unpublished as database items"
+        (mt/with-temp
+          [:model/Database   {db-id :id}       {:name "Context Test DB"}
+           :model/Collection {coll-id :id}     {:name "Context Test Collection" :location "/"}
+           :model/Table      {pub-table :id}   {:name "ContextTestTablePub"
+                                                :db_id db-id
+                                                :is_published true
+                                                :collection_id coll-id}
+           :model/Table      {unpub-table :id} {:name "ContextTestTableUnpub"
+                                                :db_id db-id
+                                                :is_published false}]
+          (search.impl/sync-reindex! {:in-place? true})
+          (doseq [engine ["in-place" "appdb"]]
+            (testing (str "with engine " engine)
+              (let [results (mt/user-http-request :crowberto :get 200 "search"
+                                                  :q "Context" :models "table" :search_engine engine)
+                    our-result (->> (filter (comp #{pub-table unpub-table} :id) (:data results))
+                                    (sort-by :id))]
+                (testing "each table appears once"
+                  (is (=? [{:id pub-table
+                            :model "table"
+                            :name "ContextTestTablePub"
+                            :collection {:id coll-id}
+                            :database_name "Context Test DB"}
+                           {:id unpub-table
+                            :model "table"
+                            :name "ContextTestTableUnpub"
+                            :collection {:id nil}
+                            :database_name "Context Test DB"}]
+                          our-result))))))))
+      (testing "Users without query permissions cannot discover tables in search"
+        (let [search-term (random-name)
+              name-1 (str search-term " 1")
+              name-2 (str search-term " 2")]
+          (mt/with-no-data-perms-for-all-users!
+            (mt/with-temp [:model/Table {unpub-table :id} {:collection_id nil :is_published false :name name-1}
+                           :model/Table {pub-table :id} {:collection_id nil :is_published true :name name-2}]
             ;; Explicitly deny both view and query perms so this assertion doesn't depend on global defaults.
-            (data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/view-data :blocked)
-            (data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/create-queries :no)
-            (doseq [engine ["in-place" "appdb"]]
-              (search/reindex! {:async? false :in-place? true})
-              (mt/with-non-admin-groups-no-root-collection-perms
-                (let [results (mt/user-http-request :rasta :get 200 "search"
-                                                    :q search-term :models "table" :search_engine engine)
-                      our-result (->> (filter (comp #{pub-table unpub-table} :id) (:data results))
-                                      (sort-by :id)
-                                      (map #(select-keys % [:id])))]
-                  (testing "no tables appear"
-                    (is (= []
-                           our-result))))))))))))
+              (data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/view-data :blocked)
+              (data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/create-queries :no)
+              (doseq [engine ["in-place" "appdb"]]
+                (search.impl/sync-reindex! {:in-place? true})
+                (mt/with-non-admin-groups-no-root-collection-perms
+                  (let [results (mt/user-http-request :rasta :get 200 "search"
+                                                      :q search-term :models "table" :search_engine engine)
+                        our-result (->> (filter (comp #{pub-table unpub-table} :id) (:data results))
+                                        (sort-by :id)
+                                        (map #(select-keys % [:id])))]
+                    (testing "no tables appear"
+                      (is (= []
+                             our-result)))))))))))))
 
 (deftest unpublished-table-visible-with-data-perms-test
-  (testing "Unpublished tables are discoverable when the user has data/query permissions"
-    (let [search-term (random-name)
-          table-name  (str search-term " unpublished")]
-      (mt/with-no-data-perms-for-all-users!
-        (mt/with-temp [:model/Table {unpub-table :id} {:collection_id nil :is_published false :name table-name}]
+  (search.tu/with-temp-index-table-for-http
+    (testing "Unpublished tables are discoverable when the user has data/query permissions"
+      (let [search-term (random-name)
+            table-name  (str search-term " unpublished")]
+        (mt/with-no-data-perms-for-all-users!
+          (mt/with-temp [:model/Table {unpub-table :id} {:collection_id nil :is_published false :name table-name}]
           ;; Explicitly grant data/query perms so this does not rely on global test defaults.
-          (data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/view-data :unrestricted)
-          (data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/create-queries :query-builder)
-          (doseq [engine ["in-place" "appdb"]]
-            (search/reindex! {:async? false :in-place? true})
-            (mt/with-non-admin-groups-no-root-collection-perms
-              (let [result-ids (->> (mt/user-http-request :rasta :get 200 "search"
-                                                          :q search-term :models "table" :search_engine engine)
-                                    :data
-                                    (map :id)
-                                    (into #{}))]
-                (is (contains? result-ids unpub-table)
-                    (format "Unpublished table should be visible with data permissions (engine=%s)" engine))))))))))
+            (data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/view-data :unrestricted)
+            (data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/create-queries :query-builder)
+            (doseq [engine ["in-place" "appdb"]]
+              (search.impl/sync-reindex! {:in-place? true})
+              (mt/with-non-admin-groups-no-root-collection-perms
+                (let [result-ids (->> (mt/user-http-request :rasta :get 200 "search"
+                                                            :q search-term :models "table" :search_engine engine)
+                                      :data
+                                      (map :id)
+                                      (into #{}))]
+                  (is (contains? result-ids unpub-table)
+                      (format "Unpublished table should be visible with data permissions (engine=%s)" engine)))))))))))

enterprise/backend/test/metabase_enterprise/metabot/tools/semantic_search_test.clj

@@ -19,9 +19,9 @@
   `(mt/with-additional-premium-features #{:semantic-search}
      (when (search.engine/supported-engine? :search.engine/semantic)
        (semantic.tu/with-mock-embeddings ~mock-embeddings
-         (binding [search.ingestion/*disable-updates* false
-                   search.ingestion/*force-sync* true]
-           ~@body)))))
+         (binding [search.ingestion/*disable-updates* false]
+           (search.tu/with-sync-search-indexing
+             ~@body))))))
 
 (defmacro with-and-without-semantic-search! [mock-embeddings & body]
   `(do ~@body (with-semantic-search-if-available! ~mock-embeddings ~@body)))

enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj

@@ -22,7 +22,7 @@
 ;; `reindex!` below is ok in a parallel test since it's not actually executing anything
 #_{:clj-kondo/ignore [:metabase/validate-deftest]}
 (use-fixtures :each (fn [f]
-                      (mt/with-dynamic-fn-redefs [search/reindex! (constantly nil)]
+                      (mt/with-dynamic-fn-redefs [search/async-reindex! (constantly nil)]
                         (test-helpers/clean-remote-sync-state f))))
 
 ;; import! tests

enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj

@@ -34,7 +34,7 @@
 
 #_{:clj-kondo/ignore [:metabase/validate-deftest]}
 (use-fixtures :each (fn [f]
-                      (mt/with-dynamic-fn-redefs [search/reindex! (constantly nil)]
+                      (mt/with-dynamic-fn-redefs [search/async-reindex! (constantly nil)]
                         (clean-remote-sync-state f))))
 
 ;;; ------------------------------------------- Event Handler Tests -------------------------------------------

enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj

@@ -23,7 +23,7 @@
 
 #_{:clj-kondo/ignore [:metabase/validate-deftest]}
 (use-fixtures :each (fn [f]
-                      (mt/with-dynamic-fn-redefs [search/reindex! (constantly nil)]
+                      (mt/with-dynamic-fn-redefs [search/async-reindex! (constantly nil)]
                         (test-helpers/clean-remote-sync-state f))))
 
 (deftest transform-event-creates-sync-object-when-setting-enabled-test

enterprise/backend/test/metabase_enterprise/semantic_search/api_test.clj

@@ -8,6 +8,8 @@
    [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
    [metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]
    [metabase-enterprise.semantic-search.test-util :as semantic.tu]
+   [metabase.mq.test-util :as mq.tu]
+   [metabase.search.impl :as search.impl]
    [metabase.search.ingestion :as search.ingestion]
    [metabase.test :as mt]))
 
@@ -61,36 +63,38 @@
   (testing "POST /api/search/re-init with semantic search support"
     (mt/with-premium-features #{:semantic-search}
       (semantic.tu/with-test-db! {:mode :mock-indexed}
-        (let [original-index      semantic.tu/mock-index
-              original-table-name (:table-name original-index)
-              new-index           (with-redefs [semantic.index/model-table-suffix (constantly 345)]
-                                    (#'semantic.pgvector-api/fresh-index semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model :force-reset? true))
-              new-table-name      (:table-name new-index)
-              pgvector (semantic.env/get-pgvector-datasource!)]
+        (mq.tu/with-sync-mq
+          (let [original-index      semantic.tu/mock-index
+                original-table-name (:table-name original-index)
+                new-index           (with-redefs [semantic.index/model-table-suffix (constantly 345)]
+                                      (#'semantic.pgvector-api/fresh-index semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model :force-reset? true))
+                new-table-name      (:table-name new-index)
+                pgvector (semantic.env/get-pgvector-datasource!)]
 
-          (is (semantic.tu/table-exists-in-db? original-table-name))
-          (is (not (semantic.tu/table-exists-in-db? new-table-name)))
-
-          (let [best-index (semantic.index-metadata/find-compatible-index! pgvector semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model)]
-            (is (=? original-index (:index best-index)))
-            (is (:active best-index)))
-
-          (testing "re-init creates the new index"
-            (with-redefs [semantic.index/model-table-suffix (constantly 345)]
-              (let [response (mt/user-http-request :crowberto :post 200 "search/re-init")]
-                (is (contains? response :message))))
-
-            (is (not= original-table-name new-table-name))
             (is (semantic.tu/table-exists-in-db? original-table-name))
-            (is (semantic.tu/table-exists-in-db? new-table-name))
+            (is (not (semantic.tu/table-exists-in-db? new-table-name)))
 
-            (is (zero? (semantic.tu/index-count new-index))))
+            (let [best-index (semantic.index-metadata/find-compatible-index! pgvector semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model)]
+              (is (=? original-index (:index best-index)))
+              (is (:active best-index)))
 
-          (let [best-index (semantic.index-metadata/find-compatible-index! pgvector semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model)]
-            (is (=? new-index (:index best-index)))
-            (is (:active best-index)))
+            (testing "re-init creates the new index"
+              (with-redefs [semantic.index/model-table-suffix (constantly 345)]
+                ;; Call init directly instead of via HTTP — with-redefs is thread-local
+                ;; and not visible to the HTTP handler thread.
+                (search.impl/sync-init-index! {:force-reset? true}))
 
-          (testing "Index can be populated after re-init"
-            (semantic.tu/upsert-index! (semantic.tu/mock-documents) :index new-index)
+              (is (not= original-table-name new-table-name))
+              (is (semantic.tu/table-exists-in-db? original-table-name))
+              (is (semantic.tu/table-exists-in-db? new-table-name))
 
-            (is (pos? (semantic.tu/index-count new-index)))))))))
+              (is (zero? (semantic.tu/index-count new-index))))
+
+            (let [best-index (semantic.index-metadata/find-compatible-index! pgvector semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model)]
+              (is (=? new-index (:index best-index)))
+              (is (:active best-index)))
+
+            (testing "Index can be populated after re-init"
+              (semantic.tu/upsert-index! (semantic.tu/mock-documents) :index new-index)
+
+              (is (pos? (semantic.tu/index-count new-index))))))))))

enterprise/backend/test/metabase_enterprise/serialization/api_test.clj

@@ -9,6 +9,7 @@
    [metabase.analytics.snowplow-test :as snowplow-test]
    [metabase.models.serialization :as serdes]
    [metabase.search.core :as search]
+   [metabase.search.impl :as search.impl]
    [metabase.search.test-util :as search.tu]
    [metabase.test :as mt]
    [metabase.util.compress :as u.compress]
@@ -94,6 +95,7 @@
                                                   :dataset_query {:type     :native
                                                                   :database (t2/select-one-pk :model/Database)
                                                                   :native   {:query "SELECT 1"}}}]
+           (search.impl/sync-reindex! {:in-place? true})
            ~@body)))))
 
 (defn- do-export
@@ -211,8 +213,8 @@
   (testing "Successful export creates valid archive with correct files and Snowplow event"
     (with-serialization-test-data! [coll dash card]
       ;; Clear entities from search index to verify export works independently
-      (search/delete! :model/Dashboard [(str (:id dash))])
-      (search/delete! :model/Card [(str (:id card))])
+      (search/async-delete! :model/Dashboard [(str (:id dash))])
+      (search/async-delete! :model/Card [(str (:id card))])
       (is (= 0 (search-result-count "dashboard" "thraddash")))
       (is (= 0 (search-result-count "dataset" "frobinate")))
 
@@ -255,8 +257,8 @@
   (testing "Import restores deleted/renamed entities and updates search index"
     (with-serialization-test-data! [coll dash card]
       ;; Clear entities from search index
-      (search/delete! :model/Dashboard [(str (:id dash))])
-      (search/delete! :model/Card [(str (:id card))])
+      (search/async-delete! :model/Dashboard [(str (:id dash))])
+      (search/async-delete! :model/Card [(str (:id card))])
 
       ;; Export the data
       (let [ba (do-export (:id coll))]
@@ -268,7 +270,7 @@
         (t2/delete! :model/Card (:id card))
 
         (let [re-indexed? (atom false)
-              res         (mt/with-dynamic-fn-redefs [search/reindex! (fn [& _] (reset! re-indexed? true) (future nil))]
+              res         (mt/with-dynamic-fn-redefs [search/async-reindex! (fn [& _] (reset! re-indexed? true) (future nil))]
                             (mt/user-http-request :crowberto :post 200 "ee/serialization/import?reindex=false"
                                                   {:request-options {:headers {"content-type" "multipart/form-data"}}}
                                                   {:file ba}))]

enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj

@@ -27,7 +27,7 @@
 ;; `reindex!` below is ok in a parallel test since it's not actually executing anything
 #_{:clj-kondo/ignore [:metabase/validate-deftest]}
 (use-fixtures :each (fn [thunk]
-                      (mt/with-dynamic-fn-redefs [search/reindex! (constantly nil)
+                      (mt/with-dynamic-fn-redefs [search/async-reindex! (constantly nil)
                                                   models.database/assert-not-h2! (constantly nil)]
                         (thunk))))
 

enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj

@@ -23,7 +23,7 @@
 ;; `reindex!` below is ok in a parallel test since it's not actually executing anything
 #_{:clj-kondo/ignore [:metabase/validate-deftest]}
 (use-fixtures :each (fn [thunk]
-                      (mt/with-dynamic-fn-redefs [search/reindex! (constantly nil)
+                      (mt/with-dynamic-fn-redefs [search/async-reindex! (constantly nil)
                                                   models.database/assert-not-h2! (constantly nil)]
                         (thunk))))
 

enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj

@@ -98,7 +98,7 @@
 (defn load-extract!
   "Perform a round-trip of loading an existing serialization, then serializing it again."
   [input-dir output-dir]
-  (mt/with-dynamic-fn-redefs [search/reindex! (constantly nil)]
+  (mt/with-dynamic-fn-redefs [search/async-reindex! (constantly nil)]
     (serdes/with-cache
       (load/load-metabase! (ingest/ingest-yaml input-dir))))
   ;; Use a separate cache to make sure there is no cross-contamination.

enterprise/backend/test/metabase_enterprise/tenants/search_test.clj

@@ -2,7 +2,6 @@
   (:require
    [clojure.test :refer :all]
    [metabase.search.appdb.core :as search.engines.appdb]
-   [metabase.search.ingestion :as search.ingestion]
    [metabase.search.test-util :as search.tu]
    [metabase.test :as mt]
    [metabase.test.fixtures :as fixtures]
@@ -13,8 +12,7 @@
   search.engines.appdb/keep-me)
 
 (use-fixtures :once (fixtures/initialize :db))
-(use-fixtures :each (fn [thunk] (binding [search.ingestion/*force-sync* true]
-                                  (search.tu/with-new-search-if-available-otherwise-legacy (thunk)))))
+(use-fixtures :each (fn [thunk] (search.tu/with-new-search-if-available-otherwise-legacy (thunk))))
 
 (deftest dedicated-tenant-collection-dataset-search-test
   (testing "Search returns datasets (models) from dedicated tenant collections"

enterprise/backend/test/metabase_enterprise/transforms/search_test.clj

@@ -36,13 +36,14 @@
 (deftest transform-query-ingestion-test
   (testing "Contents of Python transform sources are extracted and indexed for full-text search"
     (when (= (mdb/db-type) :postgres)
-      (mt/with-temp [:model/Transform _ {:target {:database (mt/id)}
-                                         :source {:type "python"
-                                                  :source-database (mt/id)
-                                                  :body "import pandas as pd\n"}
-                                         :name "Test python transform"}]
-        (let [ingested-transform (search-test/ingest-then-fetch! "transform" "Test python transform")
-              vector-value (.getValue ^PGobject (:with_native_query_vector ingested-transform))]
-          (is (string? vector-value))
-          (is (re-find #"import" vector-value))
-          (is (re-find #"panda" vector-value)))))))
+      (search.tu/with-temp-index-table
+        (mt/with-temp [:model/Transform _ {:target {:database (mt/id)}
+                                           :source {:type "python"
+                                                    :source-database (mt/id)
+                                                    :body "import pandas as pd\n"}
+                                           :name "Test python transform"}]
+          (let [ingested-transform (search-test/ingest-then-fetch! "transform" "Test python transform")
+                vector-value (.getValue ^PGobject (:with_native_query_vector ingested-transform))]
+            (is (string? vector-value))
+            (is (re-find #"import" vector-value))
+            (is (re-find #"panda" vector-value))))))))

mage/src/mage/modules.clj

@@ -118,6 +118,7 @@
      llm
      login-history
      metabot
+     mq
      notification
      permissions
      premium-features

resources/migrations/061/20260313_persistent_queue.yaml

@@ -0,0 +1,140 @@
+## Adds support for appdb persistent queue backend
+
+databaseChangeLog:
+  - changeSet:
+      id: v61.asdgew
+      author: nvoxland
+      comment: Create the table backing the message queue
+      changes:
+        - createTable:
+            tableName: queue_message_batch
+            remarks: Lightweight message queue for cross-cluster communication
+            columns:
+              - column:
+                  name: id
+                  type: BIGINT
+                  autoIncrement: true
+                  constraints:
+                    primaryKey: true
+                    nullable: false
+              - column:
+                  name: queue_name
+                  type: varchar(254)
+                  remarks: The name of queue this message is in
+                  constraints:
+                    nullable: false
+              - column:
+                  remarks: The timestamp of when the messages were created
+                  name: created_at
+                  type: ${timestamp_type}
+                  defaultValueComputed: current_timestamp
+                  constraints:
+                    nullable: false
+              - column:
+                  remarks: The status of the messages
+                  name: status
+                  type: varchar(10)
+                  defaultValue: 'pending'
+                  constraints:
+                    nullable: false
+              - column:
+                  remarks: Heartbeat to ensure status is not stuck
+                  name: status_heartbeat
+                  type: ${timestamp_type}
+                  defaultValueComputed: current_timestamp
+                  constraints:
+                    nullable: false
+              - column:
+                  remarks: Number of failures processing these messages
+                  name: failures
+                  type: int
+                  defaultValueNumeric: 0
+                  constraints:
+                    nullable: false
+              - column:
+                  remarks: Current owner of the messages
+                  name: owner
+                  type: varchar(36)
+              - column:
+                  remarks: JSON-encoded array of messages
+                  name: messages
+                  type: ${text.type}
+                  constraints:
+                    nullable: false
+
+  - changeSet:
+      id: v61.gdasfg2e
+      author: nvoxland
+      comment: Index queue_message_batch name and status
+      changes:
+        - createIndex:
+            tableName: queue_message_batch
+            indexName: idx_queue_message_batch_queue_status
+            columns:
+              - column:
+                  name: queue_name
+              - column:
+                  name: status
+
+
+  - changeSet:
+      id: v61.g9dasg4
+      author: nvoxland
+      comment: Create topic_message_batch table for pub/sub messaging
+      changes:
+        - createTable:
+            tableName: topic_message_batch
+            remarks: Append-only message log for pub/sub fan-out messaging
+            columns:
+              - column:
+                  name: id
+                  type: BIGINT
+                  autoIncrement: true
+                  constraints:
+                    primaryKey: true
+                    nullable: false
+              - column:
+                  name: topic_name
+                  type: varchar(254)
+                  remarks: The name of the topic this message was published to
+                  constraints:
+                    nullable: false
+              - column:
+                  remarks: The timestamp of when the message was created
+                  name: created_at
+                  type: ${timestamp_type}
+                  defaultValueComputed: current_timestamp
+                  constraints:
+                    nullable: false
+              - column:
+                  remarks: JSON-encoded array of messages
+                  name: messages
+                  type: ${text.type}
+                  constraints:
+                    nullable: false
+
+  - changeSet:
+      id: v61.igoiead
+      author: nvoxland
+      comment: Index topic_message_batch by topic_name and id for efficient polling
+      changes:
+        - createIndex:
+            tableName: topic_message_batch
+            indexName: idx_topic_message_batch_topic_id
+            columns:
+              - column:
+                  name: topic_name
+              - column:
+                  name: id
+
+  - changeSet:
+      id: v61.8867df
+      author: nvoxland
+      comment: Index topic_message_batch.created_at for efficient cleanup queries
+      changes:
+        - createIndex:
+            tableName: topic_message_batch
+            indexName: idx_topic_message_batch_created_at
+            columns:
+              - column:
+                  name: created_at

src/metabase/analytics/prometheus.clj

@@ -595,6 +595,54 @@
                           ;; 100ms -> 10 minutes
                           :buckets [100 500 1000 5000 10000 30000 60000 120000 300000 600000]})
 
+   ;; messaging metrics
+   (prometheus/counter :metabase-mq/appdb-cleanup-deleted
+                       {:description "Messages/batches deleted by cleanup."
+                        :labels [:transport :channel]})
+   (prometheus/gauge :metabase-mq/appdb-queue-depth
+                     {:description "Batch count per queue by status."
+                      :labels [:channel :status]})
+   (prometheus/counter :metabase-mq/appdb-queue-poll-results
+                       {:description "Queue poll results by outcome."
+                        :labels [:result]})
+   (prometheus/gauge :metabase-mq/appdb-topic-subscriber-lag
+                     {:description "Undelivered messages between subscriber offset and latest."
+                      :labels [:channel]})
+   (prometheus/counter :metabase-mq/batch-stale-recoveries
+                       {:description "Batches recovered from stale processing state."
+                        :labels [:transport :channel]})
+   (prometheus/counter :metabase-mq/batches-handled
+                       {:description "Batches handled by status."
+                        :labels [:transport :channel :status]})
+   (prometheus/counter :metabase-mq/dedup-messages-dropped
+                       {:description "Messages dropped by dedup before publishing."
+                        :labels [:channel]})
+   (prometheus/histogram :metabase-mq/handle-duration-ms
+                         {:description "Duration in milliseconds to process a batch."
+                          :labels [:transport :channel]
+                          :buckets [1 5 10 50 100 500 1000 5000 10000 30000]})
+   (prometheus/counter :metabase-mq/messages-published
+                       {:description "Total messages published."
+                        :labels [:transport :channel]})
+   (prometheus/gauge :metabase-mq/publish-buffer-depth
+                     {:description "Messages sitting in the publish buffer awaiting flush."
+                      :labels [:channel]})
+   (prometheus/counter :metabase-mq/publish-buffer-flush-errors
+                       {:description "Publish buffer flush failures where messages were re-buffered for retry."
+                        :labels [:channel]})
+   (prometheus/counter :metabase-mq/queue-batch-permanent-failures
+                       {:description "Queue batches that exhausted retries."
+                        :labels [:channel]})
+   (prometheus/counter :metabase-mq/queue-batch-retries
+                       {:description "Queue batches retried after transient failure."
+                        :labels [:channel]})
+   (prometheus/counter :metabase-mq/messages-received
+                       {:description "Individual messages delivered to handlers."
+                        :labels [:transport :channel]})
+   (prometheus/counter :metabase-mq/topic-handler-errors
+                       {:description "Errors in topic subscriber handlers."
+                        :labels [:transport :channel]})
+
    ;; release dashboard metrics
    (prometheus/counter :metabase-sync/failures
                        {:description "Number of sync operation failures."

src/metabase/app_db/connection.clj

@@ -134,11 +134,34 @@
 
 (def ^:private ^:dynamic *transaction-depth* 0)
 
+(def ^:dynamic *after-commit*
+  "When non-nil, an atom containing a vector of 0-arity fns to call after the outermost transaction commits.
+  Bound to a fresh atom at the outermost transaction boundary."
+  nil)
+
+(def ^:dynamic *transaction-state*
+  "When non-nil, an atom containing a map for arbitrary per-transaction data.
+  Any subsystem can store namespaced keys here. Bound to a fresh atom at the outermost transaction boundary."
+  nil)
+
 (defn in-transaction?
   "Whether we are currently in a transaction."
   []
   (pos? *transaction-depth*))
 
+(defn after-commit!
+  "Register `f` (a 0-arity fn) to run after the current outermost transaction commits.
+  Returns true if registered, false if not currently in a transaction."
+  [f]
+  (if *after-commit*
+    (do (swap! *after-commit* conj f) true)
+    false))
+
+(defn transaction-state
+  "Returns the current transaction state atom, or nil if not in a transaction."
+  []
+  *transaction-state*)
+
 (defn- do-transaction [^java.sql.Connection connection f]
   (letfn [(thunk []
             (let [savepoint (.setSavepoint connection)]
@@ -146,7 +169,17 @@
                 (let [result (f connection)]
                   (when (= *transaction-depth* 1)
                     ;; top-level transaction, commit
-                    (.commit connection))
+                    (.commit connection)
+                    ;; drain and execute after-commit callbacks
+                    (loop []
+                      (when-let [callbacks (seq (first (reset-vals! *after-commit* [])))]
+                        (doseq [cb callbacks]
+                          (try
+                            (cb)
+                            (catch Throwable t
+                              (log/error t "Error in after-commit callback"))))
+                        ;; If callbacks registered new callbacks, drain those too
+                        (recur))))
                   result)
                 (catch Throwable txn-e
                   (try
@@ -202,8 +235,11 @@
                     {:options options}))
 
     :else
-    (binding [*transaction-depth* (inc *transaction-depth*)]
-      (do-transaction connection f))))
+    (let [outermost? (zero? *transaction-depth*)]
+      (binding [*transaction-depth* (inc *transaction-depth*)
+                *after-commit*      (if outermost? (atom []) *after-commit*)
+                *transaction-state* (if outermost? (atom {}) *transaction-state*)]
+        (do-transaction connection f)))))
 
 (methodical/defmethod t2.pipeline/transduce-query :before :default
   "Make sure application database calls are not done inside core.async dispatch pool threads. This is done relatively

src/metabase/app_db/core.clj

@@ -37,7 +37,9 @@
   db-type
   in-transaction?
   quoting-style
-  unique-identifier]
+  unique-identifier
+  after-commit!
+  transaction-state]
 
  [mdb.connection-pool-setup
   recent-activity?]

src/metabase/cloud_migration/models/cloud_migration.clj

@@ -10,7 +10,7 @@
    [metabase.cmd.dump-to-h2 :as dump-to-h2]
    [metabase.config.core :as config]
    [metabase.models.interface :as mi]
-   [metabase.settings.core :as setting]
+
    [metabase.store-api.core :as store-api]
    [metabase.task.bootstrap :as task.bootstrap]
    [metabase.task.core :as task]
@@ -228,7 +228,8 @@
       (cloud-migration.settings/read-only-mode! true)
       (when (cluster?)
         (log/info "Cluster detected, waiting for read-only mode to propagate")
-        (Thread/sleep (int (* 1.5 setting/cache-update-check-interval-ms))))
+        ;; Wait briefly for topic-based cache invalidation to propagate to other nodes
+        (Thread/sleep 5000))
       (log/info "Stopping scheduler")
       (task/stop-scheduler!)
 

src/metabase/core/core.clj

@@ -20,6 +20,7 @@
    [metabase.events.core :as events]
    [metabase.initialization-status.core :as init-status]
    [metabase.logger.core :as logger]
+   [metabase.mq.core :as mq]
    [metabase.notification.core :as notification]
    [metabase.permissions.core :as perms]
    [metabase.plugins.core :as plugins]
@@ -34,7 +35,6 @@
    [metabase.tracing.core :as tracing]
    [metabase.util :as u]
    [metabase.util.log :as log]
-   [metabase.util.queue :as queue]
    [metabase.util.system-info :as u.system-info]
    [metabase.warehouses.models.database :as database])
   (:import
@@ -103,7 +103,7 @@
   "General application shutdown function which should be called once at application shutdown."
   []
   (log/info "Metabase Shutting Down ...")
-  (queue/stop-listeners!)
+  (mq/shutdown!)
   (task/stop-scheduler!)
   (server/stop-web-server!)
   (tracing/shutdown!)
@@ -228,7 +228,6 @@
   (setting/log-deprecated-env-var-usage!)
   (init-status/set-progress! 0.95)
   (task/start-scheduler!)
-  (queue/start-listeners!)
   (init-status/set-complete!)
   (log/info "Metabase Initialization COMPLETE"))
 

src/metabase/core/init.clj

@@ -40,6 +40,7 @@
    [metabase.metabot.init]
    [metabase.model-persistence.init]
    [metabase.models.init]
+   [metabase.mq.init]
    [metabase.notification.init]
    [metabase.oauth-server.init]
    [metabase.parameters.init]

src/metabase/driver/events/driver_notifications.clj

@@ -8,8 +8,28 @@
    [medley.core :as m]
    [metabase.driver :as driver]
    [metabase.driver-api.core :as driver-api]
+   [metabase.mq.core :as mq]
    [metabase.util.log :as log]
-   [methodical.core :as methodical]))
+   [methodical.core :as methodical]
+   ^{:clj-kondo/ignore [:discouraged-namespace]}
+   [toucan2.core :as t2]))
+
+;;; ----------------------------------------- Topic subscription -------------------------------------------
+
+(mq/def-listener! :topic/connection-pool-invalidated
+  {:sync-local-delivery true}
+  [{:keys [database-id all-databases]}]
+  (if all-databases
+    (doseq [{driver :engine, :as database} (t2/select :model/Database)]
+      (try
+        (driver/notify-database-updated driver database)
+        (catch Throwable e
+          (log/error e "Failed to notify database updated" {:id (:id database)}))))
+    (when-let [database (t2/select-one :model/Database :id database-id)]
+      (try
+        (driver/notify-database-updated (:engine database) database)
+        (catch Throwable e
+          (log/error e "Failed to notify database updated" {:id database-id}))))))
 
 (derive ::event :metabase/event)
 (derive :event/database-update ::event)
@@ -29,6 +49,7 @@
       (when (or details-changed?
                 (not= (remove-irrelevant-data database)
                       (remove-irrelevant-data previous-database)))
-        (driver/notify-database-updated (:engine database) database)))
+        (mq/with-topic :topic/connection-pool-invalidated [t]
+          (mq/put t {:database-id (:id database)}))))
     (catch Throwable e
       (log/warnf e "Failed to process driver notifications event. %s" topic))))

src/metabase/driver/events/report_timezone_updated.clj

@@ -1,30 +1,15 @@
 (ns metabase.driver.events.report-timezone-updated
   (:require
-   [metabase.driver :as driver]
    [metabase.driver-api.core :as driver-api]
-   [metabase.util.log :as log]
-   [methodical.core :as methodical]
-   ^{:clj-kondo/ignore [:discouraged-namespace]}
-   [toucan2.core :as t2]))
+   [metabase.mq.core :as mq]
+   [methodical.core :as methodical]))
 
 (derive ::event :metabase/event)
 (derive :event/report-timezone-updated ::event)
 
-(defn- notify-all-databases-updated
-  "Send notification that all Databases should immediately release cached resources (i.e., connection pools).
-
-  Currently only used below by [[report-timezone]] setter (i.e., only used when report timezone changes). Reusing
-  pooled connections with the old session timezone can have weird effects, especially if report timezone is changed to
-  `nil` (meaning subsequent queries will not attempt to change the session timezone) or something considered invalid
-  by a given Database (meaning subsequent queries will fail to change the session timezone)."
-  []
-  (doseq [{driver :engine, id :id, :as database} (t2/select :model/Database)]
-    (try
-      (driver/notify-database-updated driver database)
-      (catch Throwable e
-        (log/errorf e "Failed to notify %s Database %s updated" driver id)))))
-
 (methodical/defmethod driver-api/publish-event! ::event
-  "When the report-timezone Setting is updated, call [[metabase.driver/notify-database-updated]] for all Databases."
+  "When the report-timezone Setting is updated, publish to the connection-pool-invalidated topic
+  so all databases get their pools flushed. Local listeners fire immediately inline."
   [_topic _event]
-  (notify-all-databases-updated))
+  (mq/with-topic :topic/connection-pool-invalidated [t]
+    (mq/put t {:all-databases true})))

src/metabase/indexed_entities/models/model_index.clj

@@ -139,7 +139,7 @@
                 (t2/delete! :model/ModelIndexValue
                             :model_index_id (:id model-index)
                             :model_pk [:in (->> deletions-part (map first))])
-                (search/delete! :model/ModelIndexValue search-model-ids)))
+                (search/async-delete! :model/ModelIndexValue search-model-ids)))
             (when (seq additions)
               (doseq [additions-part (partition-all 10000 additions)]
                 (t2/insert! :model/ModelIndexValue
@@ -154,7 +154,7 @@
                        :state      (if (> (count values-to-index) max-indexed-values)
                                      "overflow"
                                      "indexed")}))
-        (run! search/update! (t2/reducible-select :model/ModelIndexValue :model_index_id (:id model-index)))
+        (run! search/async-update! (t2/reducible-select :model/ModelIndexValue :model_index_id (:id model-index)))
         (catch Exception e
           (log/errorf e "Error saving model-index values for model-index: %d, model: %d"
                       (:id model-index) (:model_id model-index))

src/metabase/models/transforms/transform.clj

@@ -260,7 +260,7 @@
 (t2/define-before-delete :model/Transform [transform]
   (ws.table/delete-orphaned-provisional-table! (:target_table_id transform) (:id transform))
   (events/publish-event! :event/delete-transform {:id (:id transform)})
-  (search.core/delete! :model/Transform [(str (:id transform))])
+  (search.core/async-delete! :model/Transform [(str (:id transform))])
   transform)
 
 (defn update-transform-tags!

src/metabase/mq/analytics.clj

@@ -0,0 +1,20 @@
+(ns metabase.mq.analytics
+  "Prometheus analytics helpers for the mq subsystem that avoids circular dependencies.
+  Lives in its own namespace to avoid further circular dependencies.")
+
+(set! *warn-on-reflection* true)
+
+(defn inc!
+  "Analytics helper that avoids circular dependencies"
+  [& args]
+  (apply (requiring-resolve 'metabase.analytics.core/inc!) args))
+
+(defn set!
+  "Analytics helper that avoids circular dependencies"
+  [& args]
+  (apply (requiring-resolve 'metabase.analytics.core/set!) args))
+
+(defn observe!
+  "Analytics helper that avoids circular dependencies"
+  [& args]
+  (apply (requiring-resolve 'metabase.analytics.core/observe!) args))

src/metabase/mq/core.clj

@@ -0,0 +1,87 @@
+(ns metabase.mq.core
+  "Unified public API for the Metabase messaging subsystem.
+
+  Two patterns are supported:
+
+  **Queue** — single-consumer, at-least-once delivery.  Each message is processed by exactly
+  one listener and removed after successful processing.  Failed messages are retried up to a
+  configurable limit.
+
+  **Topic** — single-consumer pub/sub.  Each active node receives every published message.
+  There can be up to one listener per node for each topic. Failed messages are never retried.
+
+  Listeners for both types are registered with `def-listener`
+
+      (mq/def-listener! :queue/simple-task [msg]
+        (process msg))
+
+      (mq/def-listener! :topic/my-events [msg]
+        (handle-event msg))
+
+  For queues, a config map enables batching:
+       (mq/def-listener! :queue/my-task
+         {:max-batch-messages 10}
+         [messages]
+         (process messages))
+
+  Publishing is done using `with-queue` or `with-topic` which binds a queue you can put messages on.
+  When the body finishes successfully, the message(s) in the bound queue actually publishes the messages.
+
+      (with-queue :queue/simple-task [q]
+        (put q message))
+
+      (with-topic :topic/my-events [t]
+        (put t payload))"
+  (:require
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.listener :as mq.listener]
+   [metabase.mq.publish :as mq.publish]
+   [metabase.mq.queue.appdb :as q.appdb]
+   [metabase.mq.queue.impl :as q.impl]
+   [metabase.mq.queue.memory :as q.memory]
+   [metabase.mq.queue.sync :as q.sync]
+   [metabase.mq.queue.transport-impl :as q.transport-impl]
+   [metabase.mq.topic.appdb :as topic.appdb]
+   [metabase.mq.topic.impl :as topic.impl]
+   [metabase.mq.topic.memory :as topic.memory]
+   [metabase.mq.topic.sync :as topic.sync]
+   [metabase.mq.topic.transport-impl :as topic.transport-impl]
+   [potemkin :as p]))
+
+(set! *warn-on-reflection* true)
+
+(comment
+  mq.impl/keep-me
+  mq.listener/keep-me
+  mq.publish/keep-me
+  q.appdb/keep-me
+  q.impl/keep-me
+  q.memory/keep-me
+  q.sync/keep-me
+  q.transport-impl/keep-me
+  topic.appdb/keep-me
+  topic.transport-impl/keep-me
+  topic.impl/keep-me
+  topic.memory/keep-me
+  topic.sync/keep-me)
+
+(p/import-vars
+ [mq.listener
+  batch-listen!
+  def-listener!
+  listen!
+  register-listeners!
+  unlisten!]
+
+ [mq.publish
+  put]
+
+ [mq.impl
+  last-activity
+  shutdown!]
+
+ [q.impl
+  with-queue]
+
+ [topic.impl
+  with-topic])

src/metabase/mq/impl.clj

@@ -0,0 +1,200 @@
+(ns metabase.mq.impl
+  "Backend coordination: message delivery, analytics helpers, and lifecycle management."
+  (:require
+   [metabase.mq.analytics :as mq.analytics]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.polling :as mq.polling]
+   [metabase.mq.publish-buffer :as publish-buffer]
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.util.log :as log])
+  (:import
+   (java.util.concurrent Callable ExecutorService Executors Future)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private active-handlers
+  "channel keyword → {:future Future, :metadata map}
+   Tracks which channels have an active handler running across all backends."
+  (atom {}))
+
+(def ^:private worker-pool
+  "Shared thread pool for processing messages off backend poll threads."
+  (atom nil))
+
+(def ^:private last-activity*
+  "channel keyword → nanoTime of last publish or handler completion."
+  (atom {}))
+
+(defn last-activity
+  "Returns the nanoTime of the last publish or handler completion for the given channel, or nil."
+  [channel]
+  (get @last-activity* channel))
+
+(defn channel-busy?
+  "Returns true if the given channel has an active handler running."
+  [channel]
+  (contains? @active-handlers channel))
+
+(defn busy-channels
+  "Returns the set of channels that currently have an active handler."
+  []
+  (set (keys @active-handlers)))
+
+(defn active-handler-metadata
+  "Returns the metadata map for the active handler on the given channel, or nil.
+   Backends store backend-specific data here (e.g. queue backends store :bundle-id for heartbeats)."
+  [channel]
+  (get-in @active-handlers [channel :metadata]))
+
+(defn invoke-listener!
+  "Common listener invocation skeleton for both queues and topics.
+  Looks up listener via `listener-fn`, times execution, logs errors,
+  and records metrics. Calls `on-success` / `on-error` for system-specific
+  side effects (e.g. queue ACK/NACK)."
+  [{:keys [channel listener-fn invoke-fn on-success on-error]}]
+  (let [transport (namespace channel)
+        listener    (listener-fn)
+        labels      {:transport transport :channel (name channel)}
+        start       (System/nanoTime)]
+    (try
+      (if-not listener
+        (log/debugf "No listener registered for %s %s, skipping message" transport (name channel))
+        (do
+          (invoke-fn listener)
+          (when on-success (on-success))
+          (mq.analytics/inc! :metabase-mq/batches-handled (assoc labels :status "success"))))
+      (catch Exception e
+        (log/error e (str "Error handling " transport " message") labels)
+        (when on-error (on-error e))
+        (mq.analytics/inc! :metabase-mq/batches-handled (assoc labels :status "error")))
+      (finally
+        (mq.analytics/observe! :metabase-mq/handle-duration-ms labels
+                               (/ (double (- (System/nanoTime) start)) 1e6))))))
+
+(defn handle!
+  "Handles accumulated messages for a channel by invoking the registered listener.
+   Processes each message/batch with error isolation — one failure doesn't block others.
+   For queues, bundles use all-or-nothing ACK semantics: if ANY message in the bundle
+   fails, the ENTIRE bundle is nacked and will be retried. This means successfully
+   processed messages may be re-delivered. Queue listeners MUST be idempotent.
+   The :dedup-fn option on listeners helps mitigate duplicate processing on retry."
+  [channel message-bundles messages]
+  (swap! last-activity* assoc channel (System/nanoTime))
+  (mq.analytics/inc! :metabase-mq/messages-received
+                     {:transport (namespace channel) :channel (name channel)}
+                     (count messages))
+  (let [{:keys [max-batch-messages]} (listener/get-listener channel)]
+    (invoke-listener!
+     {:channel      channel
+      :listener-fn  #(:listener (listener/get-listener channel))
+      :invoke-fn    (fn [h]
+                      (let [error (atom nil)]
+                        (if (and max-batch-messages (> max-batch-messages 1))
+                          (doseq [batch (partition-all max-batch-messages messages)]
+                            (try (h (vec batch))
+                                 (catch Exception e
+                                   (reset! error e)
+                                   (log/errorf e "Error handling %s message for %s, continuing"
+                                               (namespace channel) (name channel)))))
+                          (doseq [msg messages]
+                            (try (h msg)
+                                 (catch Exception e
+                                   (reset! error e)
+                                   (log/errorf e "Error handling %s message for %s, continuing"
+                                               (namespace channel) (name channel))))))
+                        (when-let [e @error]
+                          (throw (ex-info "One or more messages failed" {} e)))))
+      :on-success   #(doseq [[bid backend] message-bundles]
+                       (q.backend/bundle-successful! backend channel bid))
+      :on-error     (fn [_e]
+                      (doseq [[bid backend] message-bundles]
+                        (q.backend/bundle-failed! backend channel bid)))})))
+
+(defn deliver!
+  "Called by backends when messages are ready for delivery.
+   Passes messages directly to handle! for processing.
+   `bundle-id` and `backend` are optional — queues pass them for ACK/NACK, topics pass nil."
+  [channel messages bundle-id backend]
+  (handle! channel (if bundle-id {bundle-id backend} {}) messages))
+
+(defn submit-delivery!
+  "Submits a delivery to the shared worker pool for non-blocking processing.
+   Returns true if submitted, false if the channel already has an active handler.
+   `metadata` is an opaque map stored alongside the future — backends can use it for
+   backend-specific tracking (e.g. queues store {:bundle-id id} for heartbeat updates).
+   Uses bound-fn to convey dynamic bindings to the worker thread.
+   The busy-check and registration are atomic via a single swap!.
+   A generation counter prevents a completed future's cleanup from clobbering a
+   re-submission that won the slot between the finally and the dissoc."
+  [channel messages bundle-id backend metadata]
+  (let [claimed? (atom false)
+        gen      (Object.)]
+    (swap! active-handlers
+           (fn [handlers]
+             (if (contains? handlers channel)
+               handlers
+               (do (reset! claimed? true)
+                   (assoc handlers channel {:future nil :metadata metadata :gen gen})))))
+    (if-not @claimed?
+      false
+      (let [f (.submit ^ExecutorService @worker-pool
+                       ^Callable (bound-fn []
+                                   (try
+                                     (deliver! channel messages bundle-id backend)
+                                     (finally
+                                       ;; Only remove if this generation still owns the slot
+                                       (swap! active-handlers
+                                              (fn [handlers]
+                                                (if (identical? gen (:gen (get handlers channel)))
+                                                  (dissoc handlers channel)
+                                                  handlers)))
+                                       (mq.polling/notify-all!)))))]
+        ;; Only set the future if this generation still owns the slot
+        (swap! active-handlers
+               (fn [handlers]
+                 (if (identical? gen (:gen (get handlers channel)))
+                   (assoc-in handlers [channel :future] f)
+                   handlers)))
+        true))))
+
+(defn start-worker-pool!
+  "Starts the shared worker pool for non-blocking message delivery. Idempotent."
+  []
+  (when-not @worker-pool
+    (let [pool (Executors/newCachedThreadPool)]
+      (when-not (compare-and-set! worker-pool nil pool)
+        ;; Another thread won the race — shut down our pool
+        (.shutdown ^ExecutorService pool)))))
+
+(defn shutdown-worker-pool!
+  "Cancels all active handler futures and shuts down the worker pool."
+  []
+  (doseq [[_ {:keys [^Future future]}] @active-handlers]
+    (when future
+      (.cancel future true)))
+  (reset! active-handlers {})
+  (when-let [^ExecutorService pool @worker-pool]
+    (.shutdownNow pool)
+    (reset! worker-pool nil)))
+
+(defn start-transports
+  "Starts the queue/topic backends."
+  []
+  (let [transport-start! (requiring-resolve 'metabase.mq.transport/start!)]
+    (transport-start! :queue)
+    (transport-start! :topic)))
+
+(defn shutdown-transports
+  "Shuts down all queue/topic backends."
+  []
+  (let [transport-shutdown! (requiring-resolve 'metabase.mq.transport/shutdown!)]
+    (transport-shutdown! :queue)
+    (transport-shutdown! :topic)))
+
+(defn shutdown!
+  "Shuts down all mq infrastructure: publish buffer, worker pool, backends, and listener state."
+  []
+  (publish-buffer/stop-publish-buffer-flush!)
+  (shutdown-transports)
+  (shutdown-worker-pool!)
+  (reset! listener/*listeners* {}))

src/metabase/mq/init.clj

@@ -0,0 +1,37 @@
+(ns metabase.mq.init
+  "Initializes the mq subsystem at startup."
+  (:require
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.publish-buffer :as publish-buffer]
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.mq.settings :as mq.settings]
+   [metabase.mq.topic.backend :as topic.backend]
+   [metabase.startup.core :as startup]
+   [metabase.util.log :as log]))
+
+(def ^:private valid-queue-backends
+  #{:queue.backend/appdb :queue.backend/memory})
+
+(def ^:private valid-topic-backends
+  #{:topic.backend/appdb :topic.backend/memory})
+
+(defmethod startup/def-startup-logic! ::MqInit [_]
+  (let [queue-be (keyword "queue.backend" (mq.settings/queue-backend))
+        topic-be (keyword "topic.backend" (mq.settings/topic-backend))]
+    (when-not (contains? valid-queue-backends queue-be)
+      (throw (ex-info (str "Invalid queue backend: " queue-be
+                           ". Valid backends: " valid-queue-backends)
+                      {:backend queue-be :valid valid-queue-backends})))
+    (when-not (contains? valid-topic-backends topic-be)
+      (throw (ex-info (str "Invalid topic backend: " topic-be
+                           ". Valid backends: " valid-topic-backends)
+                      {:backend topic-be :valid valid-topic-backends})))
+    (alter-var-root #'q.backend/*backend* (constantly queue-be))
+    (alter-var-root #'topic.backend/*backend* (constantly topic-be))
+    (log/infof "Queue backend set to %s" queue-be)
+    (log/infof "Topic backend set to %s" topic-be)
+    (listener/register-listeners!)
+    (publish-buffer/start-publish-buffer-flush!)
+    (mq.impl/start-worker-pool!)
+    (mq.impl/start-transports)))

src/metabase/mq/listener.clj

@@ -0,0 +1,141 @@
+(ns metabase.mq.listener
+  "Listener registry: registration, lookup, instrumentation, and the `def-listener!` macro."
+  (:require
+   [metabase.mq.analytics :as mq.analytics]
+   [metabase.mq.transport :as transport]
+   [metabase.util.log :as log]
+   [metabase.util.malli.registry :as mr]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:dynamic *listeners*
+  "channel → {:listener fn :max-batch-messages int ...} for all channels."
+  (atom {}))
+
+(defn queue-names
+  "Returns the seq of queue channel names currently registered in `*listeners*`."
+  []
+  (filter #(= "queue" (namespace %)) (keys @*listeners*)))
+
+(defn topic-names
+  "Returns the seq of topic channel names currently registered in `*listeners*`."
+  []
+  (filter #(= "topic" (namespace %)) (keys @*listeners*)))
+
+(defn get-listener
+  "Returns the listener config map for `channel`, or nil if not registered."
+  [channel]
+  (get @*listeners* channel))
+
+(defn register-listener!
+  "Atomically registers a listener for the given channel, throwing if one already exists."
+  [channel listener-map]
+  (let [[old _] (swap-vals! *listeners*
+                            (fn [m]
+                              (if (contains? m channel)
+                                m
+                                (assoc m channel listener-map))))]
+    (when (contains? old channel)
+      (throw (ex-info (str "Listener already registered for " (namespace channel) " " (name channel))
+                      {:channel channel})))))
+
+(defn make-instrumented-listener
+  "Wraps a listener with Prometheus metrics instrumentation."
+  [channel listener]
+  (let [labels    {:transport (namespace channel) :channel (name channel)}
+        received  :metabase-mq/messages-received
+        errors    :metabase-mq/topic-handler-errors]
+    (fn [msg]
+      (try
+        (listener msg)
+        (mq.analytics/inc! received labels)
+        (catch Exception e
+          (mq.analytics/inc! errors labels)
+          (throw e))))))
+
+(defn listen!
+  "Registers a listener for a queue or topic.
+   `opts` is an optional map; pass nil or {} for defaults. Queues support `{:exclusive true}`."
+  [channel opts listener]
+  (let [defaults  (transport/on-listen! channel opts)
+        listener' (transport/wrap-listener channel listener)]
+    (register-listener! channel
+                        (merge defaults opts {:listener listener' :max-batch-messages 1}))))
+
+(defn batch-listen!
+  "Registers a batch listener for a queue or topic.
+   The listener will be called with a vec of messages, sized up to :max-batch-messages."
+  [channel listener config]
+  (let [defaults  (transport/on-listen! channel config)
+        listener' (transport/wrap-listener channel listener)]
+    (register-listener! channel
+                        (merge defaults config {:listener listener'}))))
+
+(defn unlisten!
+  "Removes the listener for a channel."
+  [channel]
+  (swap! *listeners* dissoc channel))
+
+(mr/def ::channel
+  [:and :keyword [:fn {:error/message "Channel must be namespaced to 'queue' or 'topic'"}
+                  #(#{"queue" "topic"} (namespace %))]])
+
+(mr/def ::listen-opts
+  [:map [:exclusive {:optional true} :boolean]])
+
+(defmulti def-listener*
+  "Multimethod backing [[def-listener!]]. Each implementation registers its listener
+   by calling [[listen!]] or [[batch-listen!]].
+   Do not call or implement directly — use [[def-listener!]] instead."
+  {:arglists '([channel])}
+  identity)
+
+(defmacro def-listener!
+  "Defines a listener for a queue or topic, detected from the keyword namespace.
+
+   **Topic listener** — receives a single message:
+
+       (mq/def-listener! :topic/settings-cache-invalidated [msg]
+         (restore-cache!))
+
+   **Queue listener** — receives a single message, with optional config:
+
+       (mq/def-listener! :queue/simple-task {:exclusive true} [msg]
+         (process msg))
+
+   **Queue batch listener** — receives a vec of messages (config must include :max-batch-messages):
+
+       (mq/def-listener! :queue/search-reindex
+         {:max-batch-messages 50 :exclusive true}
+         [messages]
+         (process-batch messages))"
+  {:arglists '([channel bindings & body]
+               [channel config bindings & body])}
+  [channel & args]
+  (let [[config & args] (if (map? (first args))
+                          args
+                          (cons nil args))
+        [bindings & body] args
+        batch?          (and config (:max-batch-messages config))]
+    (if batch?
+      `(defmethod def-listener* ~channel [~'_]
+         (batch-listen!
+          ~channel
+          (fn [~@bindings] ~@body)
+          ~(select-keys config [:max-batch-messages :exclusive :dedup-fn])))
+      `(defmethod def-listener* ~channel [~'_]
+         (listen! ~channel ~(or config {}) (fn [~@bindings] ~@body))))))
+
+(defn register-listeners!
+  "Call all [[def-listener!]] implementations to register their listeners.
+   Called at startup and in test setup (from `with-sync-mq`).
+   Throws on the first registration failure so broken listeners are caught early."
+  []
+  (doseq [[k f] (methods def-listener*)]
+    (try
+      (f k)
+      (catch Throwable e
+        (log/errorf e "Failed to register listener %s" k)
+        (throw (ex-info (str "Failed to register listener " k)
+                        {:channel k}
+                        e))))))

src/metabase/mq/memory.clj

@@ -0,0 +1,89 @@
+(ns metabase.mq.memory
+  "Shared in-memory message store for the memory queue and topic backends.
+  Each channel gets a LinkedBlockingQueue. Uses the shared polling infrastructure
+  from mq.polling to drain non-busy channels and submit messages for delivery."
+  (:require
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.polling :as mq.polling])
+  (:import
+   (java.util ArrayList)
+   (java.util.concurrent LinkedBlockingQueue)))
+
+(set! *warn-on-reflection* true)
+
+;;; ------------------------------------------- Channel state -------------------------------------------
+
+(def ^:dynamic *channels*
+  "channel-name -> LinkedBlockingQueue"
+  (atom {}))
+
+(def ^:private poll-state (mq.polling/make-poll-state))
+
+(defn- ensure-channel!
+  "Ensures a LinkedBlockingQueue exists for the channel, creating one if needed. Returns the queue."
+  ^LinkedBlockingQueue [channel-name]
+  (or (get @*channels* channel-name)
+      (let [new-q (LinkedBlockingQueue.)]
+        (-> (swap! *channels* (fn [chs]
+                                (if (contains? chs channel-name)
+                                  chs
+                                  (assoc chs channel-name new-q))))
+            (get channel-name)))))
+
+;;; ------------------------------------------- Publish / Drain -------------------------------------------
+
+(defn publish!
+  "Adds messages to the channel's queue and wakes the polling thread."
+  [channel-name messages]
+  (let [^LinkedBlockingQueue q (ensure-channel! channel-name)]
+    (doseq [msg messages]
+      (.offer q msg)))
+  (when-not (mq.impl/channel-busy? channel-name)
+    (mq.polling/notify! poll-state)))
+
+(defn- drain!
+  "Removes and returns all messages currently in the channel's queue, or nil if empty."
+  [channel-name]
+  (when-let [^LinkedBlockingQueue q (get @*channels* channel-name)]
+    (when-not (.isEmpty q)
+      (let [batch (ArrayList.)]
+        (.drainTo q batch)
+        (when-not (.isEmpty batch)
+          (vec batch))))))
+
+;;; ------------------------------------------- Polling -------------------------------------------
+
+(def ^:private bundle-registry
+  (delay (requiring-resolve 'metabase.mq.queue.memory/*bundle-registry*)))
+
+(defn- register-bundle!
+  "Registers a bundle in the queue memory backend's bundle registry for retry tracking."
+  [bundle-id messages]
+  (swap! @@bundle-registry assoc bundle-id {:messages messages :failures 0}))
+
+(defn- poll-once!
+  "Drains all non-busy channels and submits messages for delivery.
+   For queue channels, generates a bundle-id and passes :queue.backend/memory as backend.
+   For topic channels, passes nil bundle-id and nil backend (fire-and-forget)."
+  []
+  (doseq [channel-name (remove mq.impl/channel-busy?
+                               (concat (listener/queue-names) (listener/topic-names)))]
+    (when-let [messages (drain! channel-name)]
+      (if (= "queue" (namespace channel-name))
+        (let [bundle-id (str (random-uuid))]
+          (register-bundle! bundle-id messages)
+          (mq.impl/submit-delivery! channel-name messages bundle-id :queue.backend/memory
+                                    {:bundle-id bundle-id}))
+        (mq.impl/submit-delivery! channel-name messages nil nil nil)))))
+
+(defn start!
+  "Starts the memory polling thread. Idempotent — second call is a no-op."
+  []
+  (mq.polling/start-polling! poll-state "Memory" 50 poll-once!))
+
+(defn shutdown!
+  "Stops the polling thread and clears channel state."
+  []
+  (mq.polling/stop-polling! poll-state "Memory")
+  (reset! *channels* {}))

src/metabase/mq/polling.clj

@@ -0,0 +1,78 @@
+(ns metabase.mq.polling
+  "Shared polling infrastructure for appdb and memory mq backends:
+  time-gated execution and polling thread lifecycle."
+  (:require
+   [metabase.util.log :as log])
+  (:import
+   (java.util.concurrent Future)))
+
+(set! *warn-on-reflection* true)
+
+(defn periodically!
+  "Runs `f` at most once per `interval-ms`. Uses `state-atom` (an atom holding a long timestamp)
+   to track the last run time. Catches and logs exceptions using `label` for context."
+  [state-atom interval-ms label f]
+  (let [now (System/currentTimeMillis)]
+    (when (> (- now @state-atom) interval-ms)
+      (reset! state-atom now)
+      (try
+        (f)
+        (catch Exception e
+          (log/error e (str "Error in " label)))))))
+
+(def ^:private active-poll-states
+  "Set of poll states that currently have a running polling thread."
+  (atom #{}))
+
+(defn make-poll-state
+  "Creates a polling state map with an atom for the background process and a notify object."
+  []
+  {:process (atom nil)
+   :notify  (Object.)})
+
+(defn start-polling!
+  "Starts a polling thread that calls `poll-fn` in a loop, waiting on the notify object
+   for `wait-ms` between iterations. If `poll-fn` returns a truthy value (indicating work
+   was found), the next iteration runs immediately without waiting. Idempotent — second
+   call is a no-op."
+  [{:keys [process notify] :as poll-state} label wait-ms poll-fn]
+  (when-not @process
+    (log/infof "Starting %s polling thread" label)
+    (swap! active-poll-states conj poll-state)
+    (reset! process
+            (future
+              (try
+                (loop []
+                  (when @process
+                    (let [found-work? (try
+                                        (poll-fn)
+                                        (catch InterruptedException e (throw e))
+                                        (catch Exception e
+                                          (log/errorf e "Error in %s polling loop" label)
+                                          false))]
+                      (when-not found-work?
+                        (locking notify (.wait ^Object notify (long wait-ms)))))
+                    (recur)))
+                (catch InterruptedException _
+                  (log/infof "%s polling thread interrupted" label)))))))
+
+(defn stop-polling!
+  "Stops a polling thread."
+  [{:keys [process] :as poll-state} label]
+  (swap! active-poll-states disj poll-state)
+  (when-let [f @process]
+    (reset! process nil)
+    (when (instance? Future f)
+      (.cancel ^Future f true))
+    (log/infof "%s polling thread stopped" label)))
+
+(defn notify!
+  "Wakes the polling thread so it runs immediately."
+  [{:keys [notify]}]
+  (locking notify (.notifyAll ^Object notify)))
+
+(defn notify-all!
+  "Wakes all active polling threads so they run immediately."
+  []
+  (doseq [ps @active-poll-states]
+    (notify! ps)))

src/metabase/mq/publish.clj

@@ -0,0 +1,103 @@
+(ns metabase.mq.publish
+  "Publishing pipeline: buffering, transaction support, and the `with-buffer` macro."
+  (:require
+   [metabase.app-db.core :as mdb]
+   [metabase.mq.analytics :as mq.analytics]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.publish-buffer :as publish-buffer]
+   [metabase.util.log :as log]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private last-activity*
+  (delay (requiring-resolve 'metabase.mq.impl/last-activity*)))
+
+(def ^:dynamic *defer-in-transaction?*
+  "When true (the default), messages published inside a transaction are deferred
+  until after the transaction commits. Bind to false in sync/test mode where
+  listeners must process inline."
+  true)
+
+(defprotocol MessageBuffer
+  "Protocol for buffering messages before publishing."
+  (put [this msg]
+    "Put a message on the buffer."))
+
+(defn publish!
+  "Publishes messages to a channel. Applies dedup-fn if registered, buffers,
+   publishes to the appropriate backend, and records analytics."
+  [channel messages]
+  (let [{:keys [dedup-fn]} (listener/get-listener channel)
+        before-count (count messages)
+        messages     (if dedup-fn (dedup-fn messages) messages)]
+    (when (and dedup-fn (< (count messages) before-count))
+      (mq.analytics/inc! :metabase-mq/dedup-messages-dropped
+                         {:channel (name channel)}
+                         (- before-count (count messages))))
+    (when (seq messages)
+      (swap! @@last-activity* assoc channel (System/nanoTime))
+      (publish-buffer/buffered-publish! channel messages)
+      (mq.analytics/inc! :metabase-mq/messages-published
+                         {:transport (namespace channel) :channel (name channel)}
+                         (count messages)))))
+
+(defn flush-deferred-messages!
+  "Flushes all deferred messages accumulated during a transaction.
+   Called as an after-commit callback. Per-channel try/catch so one failure doesn't block others."
+  []
+  (when-let [state (mdb/transaction-state)]
+    (let [deferred (::deferred-messages @state)]
+      (doseq [[channel msgs] deferred]
+        (try
+          (publish! channel msgs)
+          (catch Exception e
+            (log/error e "Error flushing deferred messages" {:channel channel})))))))
+
+(defn defer-in-transaction!
+  "Accumulates msgs in *transaction-state* under [::deferred-messages channel].
+   Registers flush-deferred-messages! as an after-commit callback once per transaction.
+   If *defer-in-transaction?* is false (set in tests), immediately publishes"
+  [channel msgs]
+  (if *defer-in-transaction?*
+    (let [state     (mdb/transaction-state)
+          old-state (first (swap-vals! state
+                                       (fn [s]
+                                         (-> s
+                                             (update-in [::deferred-messages channel] (fnil into []) msgs)
+                                             (assoc ::flush-registered? true)))))]
+      (when-not (::flush-registered? old-state)
+        (mdb/after-commit! flush-deferred-messages!)))
+    (publish! channel msgs)))
+
+(defn run-with-buffer
+  "Runs `body-fn` with a MessageBuffer. On success, publishes collected messages —
+   deferred if inside a transaction, immediate via publish! otherwise.
+   Channels with `:sync-local-delivery true` in their listener config always publish
+   immediately, even inside a transaction.
+   On exception, discards buffered messages and rethrows."
+  [channel error-label body-fn]
+  (let [buffer     (atom [])
+        msg-buffer (reify MessageBuffer
+                     (put [_ msg] (swap! buffer conj msg)))]
+    (try
+      (let [result (body-fn msg-buffer)
+            msgs   @buffer]
+        (when (seq msgs)
+          (if (and (mdb/transaction-state)
+                   (not (:sync-local-delivery (listener/get-listener channel))))
+            (defer-in-transaction! channel msgs)
+            (publish! channel msgs)))
+        result)
+      (catch Exception e
+        (log/error e error-label)
+        (throw e)))))
+
+(defmacro with-buffer
+  "Runs the body with a MessageBuffer for the given channel.
+   Messages are buffered and only published if the body completes successfully.
+   Inside a transaction, messages are deferred until after commit."
+  [channel [binding] & body]
+  `(run-with-buffer
+    ~channel
+    (str "Error in " (namespace ~channel) " processing")
+    (fn [~binding] ~@body)))

src/metabase/mq/publish_buffer.clj

@@ -0,0 +1,94 @@
+(ns metabase.mq.publish-buffer
+  "Publish buffering: batches rapid-fire publishes into time-windowed groups
+   and flushes them on a background scheduled thread."
+  (:require
+   [metabase.mq.analytics :as mq.analytics]
+   [metabase.mq.transport :as transport]
+   [metabase.util.log :as log])
+  (:import
+   (java.util.concurrent Executors ScheduledExecutorService ThreadFactory TimeUnit)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:dynamic *publish-buffer-ms*
+  "Time window in ms to buffer rapid-fire publishes. 0 = immediate publish.
+   Resets on every new message (sliding window)."
+  100)
+
+(def ^:dynamic *publish-buffer-max-ms*
+  "Maximum time in ms since the first message before a flush is forced. 0 = no max."
+  5000)
+
+(def ^:dynamic *publish-buffer*
+  "Global buffer for buffering publishes.
+   channel → {:messages [...] :deadline-ms long :created-ms long}"
+  (atom {}))
+
+(defn buffered-publish!
+  "Routes messages through *publish-buffer* or publishes immediately based on *publish-buffer-ms*.
+   The background flush thread handles actual delivery."
+  [channel messages]
+  (if (zero? *publish-buffer-ms*)
+    (transport/publish! channel messages)
+    (swap! *publish-buffer*
+           (fn [buf]
+             (let [now (System/currentTimeMillis)]
+               (update buf channel
+                       (fn [entry]
+                         (-> (or entry {:messages [] :deadline-ms 0 :created-ms now})
+                             (update :messages into messages)
+                             (assoc :deadline-ms (+ now *publish-buffer-ms*))))))))))
+
+(defn flush-publish-buffer!
+  "Drains publish buffer entries past their deadline."
+  []
+  (doseq [[channel entry] @*publish-buffer*]
+    (mq.analytics/set! :metabase-mq/publish-buffer-depth
+                       {:channel (name channel)}
+                       (count (:messages entry))))
+  (doseq [channel (keys @*publish-buffer*)]
+    (let [drained (atom nil)]
+      (swap! *publish-buffer*
+             (fn [buf]
+               (if-let [{:keys [deadline-ms created-ms] :as entry} (get buf channel)]
+                 (let [now (System/currentTimeMillis)]
+                   (if (or (and (pos? deadline-ms) (>= now deadline-ms))
+                           (and (pos? *publish-buffer-max-ms*) (>= now (+ created-ms *publish-buffer-max-ms*))))
+                     (do (reset! drained entry) (dissoc buf channel))
+                     buf))
+                 buf)))
+      (when-let [{:keys [messages] :as entry} @drained]
+        (try (transport/publish! channel messages)
+             (catch Exception e
+               (log/error e "Error flushing publish buffer, re-buffering" {:channel channel})
+               (mq.analytics/inc! :metabase-mq/publish-buffer-flush-errors {:channel (name channel)})
+               ;; Put messages back into the buffer for retry on next flush cycle
+               (swap! *publish-buffer*
+                      (fn [buf]
+                        (update buf channel
+                                (fn [existing]
+                                  (if existing
+                                    (update existing :messages into messages)
+                                    (assoc entry :deadline-ms (+ (System/currentTimeMillis) *publish-buffer-ms*)))))))))))))
+
+(defonce ^:private publish-buffer-executor (atom nil))
+
+(defn start-publish-buffer-flush!
+  "Starts a daemon thread that flushes the publish buffer every 100ms."
+  []
+  (let [exec (Executors/newSingleThreadScheduledExecutor
+              (reify ThreadFactory
+                (newThread [_ r]
+                  (doto (Thread. r "mq-publish-buffer-flush")
+                    (.setDaemon true)))))]
+    (if (compare-and-set! publish-buffer-executor nil exec)
+      (.scheduleAtFixedRate exec ^Runnable flush-publish-buffer! 100 100 TimeUnit/MILLISECONDS)
+      (.shutdown exec))))
+
+(defn stop-publish-buffer-flush!
+  "Stops the publish buffer flush thread and drains remaining entries."
+  []
+  (when-let [^ScheduledExecutorService exec
+             (first (reset-vals! publish-buffer-executor nil))]
+    (.shutdownNow exec))
+  (flush-publish-buffer!))

src/metabase/mq/queue/appdb.clj

@@ -0,0 +1,198 @@
+(ns metabase.mq.queue.appdb
+  "Database-backed implementation of the message queue using the application database."
+  (:require
+   [metabase.app-db.core :as mdb]
+   [metabase.models.interface :as mi]
+   [metabase.mq.analytics :as mq.analytics]
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.polling :as mq.polling]
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.mq.queue.impl :as q.impl]
+   [metabase.util.json :as json]
+   [metabase.util.log :as log]
+   [toucan2.core :as t2])
+  (:import (java.sql Timestamp)
+           (java.time Instant)))
+
+(set! *warn-on-reflection* true)
+
+(defn- for-update-clause
+  "Returns the FOR UPDATE clause for the current app DB.
+  Uses SKIP LOCKED on PostgreSQL and MySQL to avoid blocking on rows locked by other nodes.
+  Falls back to plain FOR UPDATE on H2."
+  []
+  (if (#{:postgres :mysql} (mdb/db-type))
+    [:update :skip-locked]
+    [:update]))
+
+(def ^:private owner-id (str (random-uuid)))
+(def ^:private poll-state (mq.polling/make-poll-state))
+
+;;; ------------------------------------------- Fetch -------------------------------------------
+
+(defn- fetch!
+  "Fetches the oldest pending row for each of the given queue names.
+  Returns a seq of maps with :bundle-id, :queue, and :messages keys, or nil if no messages are available.
+  Selects one row per queue, marks them all as 'processing', and returns them — all within one transaction."
+  [queue-names]
+  (when (seq queue-names)
+    (let [exclusive-names (q.impl/exclusive-queue-names)]
+      (t2/with-transaction [conn]
+        (let [blocked-queues  (when (seq exclusive-names)
+                                (into #{}
+                                      (map :queue_name)
+                                      (t2/query conn {:select   [:queue_name]
+                                                      :from     [:queue_message_batch]
+                                                      :where    [:and
+                                                                 [:in :queue_name (vec exclusive-names)]
+                                                                 [:= :status "processing"]]
+                                                      :group-by [:queue_name]})))
+              fetchable-names (if (seq blocked-queues)
+                                (remove #(contains? blocked-queues (name %)) queue-names)
+                                queue-names)]
+          (when (seq fetchable-names)
+            ;; Fetch the oldest pending row per queue in a single query, locking them for update
+            (let [name-list (mapv name fetchable-names)
+                  rows      (t2/query conn {:select [:*]
+                                            :from   [:queue_message_batch]
+                                            :where  [:in :id {:select   [[[:min :id]]]
+                                                              :from     [:queue_message_batch]
+                                                              :where    [:and
+                                                                         [:in :queue_name name-list]
+                                                                         [:= :status "pending"]]
+                                                              :group-by [:queue_name]}]
+                                            :for    (for-update-clause)})]
+              (when (seq rows)
+                (t2/query conn {:update [:queue_message_batch]
+                                :set    {:status_heartbeat (mi/now)
+                                         :status           "processing"
+                                         :owner            owner-id}
+                                :where  [:in :id (mapv :id rows)]})
+                (mapv (fn [row]
+                        {:bundle-id (:id row)
+                         :queue     (keyword "queue" (:queue_name row))
+                         :messages  (json/decode (:messages row))})
+                      rows)))))))))
+
+;; Stale processing recovery
+(def ^:private stale-processing-timeout-ms
+  "Messages in 'processing' status for longer than this are considered stale and recovered."
+  (* 10 60 1000))
+
+(def ^:private last-stale-check-ms (atom 0))
+
+(defn- recover-stale-processing-batches!
+  "Recovers processing batches whose heartbeat is older than the stale timeout.
+  Returns the number of batches recovered."
+  []
+  (let [max-retries (@@q.impl/queue-max-retries)
+        threshold   (Timestamp/from (.minusMillis (Instant/now) stale-processing-timeout-ms))
+        recovered   (atom 0)]
+    (doseq [row (t2/select :queue_message_batch :status "processing" :status_heartbeat [:< threshold])]
+      (let [new-failures (inc (:failures row))]
+        (if (>= new-failures max-retries)
+          (do
+            (log/warnf "Processing batch %d on queue '%s' has reached max failures (%d), marking as failed"
+                       (:id row) (:queue_name row) max-retries)
+            (mq.analytics/inc! :metabase-mq/queue-batch-permanent-failures {:channel (:queue_name row)})
+            (t2/update! :queue_message_batch (:id row)
+                        {:status "failed" :failures new-failures :status_heartbeat (mi/now) :owner nil}))
+          (do
+            (log/warnf "Recovering processing batch %d on queue '%s' (failures: %d -> %d)"
+                       (:id row) (:queue_name row) (:failures row) new-failures)
+            (mq.analytics/inc! :metabase-mq/batch-stale-recoveries {:transport "queue" :channel (:queue_name row)})
+            (t2/update! :queue_message_batch (:id row)
+                        {:status "pending" :failures new-failures :status_heartbeat (mi/now) :owner nil})))
+        (swap! recovered inc)))
+    @recovered))
+
+;; Failed batch cleanup
+(def ^:private last-cleanup-ms (atom 0))
+
+(defn- cleanup-failed-batches! []
+  (let [threshold (Timestamp/from (.minusMillis (Instant/now) (* 7 24 60 60 1000)))
+        deleted   (t2/delete! :queue_message_batch :status "failed" :status_heartbeat [:< threshold])]
+    (when (pos? deleted)
+      (log/infof "Cleaned up %d failed queue batches" deleted)
+      (mq.analytics/inc! :metabase-mq/appdb-cleanup-deleted {:transport "queue" :channel "all"} deleted))))
+
+;; Queue depth gauge
+(def ^:private last-depth-gauge-ms (atom 0))
+
+(defn- update-depth-gauges! []
+  (doseq [{:keys [queue_name status cnt]}
+          (t2/query {:select   [:queue_name :status [[:count :*] :cnt]]
+                     :from     [:queue_message_batch]
+                     :group-by [:queue_name :status]})]
+    (mq.analytics/set! :metabase-mq/appdb-queue-depth {:channel queue_name :status status} cnt)))
+
+;; Heartbeat
+(def ^:private last-heartbeat-ms (atom 0))
+
+(defn- update-heartbeats! []
+  (doseq [channel (filter #(= "queue" (namespace %)) (mq.impl/busy-channels))]
+    (when-let [{:keys [bundle-id]} (mq.impl/active-handler-metadata channel)]
+      (try
+        (t2/update! :queue_message_batch
+                    {:id bundle-id :owner owner-id :status "processing"}
+                    {:status_heartbeat (mi/now)})
+        (catch Exception e
+          (log/warnf e "Failed to update heartbeat for bundle %d" bundle-id))))))
+
+;;; ------------------------------------------- Polling -------------------------------------------
+
+(defn- poll-iteration!
+  "One iteration of the polling loop: run periodic tasks, then try to process batches.
+  Fetches one pending row per available queue and submits each for delivery.
+  Returns true if any work was found."
+  []
+  (mq.polling/periodically! last-stale-check-ms  (* 60 1000)       "stale batch recovery"  recover-stale-processing-batches!)
+  (mq.polling/periodically! last-heartbeat-ms     (* 2 60 1000)     "heartbeat update"      update-heartbeats!)
+  (mq.polling/periodically! last-cleanup-ms       (* 24 60 60 1000) "failed batch cleanup"  cleanup-failed-batches!)
+  (mq.polling/periodically! last-depth-gauge-ms   (* 30 1000)       "queue depth gauge"     update-depth-gauges!)
+  (boolean
+   (when-let [available-queues (seq (remove mq.impl/channel-busy? (listener/queue-names)))]
+     (when-let [batches (seq (fetch! available-queues))]
+       (doseq [{:keys [bundle-id queue messages]} batches]
+         (mq.impl/submit-delivery! queue messages bundle-id :queue.backend/appdb {:bundle-id bundle-id}))
+       true))))
+
+(defmethod q.backend/start! :queue.backend/appdb [_]
+  (mq.polling/start-polling! poll-state "Queue" 5000 poll-iteration!))
+
+(defmethod q.backend/shutdown! :queue.backend/appdb [_]
+  (mq.polling/stop-polling! poll-state "Queue"))
+
+(defmethod q.backend/publish! :queue.backend/appdb
+  [_ queue messages]
+  (t2/insert! :queue_message_batch
+              {:queue_name (name queue)
+               :messages   (json/encode messages)})
+  (when-not (mq.impl/channel-busy? queue)
+    (mq.polling/notify! poll-state)))
+
+(defmethod q.backend/bundle-successful! :queue.backend/appdb
+  [_ _queue-name bundle-id]
+  (let [deleted (t2/delete! :queue_message_batch :id bundle-id :owner owner-id)]
+    (when (= 0 deleted)
+      (log/warnf "Message %d was already deleted from the queue." bundle-id))))
+
+(defmethod q.backend/bundle-failed! :queue.backend/appdb
+  [_ queue-name bundle-id]
+  (let [row     (t2/select-one :queue_message_batch :id bundle-id :owner owner-id)
+        updated (when row
+                  (if (>= (inc (:failures row)) (@@q.impl/queue-max-retries))
+                    (do
+                      (log/warnf "Message %d has reached max failures (%d), marking as failed" bundle-id (@@q.impl/queue-max-retries))
+                      (mq.analytics/inc! :metabase-mq/queue-batch-permanent-failures {:channel (name queue-name)})
+                      (t2/update! :queue_message_batch
+                                  {:id bundle-id :owner owner-id}
+                                  {:status "failed" :failures [:+ :failures 1] :status_heartbeat (mi/now) :owner nil}))
+                    (do
+                      (mq.analytics/inc! :metabase-mq/queue-batch-retries {:channel (name queue-name)})
+                      (t2/update! :queue_message_batch
+                                  {:id bundle-id :owner owner-id}
+                                  {:status "pending" :failures [:+ :failures 1] :status_heartbeat (mi/now) :owner nil}))))]
+    (when (and row (= 0 updated))
+      (log/warnf "Message %d was not found in the queue. Likely error in concurrency handling" bundle-id))))

src/metabase/mq/queue/backend.clj

@@ -0,0 +1,47 @@
+(ns metabase.mq.queue.backend
+  "Backend abstraction layer for the message queue system.
+  Defines multimethods that different queue implementations must provide."
+  (:require
+   [metabase.util.malli.registry :as mr]))
+
+(set! *warn-on-reflection* true)
+
+(mr/def ::backend
+  [:enum :queue.backend/appdb :queue.backend/memory :queue.backend/sync])
+
+(def ^:dynamic *backend*
+  "Dynamic var specifying which queue backend to use.
+  Default is `:queue.backend/appdb` for production database-backed queues."
+  :queue.backend/appdb)
+
+(defmulti publish!
+  "Publishes messages to the given queue. `messages` is always a vector of messages."
+  {:arglists '([backend queue-name messages])}
+  (fn [backend _queue-name _messages]
+    backend))
+
+(defmulti bundle-successful!
+  "Mark a bundle as successfully processed"
+  {:arglists '([backend queue-name bundle-id])}
+  (fn [backend _queue-name _bundle-id]
+    backend))
+
+(defmulti bundle-failed!
+  "Mark a bundle as failed and increment failure count"
+  {:arglists '([backend queue-name bundle-id])}
+  (fn [backend _queue-name _bundle-id]
+    backend))
+
+(defmulti start!
+  "Starts the backend polling loop. Called once at init time."
+  {:arglists '([backend])}
+  identity)
+
+(defmethod start! :default [_] nil)
+
+(defmulti shutdown!
+  "Shuts down all queue resources for this backend: stops all listeners and releases any background threads."
+  {:arglists '([backend])}
+  identity)
+
+(defmethod shutdown! :default [_] nil)

src/metabase/mq/queue/impl.clj

@@ -0,0 +1,40 @@
+(ns metabase.mq.queue.impl
+  "Internal implementation for the queue messaging system"
+  (:require
+   [metabase.mq.listener :as listener]
+   [metabase.mq.publish :as publish]
+   [metabase.util.malli.registry :as mr]))
+
+(set! *warn-on-reflection* true)
+
+(mr/def :metabase.mq.queue/queue-name
+  [:and :keyword [:fn {:error/message "Queue name must be namespaced to 'queue'"}
+                  #(= "queue" (namespace %))]])
+
+(defn exclusive?
+  "Returns true if the given queue name is registered with `:exclusive true`."
+  [queue-name]
+  (:exclusive (listener/get-listener queue-name)))
+
+(defn exclusive-queue-names
+  "Returns the set of queue names (as strings) that are registered with `:exclusive true`."
+  []
+  (into #{}
+        (comp (filter (fn [[_k v]] (:exclusive v)))
+              (map (fn [[k _v]] (name k))))
+        @listener/*listeners*))
+
+(def queue-max-retries
+  "Lazily-resolved var for `metabase.mq.settings/queue-max-retries`. Use `@@q.impl/queue-max-retries` to read the current value."
+  (delay (requiring-resolve 'metabase.mq.settings/queue-max-retries)))
+
+(defmacro with-queue
+  "Runs the body with the ability to add messages to the given queue.
+  Messages are buffered and only published if the body completes successfully.
+  If an exception occurs, no messages are published and the exception is rethrown.
+
+  When called inside a database transaction, messages are accumulated and published
+  as a single batch after the transaction commits successfully. This prevents consumers
+  from reading uncommitted data."
+  [queue-name [queue-binding] & body]
+  `(publish/with-buffer ~queue-name [~queue-binding] ~@body))

src/metabase/mq/queue/memory.clj

@@ -0,0 +1,40 @@
+(ns metabase.mq.queue.memory
+  "In-memory queue backend. Delegates storage and polling to the shared memory layer.
+  Provides queue-specific bundle tracking for retry semantics."
+  (:require
+   [metabase.mq.analytics :as mq.analytics]
+   [metabase.mq.memory :as memory]
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.mq.queue.impl :as q.impl]
+   [metabase.util.log :as log]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:dynamic *bundle-registry*
+  "Maps bundle-id -> {:messages [...] :failures n} for retry tracking."
+  (atom {}))
+
+(defmethod q.backend/publish! :queue.backend/memory [_ queue-name messages]
+  (memory/publish! queue-name messages))
+
+(defmethod q.backend/start! :queue.backend/memory [_]
+  (memory/start!))
+
+(defmethod q.backend/shutdown! :queue.backend/memory [_]
+  (memory/shutdown!))
+
+(defmethod q.backend/bundle-successful! :queue.backend/memory [_ _queue-name bundle-id]
+  (swap! *bundle-registry* dissoc bundle-id))
+
+(defmethod q.backend/bundle-failed! :queue.backend/memory [_ queue-name bundle-id]
+  (when-let [{:keys [messages failures]} (get @*bundle-registry* bundle-id)]
+    (swap! *bundle-registry* dissoc bundle-id)
+    (let [new-failures (inc failures)]
+      (if (>= new-failures (@@q.impl/queue-max-retries))
+        (do
+          (log/warnf "Bundle %s has reached max failures (%d), dropping" bundle-id (@@q.impl/queue-max-retries))
+          (mq.analytics/inc! :metabase-mq/queue-batch-permanent-failures {:channel (name queue-name)}))
+        (do
+          (mq.analytics/inc! :metabase-mq/queue-batch-retries {:channel (name queue-name)})
+          ;; Re-queue messages for retry
+          (memory/publish! queue-name messages))))))

src/metabase/mq/queue/sync.clj

@@ -0,0 +1,27 @@
+(ns metabase.mq.queue.sync
+  "Synchronous queue backend that calls the listener inline during `publish!`.
+  Useful in tests to avoid needing `*force-sync*` dynamic vars in each namespace."
+  (:require
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.util.log :as log]))
+
+(set! *warn-on-reflection* true)
+
+(defmethod q.backend/publish! :queue.backend/sync [_ queue-name messages]
+  (let [bundle-id (str (random-uuid))]
+    (if (listener/get-listener queue-name)
+      ;; Call handle! directly to bypass accumulation — the sync backend must
+      ;; deliver immediately so that tests observe side-effects inline.
+      (mq.impl/handle! queue-name {bundle-id :queue.backend/sync} messages)
+      (log/warnf "No listener registered for queue %s, dropping %d message(s)" queue-name (count messages)))))
+
+(defmethod q.backend/bundle-successful! :queue.backend/sync [_ _queue-name _bundle-id]
+  nil)
+
+(defmethod q.backend/bundle-failed! :queue.backend/sync [_ _queue-name _bundle-id]
+  nil)
+
+(defmethod q.backend/shutdown! :queue.backend/sync [_]
+  nil)

src/metabase/mq/queue/transport_impl.clj

@@ -0,0 +1,22 @@
+(ns metabase.mq.queue.transport-impl
+  "Queue implementations of the transport multimethods."
+  (:require
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.mq.transport :as transport]))
+
+(set! *warn-on-reflection* true)
+
+(defmethod transport/on-listen! :queue [_ opts]
+  {:exclusive (boolean (:exclusive opts))})
+
+(defmethod transport/publish! :queue [channel messages]
+  (q.backend/publish! q.backend/*backend* channel messages))
+
+(defmethod transport/wrap-listener :queue [_channel listener]
+  listener)
+
+(defmethod transport/start! :queue [_]
+  (q.backend/start! q.backend/*backend*))
+
+(defmethod transport/shutdown! :queue [_]
+  (q.backend/shutdown! q.backend/*backend*))

src/metabase/mq/settings.clj

@@ -0,0 +1,33 @@
+(ns metabase.mq.settings
+  (:require
+   [metabase.app-db.core :as mdb]
+   [metabase.settings.core :as setting :refer [defsetting]]
+   [metabase.util.i18n :refer [deferred-tru]]))
+
+(defsetting queue-max-retries
+  (deferred-tru "Maximum number of times a failed queue message will be retried before being dropped.")
+  :type       :integer
+  :default    5
+  :visibility :internal
+  :encryption :no
+  :export?    false)
+
+(defsetting queue-backend
+  (deferred-tru "Which queue backend to use. Valid values: appdb, memory.")
+  :type       :string
+  :default    "appdb"
+  :visibility :internal
+  :encryption :no
+  :export?    false)
+
+(defsetting topic-backend
+  (deferred-tru "Which topic backend to use. Valid values: appdb, memory.")
+  :type       :string
+  :getter     (fn []
+                (or (setting/get-value-of-type :string :topic-backend)
+                    (if (= (mdb/db-type) :postgres)
+                      "appdb"
+                      "memory")))
+  :visibility :internal
+  :encryption :no
+  :export?    false)

src/metabase/mq/topic/appdb.clj

@@ -0,0 +1,114 @@
+(ns metabase.mq.topic.appdb
+  "Database-backed implementation of the pub/sub system using the application database.
+  Messages are stored in the `topic_message_batch` table. Each subscriber on each node polls
+  independently, tracking its read offset in memory."
+  (:require
+   [metabase.mq.analytics :as mq.analytics]
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.polling :as mq.polling]
+   [metabase.mq.topic.backend :as topic.backend]
+   [metabase.util.json :as json]
+   [metabase.util.log :as log]
+   [toucan2.core :as t2])
+  (:import (java.sql Timestamp)
+           (java.time Instant)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private offsets
+  "Map of topic-name -> long offset."
+  (atom {}))
+
+(def ^:private poll-state (mq.polling/make-poll-state))
+
+(def ^:private batch-size
+  "Maximum number of messages to fetch in a single poll."
+  100)
+
+(defn- current-max-id
+  "Returns the current maximum id in topic_message_batch for the given topic, or 0."
+  [topic-name]
+  (or (t2/select-one-fn :max-id [:topic_message_batch [[:max :id] :max-id]]
+                        :topic_name (name topic-name))
+      0))
+
+(defn- poll-messages!
+  "Fetches rows from topic_message_batch with id > offset."
+  [topic-name offset]
+  (t2/query {:select   [:id :messages]
+             :from     [:topic_message_batch]
+             :where    [:and
+                        [:= :topic_name (name topic-name)]
+                        [:> :id offset]]
+             :order-by [[:id :asc]]
+             :limit    batch-size}))
+
+;;; ------------------------------------------- Periodic tasks (run on polling thread) -------------------------------------------
+
+;; Cleanup old messages
+(def ^:private last-cleanup-ms (atom 0))
+
+(defn- cleanup-old-messages! []
+  (let [threshold (Timestamp/from (.minusMillis (Instant/now) (* 60 60 1000)))
+        deleted   (t2/delete! :topic_message_batch :created_at [:< threshold])]
+    (when (pos? deleted)
+      (log/infof "Cleaned up %d old topic messages" deleted)
+      (mq.analytics/inc! :metabase-mq/appdb-cleanup-deleted {:transport "topic" :channel "all"} deleted))))
+
+(def ^:private last-lag-gauge-ms (atom 0))
+
+(defn- update-lag-gauges! []
+  (doseq [[topic-name offset] @offsets]
+    (let [max-id (current-max-id topic-name)]
+      (mq.analytics/set! :metabase-mq/appdb-topic-subscriber-lag
+                         {:channel (name topic-name)}
+                         (- max-id offset)))))
+
+(defn- poll-iteration!
+  "One iteration of the polling loop: run periodic tasks, then poll all topics.
+  Returns true if any topic had messages delivered."
+  []
+  (mq.polling/periodically! last-cleanup-ms   (* 10 60 1000) "topic cleanup"    cleanup-old-messages!)
+  (mq.polling/periodically! last-lag-gauge-ms (* 30 1000)    "topic lag gauge"  update-lag-gauges!)
+  (let [found-work? (atom false)]
+    (doseq [topic-name (remove mq.impl/channel-busy? (listener/topic-names))]
+      (let [offset (if (contains? @offsets topic-name)
+                     (get @offsets topic-name)
+                     (let [o (current-max-id topic-name)]
+                       (swap! offsets assoc topic-name o)
+                       o))
+            rows   (poll-messages! topic-name offset)]
+        (when (seq rows)
+          (let [all-messages (into [] (mapcat (comp json/decode :messages)) rows)
+                max-id       (:id (last rows))]
+            (when (mq.impl/submit-delivery! topic-name all-messages nil nil nil)
+              (swap! offsets assoc topic-name max-id)
+              (reset! found-work? true))))))
+    @found-work?))
+
+(defmethod topic.backend/start! :topic.backend/appdb [_]
+  (mq.polling/start-polling! poll-state "Topic" 2000 poll-iteration!))
+
+(defmethod topic.backend/shutdown! :topic.backend/appdb [_]
+  (mq.polling/stop-polling! poll-state "Topic")
+  (reset! offsets {}))
+
+(defmethod topic.backend/publish! :topic.backend/appdb
+  [_ topic-name messages]
+  (t2/insert-returning-pk! :topic_message_batch
+                           {:topic_name (name topic-name)
+                            :messages   (json/encode messages)})
+  (when-not (mq.impl/channel-busy? topic-name)
+    (mq.polling/notify! poll-state)))
+
+(defmethod topic.backend/subscribe! :topic.backend/appdb
+  [_ topic-name]
+  (let [offset (current-max-id topic-name)]
+    (swap! offsets assoc topic-name offset)
+    (log/infof "Subscribed to topic %s (starting offset %d)" (name topic-name) offset)))
+
+(defmethod topic.backend/unsubscribe! :topic.backend/appdb
+  [_ topic-name]
+  (swap! offsets dissoc topic-name)
+  (log/infof "Unsubscribed from topic %s" (name topic-name)))

src/metabase/mq/topic/backend.clj

@@ -0,0 +1,56 @@
+(ns metabase.mq.topic.backend
+  "Backend abstraction layer for the pub/sub system.
+  Defines multimethods that different pub/sub implementations must provide.
+
+  Topics are fire-and-forget: there is no retry logic. If a listener throws,
+  the error is logged and the batch is skipped. This matches the semantics of
+  the postgres LISTEN/NOTIFY backend where re-delivery is not possible."
+  (:require
+   [metabase.util.malli.registry :as mr]))
+
+(set! *warn-on-reflection* true)
+
+(mr/def ::backend
+  [:enum :topic.backend/appdb :topic.backend/memory :topic.backend/sync])
+
+(def ^:dynamic *backend*
+  "Dynamic var specifying which pub/sub backend to use.
+  Default is `:topic.backend/appdb` for production database-backed pub/sub.
+  Can be bound to `:topic.backend/memory` for testing."
+  :topic.backend/appdb)
+
+(defmulti publish!
+  "Publishes messages to the given topic. `messages` is a vector of values that will be JSON-encoded
+  as an array and stored in a single row for batch efficiency.
+  Return the batch id of the messages published to avoid re-publishing locally."
+  {:arglists '([backend topic-name messages])}
+  (fn [backend _topic-name _messages]
+    backend))
+
+(defmulti start!
+  "Starts the backend polling loop. Called once at init time.
+  The backend polls `mq.impl/*listeners*` dynamically to discover registered topics."
+  {:arglists '([backend])}
+  identity)
+
+(defmethod start! :default [_] nil)
+
+(defmulti subscribe!
+  "Internal operation: initializes backend state (offsets, channels) for a specific topic.
+  Called by the polling loop when it discovers a new topic in `*listeners*`, not by `listen!`."
+  {:arglists '([backend topic-name])}
+  (fn [backend _topic-name]
+    backend))
+
+(defmulti unsubscribe!
+  "Stops the polling loop for the given topic."
+  {:arglists '([backend topic-name])}
+  (fn [backend _topic-name]
+    backend))
+
+(defmulti shutdown!
+  "Shuts down all topic resources for this backend: unsubscribes all subscribers and releases any background threads."
+  {:arglists '([backend])}
+  identity)
+
+(defmethod shutdown! :default [_] nil)

src/metabase/mq/topic/impl.clj

@@ -0,0 +1,25 @@
+(ns metabase.mq.topic.impl
+  "Topic-specific implementation: on-listen! hook and topic schemas."
+  (:require
+   [metabase.mq.publish :as publish]
+   [metabase.util.malli.registry :as mr]))
+
+(set! *warn-on-reflection* true)
+
+(mr/def :metabase.mq.topic/topic-name
+  [:and :keyword [:fn {:error/message "Topic name must be namespaced to 'topic'"}
+                  #(= "topic" (namespace %))]])
+
+(defmacro with-topic
+  "Runs the body with the ability to add messages to the given topic.
+  Messages are buffered and only published if the body completes successfully.
+  If an exception occurs, no messages are published and the exception is rethrown.
+
+  Outside a transaction, publishing is best-effort and fire-and-forget — if the body
+  succeeds but `publish!` throws, the body's side effects will have already occurred.
+
+  Inside a database transaction, messages are deferred and published as a single batch
+  after the transaction commits successfully. This prevents consumers from seeing
+  uncommitted data. If the transaction rolls back, the messages are discarded."
+  [topic-name [buffer-binding] & body]
+  `(publish/with-buffer ~topic-name [~buffer-binding] ~@body))

src/metabase/mq/topic/memory.clj

@@ -0,0 +1,23 @@
+(ns metabase.mq.topic.memory
+  "In-memory topic backend. Delegates storage and polling to the shared memory layer."
+  (:require
+   [metabase.mq.memory :as memory]
+   [metabase.mq.topic.backend :as topic.backend]
+   [metabase.util.log :as log]))
+
+(set! *warn-on-reflection* true)
+
+(defmethod topic.backend/start! :topic.backend/memory [_]
+  (memory/start!))
+
+(defmethod topic.backend/publish! :topic.backend/memory [_ topic-name messages]
+  (memory/publish! topic-name messages))
+
+(defmethod topic.backend/subscribe! :topic.backend/memory [_ topic-name]
+  (log/infof "Memory subscribed to topic %s" (name topic-name)))
+
+(defmethod topic.backend/unsubscribe! :topic.backend/memory [_ topic-name]
+  (log/infof "Memory unsubscribed from topic %s" (name topic-name)))
+
+(defmethod topic.backend/shutdown! :topic.backend/memory [_]
+  (memory/shutdown!))

src/metabase/mq/topic/sync.clj

@@ -0,0 +1,18 @@
+(ns metabase.mq.topic.sync
+  "Synchronous topic backend that calls listeners inline during `publish!`.
+  Useful in tests to avoid Thread/sleep for polling-based backends."
+  (:require
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.topic.backend :as topic.backend]))
+
+(set! *warn-on-reflection* true)
+
+(defmethod topic.backend/publish! :topic.backend/sync
+  [_ topic-name messages]
+  (mq.impl/handle! topic-name {} messages)
+  nil)
+
+(defmethod topic.backend/start! :topic.backend/sync [_] nil)
+(defmethod topic.backend/subscribe! :topic.backend/sync [_ _topic-name] nil)
+(defmethod topic.backend/unsubscribe! :topic.backend/sync [_ _topic-name] nil)
+(defmethod topic.backend/shutdown! :topic.backend/sync [_] nil)

src/metabase/mq/topic/transport_impl.clj

@@ -0,0 +1,24 @@
+(ns metabase.mq.topic.transport-impl
+  "Topic implementations of the transport multimethods."
+  (:require
+   [metabase.mq.listener :as listener]
+   [metabase.mq.topic.backend :as topic.backend]
+   [metabase.mq.transport :as transport]))
+
+(set! *warn-on-reflection* true)
+
+(defmethod transport/on-listen! :topic [channel _opts]
+  (topic.backend/subscribe! topic.backend/*backend* channel)
+  {})
+
+(defmethod transport/publish! :topic [channel messages]
+  (topic.backend/publish! topic.backend/*backend* channel messages))
+
+(defmethod transport/wrap-listener :topic [channel listener]
+  (listener/make-instrumented-listener channel listener))
+
+(defmethod transport/start! :topic [_]
+  (topic.backend/start! topic.backend/*backend*))
+
+(defmethod transport/shutdown! :topic [_]
+  (topic.backend/shutdown! topic.backend/*backend*))

src/metabase/mq/transport.clj

@@ -0,0 +1,35 @@
+(ns metabase.mq.transport
+  "Multimethods dispatched by transport type (:queue or :topic).")
+
+(defn transport-type
+  "Returns :queue or :topic for a namespaced channel keyword."
+  [channel]
+  (keyword (namespace channel)))
+
+(defmulti publish!
+  "Publishes messages to the appropriate backend for the transport type."
+  {:arglists '([channel messages])}
+  (fn [channel _messages] (transport-type channel)))
+
+(defmulti on-listen!
+  "Transport-type-specific hook called during listener registration.
+   Returns a map of default opts for the transport type.
+   Implementations may also perform side effects (e.g. subscribing to a topic backend)."
+  {:arglists '([channel opts])}
+  (fn [channel _opts] (transport-type channel)))
+
+(defmulti wrap-listener
+  "Wraps a listener function with transport-type-specific behavior.
+   Queues return the listener as-is; topics add instrumentation."
+  {:arglists '([channel listener])}
+  (fn [channel _listener] (transport-type channel)))
+
+(defmulti start!
+  "Starts the backend for the given transport type."
+  {:arglists '([transport-type])}
+  identity)
+
+(defmulti shutdown!
+  "Shuts down the backend for the given transport type."
+  {:arglists '([transport-type])}
+  identity)

src/metabase/search/api.clj

@@ -6,15 +6,11 @@
    [metabase.api.common :as api]
    [metabase.api.macros :as api.macros]
    [metabase.api.open-api :as open-api]
-   [metabase.config.core :as config]
    [metabase.permissions.core :as perms]
    [metabase.request.core :as request]
    [metabase.search.config :as search.config]
    [metabase.search.core :as search]
-   [metabase.search.ingestion :as ingestion]
    [metabase.search.settings :as search.settings]
-   [metabase.search.task.search-index :as task.search-index]
-   [metabase.task.core :as task]
    [metabase.util :as u]
    [metabase.util.json :as json]
    [metabase.util.log :as log]
@@ -76,7 +72,9 @@
   []
   (api/check-superuser)
   (if (search/supports-index?)
-    {:message (search/init-index! {:force-reset? true})}
+    (do
+      (search/async-init! :force-reset? true)
+      {:message "re-init enqueued"})
     (throw (ex-info "Search index is not supported for this installation." {:status-code 501}))))
 
 ;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
@@ -88,10 +86,7 @@
   []
   (api/check-superuser)
   (if (search/supports-index?)
-    ;; The job appears to wait on the main thread when run from tests, so, unfortunately, testing this branch is hard.
-    (if (and (task/job-exists? task.search-index/reindex-job-key) (or (not ingestion/*force-sync*) config/is-test?))
-      (do (task/trigger-now! task.search-index/reindex-job-key) {:message "task triggered"})
-      (do (search/reindex!) {:message "reindex triggered"}))
+    (do (search/async-reindex!) {:message "reindex triggered"})
 
     (throw (ex-info "Search index is not supported for this installation." {:status-code 501}))))
 

src/metabase/search/appdb/core.clj

@@ -1,7 +1,6 @@
 (ns metabase.search.appdb.core
   (:require
    [clojure.string :as str]
-   [environ.core :as env]
    [honey.sql.helpers :as sql.helpers]
    [java-time.api :as t]
    [metabase.app-db.core :as mdb]
@@ -13,12 +12,12 @@
    [metabase.search.config :as search.config]
    [metabase.search.engine :as search.engine]
    [metabase.search.filter :as search.filter]
+   [metabase.search.impl :as search.impl]
    [metabase.search.ingestion :as search.ingestion]
    [metabase.search.permissions :as search.permissions]
    [metabase.search.settings :as search.settings]
    [metabase.search.spec :as search.spec]
    [metabase.search.util :as search.util]
-   [metabase.settings.core :as setting]
    [metabase.tracing.core :as tracing]
    [metabase.util :as u]
    [metabase.util.i18n :as i18n]
@@ -27,8 +26,7 @@
    [methodical.core :as methodical]
    [toucan2.core :as t2])
   (:import
-   (java.time OffsetDateTime)
-   (java.util Queue)))
+   (java.time OffsetDateTime)))
 
 ;; Register the multimethods for each specialization
 (comment
@@ -131,8 +129,7 @@
       (when init-now?
         (log/warnf "Triggering a late initialization of the %s search index." search-engine)
         (try
-          (future
-            (search.engine/init! search-engine {:force-reset? false}))
+          (search.impl/async-init! :engine search-engine :force-reset? false)
           (catch Exception e
             (log/error e))))
       ;; Even if the index exists now, return an error so that we don't obscure that there was an issue.
@@ -148,15 +145,6 @@
 
   (tracing/with-span :search "search.appdb.query" {:search/query-length (count search-string)}
     (try
-      (when (setting/string->boolean (:mb-experimental-search-block-on-queue env/env))
-        ;; wait for a bit for the queue to be drained
-        (let [pending-updates #(.size ^Queue @#'search.ingestion/queue)]
-          (when-not (u/poll {:thunk       pending-updates
-                             :done?       zero?
-                             :timeout-ms  2000
-                             :interval-ms 100})
-            (log/warn "Returning search results even though they may be stale. Queue size:" (pending-updates)))))
-
       (let [weights (search.config/weights search-ctx)
             scorers (search.scoring/scorers search-ctx)
             query   (->> (search.index/search-query search-string search-ctx [:legacy_input])
@@ -204,7 +192,7 @@
     (if (and index-created (< 3 (t/time-between (t/instant index-created) (t/instant) :days)))
       (do
         (log/info "Forcing early reindex because existing index is old")
-        (search.engine/reindex! :search.engine/appdb {}))
+        (search.impl/async-reindex! :engine :search.engine/appdb))
 
       (let [created? (search.index/ensure-ready! opts)]
         (when (or created? re-populate?)
@@ -235,6 +223,4 @@
   [_topic event]
   (when (and (= :site-locale (-> event :details :key)) (= :postgres (mdb/db-type)))
     (log/info "Reindexing appdb index because the site locale changed.")
-    (if search.ingestion/*force-sync*
-      (search.engine/reindex! :search.engine/appdb {})
-      (future (search.engine/reindex! :search.engine/appdb {})))))
+    (search.impl/async-reindex! :engine :search.engine/appdb)))

src/metabase/search/appdb/index.clj

@@ -12,7 +12,6 @@
    [metabase.search.appdb.specialization.postgres :as postgres]
    [metabase.search.config :as search.config]
    [metabase.search.engine :as search.engine]
-   [metabase.search.ingestion :as search.ingestion]
    [metabase.search.models.search-index-metadata :as search-index-metadata]
    [metabase.search.spec :as search.spec]
    [metabase.tracing.core :as tracing]
@@ -316,7 +315,7 @@
         (log/warnf "Unable to find table %s and no longer tracking it as pending", table)
         (swap! *indexes* assoc :pending nil))))
 
-  (let [reindexing? (and (= :search/reindexing context) (not search.ingestion/*force-sync*))
+  (let [reindexing? (= :search/reindexing context)
         do-writes   (fn []
                       (let [entries         (map document->entry documents)
                             ;; No need to update the active index if we are doing a full index, as this table will be
@@ -413,12 +412,10 @@
               (swap! *indexes* assoc :pending nil))
             (maybe-create-pending!)
             (activate-table!))]
-    (if search.ingestion/*force-sync*
-      (reset-logic)
-      ;; Creates and tracks tables with a unique transaction so the empty tables are available to other threads
-      ;; even while the initial startup and data load may be happening
-      (t2/with-connection [_ (mdb/data-source)]
-        (reset-logic)))))
+    ;; Creates and tracks tables with a unique transaction so the empty tables are available to other threads
+    ;; even while the initial startup and data load may be happening
+    (t2/with-connection [_ (mdb/data-source)]
+      (reset-logic))))
 
 (defn ensure-ready!
   "Ensure the index is ready to be populated. Return false if it was already ready."
@@ -453,3 +450,16 @@
            (finally
              (#'drop-table! table-name#)
              (t2/delete! :model/SearchIndexMetadata :version version#)))))))
+
+(defn expose-temp-table-to-server-threads!
+  "In tests that use HTTP requests, the server thread can't see the test's `*indexes*` binding.
+  Call this after `with-temp-index-table` to update the global atom so server threads find the temp table.
+  Returns a 0-arity fn that restores the previous state — call it in a `finally` block."
+  []
+  (let [prev-indexes @@#'*indexes*
+        prev-sync-at @@#'next-sync-at]
+    (reset! @#'*indexes* {:active (active-table)})
+    (reset! @#'next-sync-at (+ (long (#'now)) (long (* 24 60 60e9))))
+    (fn []
+      (reset! @#'*indexes* prev-indexes)
+      (reset! @#'next-sync-at prev-sync-at))))

src/metabase/search/core.clj

@@ -2,17 +2,12 @@
   "NOT the API namespace for the search module!! See [[metabase.search]] instead."
   (:require
    [metabase.analytics.core :as analytics]
-   [metabase.analytics.prometheus :as prometheus]
-   [metabase.lib-be.core :as lib-be]
    [metabase.search.config :as search.config]
    [metabase.search.engine :as search.engine]
    [metabase.search.impl :as search.impl]
    [metabase.search.ingestion :as search.ingestion]
    [metabase.search.spec :as search.spec]
    [metabase.search.util :as search.util]
-   [metabase.tracing.core :as tracing]
-   [metabase.util :as u]
-   [metabase.util.log :as log]
    [potemkin :as p]))
 
 (set! *warn-on-reflection* true)
@@ -30,7 +25,11 @@
   model-set]
 
  [search.impl
+  async-delete!
+  async-init!
+  async-reindex!
   search
+  supports-index?
   ;; We could avoid exposing this by wrapping `query-model-set` and `search` with it.
   search-context]
 
@@ -78,78 +77,6 @@
   [_ {:keys [engine]}]
   (if (search.engine/supported-engine? (keyword "search.engine" engine)) 1 0))
 
-(defn supports-index?
-  "Does this instance support a search index, of any sort?"
-  []
-  (seq (search.engine/active-engines)))
-
-(defn init-index!
-  "Ensure there is an index ready to be populated."
-  [& {:as opts}]
-  (when (supports-index?)
-    (log/info "Initializing search indexes")
-    (tracing/with-span :search "search.init-index" {}
-      (lib-be/with-metadata-provider-cache
-        ;; If there are multiple indexes, return the peak inserted for each type. In practice, they should all be the same.
-        (try
-          (let [timer    (u/start-timer)
-                report   (reduce (partial merge-with max)
-                                 nil
-                                 (for [e (search.engine/active-engines)]
-                                   (search.engine/init! e opts)))
-                duration (u/since-ms timer)]
-            (if (seq report)
-              (do
-                (analytics/inc! :metabase-search/index-reindex-ms duration)
-                (prometheus/observe! :metabase-search/index-reindex-duration-ms duration)
-                (doseq [[model cnt] report]
-                  (analytics/inc! :metabase-search/index-reindexes {:model model} cnt))
-                (log/infof "Index initialized in %.0fms %s" duration (sort-by (comp - val) report))
-                report)
-              (log/info "Found existing search index, and using it.")))
-          (catch Exception e
-            (analytics/inc! :metabase-search/index-error)
-            (throw e)))))))
-
-(defn- reindex-logic! [opts]
-  (when (supports-index?)
-    (tracing/with-span :search "search.reindex" {}
-      (lib-be/with-metadata-provider-cache
-        (try
-          (log/info "Reindexing searchable entities")
-          (let [timer    (u/start-timer)
-                report   (reduce (partial merge-with max)
-                                 nil
-                                 (for [e (search.engine/active-engines)]
-                                   (search.engine/reindex! e opts)))
-                duration (u/since-ms timer)]
-            (analytics/inc! :metabase-search/index-reindex-ms duration)
-            (prometheus/observe! :metabase-search/index-reindex-duration-ms duration)
-            (doseq [[model cnt] report]
-              (analytics/inc! :metabase-search/index-reindexes {:model model} cnt))
-            (log/infof "Done reindexing in %.0fms %s" duration (sort-by (comp - val) report))
-            report)
-          (catch Exception e
-            (analytics/inc! :metabase-search/index-error)
-            (throw e)))))))
-
-(defn reindex!
-  "Populate a new index, and make it active. Simultaneously updates the current index.
-  Returns a future that will complete when the reindexing is done.
-  Respects `search.ingestion/*force-sync*` and waits for the future if it's true.
-  Alternately, if `:async?` is false, it will also run synchronously."
-  [& {:keys [async?] :or {async? true} :as opts}]
-  (let [f (fn []
-            (try
-              (reindex-logic! opts)
-              (catch Exception e
-                (log/error e "Reindex failed")
-                (analytics/inc! :metabase-search/index-error)
-                (throw e))))]
-    (if (or search.ingestion/*force-sync* (not async?))
-      (doto (promise) (deliver (f)))
-      (future (f)))))
-
 (defn reset-tracking!
   "Stop tracking the current indexes. Used when resetting the appdb."
   []
@@ -165,7 +92,7 @@
     (doseq [e (search.engine/active-engines)]
       (search.engine/sync-from-restored-db! e))))
 
-(defn update!
+(defn async-update!
   "Given a new or updated instance, put all the corresponding search entries if needed in the queue."
   [instance & [always?]]
   (when (supports-index?)
@@ -173,14 +100,4 @@
                             (remove (comp search.util/impossible-condition? second))
                             seq)]
       ;; We need to delay execution to handle deletes, which alert us *before* updating the database.
-      (search.ingestion/ingest-maybe-async! updates))))
-
-(defn delete!
-  "Given a model and a list of model's ids, remove corresponding search entries."
-  [model ids]
-  (when (supports-index?)
-    (doseq [e            (search.engine/active-engines)
-            search-model (->> (vals (search.spec/specifications))
-                              (filter (comp #{model} :model))
-                              (map :name))]
-      (search.engine/delete! e search-model ids))))
+      (search.ingestion/async-updates updates))))

src/metabase/search/engine.clj

@@ -53,13 +53,15 @@
 
 (defmulti init!
   "Ensure that the search index exists, and is ready to take search queries.
-   Returns a map of the number of documents indexed in each model"
+   Returns a map of the number of documents indexed in each model.
+   This should normally be called through a :reindex command on :queue/search-reindex and not directly"
   {:arglists '([engine opts])}
   (fn [engine _opts]
     engine))
 
 (defmulti reindex!
-  "Perform a full refresh of the given engine's index."
+  "Perform a full refresh of the given engine's index.
+  This should normally be called through a :reindex command on :queue/search-reindex and not directly"
   {:arglists '([engine opts])}
   (fn [engine _opts] engine))
 

src/metabase/search/impl.clj

@@ -1,9 +1,13 @@
 (ns metabase.search.impl
   (:require
    [clojure.string :as str]
+   [metabase.analytics.core :as analytics]
+   [metabase.analytics.prometheus :as prometheus]
    [metabase.collections.models.collection :as collection]
    [metabase.collections.models.collection.root :as collection.root]
+   [metabase.lib-be.core :as lib-be]
    [metabase.models.interface :as mi]
+   [metabase.mq.core :as mq]
    [metabase.permissions.core :as perms]
    [metabase.premium-features.core :as premium-features]
    [metabase.search.config :as search.config :refer [SearchableModel SearchContext]]
@@ -11,6 +15,7 @@
    [metabase.search.filter :as search.filter]
    [metabase.search.in-place.filter :as search.in-place.filter]
    [metabase.search.in-place.scoring :as scoring]
+   [metabase.search.spec :as search.spec]
    [metabase.tracing.core :as tracing]
    [metabase.transforms.feature-gating :as transforms.gating]
    [metabase.util :as u]
@@ -460,3 +465,87 @@
                               true (add-collection-effective-location)
                               true (map serialize))]
       (search-results search-ctx search.engine/model-set total-results))))
+
+(defn supports-index?
+  "Does this instance support a search index, of any sort?"
+  []
+  (seq (search.engine/active-engines)))
+
+(defn sync-reindex!
+  "Synchronously populate a new index, and make it active. Simultaneously updates the current index.
+  Prefer [[reindex!]] which routes through the queue. This is for use by the queue handler itself."
+  [& {:as opts}]
+  (when (supports-index?)
+    (tracing/with-span :search "search.reindex" {}
+      (lib-be/with-metadata-provider-cache
+        (try
+          (log/info "Reindexing searchable entities")
+          (let [timer    (u/start-timer)
+                report   (reduce (partial merge-with max)
+                                 nil
+                                 (for [e (search.engine/active-engines)]
+                                   (search.engine/reindex! e opts)))
+                duration (u/since-ms timer)]
+            (analytics/inc! :metabase-search/index-reindex-ms duration)
+            (prometheus/observe! :metabase-search/index-reindex-duration-ms duration)
+            (doseq [[model cnt] report]
+              (analytics/inc! :metabase-search/index-reindexes {:model model} cnt))
+            (log/infof "Done reindexing in %.0fms %s" duration (sort-by (comp - val) report))
+            report)
+          (catch Exception e
+            (analytics/inc! :metabase-search/index-error)
+            (throw e)))))))
+
+(defn sync-init-index!
+  "Synchronously ensure there is an index ready to be populated."
+  [& {:as opts}]
+  (when (supports-index?)
+    (log/info "Initializing search indexes")
+    (tracing/with-span :search "search.init-index" {}
+      (lib-be/with-metadata-provider-cache
+        ;; If there are multiple indexes, return the peak inserted for each type. In practice, they should all be the same.
+        (try
+          (let [timer    (u/start-timer)
+                report   (reduce (partial merge-with max)
+                                 nil
+                                 (for [e (search.engine/active-engines)]
+                                   (search.engine/init! e opts)))
+                duration (u/since-ms timer)]
+            (if (seq report)
+              (do
+                (analytics/inc! :metabase-search/index-reindex-ms duration)
+                (prometheus/observe! :metabase-search/index-reindex-duration-ms duration)
+                (doseq [[model cnt] report]
+                  (analytics/inc! :metabase-search/index-reindexes {:model model} cnt))
+                (log/infof "Index initialized in %.0fms %s" duration (sort-by (comp - val) report))
+                report)
+              (log/info "Found existing search index, and using it.")))
+          (catch Exception e
+            (analytics/inc! :metabase-search/index-error)
+            (throw e)))))))
+
+(defn async-init!
+  "Enqueue an init command. The actual initialization is performed by the queue listener."
+  [& {:as opts}]
+  (mq/with-queue :queue/search-reindex [q]
+    (mq/put q (cond-> {:command :init
+                       :version (search.spec/index-version-hash)}
+                (:force-reset? opts) (assoc :force-reset? true)
+                (:re-populate? opts) (assoc :re-populate? true)
+                (:engine opts)       (assoc :engine (:engine opts))))))
+
+(defn async-reindex!
+  "Enqueue a reindex command. The actual reindexing is performed by the queue listener."
+  [& {:as opts}]
+  (mq/with-queue :queue/search-reindex [q]
+    (mq/put q (cond-> {:command :reindex}
+                (:in-place? opts) (assoc :in-place? true)
+                (:engine opts)    (assoc :engine (:engine opts))))))
+
+(defn async-delete!
+  "Enqueue a delete command. The actual deletion is performed by the queue listener."
+  [model ids]
+  (mq/with-queue :queue/search-reindex [q]
+    (mq/put q {:command :delete
+               :model   model
+               :ids     ids})))

src/metabase/search/ingestion.clj

@@ -1,5 +1,6 @@
 (ns metabase.search.ingestion
   (:require
+   [clojure.edn :as edn]
    [clojure.string :as str]
    [honey.sql.helpers :as sql.helpers]
    [medley.core :as m]
@@ -7,34 +8,18 @@
    [metabase.analytics.prometheus :as prometheus]
    [metabase.app-db.core :as mdb]
    [metabase.lib-be.core :as lib-be]
+   [metabase.mq.core :as mq]
    [metabase.search.engine :as search.engine]
    [metabase.search.spec :as search.spec]
    [metabase.tracing.core :as tracing]
    [metabase.util :as u]
    [metabase.util.log :as log]
    [metabase.util.performance :as perf]
-   [metabase.util.queue :as queue]
    [toucan2.core :as t2]
-   [toucan2.realize :as t2.realize])
-  (:import
-   (java.util.concurrent DelayQueue)))
+   [toucan2.realize :as t2.realize]))
 
 (set! *warn-on-reflection* true)
 
-;; Currently we use a single queue, even if multiple engines are enabled, but may want to revisit this.
-(defonce ^:private ^DelayQueue queue (queue/delay-queue))
-
-;; Perhaps this config move up somewhere more visible? Conversely, we may want to specialize it per engine.
-
-(def ^:private message-delay-ms
-  "The time a message should wait before coming off the queue.
-  This delay exists to ensure the data is fully committed before indexing."
-  100)
-
-(def ^:private listener-name
-  "The name of the listener that consumes the queue"
-  "search-index-update")
-
 (def max-searchable-value-length
   "The maximum length of a searchable value. This is mostly driven by postgresql max-lengths on tsvector columns.
   And is about half of postgresql's max, since we concat two values together often. That is likely aggressive, but being safe until we can better understand normal data shapes"
@@ -222,10 +207,6 @@
        (filter some?)
        (reduce + 0)))
 
-(def ^:dynamic *force-sync*
-  "Force ingestion to happen immediately, on the same thread."
-  false)
-
 (def ^:dynamic *disable-updates*
   "Used by tests to disable updates, for example when testing migrations, where the schema is wrong."
   false)
@@ -275,14 +256,22 @@
 
        :and (first (keep (partial extract-model-and-id model) values))))))
 
+(defn- deserialize-where-clause
+  "Deserialize a where-clause that may have been EDN-encoded for safe JSON round-tripping."
+  [where-clause]
+  (if (string? where-clause)
+    (edn/read-string where-clause)
+    where-clause))
+
 (defn bulk-ingest!
   "Process the given search model updates."
   [updates]
   (tracing/with-span :search "search.ingestion.bulk-ingest" {:search/update-count (count updates)}
     (lib-be/with-metadata-provider-cache
       (if (seq (search.engine/active-engines))
-        (let [documents (->> (for [[search-model where-clauses] (u/group-by first second updates)]
-                               (spec-index-reducible search-model (into [:or] (distinct where-clauses))))
+        (let [updates  (mapv (fn [[model where]] [model (deserialize-where-clause where)]) updates)
+              documents (->> (for [[search-model where-clauses] (u/group-by first second updates)]
+                               (spec-index-reducible search-model (into [:or] where-clauses)))
                              ;; init collection is only for clj-kondo, as we know that the list is non-empty
                              (reduce u/rconcat [])
                              query->documents)
@@ -298,72 +287,38 @@
           (update! documents to-delete))
         {}))))
 
-(defn- track-queue-size! []
-  (analytics/set! :metabase-search/queue-size (.size queue)))
-
-(defn- index-worker-exists? []
-  (queue/listener-exists? listener-name))
-
-(defn ingest-maybe-async!
+(defn async-updates
   "Update or create any search index entries related to the given updates.
-  Will be async if the worker exists, otherwise it will be done synchronously on the calling thread.
-  Can also be forced to run synchronously for testing."
-  ([updates]
-   (ingest-maybe-async! updates (or *force-sync* (not (index-worker-exists?)))))
-  ([updates sync?]
-   (when-not *disable-updates*
-     (if sync?
-       (bulk-ingest! updates)
-       (do
-         (doseq [update updates]
-           (log/trace "Queuing update" update)
-           (queue/put-with-delay! queue message-delay-ms update))
-         (track-queue-size!)
-         true)))))
+  Enqueues updates for async processing via the persistent queue."
+  [updates]
+  (when-not *disable-updates*
+    ;; Serialize where-clauses as EDN strings so they survive JSON round-tripping
+    ;; through the appdb queue (keywords would otherwise become strings).
+    (let [serialized (mapv (fn [[model where]] [model (pr-str where)]) updates)]
+      (mq/with-queue :queue/search-reindex [q]
+        (mq/put q serialized)))))
 
 (defn wait-for-idle!
-  "Block until the ingestion queue has been drained and no more items arrive.
-   Polls the queue and considers it idle when it has been empty for `settle-ms`
-   (default 500ms). Gives up after `timeout-ms` (default 30s)."
+  "Block until the search reindex queue has settled — no publishes or handler completions
+  for `settle-ms` (default 500ms). Gives up after `timeout-ms` (default 30s).
+  Uses an in-memory activity timestamp rather than polling the database."
   [& {:keys [settle-ms timeout-ms poll-ms]
       :or   {settle-ms 500, timeout-ms 30000, poll-ms 50}}]
-  (let [deadline-ns (+ (System/nanoTime) (* (long timeout-ms) 1000000))
-        settle-ns   (* (long settle-ms) 1000000)
-        poll-ms     (long poll-ms)
-        empty-since (atom nil)]
+  (let [deadline-ms (+ (System/currentTimeMillis) timeout-ms)
+        channel     :queue/search-reindex]
     (loop []
-      (let [now-ns       (System/nanoTime)
-            sz           (.size queue)
-            remaining-ns (- settle-ns (- now-ns (or @empty-since now-ns)))]
+      (let [now-ms  (System/currentTimeMillis)
+            last-ns (mq/last-activity channel)
+            idle-ms (if last-ns
+                      (/ (double (- (System/nanoTime) last-ns)) 1e6)
+                      (double settle-ms))]
         (cond
-          (>= now-ns deadline-ns)
-          (log/warnf "wait-for-idle! timed out with %d items still in the queue" sz)
+          (>= now-ms deadline-ms)
+          (log/warn "wait-for-idle! timed out waiting for search reindex queue to settle")
 
-          (pos? sz)
-          (do (reset! empty-since nil)
-              (Thread/sleep poll-ms)
-              (recur))
+          (>= idle-ms settle-ms)
+          nil ;; settled
 
-          (pos? remaining-ns)
-          (do (when-not @empty-since (reset! empty-since now-ns))
-              (Thread/sleep (long (max poll-ms (quot remaining-ns 1000000))))
+          :else
+          (do (Thread/sleep (long poll-ms))
               (recur)))))))
-
-(defn start-listener!
-  "Starts the ingestion listener on the queue"
-  []
-  (when (seq (search.engine/active-engines))
-    (queue/listen! listener-name queue bulk-ingest!
-                   {:success-handler     (fn [_result _duration _]
-                                           (track-queue-size!))
-                    :err-handler        (fn [err _]
-                                          (log/error err "Error indexing search entries")
-                                          (analytics/inc! :metabase-search/index-error)
-                                          (track-queue-size!))
-                    ;; Note that each message can correspond to multiple documents,
-                    ;; for example there would be 1 message for updating all
-                    ;; the tables within a given database when it is renamed.
-                    ;; Messages can also correspond to zero documents,
-                    ;; such as when updating a table that is marked as not visible.
-                    :max-batch-messages 50
-                    :max-next-ms       100})))

src/metabase/search/models.clj

@@ -10,12 +10,12 @@
 
 (t2/define-after-insert :hook/search-index
   [instance]
-  (search/update! instance true)
+  (search/async-update! instance true)
   instance)
 
 (t2/define-after-update :hook/search-index
   [instance]
-  (search/update! instance)
+  (search/async-update! instance)
   nil)
 
 ;; Too much of a performance risk.

src/metabase/search/task/search_index.clj

@@ -1,15 +1,18 @@
 (ns metabase.search.task.search-index
   (:require
+   [clojure.walk :as walk]
    [clojurewerkz.quartzite.jobs :as jobs]
    [clojurewerkz.quartzite.schedule.simple :as simple]
    [clojurewerkz.quartzite.triggers :as triggers]
-   [metabase.app-db.cluster-lock :as cluster-lock]
+   [metabase.analytics.core :as analytics]
+   [metabase.mq.core :as mq]
    [metabase.search.core :as search]
+   [metabase.search.engine :as search.engine]
+   [metabase.search.impl :as search.impl]
    [metabase.search.ingestion :as ingestion]
+   [metabase.search.spec :as search.spec]
    [metabase.startup.core :as startup]
-   [metabase.task.core :as task]
-   [metabase.tracing.core :as tracing]
-   [metabase.util.queue :as queue])
+   [metabase.task.core :as task])
   (:import
    (java.time Instant)
    (java.util Date)
@@ -19,7 +22,6 @@
 
 (def ^:private init-stem "metabase.task.search-index.init")
 (def ^:private reindex-stem "metabase.task.search-index.reindex")
-(def ^:private cluster-lock-name ::search-index-lock)
 
 (def init-job-key
   "Key used to define and trigger a job that ensures there is an active index."
@@ -31,19 +33,11 @@
 
 ;; We define the job bodies outside the defrecord, so that we can redefine them live from the REPL
 
-(defn init!
-  "Create a new index, if necessary"
-  []
-  (when (search/supports-index?)
-    (tracing/with-span :search "search.task.init" {}
-      (cluster-lock/with-cluster-lock cluster-lock-name
-        (search/init-index! {:force-reset? false, :re-populate? false})))))
-
 (task/defjob ^{DisallowConcurrentExecution true
                :doc                        "Populate a new Search Index"}
   SearchIndexReindex [_ctx]
-  (cluster-lock/with-cluster-lock cluster-lock-name
-    (search/reindex! {:async? false})))
+  (when (search/supports-index?)
+    (search/async-reindex!)))
 
 ;; Atom holding a promise that is delivered when the background init thread finishes.
 ;; nil when no init has been started — [[wait-for-init!]] returns immediately in that case.
@@ -56,14 +50,8 @@
   (some-> @init-promise deref))
 
 (defmethod startup/def-startup-logic! ::SearchIndexInit [_]
-  (let [p (promise)]
-    (reset! init-promise p)
-    (doto (Thread. ^Runnable (fn []
-                               (try
-                                 (init!)
-                                 (finally
-                                   (deliver p true)))))
-      .start)))
+  (when (search/supports-index?)
+    (search/async-init!)))
 
 (defmethod task/init! ::SearchIndexReindex [_]
   (let [job         (jobs/build
@@ -81,8 +69,69 @@
                        (simple/repeat-forever))))]
     (task/schedule-task! job trigger)))
 
-(defmethod queue/init-listener! ::SearchIndexUpdate [_]
-  (ingestion/start-listener!))
+(defn handle-command-message!
+  "Handle a command message from the search reindex queue."
+  [msg]
+  (condp = (keyword (:command msg))
+    :init (if (= (:version msg) (search.spec/index-version-hash))
+            (if-let [engine (:engine msg)]
+              (search.engine/init! engine (select-keys msg [:force-reset? :re-populate?]))
+              (search.impl/sync-init-index! {:force-reset? (:force-reset? msg false)
+                                             :re-populate?      false}))
+            (throw (ex-info "Cannot handle init for different index version. Will retry on a different node."
+                            {:expected (search.spec/index-version-hash)
+                             :received (:version msg)})))
+    :reindex (if-let [engine (:engine msg)]
+               (search.engine/reindex! engine (select-keys msg [:in-place?]))
+               (search.impl/sync-reindex! (select-keys msg [:in-place?])))
+    :delete (doseq [e            (search.engine/active-engines)
+                    search-model (->> (vals (search.spec/specifications))
+                                      (filter (comp #{(:model msg)} :model))
+                                      (map :name))]
+              (search.engine/delete! e search-model (:ids msg)))))
+
+(defn- command-key
+  "Extract the command value from a message map, handling both string and keyword keys
+  (messages may not be keywordized yet when dedup runs)."
+  [msg]
+  (when (map? msg)
+    (some-> (or (:command msg) (get msg "command")) keyword)))
+
+(defn dedup-search-messages
+  "Dedup function for search reindex queue messages.
+  - Update vectors: flatten to individual [model where] pairs, dedup by identity, re-group
+  - Command messages: collapse duplicate :init and :reindex (keep last), pass through :delete"
+  [messages]
+  (let [{commands true updates false} (group-by #(boolean (command-key %)) messages)
+        ;; For commands: collapse duplicate :init and :reindex, keep last occurrence
+        deduped-commands (let [grouped (group-by command-key commands)]
+                           (concat
+                            (when-let [inits (seq (:init grouped))]
+                              [(last inits)])
+                            (when-let [reindexes (seq (:reindex grouped))]
+                              [(last reindexes)])
+                            (:delete grouped)))
+        ;; For updates: flatten all update vecs, dedup individual pairs, re-wrap
+        deduped-updates (when (seq updates)
+                          (let [all-pairs (into [] cat updates)]
+                            (when (seq all-pairs)
+                              [(into [] (distinct) all-pairs)])))]
+    (vec (concat deduped-commands deduped-updates))))
+
+(mq/def-listener! :queue/search-reindex
+  {:max-batch-messages 50 :exclusive true
+   :dedup-fn dedup-search-messages}
+  [messages]
+  (try
+    (let [messages (map #(cond-> % (map? %) walk/keywordize-keys) messages)
+          {commands true updates false} (group-by #(boolean (and (map? %) (:command %))) messages)]
+      (doseq [cmd commands]
+        (handle-command-message! cmd))
+      (when (seq updates)
+        (ingestion/bulk-ingest! (into [] cat updates))))
+    (catch Exception e
+      (analytics/inc! :metabase-search/index-error)
+      (throw e))))
 
 (comment
   (task/job-exists? reindex-job-key)

src/metabase/server/handler.clj

@@ -16,7 +16,7 @@
    [metabase.server.middleware.request-id :as mw.request-id]
    [metabase.server.middleware.security :as mw.security]
    [metabase.server.middleware.session :as mw.session]
-   [metabase.server.middleware.settings-cache :as mw.settings-cache]
+
    [metabase.server.middleware.ssl :as mw.ssl]
    [metabase.server.middleware.trace :as mw.trace]
    [metabase.util.log :as log]
@@ -87,7 +87,6 @@
         #'mw.session/bind-current-user               ; Binds *current-user* and *current-user-id* if :metabase-user-id is non-nil
         #'mw.session/wrap-current-user-info          ; looks for :metabase-session-key and sets :metabase-user-id and other info if Session ID is valid
         #'mw.pf-cache/wrap-premium-features-cache-check ; check cookie to refresh premium features cache if needed
-        #'mw.settings-cache/wrap-settings-cache-check ; check cookie to refresh settings cache if needed
         #'analytics/embedding-mw                     ; reads sdk client headers, binds them to *client* and *version*, and tracks sdk-response metrics
         #'mw.session/wrap-session-key                ; looks for a Metabase Session ID and assoc as :metabase-session-key
         #'mw.auth/wrap-static-api-key                ; looks for a static Metabase API Key on the request and assocs as :metabase-api-key

src/metabase/server/middleware/settings_cache.clj

@@ -1,50 +0,0 @@
-(ns metabase.server.middleware.settings-cache
-  "Ring middleware to check for settings updates based on a cookie value."
-  (:require
-   [metabase.settings.core :as setting]
-   [metabase.util.log :as log]
-   [ring.util.response :as response]))
-
-(def ^:private settings-last-updated-cookie-name "metabase.SETTINGS_LAST_UPDATED")
-
-(defn- check-and-update-settings-cache
-  "Check if the cookie timestamp is newer than our cached timestamp, and if so, update the cache."
-  [request]
-  (when-let [cookie-timestamp (get-in request [:cookies settings-last-updated-cookie-name :value])]
-    (let [last-known-update   (setting/cache-last-updated-at)]
-      ;; If the cookie timestamp is newer than our cached timestamp, restore the cache
-      (when (and last-known-update
-                 (try (pos? (compare cookie-timestamp last-known-update))
-                      (catch Exception _e
-                        (log/infof "Strange last known update cookie: %s" cookie-timestamp)
-                        false)))
-        (log/info "Settings cookie indicates cache is out of date. Refreshing...")
-        (setting/restore-cache!)
-        ::restored))))
-
-(defn- maybe-set-settings-last-updated-cookie
-  "Set the most recent settings cache update timestamp if this request changed a setting (it will have
-  the :cookie/settings-cache-timestamp) or if this request caused a cache to be updated."
-  [restored? response]
-  (if (or restored?
-          (-> response :mb/cookies :cookie/settings-cache-timestamp))
-    (if-let [timestamp (setting/cache-last-updated-at)]
-      (response/set-cookie (update response :mb/cookies dissoc :cookie/settings-cache-timestamp)
-                           settings-last-updated-cookie-name
-                           timestamp
-                           {:path      "/"
-                            :max-age   (* 5 60)
-                            :same-site :lax})
-      response)
-    response))
-
-(defn wrap-settings-cache-check
-  "Middleware that checks if settings need to be refreshed based on a cookie value.
-
-  When a setting is updated, the API sets a cookie with the settings-last-updated timestamp.
-  This middleware checks that cookie on subsequent requests and updates the local cache if needed,
-  ensuring consistency in multi-instance deployments without waiting for the 60-second polling interval."
-  [handler]
-  (fn [request respond raise]
-    (let [restored? (check-and-update-settings-cache request)]
-      (handler request (comp respond #(maybe-set-settings-last-updated-cookie restored? %)) raise))))

src/metabase/settings/core.clj

@@ -120,7 +120,6 @@
   validate-settable-for-db!
   writable-settings]
  [metabase.settings.models.setting.cache
-  cache-update-check-interval-ms
   cache-last-updated-at
   restore-cache!]
  [metabase.settings.models.setting.multi-setting

src/metabase/settings/models/setting.clj

@@ -539,8 +539,8 @@
   (if config/*disable-setting-cache*
     (t2/select-one-fn :value :model/Setting :key setting-name-str)
     (do
-      ;; gotcha - returns immediately if another process is restoring it, i.e. before it's been populated
-      (setting.cache/restore-cache-if-needed!)
+      (when (nil? (setting.cache/cache))
+        (setting.cache/restore-cache!))
       (let [cache (setting.cache/cache)]
         (if (nil? cache)
           ;; nil if we returned early above, and the cache is still being restored - in that case hit the db
@@ -866,11 +866,9 @@
               (set-new-setting! setting-name new-value))
             ;; update cached value
             (setting.cache/update-cache! setting-name new-value)
-            ;; Record the fact that a Setting has been updated so eventually other instances (if applicable) find out
-            ;; about it (For Settings that don't use the Cache, don't update the `last-updated` value, because it will
-            ;; cause other instances to do needless reloading of the cache from the DB)
+            ;; Broadcast to other nodes so they invalidate their caches too.
             (when-not config/*disable-setting-cache*
-              (setting.cache/update-settings-last-updated!))))
+              (setting.cache/broadcast-cache-invalidation!))))
         ;; Now return the `new-value`.
         new-value))))
 

src/metabase/settings/models/setting/cache.clj

@@ -2,16 +2,12 @@
   "Settings cache. Cache is a 1:1 mapping of what's in the DB. Cached lookup time is ~60µs, compared to ~1800µs for DB
   lookup."
   (:require
-   [clojure.core :as core]
    [clojure.java.jdbc :as jdbc]
    [metabase.app-db.core :as mdb]
-   [metabase.util :as u]
+   [metabase.mq.core :as mq]
    [metabase.util.honey-sql-2 :as h2x]
    [metabase.util.log :as log]
-   [toucan2.core :as t2])
-  (:import
-   (java.util.concurrent.atomic AtomicLong)
-   (java.util.concurrent.locks ReentrantLock)))
+   [toucan2.core :as t2]))
 
 (set! *warn-on-reflection* true)
 
@@ -46,29 +42,11 @@
     (swap! (cache*) assoc  setting-name new-value)
     (swap! (cache*) dissoc setting-name)))
 
-;; CACHE SYNCHRONIZATION
-;;
-;; When running multiple Metabase instances (horizontal scaling), it is of course possible for one instance to update
-;; a Setting, and, since Settings are cached (to avoid tons of DB calls), for the other instances to then have an
-;; out-of-date cache. Thus we need a way for instances to know when their caches are out of date, so they can update
-;; them accordingly. Here is our solution:
-;;
-;; We will record the last time *any* Setting was updated in a special Setting called `settings-last-updated`.
-;;
-;; Since `settings-last-updated` itself is a Setting, it will get fetched as part of each instance's local cache; we
-;; can then periodically compare the locally cached value of `settings-last-updated` with the value in the DB. If our
-;; locally cached value is older than the one in the DB, we will flush our cache. When the cache is fetched again, it
-;; will have the up-to-date value.
-;;
-;; Because different machines can have out-of-sync clocks, we'll rely entirely on the application DB for calculating
-;; and comparing values of `settings-last-updated`. Because the Setting table itself only stores text values, we'll
-;; need to cast it between TEXT and TIMESTAMP SQL types as needed.
-
 (def ^String settings-last-updated-key
   "Internal key used to store the last updated timestamp for Settings."
   "settings-last-updated")
 
-(defn update-settings-last-updated!
+(defn- update-settings-last-updated-in-db!
   "Update the value of `settings-last-updated` in the DB; if the row does not exist, insert one."
   []
   (log/debug "Updating value of settings-last-updated in DB...")
@@ -92,51 +70,17 @@
   ;; changes
   (swap! (cache*) assoc settings-last-updated-key (t2/select-one-fn :value :model/Setting :key settings-last-updated-key)))
 
+(defn broadcast-cache-invalidation!
+  "Update the `settings-last-updated` timestamp in the DB and broadcast a cache invalidation signal to all nodes."
+  []
+  (update-settings-last-updated-in-db!)
+  (mq/with-topic :topic/settings-cache-invalidated [t]
+    (mq/put t {:invalidated-at (System/currentTimeMillis)})))
+
 (defn cache-last-updated-at
   "Fetch the value of `settings-last-updated`, indicating the timestamp of the settings cache. Possibly null."
   []
-  (let [current-cache (cache)]
-    (core/get current-cache settings-last-updated-key)))
-
-(defn- cache-out-of-date?
-  "Check whether our Settings cache is out of date. We know the cache is out of date if either of the following
-  conditions is true:
-
-   *  The cache is empty (the `(cache*` atom is `nil`), which of course means it needs to be updated
-   *  There is a value of `settings-last-updated` in the cache, and it is older than the value of in the DB. (There
-      will be no value until the first time a normal Setting is updated; thus if it is not yet set, we do not yet need
-      to invalidate our cache.)"
-  []
-  (log/debug "Checking whether settings cache is out of date (requires DB call)...")
-  (let [current-cache (cache)]
-    (boolean
-     (or
-      ;; is the cache empty?
-      (not current-cache)
-      ;; if not, get the cached value of `settings-last-updated`, and if it exists...
-      (when-let [last-known-update (cache-last-updated-at)]
-        ;; compare it to the value in the DB. This is done be seeing whether a row exists
-        ;; WHERE value > <local-value>
-        (u/prog1 (t2/select-one-fn :value :model/Setting
-                                   {:where [:and
-                                            [:= :key settings-last-updated-key]
-                                            [:> :value last-known-update]]})
-          (log/trace "last known Settings update: " (pr-str last-known-update))
-          (log/trace "actual last Settings update:" (pr-str <>))
-          (when <>
-            (log/info (u/format-color :red "Settings have been changed on another instance, and will be reloaded here.")))))))))
-
-(def ^:const cache-update-check-interval-ms
-  "How often we should check whether the Settings cache is out of date (which requires a DB call)?"
-  (u/minutes->ms 1))
-
-(defonce ^:private ^AtomicLong last-update-check (AtomicLong. 0))
-
-(defn- time-for-another-update-check?
-  "Has it has been more than a minute since the last time we checked for updates?"
-  []
-  (> (quot (- (System/nanoTime) (.get last-update-check)) 1000000)
-     cache-update-check-interval-ms))
+  (get (cache) settings-last-updated-key))
 
 (defn restore-cache!
   "Populate cache with the latest hotness from the db"
@@ -144,29 +88,6 @@
   (log/debug "Refreshing Settings cache...")
   (reset! (cache*) (t2/select-fn->fn :key :value :model/Setting)))
 
-(defonce ^:private ^ReentrantLock restore-cache-lock (ReentrantLock.))
-
-(defn restore-cache-if-needed!
-  "Check whether we need to repopulate the cache with fresh values from the DB (because the cache is either empty or
-  known to be out-of-date), and do so if needed. This is intended to be called every time a Setting value is
-  retrieved, so it should be efficient; thus the calculation (`should-restore-cache?`) is itself TTL-memoized."
-  []
-  ;; There's a potential race condition here where two threads both call this at the exact same moment, and both get
-  ;; `true` when they call `should-restore-cache`, and then both simultaneously try to update the cache (or, one
-  ;; updates the cache, but the other calls `should-restore-cache?` and gets `true` before the other calls
-  ;; `memo-swap!` (see below))
-  ;;
-  ;; This is not desirable, since either situation would result in duplicate work. Better to just add a quick lock
-  ;; here so only one of them does it, since at any rate waiting for the other thread to finish the task in progress is
-  ;; certainly quicker than starting the task ourselves from scratch
-  (when (time-for-another-update-check?)
-    ;; if the lock is not already held by any thread, including this one...
-    (when-not (.isLocked restore-cache-lock)
-      ;; attempt to acquire the lock. Returns immediately if lock is already held.
-      (when (.tryLock restore-cache-lock)
-        (try
-          (.set last-update-check (System/nanoTime))
-          (when (cache-out-of-date?)
-            (restore-cache!))
-          (finally
-            (.unlock restore-cache-lock)))))))
+(mq/def-listener! :topic/settings-cache-invalidated [_msg]
+  (log/debug "Received settings cache invalidation signal")
+  (restore-cache!))

src/metabase/settings_rest/api.clj

@@ -23,11 +23,6 @@
   [& body]
   `(do-with-setting-access-control (fn [] ~@body)))
 
-(defn- add-settings-last-updated-cookie
-  "Add a cookie with the current settings-last-updated timestamp to the response."
-  [response]
-  (assoc-in response [:mb/cookies :cookie/settings-cache-timestamp] true))
-
 ;; TODO: deprecate /api/session/properties and have a single endpoint for listing settings
 ;;
 ;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
@@ -56,7 +51,7 @@
    settings :- [:map-of kebab-cased-keyword :any]]
   (with-setting-access-control
     (setting/set-many! settings))
-  (add-settings-last-updated-cookie api/generic-204-no-content))
+  api/generic-204-no-content)
 
 ;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
 ;; use our API + we will need it when we make auto-TypeScript-signature generation happen
@@ -82,4 +77,4 @@
    {:keys [value]} :- [:map [:value :any]]]
   (with-setting-access-control
     (setting/set! key value))
-  (add-settings-last-updated-cookie api/generic-204-no-content))
+  api/generic-204-no-content)

src/metabase/testing_api/api.clj

@@ -27,7 +27,6 @@
    [toucan2.core :as t2])
   (:import
    (com.mchange.v2.c3p0 PoolBackedDataSource)
-   (java.util Queue)
    (java.util.concurrent.locks ReentrantReadWriteLock)))
 
 (set! *warn-on-reflection* true)
@@ -140,7 +139,6 @@
                              [:name ms/NonBlankString]]]
   ;; reset the system clock, in case `/set-time` was called without cleanup
   (alter-var-root #'java-time.clock/*clock* (constantly nil))
-  (.clear ^Queue @#'search.ingestion/queue)
   (restore-snapshot! snapshot-name)
   (search/sync-from-restored-db!)
   nil)

src/metabase/util/queue.clj

@@ -1,263 +0,0 @@
-(ns metabase.util.queue
-  "Functionality for working with queues.
-   There are two main blocks of functionality: a custom BoundedTransferQueue and a queue listener.
-
-  The BoundedTransferQueue allows for callers to decide whether to block the previous synchronous put to complete
-  before adding another message, or attempt to add it to a fixed-sized async queue if there is space.
-  See `bounded-transfer-queue` for more details.
-
-  The queue listener allows the creation and management of (possibly) multithreaded queue listeners that can
-  process messages off the queue in batches.
-  Listeners should generally be managed with `init-listener! which calls `listen!`.
-  "
-  (:require
-   [com.climate.claypoole :as cp]
-   [metabase.util :as u]
-   [metabase.util.log :as log]
-   [metabase.util.malli :as mu]
-   [metabase.util.malli.registry :as mr]
-   [metabase.util.malli.schema :as ms])
-  (:import
-   (java.time Duration Instant)
-   (java.util.concurrent
-    ArrayBlockingQueue
-    BlockingQueue
-    DelayQueue
-    Delayed
-    ExecutorService
-    ScheduledExecutorService
-    SynchronousQueue
-    TimeUnit)))
-
-(set! *warn-on-reflection* true)
-
-(defprotocol BoundedTransferQueue
-  (maybe-put! [queue msg]
-    "Put a message on the queue if there is space for it, otherwise drop it.
-     Returns whether the item was enqueued.")
-  (blocking-put! [queue timeout msg]
-    "Put a message on the queue. If necessary, block until there is space for it.")
-  (blocking-take! [queue timeout]
-    "Take a message off the queue, blocking if necessary.")
-  (clear! [queue]
-    "Discard all messages on the given queue."))
-
-;; Similar to java.util.concurrent.LinkedTransferQueue, but bounded.
-(deftype ^:private ArrayTransferQueue [^ArrayBlockingQueue async-queue
-                                       ^SynchronousQueue sync-queue
-                                       ^long block-ms
-                                       ^long sleep-ms]
-  BoundedTransferQueue
-  (maybe-put! [_ msg]
-    (.offer async-queue msg))
-  (blocking-put! [_ timeout msg]
-    (.offer sync-queue msg timeout TimeUnit/MILLISECONDS))
-  (blocking-take! [_ timeout]
-    (loop [time-remaining timeout]
-      (when (pos? time-remaining)
-        ;; Async messages are given higher priority, as sync messages will never be dropped.
-        (or (.poll async-queue)
-            (.poll sync-queue block-ms TimeUnit/MILLISECONDS)
-            (do (Thread/sleep ^long sleep-ms)
-                ;; This is an underestimate, as the thread may have taken a while to wake up. That's OK.
-                (recur (- time-remaining block-ms sleep-ms)))))))
-  (clear! [_]
-    (.clear sync-queue)
-    (.clear async-queue)))
-
-(defn bounded-transfer-queue
-  "Create a bounded transfer queue, specialized based on the high-level options."
-  [capacity & {:keys [block-ms sleep-ms]
-               :or   {block-ms 100
-                      sleep-ms 100}}]
-  (->ArrayTransferQueue (ArrayBlockingQueue. capacity)
-                        (SynchronousQueue.)
-                        block-ms
-                        sleep-ms))
-
-(defrecord DelayValue [value ^Instant ready-at]
-  Delayed
-  (getDelay [_ unit]
-    (.convert unit (- (.toEpochMilli ready-at) (System/currentTimeMillis)) TimeUnit/MILLISECONDS))
-  (compareTo [this other]
-    (Long/compare (.getDelay this TimeUnit/MILLISECONDS)
-                  (.getDelay ^Delayed other TimeUnit/MILLISECONDS))))
-
-(defn delay-queue
-  "Return an unbounded queue that returns each item only after some specified delay."
-  ^DelayQueue []
-  (DelayQueue.))
-
-(defn put-with-delay!
-  "Put an item on a delay queue, with a delay given in milliseconds."
-  [^DelayQueue queue delay-ms value]
-  (.offer queue (->DelayValue value (.plus (Instant/now) (Duration/ofMillis delay-ms)))))
-
-(defn- take-batch* [^BlockingQueue queue ^long max-messages ^long poll-ms acc]
-  (loop [acc acc]
-    (if (>= (count acc) max-messages)
-      acc
-      (if-let [item (if (pos? poll-ms)
-                      (.poll queue poll-ms TimeUnit/MILLISECONDS)
-                      (.poll queue))]
-        (recur (conj acc (if (instance? DelayQueue queue)
-                           (:value item)
-                           item)))
-        (not-empty acc)))))
-
-(defn- take-batch!
-  "Get a batch of messages off the given queue.
-  Will wait max-first-ms for the first message to be available, then will collect up to max-message, waiting up to max-next-ms for each additional message, whichever comes first.
-  For convenience, if the queue is a DelayQueue, the returned values will be the actual values, not the Delay objects."
-  ([^BlockingQueue queue ^long max-first-ms ^long max-batch-messages ^long max-next-ms]
-   (when-let [fst (.poll queue max-first-ms TimeUnit/MILLISECONDS)]
-     (take-batch* queue max-batch-messages max-next-ms [(if (instance? DelayQueue queue) (:value fst) fst)]))))
-
-(mr/def ::listener-options [:map [:success-handler {:optional true} [:=> [:cat :any :double :string] :any]
-                                  :err-handler {:optional true} [:=> [:cat [:fn (ms/InstanceOfClass Throwable) :string]] :any]
-                                  :pool-size {:optional true} number?
-                                  :max-batch-messages {:optional true} number?
-                                  :max-next-ms {:optional true} number?]])
-
-(defonce
-  ^:private
-  ^{:malli/schema [:map-of :string (ms/InstanceOfClass ScheduledExecutorService)]}
-  listeners (atom {}))
-
-(defn listener-exists?
-  "Returns true if there is a running listener with the given name"
-  [listener-name]
-  (when-let [^ExecutorService executor (get @listeners listener-name)]
-    (not (.isTerminated executor))))
-
-(mu/defn- listener-thread [listener-name :- :string
-                           queue :- (ms/InstanceOfClass BlockingQueue)
-                           handler :- [:=> [:cat [:sequential :any]] :any]
-                           {:keys [success-handler err-handler max-batch-messages max-next-ms]} :- ::listener-options]
-  (log/debugf "Thread for listener %s started" listener-name)
-  (while true
-    (try
-      (log/debugf "Listener %s waiting for next batch..." listener-name)
-      (let [batch (take-batch! queue Long/MAX_VALUE max-batch-messages max-next-ms)]
-        (if (seq batch)
-          (do
-            (log/debugf "Listener %s processing batch of %d" listener-name (count batch))
-            (log/tracef "Listener %s processing batch: %s" listener-name batch)
-            (let [timer (u/start-timer)
-                  output (handler batch)
-                  duration (u/since-ms timer)]
-              (log/debugf "Listener %s processed batch in %.0fms" listener-name duration)
-              (success-handler output duration listener-name)))
-          (log/debugf "Listener %s found no items to process" listener-name)))
-      (catch InterruptedException e
-        (log/debugf "Listener thread %s stopped" listener-name)
-        (throw e))
-      (catch Exception e
-        (err-handler e listener-name)
-        (log/errorf e "Error in %s while processing batch" listener-name))))
-  (log/infof "Listener %s stopped" listener-name))
-
-(def ^:private ^:const max-restart-backoff-ms 30000)
-(def ^:private ^:const initial-restart-backoff-ms 500)
-
-(defn- listener-thread-with-restart
-  "Wraps `listener-thread` in a restart loop with exponential backoff.
-  If `listener-thread` exits unexpectedly (e.g. because the inner catch block itself throws),
-  this will restart it as long as the executor has not been shut down."
-  [listener-name queue handler options]
-  (loop [backoff-ms initial-restart-backoff-ms]
-    (try
-      (listener-thread listener-name queue handler options)
-      (catch InterruptedException _e
-        (log/debugf "Listener thread %s stopped" listener-name)
-        (throw (InterruptedException.)))
-      (catch Throwable e
-        (log/errorf e "Listener thread %s crashed, restarting in %dms" listener-name backoff-ms)))
-    (Thread/sleep ^long backoff-ms)
-    (when-not (.isShutdown ^ExecutorService (get @listeners listener-name))
-      (recur (min max-restart-backoff-ms (* 2 backoff-ms))))))
-
-(mu/defn listen!
-  "Starts an async listener on the given queue. This should generally be called from `init-listener!` in a task namespace.
-
-  Arguments:
-  - listener-name: A unique string. Calls to register another listener with the same name will be a no-op
-  - queue: The queue to listen on
-  - handler: A function taking a list of 1 or more values that have been sent to the queue.
-
-  Options:
-  - success-handler: A function called when handler does not throw an exception. Accepts [result-of-handler, duration-in-ms, listener-name]
-  - err-handler: A function called when the handler throws an exception. Accepts [exception, duration-in-ms, listener-name]
-  - pool-size: Number of threads in the listener. Default: 1
-  - max-batch-messages: Max number of items to batch up before calling handler. Default 50
-  - max-next-ms: Max number of ms to wait for each additional message before calling the handler. Default 100"
-  [listener-name :- :string
-   queue :- (ms/InstanceOfClass BlockingQueue)
-   handler :- [:=> [:cat [:sequential :any]] :any]
-   {:keys [success-handler
-           err-handler
-           pool-size
-           max-batch-messages
-           max-next-ms]
-    :or   {success-handler    (constantly nil)
-           err-handler        (constantly nil)
-           pool-size          1
-           max-batch-messages 50
-           max-next-ms        100}} :- ::listener-options]
-  (if (listener-exists? listener-name)
-    (log/errorf "Listener %s already exists" listener-name)
-
-    (let [executor (cp/threadpool pool-size {:name (str "queue-" listener-name)})]
-      (log/infof "Starting listener %s with %d threads %s" (u/format-color 'green listener-name) pool-size (u/emoji "\uD83C\uDFA7"))
-      (dotimes [_ pool-size]
-        (cp/future executor (listener-thread-with-restart listener-name queue handler
-                                                          {:success-handler    success-handler
-                                                           :err-handler        err-handler
-                                                           :max-batch-messages max-batch-messages
-                                                           :max-next-ms        max-next-ms})))
-
-      (swap! listeners assoc listener-name executor))))
-
-(mu/defn stop-listening!
-  "Stops the listener previously started with (listen!).
-  If there is no running listener with the given name, it is a no-op"
-  [listener-name :- :string]
-  (if-let [^ExecutorService executor (get @listeners listener-name)]
-    (do
-      (log/infof "Stopping listener %s..." listener-name)
-      (cp/shutdown! executor)
-      ;; wait up to 10 seconds for executor to stop. Largely for CI/tests FAIL in (listener-handler-test) (queue_test.clj:178)
-      (.awaitTermination executor 10 TimeUnit/SECONDS)
-
-      (swap! listeners dissoc listener-name)
-      (log/infof "Stopping listener %s...done" listener-name))
-    (log/infof "No running listener named %s" listener-name)))
-
-(defmulti init-listener!
-  "Initialize a listener with a given name. All implementations of this method are called once and only
-  once when the server is starting up. Task namespaces (`metabase.*.task`) should add new
-  implementations of this method to start new listeners they define (i.e., with a call to `queue/listen!`.)
-
-  The dispatch value for this function can be any unique keyword, but by convention is a namespaced keyword version of
-  the name of the listener being initialized; for sake of consistency with the listener name itself, the keyword should be left
-  CamelCased.
-
-    (defmethod queue/init-listener! ::ExampleListener [_]
-      (queue/listen! \"example-listener\" queue handler)"
-  {:arglists '([job-name-string])}
-  keyword)
-
-(defn start-listeners!
-  "Call all implementations of `init-listener!`. Called by metabase.core/init!"
-  []
-  (doseq [[k f] (methods init-listener!)]
-    (try
-      (f k)
-      (catch Throwable e
-        (log/errorf e "Error initializing listener %s" k)))))
-
-(defn stop-listeners!
-  "Stops all running listeners"
-  []
-  (doseq [[name _] @listeners]
-    (stop-listening! name)))

src/metabase/warehouses/models/database.clj

@@ -427,7 +427,7 @@
                                :where  [:= :database_id id]})))
        (run! (fn [batch]
                ;; damn circular deps
-               ((requiring-resolve 'metabase.search.core/delete!) :model/Card (map (comp str :id) batch)))))
+               ((requiring-resolve 'metabase.search.core/async-delete!) :model/Card (map (comp str :id) batch)))))
   (when (not= :postgres (mdb/db-type))
     (t2/query {:delete-from (t2/table-name :model/Card)
                :where       [:= :database_id id]}))

test/metabase/agent_api/api_test.clj

@@ -10,7 +10,6 @@
    [metabase.lib.metadata :as lib.metadata]
    [metabase.lib.normalize :as lib.normalize]
    [metabase.metabot.tools.util :as metabot.tools.u]
-   [metabase.search.ingestion :as search.ingestion]
    [metabase.search.test-util :as search.tu]
    [metabase.session.models.session :as session.models]
    [metabase.test :as mt]
@@ -180,14 +179,13 @@
              (mt/user-http-request :crowberto :get 400 (format "agent/v1/table/%d/field/not-a-valid-id/values" table-id)))))))
 
 (deftest search-test
-  (binding [search.ingestion/*force-sync* true]
-    (search.tu/with-new-search-if-available-otherwise-legacy
-      (mt/with-temp [:model/Table _ {:name "AgentSearchTestTable"}]
-        (testing "Returns search results for term queries"
-          (is (=? {:data        [{:type "table" :name "AgentSearchTestTable"}]
-                   :total_count 1}
-                  (mt/user-http-request :rasta :post 200 "agent/v1/search"
-                                        {:term_queries ["AgentSearchTestTable"]}))))))))
+  (search.tu/with-new-search-if-available-otherwise-legacy
+    (mt/with-temp [:model/Table _ {:name "AgentSearchTestTable"}]
+      (testing "Returns search results for term queries"
+        (is (=? {:data        [{:type "table" :name "AgentSearchTestTable"}]
+                 :total_count 1}
+                (mt/user-http-request :rasta :post 200 "agent/v1/search"
+                                      {:term_queries ["AgentSearchTestTable"]})))))))
 
 (defn- decode-query
   "Decode a base64-encoded query response to a Clojure map, then normalize it so lib functions work."
@@ -415,14 +413,13 @@
                                     {:metric_id (:id metric)}))))))
 
 (deftest search-finds-metrics-test
-  (binding [search.ingestion/*force-sync* true]
-    (search.tu/with-new-search-if-available-otherwise-legacy
-      (mt/with-temp [:model/Card _metric {:name          "AgentSearchTestMetric"
-                                          :type          :metric
-                                          :database_id   (mt/id)
-                                          :dataset_query (orders-count-query)}]
-        (testing "Returns metrics in search results"
-          (is (=? {:data        [{:type "metric" :name "AgentSearchTestMetric"}]
-                   :total_count 1}
-                  (mt/user-http-request :rasta :post 200 "agent/v1/search"
-                                        {:term_queries ["AgentSearchTestMetric"]}))))))))
+  (search.tu/with-new-search-if-available-otherwise-legacy
+    (mt/with-temp [:model/Card _metric {:name          "AgentSearchTestMetric"
+                                        :type          :metric
+                                        :database_id   (mt/id)
+                                        :dataset_query (orders-count-query)}]
+      (testing "Returns metrics in search results"
+        (is (=? {:data        [{:type "metric" :name "AgentSearchTestMetric"}]
+                 :total_count 1}
+                (mt/user-http-request :rasta :post 200 "agent/v1/search"
+                                      {:term_queries ["AgentSearchTestMetric"]})))))))

test/metabase/app_db/connection_test.clj

@@ -134,6 +134,71 @@
             (when-not (is-transaction-exception? e)
               (throw e))))))))
 
+(deftest after-commit-returns-false-outside-transaction-test
+  (testing "after-commit! returns false when not in a transaction"
+    (is (false? (mdb.connection/after-commit! (fn []))))))
+
+(deftest after-commit-returns-true-inside-transaction-test
+  (testing "after-commit! returns true when inside a transaction"
+    (t2/with-transaction []
+      (is (true? (mdb.connection/after-commit! (fn [])))))))
+
+(deftest after-commit-callbacks-execute-after-commit-test
+  (testing "Callbacks registered via after-commit! execute after successful commit"
+    (let [result (atom [])]
+      (t2/with-transaction []
+        (mdb.connection/after-commit! (fn [] (swap! result conj :callback-1)))
+        (mdb.connection/after-commit! (fn [] (swap! result conj :callback-2)))
+        (is (= [] @result) "Callbacks should not have executed yet"))
+      (is (= [:callback-1 :callback-2] @result) "Callbacks should execute after commit"))))
+
+(deftest after-commit-callbacks-not-executed-on-rollback-test
+  (testing "Callbacks are not executed when a transaction is rolled back"
+    (let [result (atom [])]
+      (try
+        (t2/with-transaction []
+          (mdb.connection/after-commit! (fn [] (swap! result conj :should-not-run)))
+          (throw (ex-info "force rollback" {})))
+        (catch Exception _))
+      (is (= [] @result) "Callbacks should not execute on rollback"))))
+
+(deftest after-commit-callback-registers-new-callback-test
+  (testing "A callback that registers another callback — both execute"
+    (let [result (atom [])]
+      (t2/with-transaction []
+        (mdb.connection/after-commit!
+         (fn []
+           (swap! result conj :first)
+           ;; Register a new callback from within a callback
+           (mdb.connection/after-commit! (fn [] (swap! result conj :second))))))
+      (is (= [:first :second] @result) "Both callbacks should execute")))
+
+  (testing "Deeply chained callbacks all execute"
+    (let [result (atom [])]
+      (t2/with-transaction []
+        (mdb.connection/after-commit!
+         (fn []
+           (swap! result conj :a)
+           (mdb.connection/after-commit!
+            (fn []
+              (swap! result conj :b)
+              (mdb.connection/after-commit!
+               (fn [] (swap! result conj :c))))))))
+      (is (= [:a :b :c] @result)))))
+
+(deftest nested-transactions-share-after-commit-and-state-test
+  (testing "Nested transactions share the same *after-commit* and *transaction-state*"
+    (let [result (atom [])]
+      (t2/with-transaction []
+        (mdb.connection/after-commit! (fn [] (swap! result conj :outer)))
+        (let [outer-state mdb.connection/*transaction-state*]
+          (t2/with-transaction []
+            (is (identical? outer-state mdb.connection/*transaction-state*)
+                "*transaction-state* should be the same atom in nested transactions")
+            (mdb.connection/after-commit! (fn [] (swap! result conj :inner)))))
+        (is (= [] @result) "Callbacks should not have executed yet"))
+      (is (= [:outer :inner] @result) "Both callbacks should execute after outermost commit"))))
+
 (deftest ^:parallel transaction-isolation-level-test
   (testing "We should always use READ_COMMITTED for the app DB (#44505)"
     (with-open [conn (.getConnection mdb.connection/*application-db*)]

test/metabase/documents/search_test.clj

@@ -5,6 +5,7 @@
    [metabase.events.core :as events]
    [metabase.permissions.core :as perms]
    [metabase.permissions.models.permissions-group :as perms-group]
+   [metabase.search.impl :as search.impl]
    [metabase.search.test-util :as search.tu]
    [metabase.test :as mt]
    [metabase.test.fixtures :as fixtures]
@@ -36,13 +37,13 @@
   (testing "Cards with document_id are excluded from search indexing"
     (let [card-name (mt/random-name)
           regular-card-name (str card-name "-regular")]
-      (search.tu/with-temp-index-table
+      (search.tu/with-temp-index-table-for-http
         (mt/with-temp [:model/Document document {:name "Test Document"
                                                  :document (text->prose-mirror-ast "")}
                        :model/Card document-card (-> (card-with-name-and-query card-name)
                                                      (assoc :document_id (u/the-id document)))
                        :model/Card regular-card (card-with-name-and-query regular-card-name)]
-
+          (search.impl/sync-reindex! {:in-place? true})
           (testing "Search API excludes document cards"
             (let [results (mt/user-http-request :crowberto :get 200 "search" :q card-name :models "card")]
               (is (not (some #(= (:id %) (u/the-id document-card)) (:data results)))
@@ -53,51 +54,57 @@
 
 (deftest document-permission-filtering-test
   (testing "Document search respects collection-based permissions"
-    (mt/with-non-admin-groups-no-root-collection-perms
-      (search.tu/with-temp-index-table
-        (mt/with-temp [:model/Collection {coll-id :id} {}
-                       :model/Collection {private-coll-id :id} {}
-                       :model/Document {doc-in-public-coll :id} {:name "Public Document"
-                                                                 :collection_id coll-id
-                                                                 :document (text->prose-mirror-ast "content")}
-                       :model/Document {doc-in-private-coll :id} {:name "Private Document"
-                                                                  :collection_id private-coll-id
-                                                                  :document (text->prose-mirror-ast "content")}
-                       :model/Document {doc-archived :id} {:name "Archived Document"
-                                                           :collection_id coll-id
-                                                           :archived true
-                                                           :document (text->prose-mirror-ast "content")}]
-          ;; Give user read access to first collection only
-          (perms/grant-collection-read-permissions! (perms-group/all-users) coll-id)
+    ;; Use unique search terms to avoid matching documents from other tests
+    (let [search-id   (mt/random-name)
+          public-name (str search-id " Public")
+          private-name (str search-id " Private")
+          archived-name (str search-id " Archived")]
+      (mt/with-non-admin-groups-no-root-collection-perms
+        (search.tu/with-temp-index-table-for-http
+          (mt/with-temp [:model/Collection {coll-id :id} {}
+                         :model/Collection {private-coll-id :id} {}
+                         :model/Document {doc-in-public-coll :id} {:name public-name
+                                                                   :collection_id coll-id
+                                                                   :document (text->prose-mirror-ast "content")}
+                         :model/Document {doc-in-private-coll :id} {:name private-name
+                                                                    :collection_id private-coll-id
+                                                                    :document (text->prose-mirror-ast "content")}
+                         :model/Document {doc-archived :id} {:name archived-name
+                                                             :collection_id coll-id
+                                                             :archived true
+                                                             :document (text->prose-mirror-ast "content")}]
+            (search.impl/sync-reindex! {:in-place? true})
+            ;; Give user read access to first collection only
+            (perms/grant-collection-read-permissions! (perms-group/all-users) coll-id)
 
-          (testing "Regular user sees only documents in accessible collections"
-            (let [results (mt/user-http-request :rasta :get 200 "search" :q "Document" :models "document")]
-              (is (= #{doc-in-public-coll}
-                     (set (map :id (:data results))))
-                  "Should only see document in accessible collection")))
+            (testing "Regular user sees only documents in accessible collections"
+              (let [results (mt/user-http-request :rasta :get 200 "search" :q search-id :models "document")]
+                (is (= #{doc-in-public-coll}
+                       (set (map :id (:data results))))
+                    "Should only see document in accessible collection")))
 
-          (testing "Regular user cannot see archived documents (no write perms)"
-            (let [results (mt/user-http-request :rasta :get 200 "search" :q "Archived" :archived true :models "document")]
-              (is (empty? (:data results))
-                  "Should not see archived document without write permissions")))
+            (testing "Regular user cannot see archived documents (no write perms)"
+              (let [results (mt/user-http-request :rasta :get 200 "search" :q archived-name :archived true :models "document")]
+                (is (empty? (:data results))
+                    "Should not see archived document without write permissions")))
 
-          (testing "Admin can see all documents including archived"
-            (let [regular-results (mt/user-http-request :crowberto :get 200 "search" :q "Document" :models "document")
-                  archived-results (mt/user-http-request :crowberto :get 200 "search" :q "Archived" :archived true :models "document")]
-              (is (= #{doc-in-public-coll doc-in-private-coll}
-                     (set (map :id (:data regular-results))))
-                  "Admin should see all non-archived documents")
-              (is (= #{doc-archived}
-                     (set (map :id (:data archived-results))))
-                  "Admin should see archived documents")))
+            (testing "Admin can see all documents including archived"
+              (let [regular-results (mt/user-http-request :crowberto :get 200 "search" :q search-id :models "document")
+                    archived-results (mt/user-http-request :crowberto :get 200 "search" :q archived-name :archived true :models "document")]
+                (is (= #{doc-in-public-coll doc-in-private-coll}
+                       (set (map :id (:data regular-results))))
+                    "Admin should see all non-archived documents")
+                (is (= #{doc-archived}
+                       (set (map :id (:data archived-results))))
+                    "Admin should see archived documents")))
 
-          (testing "User with write permissions can see archived documents"
-            ;; Give user write access to first collection
-            (perms/grant-collection-readwrite-permissions! (perms-group/all-users) coll-id)
-            (let [archived-results (mt/user-http-request :rasta :get 200 "search" :q "Archived" :archived true :models "document")]
-              (is (= #{doc-archived}
-                     (set (map :id (:data archived-results))))
-                  "User with write permissions should see archived documents in accessible collections"))))))))
+            (testing "User with write permissions can see archived documents"
+              ;; Give user write access to first collection
+              (perms/grant-collection-readwrite-permissions! (perms-group/all-users) coll-id)
+              (let [archived-results (mt/user-http-request :rasta :get 200 "search" :q archived-name :archived true :models "document")]
+                (is (= #{doc-archived}
+                       (set (map :id (:data archived-results))))
+                    "User with write permissions should see archived documents in accessible collections")))))))))
 
 (deftest document-view-tracking-integration-test
   (testing "Document view tracking integrates with search"

test/metabase/driver/test_util.clj

@@ -1,7 +1,7 @@
 (ns metabase.driver.test-util
   (:require
    [mb.hawk.parallel]
-   [metabase.driver.events.report-timezone-updated]
+   [metabase.driver-api.core :as driver-api]
    [metabase.test.initialize :as initialize])
   (:import
    (org.apache.sshd.server SshServer)
@@ -14,7 +14,7 @@
   ;; It makes sense to notify databases only if app db is initialized.
   (when (initialize/initialized? :db)
     (initialize/initialize-if-needed! :plugins)
-    (#'metabase.driver.events.report-timezone-updated/notify-all-databases-updated)))
+    (driver-api/publish-event! :event/report-timezone-updated {})))
 
 (defmacro wrap-notify-all-databases-updated!
   [& body]

test/metabase/metabot/agent/tool_output_test.clj

@@ -11,6 +11,7 @@
    [metabase.metabot.tools.metadata :as metadata-tools]
    [metabase.metabot.tools.resources :as resource-tools]
    [metabase.metabot.tools.search :as search-tools]
+   [metabase.search.test-util :as search.tu]
    [metabase.test :as mt]
    [metabase.test.fixtures :as fixtures]
    [metabase.warehouse-schema.models.field-values :as field-values]
@@ -194,31 +195,33 @@
 (deftest search-tool-structured-output-formats-correctly-test
   (testing "search tool :structured-output formats to clean XML via format-structured-result"
     (mt/test-driver :h2
-      (mt/with-current-user (mt/user->id :crowberto)
-        (let [result (search-tools/search-tool
-                      {:semantic_queries ["orders"]
-                       :keyword_queries  ["orders"]
-                       :entity_types     ["table"]})]
-          (assert-formatted-structured result "search: tables" #"<search-results\b"))))))
+      (search.tu/with-temp-index-table
+        (mt/with-current-user (mt/user->id :crowberto)
+          (let [result (search-tools/search-tool
+                        {:semantic_queries ["orders"]
+                         :keyword_queries  ["orders"]
+                         :entity_types     ["table"]})]
+            (assert-formatted-structured result "search: tables" #"<search-results\b")))))))
 
 (deftest search-tool-with-models-structured-output-test
   (testing "search tool with model results formats correctly"
     (mt/test-driver :h2
-      (mt/with-current-user (mt/user->id :crowberto)
-        (let [model-query (-> (lib/query (mt/metadata-provider)
-                                         (lib.metadata/table (mt/metadata-provider) (mt/id :orders)))
-                              (lib/with-fields [(lib.metadata/field (mt/metadata-provider) (mt/id :orders :id))
-                                                (lib.metadata/field (mt/metadata-provider) (mt/id :orders :total))]))]
-          (mt/with-temp [:model/Card {_model-id :id} {:dataset_query model-query
-                                                      :database_id   (mt/id)
-                                                      :name          "Searchable Test Model"
-                                                      :type          :model}]
-            (let [result (search-tools/search-tool
-                          {:semantic_queries ["Searchable Test Model"]
-                           :keyword_queries  ["Searchable Test Model"]
-                           :entity_types     ["model"]})]
-              (assert-formatted-structured
-               result "search: model" #"<search-results\b"))))))))
+      (search.tu/with-temp-index-table
+        (mt/with-current-user (mt/user->id :crowberto)
+          (let [model-query (-> (lib/query (mt/metadata-provider)
+                                           (lib.metadata/table (mt/metadata-provider) (mt/id :orders)))
+                                (lib/with-fields [(lib.metadata/field (mt/metadata-provider) (mt/id :orders :id))
+                                                  (lib.metadata/field (mt/metadata-provider) (mt/id :orders :total))]))]
+            (mt/with-temp [:model/Card {_model-id :id} {:dataset_query model-query
+                                                        :database_id   (mt/id)
+                                                        :name          "Searchable Test Model"
+                                                        :type          :model}]
+              (let [result (search-tools/search-tool
+                            {:semantic_queries ["Searchable Test Model"]
+                             :keyword_queries  ["Searchable Test Model"]
+                             :entity_types     ["model"]})]
+                (assert-formatted-structured
+                 result "search: model" #"<search-results\b")))))))))
 
 (deftest list-available-fields-structured-output-test
   (testing "list_available_fields :structured-output formats to clean XML"

test/metabase/metabot/tools/search_test.clj

@@ -9,6 +9,7 @@
    [metabase.permissions.core :as perms]
    [metabase.permissions.models.permissions-group :as perms-group]
    [metabase.search.core :as search-core]
+   [metabase.search.impl :as search.impl]
    [metabase.search.test-util :as search.tu]
    [metabase.test :as mt]
    [metabase.util :as u]
@@ -277,6 +278,7 @@
                          :model/Dashboard  {dash-id-1 :id}      {:name "Our Dashboard",  :collection_id public-coll-id}
                          :model/Dashboard  {dash-id-2 :id}      {:name "My Dashboard",   :collection_id admins-coll-id}
                          :model/Dashboard  {dash-id-3 :id}      {:name "Your Dashboard", :collection_id others-coll-id}]
+            (search.impl/sync-reindex! {:in-place? true})
             (let [test-dashboard-ids #{dash-id-1 dash-id-2 dash-id-3}]
               (is (= #{"Our Dashboard" "My Dashboard"}
                      (->> (search/search {:term-queries ["Dashboard"]})
@@ -299,6 +301,7 @@
                                                          :collection_id analytics-coll-id}
                        :model/Dashboard {dash-3-id :id} {:name "No Desc Dashboard"
                                                          :collection_id no-desc-coll-id}]
+          (search.impl/sync-reindex! {:in-place? true})
           (testing "search results include collection descriptions"
             (let [results (search/search {:term-queries ["Dashboard"]})
                   test-dashboard-ids #{dash-1-id dash-2-id dash-3-id}
@@ -350,6 +353,7 @@
                        :model/Dashboard  {id-1 :id}    {:name "Regular Dash (sh1b0le#h)",    :collection_id coll-id}
                        :model/Dashboard  {id-2 :id}    {:name "Bookmarked Dash (sh1b0le#h)", :collection_id coll-id}
                        :model/DashboardBookmark _      {:dashboard_id id-2, :user_id api/*current-user-id*}]
+          (search.impl/sync-reindex! {:in-place? true})
           (let [base-query   {:term-queries ["sh1b0le#h"], :entity-types ["dashboard"]}
                 test-entity? (comp #{id-1 id-2} :id)
                 query        (fn [& [weights]]

test/metabase/mq/publish_buffer_test.clj

@@ -0,0 +1,60 @@
+(ns metabase.mq.publish-buffer-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase.mq.publish-buffer :as publish-buffer]
+   [metabase.mq.transport :as transport]))
+
+(set! *warn-on-reflection* true)
+
+(deftest flush-re-buffers-on-failure-test
+  (testing "When transport/publish! throws, messages are re-buffered (not lost)"
+    (let [published (atom [])
+          call-count (atom 0)]
+      (binding [publish-buffer/*publish-buffer* (atom {})
+                publish-buffer/*publish-buffer-ms* 0
+                publish-buffer/*publish-buffer-max-ms* 0]
+        ;; Directly populate the buffer with an entry past its deadline
+        (reset! publish-buffer/*publish-buffer*
+                {:queue/test {:messages ["msg1" "msg2"]
+                              :deadline-ms 1
+                              :created-ms 1}})
+        (with-redefs [transport/publish! (fn [channel messages]
+                                           (swap! call-count inc)
+                                           (if (= 1 @call-count)
+                                             (throw (ex-info "publish failed" {}))
+                                             (swap! published conj {:channel channel :messages messages})))]
+          ;; First flush — publish throws, messages should be re-buffered
+          (publish-buffer/flush-publish-buffer!)
+          (testing "Messages are back in the buffer after failure"
+            (is (contains? @publish-buffer/*publish-buffer* :queue/test))
+            (is (= ["msg1" "msg2"]
+                   (get-in @publish-buffer/*publish-buffer* [:queue/test :messages]))))
+
+          ;; Second flush — publish succeeds
+          (publish-buffer/flush-publish-buffer!)
+          (testing "Messages are delivered on retry"
+            (is (= [{:channel :queue/test :messages ["msg1" "msg2"]}] @published))
+            (is (empty? @publish-buffer/*publish-buffer*))))))))
+
+(deftest flush-re-buffer-merges-with-new-messages-test
+  (testing "Re-buffered messages merge with new messages that arrived during flush"
+    (binding [publish-buffer/*publish-buffer* (atom {})
+              publish-buffer/*publish-buffer-ms* 100
+              publish-buffer/*publish-buffer-max-ms* 0]
+      ;; Populate buffer with messages past deadline
+      (reset! publish-buffer/*publish-buffer*
+              {:queue/test {:messages ["old1" "old2"]
+                            :deadline-ms 1
+                            :created-ms 1}})
+      (with-redefs [transport/publish! (fn [_channel _messages]
+                                         ;; Simulate new messages arriving during flush
+                                         (swap! publish-buffer/*publish-buffer*
+                                                assoc :queue/test
+                                                {:messages ["new1"]
+                                                 :deadline-ms (+ (System/currentTimeMillis) 100)
+                                                 :created-ms (System/currentTimeMillis)})
+                                         (throw (ex-info "publish failed" {})))]
+        (publish-buffer/flush-publish-buffer!)
+        (testing "Re-buffered messages are merged with new messages"
+          (let [messages (get-in @publish-buffer/*publish-buffer* [:queue/test :messages])]
+            (is (= ["new1" "old1" "old2"] messages))))))))

test/metabase/mq/queue/appdb_test.clj

@@ -0,0 +1,320 @@
+(ns metabase.mq.queue.appdb-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase.mq.analytics :as mq.analytics]
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.queue.appdb :as q.appdb]
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.mq.settings :as mq.settings]
+   [metabase.util.json :as json]
+   [toucan2.core :as t2])
+  (:import (java.sql Timestamp)
+           (java.time Instant)))
+
+(set! *warn-on-reflection* true)
+
+(deftest publish-test
+  (let [queue-name (keyword "queue" (str (gensym "publish-test-")))]
+    (try
+      (q.backend/publish! :queue.backend/appdb queue-name ["test message"])
+      (testing "Messages are persisted in the queue"
+        (let [row (t2/select-one :queue_message_batch :queue_name (name queue-name))]
+          (is (= ["test message"] (json/decode (:messages row))))
+          (is (= (name queue-name) (:queue_name row)))
+          (is (= "pending" (:status row)))
+          (is (not (nil? (:status_heartbeat row))))
+          (is (= (:created_at row) (:status_heartbeat row)))
+          (is (= 0 (:failures row)))
+          (is (nil? (:owner row)))))
+      (finally
+        (t2/delete! :queue_message_batch :queue_name (name queue-name))))))
+
+(deftest fetch-test
+  (let [queue1 (keyword "queue" (str "queue1-" (gensym)))
+        queue2 (keyword "queue" (str "queue2-" (gensym)))
+        invalid-queue (keyword "queue" (str "queue-invalid-" (gensym)))]
+    (binding [listener/*listeners* (atom {queue1 {:listener identity} queue2 {:listener identity}})]
+      (try
+        (t2/with-connection [_conn]
+          (testing "Returns nil if no rows are found"
+            (is (nil? (#'q.appdb/fetch! (listener/queue-names)))))
+
+          (t2/insert! :queue_message_batch
+                      {:queue_name (name invalid-queue)
+                       :messages   (json/encode ["invalid"])})
+          (t2/insert! :queue_message_batch
+                      {:queue_name (name queue1)
+                       :messages   (json/encode ["data1"])})
+          (t2/insert! :queue_message_batch
+                      {:queue_name (name queue2)
+                       :messages   (json/encode ["data2"])})
+
+          (testing "Returns nil if no queues are defined"
+            (binding [listener/*listeners* (atom {})]
+              (is (nil? (#'q.appdb/fetch! (listener/queue-names))))))
+          (testing "Fetches one row per queue in a single call"
+            (let [results (#'q.appdb/fetch! (listener/queue-names))
+                  by-queue (into {} (map (juxt :queue identity)) results)]
+              (is (= 2 (count results)))
+              (testing "queue1 row"
+                (let [{:keys [bundle-id messages]} (get by-queue queue1)]
+                  (is (pos-int? bundle-id))
+                  (is (= ["data1"] messages))
+                  (testing "Fetched row gets marked as processing"
+                    (let [updated-row (t2/select-one :queue_message_batch :id bundle-id)]
+                      (is (= "processing" (:status updated-row)))
+                      (is (not= (:created_at updated-row) (:status_heartbeat updated-row)))
+                      (is (= 0 (:failures updated-row)))
+                      (is (= @#'q.appdb/owner-id (:owner updated-row)))))))
+              (testing "queue2 row"
+                (let [{:keys [bundle-id messages]} (get by-queue queue2)]
+                  (is (pos-int? bundle-id))
+                  (is (= ["data2"] messages))))))
+          (testing "When everything valid is processing, return nil"
+            (is (nil? (#'q.appdb/fetch! (listener/queue-names))))))
+        (finally
+          (t2/delete! :queue_message_batch :queue_name [:in [(name queue1) (name queue2) (name invalid-queue)]]))))))
+
+(deftest exclusive-fetch-test
+  (let [exclusive-q (keyword "queue" (str "exclusive-" (random-uuid)))
+        normal-q    (keyword "queue" (str "normal-" (random-uuid)))]
+    (binding [listener/*listeners* (atom {exclusive-q {:listener identity :exclusive true}
+                                          normal-q    {:listener identity :exclusive false}})]
+      (try
+        (t2/with-connection [_conn]
+          (testing "With no processing rows, exclusive queue messages are fetched normally"
+            (t2/insert! :queue_message_batch
+                        {:queue_name (name exclusive-q)
+                         :messages   (json/encode ["exclusive-msg1"])})
+            (let [results (#'q.appdb/fetch! (listener/queue-names))
+                  {:keys [queue messages]} (first results)]
+              (is (= 1 (count results)))
+              (is (= exclusive-q queue))
+              (is (= ["exclusive-msg1"] messages))))
+
+          (testing "With a processing row on exclusive queue, fetch skips it"
+            (t2/insert! :queue_message_batch
+                        {:queue_name (name exclusive-q)
+                         :messages   (json/encode ["exclusive-msg2"])})
+            ;; The first message is now 'processing', so the second should be skipped
+            (is (nil? (#'q.appdb/fetch! (listener/queue-names)))
+                "Should return nil because exclusive queue has a processing row"))
+
+          (testing "Non-exclusive queue is still fetchable even when exclusive queue is blocked"
+            (t2/insert! :queue_message_batch
+                        {:queue_name (name normal-q)
+                         :messages   (json/encode ["normal-msg1"])})
+            (let [results (#'q.appdb/fetch! (listener/queue-names))
+                  {:keys [queue messages]} (first results)]
+              (is (= 1 (count results)))
+              (is (= normal-q queue))
+              (is (= ["normal-msg1"] messages))))
+
+          (testing "After exclusive processing completes, next message can be fetched"
+            ;; Mark the processing row as done by deleting it (simulating bundle-successful!)
+            (t2/delete! :queue_message_batch :queue_name (name exclusive-q) :status "processing")
+            (let [results (#'q.appdb/fetch! (listener/queue-names))
+                  {:keys [queue messages]} (first results)]
+              (is (= exclusive-q queue))
+              (is (= ["exclusive-msg2"] messages)))))
+        (finally
+          (t2/delete! :queue_message_batch :queue_name [:in [(name exclusive-q) (name normal-q)]]))))))
+
+(deftest message-successful-test
+  (let [queue-name (keyword "queue" (str (gensym "successful-test-")))]
+    (testing "Message successful deletes the message"
+      (let [message-id (t2/insert-returning-pk! :queue_message_batch
+                                                {:queue_name (name queue-name)
+                                                 :messages (json/encode ["test-message"])
+                                                 :status "processing"
+                                                 :owner @#'q.appdb/owner-id})]
+        (q.backend/bundle-successful! :queue.backend/appdb queue-name message-id)
+        (is (nil? (t2/select-one :queue_message_batch :id message-id)))))
+
+    (testing "Message successful on non-existent message does not fail"
+      (is (nil? (q.backend/bundle-successful! :queue.backend/appdb queue-name 99999))))))
+
+;; Note: no cleanup needed — bundle-successful! deletes the row
+
+(deftest message-failed-test
+  (let [queue-name (keyword "queue" (str (gensym "failed-test-")))]
+    (try
+      (testing "Message failed resets to pending and increments failures"
+        (let [message-id (t2/insert-returning-pk! :queue_message_batch
+                                                  {:queue_name (name queue-name)
+                                                   :messages (json/encode ["test-message"])
+                                                   :status "processing"
+                                                   :failures 0
+                                                   :owner @#'q.appdb/owner-id})]
+          (q.backend/bundle-failed! :queue.backend/appdb queue-name message-id)
+          (let [updated-message (t2/select-one :queue_message_batch :id message-id)]
+            (is (= "pending" (:status updated-message)))
+            (is (= 1 (:failures updated-message)))
+            (is (nil? (:owner updated-message)))
+            (is (not (nil? (:status_heartbeat updated-message)))))))
+
+      (testing "Message failed increments failures multiple times"
+        (let [message-id (t2/insert-returning-pk! :queue_message_batch
+                                                  {:queue_name (name queue-name)
+                                                   :messages (json/encode ["test-message"])
+                                                   :status "processing"
+                                                   :failures 2
+                                                   :owner @#'q.appdb/owner-id})]
+          (q.backend/bundle-failed! :queue.backend/appdb queue-name message-id)
+          (let [updated-message (t2/select-one :queue_message_batch :id message-id)]
+            (is (= "pending" (:status updated-message)))
+            (is (= 3 (:failures updated-message)))
+            (is (nil? (:owner updated-message))))))
+
+      (testing "Message moved to failed status after max failures"
+        (let [message-id (t2/insert-returning-pk! :queue_message_batch
+                                                  {:queue_name (name queue-name)
+                                                   :messages (json/encode ["test-message"])
+                                                   :status "processing"
+                                                   :failures (dec (mq.settings/queue-max-retries))
+                                                   :owner @#'q.appdb/owner-id})]
+          (q.backend/bundle-failed! :queue.backend/appdb queue-name message-id)
+          (let [updated-message (t2/select-one :queue_message_batch :id message-id)]
+            (is (= "failed" (:status updated-message)))
+            (is (= (mq.settings/queue-max-retries) (:failures updated-message)))
+            (is (nil? (:owner updated-message))))))
+
+      (testing "Message failed on non-existent message does not fail"
+        (is (nil? (q.backend/bundle-failed! :queue.backend/appdb queue-name 99999))))
+
+      (testing "Message failed when being processed by another node is a no-op"
+        (let [message-id (t2/insert-returning-pk! :queue_message_batch
+                                                  {:queue_name (name queue-name)
+                                                   :messages (json/encode ["test-message"])
+                                                   :status "processing"
+                                                   :owner "another-node"
+                                                   :failures 0})]
+          (q.backend/bundle-failed! :queue.backend/appdb queue-name message-id)
+          (let [updated-message (t2/select-one :queue_message_batch :id message-id)]
+            (is (= "processing" (:status updated-message)))
+            (is (= 0 (:failures updated-message)))
+            (is (= "another-node" (:owner updated-message))))))
+      (finally
+        (t2/delete! :queue_message_batch :queue_name (name queue-name))))))
+
+(deftest recover-stale-processing-batches-test
+  (let [queue-name (keyword "queue" (str "stale-test-" (random-uuid)))
+        stale-heartbeat (Timestamp/from (.minusMillis (Instant/now) (+ @#'q.appdb/stale-processing-timeout-ms 60000)))]
+    (try
+      (testing "Stale processing batch is reset to pending with incremented failures"
+        (let [message-id (t2/insert-returning-pk! :queue_message_batch
+                                                  {:queue_name       (name queue-name)
+                                                   :messages         (json/encode ["stale-msg"])
+                                                   :status           "processing"
+                                                   :status_heartbeat stale-heartbeat
+                                                   :failures         1
+                                                   :owner            "dead-owner"})]
+          (is (pos? (#'q.appdb/recover-stale-processing-batches!)))
+          (let [row (t2/select-one :queue_message_batch :id message-id)]
+            (is (= "pending" (:status row)))
+            (is (= 2 (:failures row)))
+            (is (nil? (:owner row))))))
+
+      (testing "Stale batch at max retries is marked as failed"
+        (let [message-id (t2/insert-returning-pk! :queue_message_batch
+                                                  {:queue_name       (name queue-name)
+                                                   :messages         (json/encode ["stale-msg-max"])
+                                                   :status           "processing"
+                                                   :status_heartbeat stale-heartbeat
+                                                   :failures         (dec (mq.settings/queue-max-retries))
+                                                   :owner            "dead-owner"})]
+          (is (pos? (#'q.appdb/recover-stale-processing-batches!)))
+          (let [row (t2/select-one :queue_message_batch :id message-id)]
+            (is (= "failed" (:status row)))
+            (is (= (mq.settings/queue-max-retries) (:failures row)))
+            (is (nil? (:owner row))))))
+
+      (testing "Recent processing batch is not recovered by stale check"
+        (let [message-id (t2/insert-returning-pk! :queue_message_batch
+                                                  {:queue_name       (name queue-name)
+                                                   :messages         (json/encode ["recent-msg"])
+                                                   :status           "processing"
+                                                   :failures         0
+                                                   :owner            "active-owner"})]
+          (#'q.appdb/recover-stale-processing-batches!)
+          (let [row (t2/select-one :queue_message_batch :id message-id)]
+            (is (= "processing" (:status row)))
+            (is (= 0 (:failures row))))))
+      (finally
+        (t2/delete! :queue_message_batch :queue_name (name queue-name))))))
+
+(deftest heartbeat-test
+  (let [queue-name (keyword "queue" (str "heartbeat-test-" (random-uuid)))]
+    (try
+      (let [message-id         (t2/insert-returning-pk! :queue_message_batch
+                                                        {:queue_name (name queue-name)
+                                                         :messages   (json/encode ["msg"])
+                                                         :status     "processing"
+                                                         :owner      @#'q.appdb/owner-id})
+            original-heartbeat (:status_heartbeat (t2/select-one :queue_message_batch :id message-id))]
+        (Thread/sleep 100) ; Ensure enough time passes for timestamp to differ
+        (with-redefs [mq.impl/busy-channels          (constantly #{queue-name})
+                      mq.impl/active-handler-metadata (fn [_ch] {:bundle-id message-id})]
+          (#'q.appdb/update-heartbeats!))
+        (testing "Heartbeat is updated after update-heartbeats!"
+          (let [updated-heartbeat (:status_heartbeat (t2/select-one :queue_message_batch :id message-id))]
+            (is (not= original-heartbeat updated-heartbeat)
+                "Heartbeat timestamp should be updated"))))
+      (finally
+        (t2/delete! :queue_message_batch :queue_name (name queue-name))))))
+
+(deftest cleanup-failed-batches-test
+  (let [queue-name (keyword "queue" (str "cleanup-test-" (random-uuid)))
+        old-ts     (Timestamp/from (.minusMillis (Instant/now) (+ (* 7 24 60 60 1000) 60000)))]
+    (try
+      (testing "Failed batch older than 7 days is deleted"
+        (let [old-id (t2/insert-returning-pk! :queue_message_batch
+                                              {:queue_name       (name queue-name)
+                                               :messages         (json/encode ["old-msg"])
+                                               :status           "failed"
+                                               :status_heartbeat old-ts})]
+          (#'q.appdb/cleanup-failed-batches!)
+          (is (nil? (t2/select-one :queue_message_batch :id old-id))
+              "Old failed batch should be deleted")))
+
+      (testing "Recent failed batch is not deleted"
+        (let [recent-id (t2/insert-returning-pk! :queue_message_batch
+                                                 {:queue_name (name queue-name)
+                                                  :messages   (json/encode ["recent-msg"])
+                                                  :status     "failed"})]
+          (#'q.appdb/cleanup-failed-batches!)
+          (is (some? (t2/select-one :queue_message_batch :id recent-id))
+              "Recent failed batch should not be deleted")
+          (t2/delete! :queue_message_batch :id recent-id)))
+
+      (testing "Non-failed batch with old heartbeat is not cleaned up"
+        (let [pending-id (t2/insert-returning-pk! :queue_message_batch
+                                                  {:queue_name       (name queue-name)
+                                                   :messages         (json/encode ["pending-msg"])
+                                                   :status           "pending"
+                                                   :status_heartbeat old-ts})]
+          (#'q.appdb/cleanup-failed-batches!)
+          (is (some? (t2/select-one :queue_message_batch :id pending-id))
+              "Non-failed batch should not be deleted")
+          (t2/delete! :queue_message_batch :id pending-id)))
+      (finally
+        (t2/delete! :queue_message_batch :queue_name (name queue-name))))))
+
+(deftest update-depth-gauges-test
+  (let [queue-name  (keyword "queue" (str "depth-gauge-test-" (random-uuid)))
+        gauge-calls (atom [])]
+    (try
+      (t2/insert! :queue_message_batch {:queue_name (name queue-name) :messages (json/encode ["m1"]) :status "pending"})
+      (t2/insert! :queue_message_batch {:queue_name (name queue-name) :messages (json/encode ["m2"]) :status "failed"})
+      (with-redefs [mq.analytics/set! (fn [metric labels value]
+                                        (when (= metric :metabase-mq/appdb-queue-depth)
+                                          (swap! gauge-calls conj {:labels labels :value value})))]
+        (#'q.appdb/update-depth-gauges!))
+      (testing "Gauge is emitted for each queue/status combination"
+        (let [calls-for-queue (filter #(= (name queue-name) (-> % :labels :channel)) @gauge-calls)
+              by-status       (into {} (map (juxt #(-> % :labels :status) :value)) calls-for-queue)]
+          (is (= 1 (get by-status "pending")) "Should count 1 pending message")
+          (is (= 1 (get by-status "failed")) "Should count 1 failed message")))
+      (finally
+        (t2/delete! :queue_message_batch :queue_name (name queue-name))))))

test/metabase/mq/queue/core_test.clj

@@ -0,0 +1,263 @@
+(ns metabase.mq.queue.core-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase.app-db.connection :as app-db.conn]
+   [metabase.mq.core :as mq]
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.publish :as mq.publish]
+   [metabase.mq.publish-buffer :as publish-buffer]
+   [metabase.mq.test-util :as mq.tu])
+  (:import (clojure.lang ExceptionInfo)
+           (java.util.concurrent CountDownLatch CyclicBarrier)))
+
+(set! *warn-on-reflection* true)
+
+(deftest e2e-test
+  (let [heard-messages (atom [])]
+    (mq.tu/with-sync-mq {:queue/test (fn [message]
+                                       (swap! heard-messages conj message))}
+      (testing "The messages are heard and processed"
+        (mq/with-queue :queue/test [q]
+          (mq/put q "test message 1")
+          (mq/put q "test message 2"))
+        (is (= ["test message 1" "test message 2"] @heard-messages))))))
+
+(deftest publish-to-unlistened-queue-test
+  (mq.tu/with-sync-mq
+    (testing "with-queue on a queue with no listener still succeeds"
+      (mq/with-queue :queue/nonexistent [q]
+        (mq/put q "msg")))))
+
+(deftest with-queue-success-test
+  (let [heard (atom [])]
+    (mq.tu/with-sync-mq {:queue/test (fn [msg] (swap! heard conj msg))}
+      (testing "with-queue publishes buffered messages on success"
+        (let [result (mq/with-queue :queue/test [q]
+                       (mq/put q "a")
+                       (mq/put q "b")
+                       :done)]
+          (is (= :done result))
+          (is (= ["a" "b"] @heard)))))))
+
+(deftest with-queue-exception-discards-test
+  (let [heard (atom [])]
+    (mq.tu/with-sync-mq {:queue/test (fn [msg] (swap! heard conj msg))}
+      (testing "with-queue discards buffered messages on exception"
+        (is (thrown? Exception
+                     (mq/with-queue :queue/test [q]
+                       (mq/put q "should-be-discarded")
+                       (throw (ex-info "boom" {})))))
+        (is (empty? @heard))))))
+
+(deftest with-queue-no-listener-test
+  (mq.tu/with-sync-mq
+    (testing "with-queue on queue with no listener still succeeds"
+      (mq/with-queue :queue/nonexistent [q]
+        (mq/put q "msg")))))
+
+(deftest double-listen-throws-test
+  (mq.tu/with-sync-mq {:queue/test (fn [_] nil)}
+    (testing "Registering a second listener on the same queue throws"
+      (is (thrown-with-msg? ExceptionInfo #"Listener already registered"
+                            (mq/listen! :queue/test {} (fn [_] nil)))))))
+
+(deftest concurrent-listen-throws-test
+  (mq.tu/with-sync-mq
+    (let [queue-name :queue/test
+          n          10
+          barrier    (CyclicBarrier. n)
+          results    (atom [])]
+      (testing "Concurrent listen! calls: exactly one succeeds, rest throw"
+        (let [threads (mapv (fn [_]
+                              (let [f (bound-fn []
+                                        (.await barrier)
+                                        (try
+                                          (mq/listen! queue-name {} (fn [_] nil))
+                                          (swap! results conj :ok)
+                                          (catch ExceptionInfo _
+                                            (swap! results conj :error))))]
+                                (Thread. ^Runnable f)))
+                            (range n))]
+          (run! (fn [^Thread t] (.start t)) threads)
+          (run! (fn [^Thread t] (.join t 5000)) threads)
+          (is (= 1 (count (filter #{:ok} @results))))
+          (is (= (dec n) (count (filter #{:error} @results))))))
+      (mq/unlisten! queue-name))))
+
+(deftest exclusive-listen-test
+  (let [heard-messages (atom [])]
+    (mq.tu/with-sync-mq {:queue/exclusive-test {:listener           (fn [message]
+                                                                      (swap! heard-messages conj message))
+                                                :max-batch-messages 1
+
+                                                :exclusive          true}}
+      (testing "Exclusive queue processes messages normally via sync backend"
+        (mq/with-queue :queue/exclusive-test [q]
+          (mq/put q "msg1")
+          (mq/put q "msg2"))
+        (is (= ["msg1" "msg2"] @heard-messages))))))
+
+(deftest batch-listen-exclusive-test
+  (let [heard-batches (atom [])]
+    (mq.tu/with-sync-mq
+      (mq/batch-listen! :queue/batch-exclusive
+                        (fn [batch] (swap! heard-batches conj batch))
+                        {:max-batch-messages 10 :exclusive true})
+      (testing "Exclusive batch-listen! registers and processes"
+        (mq/with-queue :queue/batch-exclusive [q]
+          (mq/put q "a")
+          (mq/put q "b"))
+        (is (= [["a" "b"]] @heard-batches))))))
+
+(deftest transaction-defers-publish-test
+  (let [heard (atom [])]
+    (mq.tu/with-sync-mq {:queue/test (fn [msg] (swap! heard conj msg))}
+      (binding [app-db.conn/*after-commit*  (atom [])
+                app-db.conn/*transaction-state* (atom {})]
+        (testing "Inside a transaction, messages are accumulated, not published immediately"
+          (mq/with-queue :queue/test [q]
+            (mq/put q "msg1")
+            (mq/put q "msg2"))
+          (mq/with-queue :queue/test [q]
+            (mq/put q "msg3"))
+          (testing "Messages are deferred in transaction state"
+            (is (= ["msg1" "msg2" "msg3"]
+                   (get-in @app-db.conn/*transaction-state*
+                           [::mq.publish/deferred-messages :queue/test]))))
+          (testing "Listener has not been called yet"
+            (is (= [] @heard))))
+        (testing "After flush (simulating commit), all messages are delivered"
+          (mq.publish/flush-deferred-messages!)
+          (is (= ["msg1" "msg2" "msg3"] @heard)))))))
+
+(deftest transaction-rollback-discards-messages-test
+  (mq.tu/with-sync-mq {:queue/test (fn [_] nil)}
+    (binding [app-db.conn/*after-commit*  (atom [])
+              app-db.conn/*transaction-state* (atom {})]
+      (testing "Messages accumulated during a failed transaction are discarded"
+        (is (thrown? Exception
+                     (mq/with-queue :queue/test [q]
+                       (mq/put q "should-be-discarded")
+                       (throw (ex-info "boom" {})))))
+        (is (nil? (get-in @app-db.conn/*transaction-state*
+                          [::mq.publish/deferred-messages :queue/test]))
+            "No messages should be in transaction state after exception")))))
+
+(deftest outside-transaction-publishes-immediately-test
+  (let [heard (atom [])]
+    (mq.tu/with-sync-mq {:queue/test (fn [msg] (swap! heard conj msg))}
+      (testing "Outside a transaction, messages are published immediately"
+        (is (nil? app-db.conn/*transaction-state*))
+        (mq/with-queue :queue/test [q]
+          (mq/put q "msg1"))
+        (is (= ["msg1"] @heard))))))
+
+(deftest multiple-queues-in-transaction-test
+  (let [heard-a (atom [])
+        heard-b (atom [])]
+    (mq.tu/with-sync-mq {:queue/test-a (fn [msg] (swap! heard-a conj msg))
+                         :queue/test-b (fn [msg] (swap! heard-b conj msg))}
+      (binding [app-db.conn/*after-commit*  (atom [])
+                app-db.conn/*transaction-state* (atom {})]
+        (mq/with-queue :queue/test-a [q]
+          (mq/put q "a1")
+          (mq/put q "a2"))
+        (mq/with-queue :queue/test-b [q]
+          (mq/put q "b1"))
+        (testing "Both queues have deferred messages"
+          (is (= ["a1" "a2"]
+                 (get-in @app-db.conn/*transaction-state*
+                         [::mq.publish/deferred-messages :queue/test-a])))
+          (is (= ["b1"]
+                 (get-in @app-db.conn/*transaction-state*
+                         [::mq.publish/deferred-messages :queue/test-b]))))
+        (testing "After flush, both queues receive their messages"
+          (mq.publish/flush-deferred-messages!)
+          (is (= ["a1" "a2"] @heard-a))
+          (is (= ["b1"] @heard-b)))))))
+
+(deftest buffering-combines-messages-test
+  (let [heard (atom [])]
+    (mq.tu/with-sync-mq
+      (mq/batch-listen! :queue/test
+                        (fn [batch] (swap! heard into batch))
+                        {:max-batch-messages 50})
+      (binding [publish-buffer/*publish-buffer-ms* 100
+                publish-buffer/*publish-buffer*    (atom {})]
+        (testing "buffered-publish! buffers when *publish-buffer-ms* > 0"
+          (mq.publish/publish! :queue/test ["msg1"])
+          (mq.publish/publish! :queue/test ["msg2"])
+          (is (empty? @heard)
+              "Nothing delivered yet because buffer window hasn't elapsed")
+          (is (= ["msg1" "msg2"]
+                 (:messages (get @publish-buffer/*publish-buffer* :queue/test)))
+              "Messages are buffered"))
+        (testing "After flush, all messages are delivered"
+          (swap! publish-buffer/*publish-buffer*
+                 update :queue/test assoc :deadline-ms 1)
+          (publish-buffer/flush-publish-buffer!)
+          (is (= ["msg1" "msg2"] @heard)
+              "Both messages delivered after flush"))))))
+
+(deftest buffering-immediate-when-zero-test
+  (let [heard (atom [])]
+    (mq.tu/with-sync-mq {:queue/test (fn [msg] (swap! heard conj msg))}
+      ;; *publish-buffer-ms* is already 0 from with-sync-mq, but be explicit
+      (binding [publish-buffer/*publish-buffer-ms* 0
+                publish-buffer/*publish-buffer*    (atom {})]
+        (testing "When *publish-buffer-ms* is 0, messages publish immediately"
+          (mq/with-queue :queue/test [q]
+            (mq/put q "msg1"))
+          (is (= ["msg1"] @heard))
+          (mq/with-queue :queue/test [q]
+            (mq/put q "msg2"))
+          (is (= ["msg1" "msg2"] @heard)))))))
+
+(deftest buffering-flush-delivers-test
+  (let [heard (atom [])]
+    (mq.tu/with-sync-mq
+      (mq/batch-listen! :queue/test
+                        (fn [batch] (swap! heard into batch))
+                        {:max-batch-messages 3})
+      (binding [publish-buffer/*publish-buffer-ms*  1
+                publish-buffer/*publish-buffer-max-ms* 0
+                publish-buffer/*publish-buffer*     (atom {})]
+        (testing "Messages are buffered until flushed"
+          (mq.publish/publish! :queue/test ["a" "b"])
+          (is (empty? @heard) "not yet flushed")
+          (mq.publish/publish! :queue/test ["c"])
+          (is (empty? @heard) "still buffered, no proactive flush")
+          (Thread/sleep 5)
+          (publish-buffer/flush-publish-buffer!)
+          (is (= ["a" "b" "c"] @heard) "all delivered after flush"))))))
+
+(deftest fifo-ordering-test
+  (let [received (atom [])]
+    (mq.tu/with-sync-mq {:queue/test (fn [message]
+                                       (swap! received conj message))}
+      (doseq [i (range 10)]
+        (mq/with-queue :queue/test [q]
+          (mq/put q i)))
+      (testing "All messages are delivered in order"
+        (is (= (range 10) @received))))))
+
+(deftest exclusive-queue-single-active-handler-test
+  "Verifies that submit-delivery! returns false for a channel that already has an
+   active handler, enforcing at-most-one concurrent delivery per channel."
+  (mq.impl/start-worker-pool!)
+  (mq.tu/with-sync-mq
+    (let [queue-name :queue/exclusive-concurrency-test
+          latch      (CountDownLatch. 1)]
+      (mq/listen! queue-name {:exclusive true}
+                  (fn [_] (.await latch))) ; Block listener until released
+      (testing "First submission succeeds and marks the channel as busy"
+        (is (true? (mq.impl/submit-delivery! queue-name ["msg1"] nil nil nil)))
+        (Thread/sleep 50) ; Give worker thread time to start and block on latch
+        (is (true? (mq.impl/channel-busy? queue-name))))
+      (testing "Second submission returns false while channel is busy"
+        (is (false? (mq.impl/submit-delivery! queue-name ["msg2"] nil nil nil))))
+      (.countDown latch) ; Release the listener
+      (Thread/sleep 100) ; Wait for delivery to complete and active-handlers to clear
+      (testing "Channel is no longer busy after delivery completes"
+        (is (false? (mq.impl/channel-busy? queue-name))))
+      (mq/unlisten! queue-name))))

test/metabase/mq/queue/memory_test.clj

@@ -0,0 +1,59 @@
+(ns metabase.mq.queue.memory-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase.mq.core :as mq]
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.memory :as memory]
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.mq.queue.memory :as q.memory]))
+
+(set! *warn-on-reflection* true)
+
+(defmacro ^:private with-memory-queue
+  [& body]
+  `(binding [q.backend/*backend*        :queue.backend/memory
+             listener/*listeners*        (atom {})
+             memory/*channels*           (atom {})
+             q.memory/*bundle-registry*  (atom {})]
+     ~@body))
+
+(defn- deliver-pending!
+  "Drains all pending memory channels and delivers messages synchronously on the current thread.
+  For queue channels, also registers bundles so ACK/NACK works."
+  []
+  (doseq [channel-name (concat (listener/queue-names) (listener/topic-names))]
+    (when-let [messages (#'memory/drain! channel-name)]
+      (if (= "queue" (namespace channel-name))
+        (let [bundle-id (str (random-uuid))]
+          (#'memory/register-bundle! bundle-id messages)
+          (mq.impl/deliver! channel-name messages bundle-id :queue.backend/memory))
+        (mq.impl/deliver! channel-name messages nil nil)))))
+
+(deftest publish-test
+  (with-memory-queue
+    (let [queue-name (keyword "queue" (str "publish-test-" (gensym)))]
+      (testing "Publish lazily creates channel and adds messages"
+        (q.backend/publish! :queue.backend/memory queue-name ["test-message"])
+        (q.backend/publish! :queue.backend/memory queue-name ["test-message2"]))
+
+      (testing "Publish to new queue lazily creates it"
+        (q.backend/publish! :queue.backend/memory :queue/new-queue ["test-message"])))))
+
+(deftest exclusive-memory-test
+  (with-memory-queue
+    (let [queue-name :queue/exclusive-test
+          call-count (atom 0)]
+      (testing "Exclusive queue processes all messages"
+        (mq/listen! queue-name
+                    {:exclusive true}
+                    (fn [_msg]
+                      (swap! call-count inc)))
+        (q.backend/publish! :queue.backend/memory queue-name ["msg1"])
+        (q.backend/publish! :queue.backend/memory queue-name ["msg2"])
+        (q.backend/publish! :queue.backend/memory queue-name ["msg3"])
+        (q.backend/publish! :queue.backend/memory queue-name ["msg4"])
+        (q.backend/publish! :queue.backend/memory queue-name ["msg5"])
+        ;; Deliver synchronously — each drain picks up all pending messages
+        (deliver-pending!)
+        (is (= 5 @call-count) "All messages should be processed")))))

test/metabase/mq/test_util.clj

@@ -0,0 +1,42 @@
+(ns metabase.mq.test-util
+  (:require
+   [metabase.mq.listener :as listener]
+   [metabase.mq.publish-buffer :as publish-buffer]
+   [metabase.mq.queue.backend :as q.backend]
+   [metabase.mq.queue.sync]
+   [metabase.mq.topic.backend :as topic.backend]
+   [metabase.mq.topic.sync]))
+
+(defmacro with-sync-mq
+  "Binds both queue and topic systems to their synchronous backends, which
+   invoke listeners inline during `publish!`. Optionally accepts a queue
+   listeners map keyed by queue name. Values can be plain fns (wrapped in
+   standard listener config) or full config maps with :listener and :max-batch-messages.
+   Listeners in the map are merged with any already-registered
+   listeners; for existing keys a plain fn replaces only the :listener entry."
+  [& body]
+  (let [[listeners & body] (if (map? (first body))
+                             body
+                             (cons {} body))]
+    `(binding [q.backend/*backend*              :queue.backend/sync
+               listener/*listeners*              (atom {})
+               publish-buffer/*publish-buffer*    (atom {})
+               publish-buffer/*publish-buffer-ms* 0
+               topic.backend/*backend*            :topic.backend/sync]
+       ;; Register all listen!/batch-listen! implementations into the fresh test atoms
+       (listener/register-listeners!)
+       ;; Merge any explicitly-provided listeners on top
+       (let [listeners# ~listeners]
+         (when (seq listeners#)
+           (swap! listener/*listeners*
+                  (fn [current#]
+                    (reduce-kv (fn [m# k# v#]
+                                 (assoc m# k# (if (fn? v#)
+                                                (if-let [existing# (get current# k#)]
+                                                  (assoc existing# :listener v#)
+                                                  {:listener           v#
+                                                   :max-batch-messages 1})
+                                                v#)))
+                               current#
+                               listeners#)))))
+       ~@body)))

test/metabase/mq/topic/appdb_test.clj

@@ -0,0 +1,126 @@
+(ns metabase.mq.topic.appdb-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase.mq.analytics :as mq.analytics]
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.topic.appdb]
+   [metabase.mq.topic.backend :as topic.backend]
+   [metabase.mq.topic.transport-impl]
+   [metabase.util.json :as json]
+   [toucan2.core :as t2])
+  (:import (java.sql Timestamp)
+           (java.time Instant)))
+
+(set! *warn-on-reflection* true)
+
+(deftest publish-test
+  (let [topic-name :topic/publish-test]
+    (topic.backend/publish! :topic.backend/appdb topic-name ["test message"])
+    (testing "Message is persisted in topic_message"
+      (let [row (t2/select-one :topic_message_batch :topic_name (name topic-name))]
+        (is (some? row))
+        (is (= (name topic-name) (:topic_name row)))
+        (is (= ["test message"] (json/decode (:messages row))))
+        (is (some? (:created_at row)))))
+    ;; cleanup
+    (t2/delete! :topic_message_batch :topic_name (name topic-name))))
+
+(deftest batch-publish-test
+  (let [topic-name :topic/batch-publish-test]
+    (topic.backend/publish! :topic.backend/appdb topic-name ["msg-1" "msg-2" "msg-3"])
+    (testing "Batch of messages stored in single row"
+      (let [row (t2/select-one :topic_message_batch :topic_name (name topic-name))]
+        (is (= ["msg-1" "msg-2" "msg-3"] (json/decode (:messages row))))))
+    ;; cleanup
+    (t2/delete! :topic_message_batch :topic_name (name topic-name))))
+
+(deftest subscribe-and-receive-test
+  (let [topic-name (keyword "topic" (str "sub-receive-test-" (random-uuid)))
+        received   (atom [])]
+    (try
+      ;; Register listener (this also subscribes, setting the offset via the current *backend*)
+      (binding [topic.backend/*backend* :topic.backend/appdb]
+        (listener/listen! topic-name
+                          {}
+                          (fn [message]
+                            (swap! received conj message))))
+      ;; Publish then deliver synchronously via poll-iteration! + deliver!
+      (topic.backend/publish! :topic.backend/appdb topic-name ["hello-appdb"])
+      ;; poll-iteration! fetches rows and calls submit-delivery! which is async.
+      ;; Instead, poll the messages and deliver synchronously on this thread.
+      (let [offset (or (get @@#'metabase.mq.topic.appdb/offsets topic-name) 0)
+            rows   (#'metabase.mq.topic.appdb/poll-messages! topic-name offset)]
+        (when (seq rows)
+          (let [all-messages (into [] (mapcat (comp json/decode :messages)) rows)]
+            (mq.impl/deliver! topic-name all-messages nil nil))))
+
+      (testing "Subscriber receives the published message"
+        (is (= ["hello-appdb"] @received)))
+      (finally
+        (listener/unlisten! topic-name)
+        (t2/delete! :topic_message_batch :topic_name (name topic-name))))))
+
+(deftest no-messages-without-subscribers-test
+  (let [topic-name :topic/no-sub-test]
+    (topic.backend/publish! :topic.backend/appdb topic-name ["orphan-msg"])
+    (testing "Message exists in table even with no subscribers"
+      (is (pos? (t2/count :topic_message_batch :topic_name (name topic-name)))))
+    ;; cleanup
+    (t2/delete! :topic_message_batch :topic_name (name topic-name))))
+
+(deftest cleanup-old-messages-test
+  (let [topic-name (str "cleanup-test-" (random-uuid))
+        old-ts     (Timestamp/from (.minusMillis (Instant/now) (+ (* 60 60 1000) 60000)))]
+    (try
+      (testing "Topic messages older than 1 hour are deleted"
+        (t2/query {:insert-into :topic_message_batch
+                   :values      [{:topic_name topic-name
+                                  :messages   (json/encode ["old-msg"])
+                                  :created_at old-ts}]})
+        (let [old-row (t2/select-one :topic_message_batch :topic_name topic-name)]
+          (#'metabase.mq.topic.appdb/cleanup-old-messages!)
+          (is (nil? (t2/select-one :topic_message_batch :id (:id old-row)))
+              "Old topic message should be deleted")))
+
+      (testing "Recent topic messages are not deleted"
+        (t2/insert! :topic_message_batch {:topic_name topic-name :messages (json/encode ["recent-msg"])})
+        (let [recent-row (t2/select-one :topic_message_batch :topic_name topic-name)]
+          (#'metabase.mq.topic.appdb/cleanup-old-messages!)
+          (is (some? (t2/select-one :topic_message_batch :id (:id recent-row)))
+              "Recent topic message should not be deleted")))
+      (finally
+        (t2/delete! :topic_message_batch :topic_name topic-name)))))
+
+(deftest lag-gauge-test
+  (let [topic-name  (keyword "topic" (str "lag-gauge-test-" (random-uuid)))
+        gauge-calls (atom [])]
+    (try
+      ;; Publish two messages and look up their IDs from the DB
+      (topic.backend/publish! :topic.backend/appdb topic-name ["m1"])
+      (let [id1 (:id (first (t2/query {:select   [:id]
+                                       :from     [:topic_message_batch]
+                                       :where    [:= :topic_name (name topic-name)]
+                                       :order-by [[:id :desc]]
+                                       :limit    1})))]
+        ;; Subscriber has read up to id1
+        (swap! @#'metabase.mq.topic.appdb/offsets assoc topic-name id1)
+        ;; Publish a second message (unread)
+        (topic.backend/publish! :topic.backend/appdb topic-name ["m2"])
+        (let [id2 (:id (first (t2/query {:select   [:id]
+                                         :from     [:topic_message_batch]
+                                         :where    [:= :topic_name (name topic-name)]
+                                         :order-by [[:id :desc]]
+                                         :limit    1})))]
+          (with-redefs [mq.analytics/set! (fn [metric labels value]
+                                            (when (= metric :metabase-mq/appdb-topic-subscriber-lag)
+                                              (swap! gauge-calls conj {:labels labels :value value})))]
+            (#'metabase.mq.topic.appdb/update-lag-gauges!))
+          (testing "Lag gauge is emitted with the correct unread count"
+            (let [call (first (filter #(= (name topic-name) (-> % :labels :channel)) @gauge-calls))]
+              (is (some? call) "Gauge should be recorded for the test topic")
+              (is (= (- id2 id1) (:value call))
+                  "Lag should equal the difference between max-id and current offset")))))
+      (finally
+        (swap! @#'metabase.mq.topic.appdb/offsets dissoc topic-name)
+        (t2/delete! :topic_message_batch :topic_name (name topic-name))))))

test/metabase/mq/topic/core_test.clj

@@ -0,0 +1,256 @@
+(ns metabase.mq.topic.core-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase.app-db.connection :as app-db.conn]
+   [metabase.mq.core :as mq]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.publish :as mq.publish]
+   [metabase.mq.publish-buffer :as publish-buffer]
+   [metabase.mq.test-util :as mq.tu])
+  (:import (clojure.lang ExceptionInfo)
+           (java.util.concurrent CyclicBarrier)))
+
+(set! *warn-on-reflection* true)
+
+(deftest e2e-publish-subscribe-test
+  (mq.tu/with-sync-mq
+    (let [received (atom [])]
+      (mq/listen! :topic/e2e {}
+                  (fn [message]
+                    (swap! received conj message)))
+
+      (testing "Messages are received by subscriber"
+        (mq/with-topic :topic/e2e [t]
+          (mq/put t "message-1"))
+        (mq/with-topic :topic/e2e [t]
+          (mq/put t "message-2"))
+        (is (= ["message-1" "message-2"] @received)))
+
+      (testing "Unsubscribe stops delivery"
+        (mq/unlisten! :topic/e2e)
+        (mq/with-topic :topic/e2e [t]
+          (mq/put t "message-3"))
+        (is (= ["message-1" "message-2"] @received))))))
+
+(deftest batch-publish-e2e-test
+  (mq.tu/with-sync-mq
+    (let [received (atom [])]
+      (mq/listen! :topic/batch {}
+                  (fn [message]
+                    (swap! received conj message)))
+
+      (mq/with-topic :topic/batch [t]
+        (mq/put t "a")
+        (mq/put t "b")
+        (mq/put t "c"))
+
+      (testing "Batch of messages delivered together"
+        (is (= ["a" "b" "c"] @received)))
+
+      (mq/unlisten! :topic/batch))))
+
+(deftest error-handling-e2e-test
+  (mq.tu/with-sync-mq
+    (let [received (atom [])]
+      (mq/listen! :topic/errors {}
+                  (fn [message]
+                    (when (= "fail" message)
+                      (throw (ex-info "Handler error" {})))
+                    (swap! received conj message)))
+
+      (mq/with-topic :topic/errors [t]
+        (mq/put t "ok-1"))
+      (mq/with-topic :topic/errors [t]
+        (mq/put t "fail"))
+      (mq/with-topic :topic/errors [t]
+        (mq/put t "ok-2"))
+
+      (testing "Non-error messages are delivered"
+        (is (= ["ok-1" "ok-2"] @received)))
+
+      (mq/unlisten! :topic/errors))))
+
+(deftest batch-partial-failure-test
+  (testing "When listener throws on one message in a batch, remaining messages are still delivered"
+    (mq.tu/with-sync-mq
+      (let [received (atom [])]
+        (mq/listen! :topic/batch-fail {}
+                    (fn [message]
+                      (when (= "boom" message)
+                        (throw (ex-info "Handler error" {})))
+                      (swap! received conj message)))
+
+        (mq/with-topic :topic/batch-fail [t]
+          (mq/put t "msg-1")
+          (mq/put t "boom")
+          (mq/put t "msg-3")
+          (mq/put t "msg-4"))
+
+        (is (= ["msg-1" "msg-3" "msg-4"] @received))
+
+        (mq/unlisten! :topic/batch-fail)))))
+
+(deftest concurrent-subscribe-throws-test
+  (mq.tu/with-sync-mq
+    (let [topic-name :topic/concurrent-sub-test
+          n          10
+          barrier    (CyclicBarrier. n)
+          results    (atom [])]
+      (testing "Concurrent listen! calls: exactly one succeeds, rest throw"
+        (let [threads (mapv (fn [_]
+                              (let [f (bound-fn []
+                                        (.await barrier)
+                                        (try
+                                          (listener/listen! topic-name {} (fn [_] nil))
+                                          (swap! results conj :ok)
+                                          (catch ExceptionInfo _
+                                            (swap! results conj :error))))]
+                                (Thread. ^Runnable f)))
+                            (range n))]
+          (run! (fn [^Thread t] (.start t)) threads)
+          (run! (fn [^Thread t] (.join t 5000)) threads)
+          (is (= 1 (count (filter #{:ok} @results))))
+          (is (= (dec n) (count (filter #{:error} @results))))))
+      (mq/unlisten! topic-name))))
+
+(deftest late-subscriber-test
+  (mq.tu/with-sync-mq
+    (mq/with-topic :topic/late [t]
+      (mq/put t "old-message"))
+
+    (let [received (atom [])]
+      (mq/listen! :topic/late {}
+                  (fn [message]
+                    (swap! received conj message)))
+
+      (mq/with-topic :topic/late [t]
+        (mq/put t "new-message"))
+
+      (testing "Late subscriber only sees new messages"
+        (is (= ["new-message"] @received)))
+
+      (mq/unlisten! :topic/late))))
+
+(deftest transaction-defers-topic-publish-test
+  (mq.tu/with-sync-mq
+    (let [received (atom [])]
+      (mq/listen! :topic/txn-test {}
+                  (fn [message]
+                    (swap! received conj message)))
+      (binding [app-db.conn/*after-commit*      (atom [])
+                app-db.conn/*transaction-state* (atom {})]
+        (testing "Inside a transaction, messages are accumulated, not published immediately"
+          (mq/with-topic :topic/txn-test [t]
+            (mq/put t "msg1")
+            (mq/put t "msg2"))
+          (mq/with-topic :topic/txn-test [t]
+            (mq/put t "msg3"))
+          (testing "Messages are deferred in transaction state"
+            (is (= ["msg1" "msg2" "msg3"]
+                   (get-in @app-db.conn/*transaction-state*
+                           [::mq.publish/deferred-messages :topic/txn-test]))))
+          (testing "Listener has not been called yet"
+            (is (= [] @received))))
+        (testing "After flush (simulating commit), all messages are delivered"
+          (mq.publish/flush-deferred-messages!)
+          (is (= ["msg1" "msg2" "msg3"] @received))))
+      (mq/unlisten! :topic/txn-test))))
+
+(deftest transaction-rollback-discards-topic-messages-test
+  (mq.tu/with-sync-mq
+    (let [received (atom [])]
+      (mq/listen! :topic/txn-rollback {}
+                  (fn [message]
+                    (swap! received conj message)))
+      (binding [app-db.conn/*after-commit*      (atom [])
+                app-db.conn/*transaction-state* (atom {})]
+        (testing "Messages accumulated during a failed transaction are discarded"
+          (is (thrown? Exception
+                       (mq/with-topic :topic/txn-rollback [t]
+                         (mq/put t "should-be-discarded")
+                         (throw (ex-info "boom" {})))))
+          (is (nil? (get-in @app-db.conn/*transaction-state*
+                            [::mq.publish/deferred-messages :topic/txn-rollback]))
+              "No messages should be in transaction state after exception")))
+      (mq/unlisten! :topic/txn-rollback))))
+
+(deftest outside-transaction-publishes-topic-immediately-test
+  (mq.tu/with-sync-mq
+    (let [received (atom [])]
+      (mq/listen! :topic/txn-immediate {}
+                  (fn [message]
+                    (swap! received conj message)))
+      (testing "Outside a transaction, messages are published immediately"
+        (is (nil? app-db.conn/*transaction-state*))
+        (mq/with-topic :topic/txn-immediate [t]
+          (mq/put t "msg1"))
+        (is (= ["msg1"] @received)))
+      (mq/unlisten! :topic/txn-immediate))))
+
+(deftest multiple-topics-in-transaction-test
+  (mq.tu/with-sync-mq
+    (let [received-a (atom [])
+          received-b (atom [])]
+      (mq/listen! :topic/txn-multi-a {}
+                  (fn [message] (swap! received-a conj message)))
+      (mq/listen! :topic/txn-multi-b {}
+                  (fn [message] (swap! received-b conj message)))
+      (binding [app-db.conn/*after-commit*      (atom [])
+                app-db.conn/*transaction-state* (atom {})]
+        (mq/with-topic :topic/txn-multi-a [t]
+          (mq/put t "a1")
+          (mq/put t "a2"))
+        (mq/with-topic :topic/txn-multi-b [t]
+          (mq/put t "b1"))
+        (testing "Both topics have deferred messages"
+          (is (= ["a1" "a2"]
+                 (get-in @app-db.conn/*transaction-state*
+                         [::mq.publish/deferred-messages :topic/txn-multi-a])))
+          (is (= ["b1"]
+                 (get-in @app-db.conn/*transaction-state*
+                         [::mq.publish/deferred-messages :topic/txn-multi-b]))))
+        (testing "After flush, both topics receive their messages"
+          (mq.publish/flush-deferred-messages!)
+          (is (= ["a1" "a2"] @received-a))
+          (is (= ["b1"] @received-b))))
+      (mq/unlisten! :topic/txn-multi-a)
+      (mq/unlisten! :topic/txn-multi-b))))
+
+(deftest buffering-combines-topic-messages-test
+  (mq.tu/with-sync-mq
+    (let [received (atom [])]
+      (mq/listen! :topic/buffer-test {}
+                  (fn [message] (swap! received conj message)))
+      (binding [publish-buffer/*publish-buffer-ms* 100
+                publish-buffer/*publish-buffer*    (atom {})]
+        (testing "buffered-publish! buffers topic messages when *publish-buffer-ms* > 0"
+          (mq/with-topic :topic/buffer-test [t]
+            (mq/put t "msg1"))
+          (mq/with-topic :topic/buffer-test [t]
+            (mq/put t "msg2"))
+          (is (empty? @received)
+              "Nothing delivered yet because buffer window hasn't elapsed")
+          (is (= ["msg1" "msg2"]
+                 (:messages (get @publish-buffer/*publish-buffer* :topic/buffer-test)))
+              "Messages are buffered"))
+        (testing "After flush, all messages are delivered"
+          (swap! publish-buffer/*publish-buffer*
+                 update :topic/buffer-test assoc :deadline-ms 1)
+          (publish-buffer/flush-publish-buffer!)
+          (is (= ["msg1" "msg2"] @received)
+              "Both messages delivered after flush")))
+      (mq/unlisten! :topic/buffer-test))))
+
+(deftest buffering-immediate-when-zero-test
+  (mq.tu/with-sync-mq
+    (let [received (atom [])]
+      (mq/listen! :topic/buffer-zero {}
+                  (fn [message] (swap! received conj message)))
+      (testing "With *publish-buffer-ms* 0 (default in with-sync-mq), topic messages publish immediately"
+        (mq/with-topic :topic/buffer-zero [t]
+          (mq/put t "msg1"))
+        (is (= ["msg1"] @received))
+        (mq/with-topic :topic/buffer-zero [t]
+          (mq/put t "msg2"))
+        (is (= ["msg1" "msg2"] @received)))
+      (mq/unlisten! :topic/buffer-zero))))

test/metabase/mq/topic/memory_test.clj

@@ -0,0 +1,166 @@
+(ns metabase.mq.topic.memory-test
+  (:require
+   [clojure.test :refer :all]
+   [metabase.mq.core :as mq]
+   [metabase.mq.impl :as mq.impl]
+   [metabase.mq.listener :as listener]
+   [metabase.mq.memory :as memory]
+   [metabase.mq.publish-buffer :as publish-buffer]
+   [metabase.mq.topic.backend :as topic.backend])
+  (:import (clojure.lang ExceptionInfo)
+           (java.util.concurrent CyclicBarrier)))
+
+(set! *warn-on-reflection* true)
+
+(defmacro ^:private with-memory-topics
+  "Sets up an isolated memory topic environment. Does NOT start the background poll thread —
+  tests call `deliver-pending!` to drive delivery synchronously."
+  [& body]
+  `(binding [topic.backend/*backend*              :topic.backend/memory
+             listener/*listeners*                  (atom {})
+             publish-buffer/*publish-buffer-ms*    0
+             publish-buffer/*publish-buffer*       (atom {})
+             memory/*channels*                     (atom {})]
+     ~@body))
+
+(defn- deliver-pending!
+  "Drains all pending memory channels and delivers messages synchronously on the current thread."
+  []
+  (doseq [channel-name (concat (listener/queue-names) (listener/topic-names))]
+    (when-let [messages (#'memory/drain! channel-name)]
+      (mq.impl/deliver! channel-name messages nil nil))))
+
+(deftest publish-and-subscribe-test
+  (with-memory-topics
+    (let [received (atom [])]
+      (mq/listen! :topic/test {}
+                  (fn [message]
+                    (swap! received conj message)))
+      (mq/with-topic :topic/test [t]
+        (mq/put t "hello"))
+      (mq/with-topic :topic/test [t]
+        (mq/put t "world"))
+      (deliver-pending!)
+
+      (testing "Subscriber receives published messages"
+        (is (= ["hello" "world"] @received)))
+
+      (mq/unlisten! :topic/test))))
+
+(deftest batch-publish-test
+  (with-memory-topics
+    (let [received (atom [])]
+      (mq/listen! :topic/batch {}
+                  (fn [message]
+                    (swap! received conj message)))
+      (mq/with-topic :topic/batch [t]
+        (mq/put t "msg-1")
+        (mq/put t "msg-2")
+        (mq/put t "msg-3"))
+      (deliver-pending!)
+
+      (testing "Batch of messages received in one row"
+        (is (= ["msg-1" "msg-2" "msg-3"] @received)))
+
+      (mq/unlisten! :topic/batch))))
+
+(deftest unsubscribe-stops-delivery-test
+  (with-memory-topics
+    (let [received (atom [])]
+      (mq/listen! :topic/unsub {}
+                  (fn [message]
+                    (swap! received conj message)))
+
+      (mq/with-topic :topic/unsub [t]
+        (mq/put t "before"))
+      (deliver-pending!)
+      (is (= ["before"] @received))
+
+      (mq/unlisten! :topic/unsub)
+
+      (mq/with-topic :topic/unsub [t]
+        (mq/put t "after"))
+      (deliver-pending!)
+
+      (testing "No messages received after unsubscribe"
+        (is (= ["before"] @received))))))
+
+(deftest error-handling-test
+  (with-memory-topics
+    (let [received (atom [])]
+      (mq/listen! :topic/errors {}
+                  (fn [message]
+                    (when (= "error!" message)
+                      (throw (ex-info "Test error" {})))
+                    (swap! received conj message)))
+
+      (mq/with-topic :topic/errors [t]
+        (mq/put t "good"))
+      (mq/with-topic :topic/errors [t]
+        (mq/put t "error!"))
+      (mq/with-topic :topic/errors [t]
+        (mq/put t "also-good"))
+      (deliver-pending!)
+
+      (testing "Good messages are received"
+        (is (= ["good" "also-good"] @received)))
+
+      (mq/unlisten! :topic/errors))))
+
+(deftest double-subscribe-throws-test
+  (with-memory-topics
+    (mq/listen! :topic/double {} (fn [_] nil))
+    (testing "Subscribing twice to the same topic throws"
+      (is (thrown-with-msg? ExceptionInfo #"Listener already registered"
+                            (mq/listen! :topic/double {} (fn [_] nil)))))
+    (mq/unlisten! :topic/double)))
+
+(deftest concurrent-publish-ordering-test
+  (testing "Concurrent publishes are all delivered"
+    (with-memory-topics
+      (let [received (atom [])
+            n        20
+            barrier  (CyclicBarrier. n)]
+        (mq/listen! :topic/concurrent-order {}
+                    (fn [message]
+                      (swap! received conj message)))
+        ;; Publish concurrently from n threads
+        (let [threads (mapv (fn [i]
+                              (let [f (bound-fn []
+                                        (.await barrier)
+                                        (mq/with-topic :topic/concurrent-order [t]
+                                          (mq/put t (str "msg-" i))))]
+                                (Thread. ^Runnable f)))
+                            (range n))]
+          (run! (fn [^Thread t] (.start t)) threads)
+          (run! (fn [^Thread t] (.join t 5000)) threads))
+        (deliver-pending!)
+        (testing "All messages delivered exactly once"
+          (is (= n (count @received))))
+        (testing "No duplicate messages"
+          (is (= (count @received) (count (distinct @received)))))
+        (mq/unlisten! :topic/concurrent-order)))))
+
+(deftest topic-isolation-test
+  (with-memory-topics
+    (let [received-a (atom [])
+          received-b (atom [])]
+      (mq/listen! :topic/isolated-a {}
+                  (fn [message]
+                    (swap! received-a conj message)))
+      (mq/listen! :topic/isolated-b {}
+                  (fn [message]
+                    (swap! received-b conj message)))
+
+      (mq/with-topic :topic/isolated-a [t]
+        (mq/put t "for-a"))
+      (mq/with-topic :topic/isolated-b [t]
+        (mq/put t "for-b"))
+      (deliver-pending!)
+
+      (testing "Messages on topic A don't appear on topic B"
+        (is (= ["for-a"] @received-a))
+        (is (= ["for-b"] @received-b)))
+
+      (mq/unlisten! :topic/isolated-a)
+      (mq/unlisten! :topic/isolated-b))))

test/metabase/search/api_test.clj

@@ -18,7 +18,7 @@
    [metabase.search.appdb.index :as search.index]
    [metabase.search.config :as search.config]
    [metabase.search.core :as search]
-   [metabase.search.ingestion :as search.ingestion]
+   [metabase.search.impl :as search.impl]
    [metabase.search.test-util :as search.tu]
    [metabase.test :as mt]
    [metabase.test.fixtures :as fixtures]
@@ -36,8 +36,7 @@
 
 (def ^:private default-collection {:id false :name nil :authority_level nil :type nil})
 
-(use-fixtures :each (fn [thunk] (binding [search.ingestion/*force-sync* true]
-                                  (search.tu/with-new-search-if-available-otherwise-legacy (thunk)))))
+(use-fixtures :each (fn [thunk] (search.tu/with-new-search-if-available-otherwise-legacy (thunk))))
 
 (def ^:private default-search-row
   {:archived                   false
@@ -370,20 +369,20 @@
 (deftest custom-engine-test
   (when (search/supports-index?)
     (testing "It can use an alternate search engine"
-      (search/init-index! {:force-reset? false :re-populate? false})
+      (search.impl/sync-init-index! {:force-reset? false :re-populate? false})
       (with-search-items-in-root-collection "test"
         (let [resp (search-request :crowberto :q "test" :search_engine "appdb" :limit 1)]
           ;; The index is not populated here, so there's not much interesting to assert.
           (is (= "search.engine/appdb" (:engine resp))))))
 
     (testing "It can use the old search engine name, e.g. for old cookies"
-      (search/init-index! {:force-reset? false :re-populate? false})
+      (search.impl/sync-init-index! {:force-reset? false :re-populate? false})
       (with-search-items-in-root-collection "test"
         (let [resp (search-request :crowberto :q "test" :search_engine "fulltext" :limit 1)]
           (is (= "search.engine/fulltext" (:engine resp))))))
 
     (testing "It will not use an unknown search engine"
-      (search/init-index! {:force-reset? false :re-populate? false})
+      (search.impl/sync-init-index! {:force-reset? false :re-populate? false})
       (with-search-items-in-root-collection "test"
         (let [resp (search-request :crowberto :q "test" :search_engine "wut" :limit 1)]
           (is (#{"search.engine/in-place"
@@ -468,7 +467,7 @@
                    :model/DashboardCard _               {:card_id card-id-5 :dashboard_id dash-id}
                    :model/DashboardCard _               {:card_id card-id-5 :dashboard_id dash-id}]
       ;; We do not synchronously update dashboard count
-      (search/reindex! {:async? false :in-place? true})
+      (search.impl/sync-reindex! {:in-place? true})
       (is (= (sort-by :dashboardcard_count (cleaned-results dashboard-count-results))
              (sort-by :dashboardcard_count (unsorted-search-request-data :rasta :q "dashboard-count")))))))
 
@@ -676,7 +675,7 @@
                (search-request-data :crowberto :q "test"))))))
 
   ;; TODO need to isolate these two tests properly, they're sharing  temp index
-  (search/reindex! {:async? false :in-place? true})
+  (search.impl/sync-reindex! {:in-place? true})
 
   (testing "Basic search, should find 1 of each entity type and include bookmarks when available"
     (with-search-items-in-collection {:keys [card dashboard]} "test"
@@ -725,6 +724,7 @@
                 search!     (fn [search-term]
                               (:data (make-search-request :crowberto [:q search-term])))]
             (model-index/add-values! model-index)
+            (search.impl/sync-reindex! {:in-place? true})
 
             (is (= #{"Dallas-Fort Worth" "Fort Lauderdale" "Fort Myers"
                      "Fort Worth" "Fort Smith" "Fort Wayne"}
@@ -770,6 +770,7 @@
               normalize     (fn [x] (-> x (update :pk_ref mbql.normalize/normalize)))]
           (model-index/add-values! model-index-1)
           (model-index/add-values! model-index-2)
+          (search.impl/sync-reindex! {:in-place? true})
 
           (testing "Indexed entities returned if a non-admin user has full data perms and collection access"
             (mt/with-all-users-data-perms-graph! {(mt/id) {:view-data :unrestricted
@@ -1446,7 +1447,7 @@
         {http-action :action-id}  {:type :http :name search-term}
         {query-action :action-id} {:type :query :dataset_query (mt/native-query {:query (format "delete from %s" search-term)})}]
        ;; TODO investigate why the actions don't get indexed automatically
-        (search/reindex! {:async? false :in-place? true})
+        (search.impl/sync-reindex! {:in-place? true})
         (testing "by default do not search for native content"
           (is (= #{["card" mbql-card]
                    ["card" native-card-in-name]
@@ -1808,52 +1809,53 @@
         (is (= 5.0 (search.config/scorer-param search-ctx :model :dataset)))))))
 
 (deftest ^:synchronized dashboard-questions
-  (testing "Dashboard questions get a dashboard_id when searched"
-    (let [search-name (random-uuid)
-          named #(str search-name "-" %)]
-      (mt/with-temp [:model/Dashboard {dash-id :id} {:name (named "dashboard")}
-                     :model/Card {card-id :id} {:dashboard_id dash-id :name (named "dashboard card")}
-                     :model/Card {reg-card-id :id} {:name (named "regular card")}
-                     ;; DQs aren't searchable without a DashboardCard (see later test)
-                     :model/DashboardCard _ {:dashboard_id dash-id :card_id card-id}]
+  (search.tu/with-sync-search-indexing
+    (testing "Dashboard questions get a dashboard_id when searched"
+      (let [search-name (random-uuid)
+            named       #(str search-name "-" %)]
+        (mt/with-temp [:model/Dashboard {dash-id :id} {:name (named "dashboard")}
+                       :model/Card {card-id :id} {:dashboard_id dash-id :name (named "dashboard card")}
+                       :model/Card {reg-card-id :id} {:name (named "regular card")}
+                       ;; DQs aren't searchable without a DashboardCard (see later test)
+                       :model/DashboardCard _ {:dashboard_id dash-id :card_id card-id}]
 
-        ;; We need to update the entry for the card once the join is created.
-        ;; This is not necessary in the real app because of how the index updates are batched.
-        ;; Another solution would be to explicitly mark this data dependency, which we explicitly chose not to do for
-        ;; now (see note of the Card spec).
-        (search/update! (t2/instance :model/Card {:id card-id}))
+          ;; We need to update the entry for the card once the join is created.
+          ;; This is not necessary in the real app because of how the index updates are batched.
+          ;; Another solution would be to explicitly mark this data dependency, which we explicitly chose not to do for
+          ;; now (see note of the Card spec).
+          (search/async-update! (t2/instance :model/Card {:id card-id}))
 
-        (testing "The card data also include `dashboard` info"
-          (is (= {:id dash-id
-                  :name (named "dashboard")
-                  :moderation_status nil}
+          (testing "The card data also include `dashboard` info"
+            (is (= {:id                dash-id
+                    :name              (named "dashboard")
+                    :moderation_status nil}
+                   (->> (mt/user-http-request :crowberto :get 200 "/search" :q search-name :include_dashboard_questions "true")
+                        :data
+                        (filter #(= card-id (:id %)))
+                        first
+                        :dashboard))))
+          (testing "Regular cards don't have it"
+            (is (nil?
                  (->> (mt/user-http-request :crowberto :get 200 "/search" :q search-name :include_dashboard_questions "true")
                       :data
-                      (filter #(= card-id (:id %)))
+                      (filter #(= reg-card-id (:id %)))
                       first
                       :dashboard))))
-        (testing "Regular cards don't have it"
-          (is (nil?
-               (->> (mt/user-http-request :crowberto :get 200 "/search" :q search-name :include_dashboard_questions "true")
-                    :data
-                    (filter #(= reg-card-id (:id %)))
-                    first
-                    :dashboard))))
-        (testing "Dashboard questions are only returned if you pass `include_dashboard_questions=true`"
-          (is (= []
-                 (->> (mt/user-http-request :crowberto :get 200 "/search" :q search-name :include_dashboard_questions "false")
-                      :data
-                      (filter #(= card-id (:id %))))))))))
-  (testing "Dashboard questions aren't searchable without a DashboardCard"
-    (let [search-name (random-uuid)
-          named #(str search-name "-" %)]
-      (mt/with-temp [:model/Dashboard {dash-id :id} {:name "Dashboard"}
-                     :model/Card _ {:dashboard_id dash-id :name (named "dashboard card")}]
-        (is (= {:total 0
-                :data []}
-               (select-keys
-                (mt/user-http-request :crowberto :get 200 "/search" :q search-name :include_dashboard_questions "true")
-                [:total :data])))))))
+          (testing "Dashboard questions are only returned if you pass `include_dashboard_questions=true`"
+            (is (= []
+                   (->> (mt/user-http-request :crowberto :get 200 "/search" :q search-name :include_dashboard_questions "false")
+                        :data
+                        (filter #(= card-id (:id %))))))))))
+    (testing "Dashboard questions aren't searchable without a DashboardCard"
+      (let [search-name (random-uuid)
+            named       #(str search-name "-" %)]
+        (mt/with-temp [:model/Dashboard {dash-id :id} {:name "Dashboard"}
+                       :model/Card _ {:dashboard_id dash-id :name (named "dashboard card")}]
+          (is (= {:total 0
+                  :data  []}
+                 (select-keys
+                  (mt/user-http-request :crowberto :get 200 "/search" :q search-name :include_dashboard_questions "true")
+                  [:total :data]))))))))
 
 (deftest ^:synchronized include-metadata
   (testing "Include card result_metadata if include-metadata is set"
@@ -1901,41 +1903,41 @@
             (is (= 1 (count (filter #{:metabase-search/response-error} @calls))))))))))
 
 (deftest ^:synchronized multiple-limits-test
-  (when (search/supports-index?)
-    ;; This test is failing with "no index" for some reason, forcing the reindex
-    (mt/user-real-request :crowberto :post 200 "search/force-reindex"))
   (testing "Multiple `limit` query args should be handled correctly (#45345)"
-    (let [total-count (-> (mt/user-real-request :crowberto :get 200 "search?q=product")
-                          :data count)
-          result-count (-> (mt/user-real-request :crowberto :get 200 "search?q=product&limit=1&limit=3")
+    ;; Uses search_engine=in-place to avoid needing an appdb index (this test verifies limit parsing, not search content).
+    ;; Uses user-real-request to support duplicate limit query params.
+    (let [total-count  (-> (mt/user-real-request :crowberto :get 200 "search?q=product&search_engine=in-place")
+                           :data count)
+          result-count (-> (mt/user-real-request :crowberto :get 200 "search?q=product&limit=1&limit=3&search_engine=in-place")
                            :data count)]
       (is (>= total-count result-count))
       (is (= 1 result-count)))))
 
 (deftest ^:synchronized delete-database-hides-cards-from-search-test
-  (testing "When deleting a database, cards referring to that database should be hidden from search"
-    (let [card-name (str (random-uuid))]
-      (mt/with-temp [:model/Database {db-id :id} {:name "Test Database"}
-                     :model/Table {table-id :id} {:db_id db-id :name "Test Table"}
-                     :model/Card {card-id :id} {:name card-name
-                                                :database_id db-id
-                                                :table_id table-id
-                                                :dataset_query {:database db-id
-                                                                :type :query
-                                                                :query {:source-table table-id}}}]
-        (search/reindex! {:async? false :in-place? true})
-        (testing "Card should be visible in search before database deletion"
-          (let [search-results (mt/user-http-request :crowberto :get 200 "search" :q card-name)]
-            (is (some #(= (:id %) card-id) (:data search-results))
-                "Card should be found in search results before database deletion")))
+  (search.tu/with-sync-search-indexing
+    (testing "When deleting a database, cards referring to that database should be hidden from search"
+      (let [card-name (str (random-uuid))]
+        (mt/with-temp [:model/Database {db-id :id} {:name "Test Database"}
+                       :model/Table {table-id :id} {:db_id db-id :name "Test Table"}
+                       :model/Card {card-id :id} {:name          card-name
+                                                  :database_id   db-id
+                                                  :table_id      table-id
+                                                  :dataset_query {:database db-id
+                                                                  :type     :query
+                                                                  :query    {:source-table table-id}}}]
+          (search/async-reindex! {:in-place? true})
+          (testing "Card should be visible in search before database deletion"
+            (let [search-results (mt/user-http-request :crowberto :get 200 "search" :q card-name)]
+              (is (some #(= (:id %) card-id) (:data search-results))
+                  "Card should be found in search results before database deletion")))
 
-        (testing "Card should be hidden from search after database deletion"
-          (t2/delete! :model/Database :id db-id)
-          (is (not (t2/exists? :model/Card :id card-id)))
-          (let [search-results (mt/user-http-request :crowberto :get 200 "search" :q card-name)]
-            (is (not (some #{card-id}
-                           (mapv :id (:data search-results))))
-                "Card should not be found in search results after database deletion")))))))
+          (testing "Card should be hidden from search after database deletion"
+            (t2/delete! :model/Database :id db-id)
+            (is (not (t2/exists? :model/Card :id card-id)))
+            (let [search-results (mt/user-http-request :crowberto :get 200 "search" :q card-name)]
+              (is (not (some #{card-id}
+                             (mapv :id (:data search-results))))
+                  "Card should not be found in search results after database deletion"))))))))
 
 (deftest collection-filter-test
   (mt/with-temp
@@ -2091,7 +2093,7 @@
         (search.tu/with-temp-index-table
           (mt/with-temp [:model/Measure _ {:name     measure-name
                                            :table_id (mt/id :venues)}]
-            (search/reindex! {:async? false :in-place? true})
+            (search.impl/sync-reindex! {:async? false :in-place? true})
             (is (=? [{:name measure-name :model "measure"}]
                     (search-request-data :crowberto :q measure-name
                                          :search_engine "appdb"

test/metabase/search/appdb/index_test.clj

@@ -47,23 +47,22 @@
   "Ensure a clean, small index."
   [& body]
   `(search.tu/with-temp-index-table
-     (binding [search.ingestion/*force-sync* true]
-       (mt/dataset ~(symbol "test-data")
+     (mt/dataset ~(symbol "test-data")
          ;; Sneaky trick so make sure we have a user with ID 1
-         (mt/with-temp [:model/User       {}            (when-not (t2/exists? :model/User 1) {:id 1})
-                        :model/Collection {col-id# :id} {:name "Collection"}
-                        :model/Card       {}            {:name "Customer Satisfaction"          :collection_id col-id#}
-                        :model/Card       {}            {:name "The Latest Revenue Projections" :collection_id col-id#}
-                        :model/Card       {}            {:name "Projected Revenue"              :collection_id col-id#}
-                        :model/Card       {}            {:name "Employee Satisfaction"          :collection_id col-id#}
-                        :model/Card       {}            {:name "Projected Satisfaction"         :collection_id col-id#}
-                        :model/Card       {}            {:name "Organization Reference"         :collection_id col-id#}
+       (mt/with-temp [:model/User       {}            (when-not (t2/exists? :model/User 1) {:id 1})
+                      :model/Collection {col-id# :id} {:name "Collection"}
+                      :model/Card       {}            {:name "Customer Satisfaction"          :collection_id col-id#}
+                      :model/Card       {}            {:name "The Latest Revenue Projections" :collection_id col-id#}
+                      :model/Card       {}            {:name "Projected Revenue"              :collection_id col-id#}
+                      :model/Card       {}            {:name "Employee Satisfaction"          :collection_id col-id#}
+                      :model/Card       {}            {:name "Projected Satisfaction"         :collection_id col-id#}
+                      :model/Card       {}            {:name "Organization Reference"         :collection_id col-id#}
                         ;; this is to show off that our storage can give false positives (stemming etc)
-                        :model/Card       {}            {:name "Nobody Expects Organi"          :collection_id col-id#}
-                        :model/Database   {db-id# :id}  {:name "Indexed Database"}
-                        :model/Table      {}            {:name "Indexed Table", :db_id db-id#}]
-           (search.engine/reindex! :search.engine/appdb {:in-place? true})
-           ~@body)))))
+                      :model/Card       {}            {:name "Nobody Expects Organi"          :collection_id col-id#}
+                      :model/Database   {db-id# :id}  {:name "Indexed Database"}
+                      :model/Table      {}            {:name "Indexed Table", :db_id db-id#}]
+         (search.engine/reindex! :search.engine/appdb {:in-place? true})
+         ~@body))))
 
 (deftest idempotent-test
   (with-index

test/metabase/search/filter_test.clj

@@ -6,15 +6,13 @@
    [metabase.search.config :as search.config]
    [metabase.search.filter :as search.filter]
    [metabase.search.in-place.filter :as search.in-place.filter]
-   [metabase.search.ingestion :as search.ingestion]
    [metabase.search.test-util :as search.tu]
    [metabase.test :as mt]
    [metabase.test.fixtures :as fixtures]))
 
 (use-fixtures :once (fixtures/initialize :db))
 
-(use-fixtures :each (fn [thunk] (binding [search.ingestion/*force-sync* true]
-                                  (search.tu/with-new-search-if-available-otherwise-legacy (thunk)))))
+(use-fixtures :each (fn [thunk] (search.tu/with-new-search-if-available-otherwise-legacy (thunk))))
 
 (defn- filter-keys []
   (remove #{:ids} (map :context-key (vals search.config/filters))))

test/metabase/search/impl_test.clj

@@ -13,7 +13,7 @@
    [metabase.search.core :as search]
    [metabase.search.impl :as search.impl]
    [metabase.search.in-place.legacy :as search.legacy]
-   [metabase.search.ingestion :as search.ingestion]
+   [metabase.search.test-util :as search.tu]
    [metabase.test :as mt]
    [metabase.transforms.feature-gating :as transforms.gating]
    [toucan2.core :as t2]))
@@ -95,7 +95,7 @@
                                                  :enabled-transform-source-types (transforms.gating/enabled-source-types)}))]
             ;; warm it up, in case the DB call depends on the order of test execution and it needs to
             ;; do some initialization
-            (search/init-index!)
+            (search.impl/sync-init-index!)
             (do-search)
             (t2/with-call-count [call-count]
               (do-search)
@@ -287,7 +287,7 @@
   (when (search/supports-index?)
     (#'search.index/sync-tracking-atoms!)
     (let [search-term (str (random-uuid))]
-      (binding [search.ingestion/*force-sync* true]
+      (search.tu/with-sync-search-indexing
         (mt/with-temp
           [:model/Card {card-id :id} {:name search-term}]
           (mt/with-current-user (mt/user->id :crowberto)

test/metabase/search/ingestion_test.clj

@@ -4,8 +4,8 @@
    [clojure.test :refer :all]
    [metabase.lib-be.metadata.jvm :as metadata.jvm]
    [metabase.lib.metadata.protocols :as lib.metadata.protocols]
-   [metabase.search.core :as search]
    [metabase.search.engine :as search.engine]
+   [metabase.search.impl :as search.impl]
    [metabase.search.ingestion :as search.ingestion]
    [metabase.search.spec :as search.spec]
    [metabase.test :as mt]))
@@ -176,6 +176,6 @@
                                                    (metadata.jvm/application-database-metadata-provider 3)
                                                    {"card" 6})
                     metadata.jvm/application-database-metadata-provider-factory (counting-factory factory-calls)]
-        (search/init-index!)
+        (search.impl/sync-init-index!)
         (is (= 3 @factory-calls)
             "Factory should be called once per unique database-id, not once per lookup")))))

test/metabase/search/task/search_index_test.clj

@@ -3,8 +3,7 @@
    [clojure.test :refer :all]
    [honey.sql.helpers :as sql.helpers]
    [metabase.search.appdb.index :as search.index]
-   [metabase.search.core :as search]
-   [metabase.search.task.search-index :as task]
+   [metabase.search.impl :as search.impl]
    [metabase.search.test-util :as search.tu]
    [toucan2.core :as t2]))
 
@@ -20,22 +19,22 @@
     (t2/query (sql.helpers/drop-table (search.index/active-table)))
     (testing "It can recreate the index from scratch"
      ;; May return falsey if there is nothing to index.
-      (is (task/init!))
+      (is (search.impl/sync-init-index! {:force-reset? false :re-populate? false}))
       (is (pos? (index-size))))
     (testing "It will reuse an existing index"
-      (is (not (task/init!))))))
+      (is (not (search.impl/sync-init-index! {:force-reset? false :re-populate? false}))))))
 
 (deftest reindex!-test
   (search.tu/with-temp-index-table
    ;; TODO this is coupled to appdb engines at the moment
     (t2/query (sql.helpers/drop-table (search.index/active-table)))
     (testing "It can recreate the index from scratch"
-      (is (search/reindex! {:async? false}))
+      (is (search.impl/sync-reindex!))
       (let [initial-size (index-size)
             table-name (search.index/active-table)]
         (is (pos? initial-size))
         (t2/delete! table-name (t2/select-one-pk table-name))
         (is (= (dec initial-size) (index-size)))
         (testing "It can cycle the index gracefully"
-          (is (search/reindex! {:async? false}))
+          (is (search.impl/sync-reindex!))
           (is (= initial-size (index-size))))))))

test/metabase/search/test_util.clj

@@ -1,6 +1,8 @@
 (ns metabase.search.test-util
   (:require
    [metabase.api.common :as api]
+   [metabase.mq.publish :as mq.publish]
+   [metabase.mq.test-util :as mq.tu]
    [metabase.permissions.util :as perms-util]
    [metabase.request.core :as request] ;; For now, this is specialized to the appdb engine, but we should be able to generalize it to all engines.
    [metabase.search.appdb.index :as search.index]
@@ -15,10 +17,12 @@
 
 #_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
 (defmacro with-sync-search-indexing
-  "Perform all search indexing synchronously."
+  "Perform all search indexing synchronously.
+  Also disables transaction deferral so that after-insert hooks index entities inline."
   [& body]
-  `(binding [metabase.search.ingestion/*force-sync* true]
-     ~@body))
+  `(mq.tu/with-sync-mq
+     (binding [mq.publish/*defer-in-transaction?* false]
+       ~@body)))
 
 #_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
 (defmacro with-temp-index-table
@@ -30,14 +34,35 @@
        (with-sync-search-indexing
          ~@body))))
 
+#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
+(defmacro with-temp-index-table-for-http
+  "Like [[with-temp-index-table]] but also exposes the temp table to server threads (HTTP handlers).
+  Use this when your test makes HTTP search requests via `mt/user-http-request`.
+  Includes a `sync-reindex!` to populate the index."
+  [& body]
+  `(when (search/supports-index?)
+     (search.index/with-temp-index-table
+       (with-sync-search-indexing
+         (let [restore-fn# (search.index/expose-temp-table-to-server-threads!)]
+           (try
+             (search.impl/sync-reindex! {:in-place? true})
+             ~@body
+             (finally
+               (restore-fn#))))))))
+
 #_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
 (defmacro with-new-search-if-available*
-  "Create a temporary index table for the duration of the body."
+  "Create a temporary index table for the duration of the body.
+  Also exposes the temp table to server threads so HTTP-based tests work."
   [& body]
   `(mt/with-dynamic-fn-redefs [search.engine/default-engine (constantly :search.engine/appdb)]
      (with-temp-index-table
-       (search/reindex! {:async? false :in-place? true})
-       ~@body)))
+       (let [restore-fn# (search.index/expose-temp-table-to-server-threads!)]
+         (try
+           (search.impl/sync-reindex! {:in-place? true})
+           ~@body
+           (finally
+             (restore-fn#)))))))
 
 #_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
 (defmacro with-new-search-if-available-otherwise-legacy

test/metabase/server/middleware/settings_cache_test.clj

@@ -1,60 +0,0 @@
-(ns metabase.server.middleware.settings-cache-test
-  (:require
-   [clj-http.cookies :as cookies]
-   [clojure.test :refer :all]
-   [metabase.server.middleware.settings-cache :as mw.settings-cache]
-   [metabase.settings.core :as setting]
-   [metabase.test :as mt]
-   [ring.util.codec :as codec])
-  (:import
-   (org.apache.http.client CookieStore)
-   (org.apache.http.cookie Cookie)
-   (org.apache.http.impl.cookie BasicClientCookie)))
-
-(set! *warn-on-reflection* true)
-
-(def cookie-name (var-get #'mw.settings-cache/settings-last-updated-cookie-name))
-
-(defn- update-cookie
-  "Update the cookie in a cookie store with name `k` giving it value `v`. Keeps other properties like expiration and
-  domain. Throws an error if the cookie is not found"
-  [^CookieStore cs k v]
-  (let [cookies (.getCookies cs)
-        ^BasicClientCookie cookie (or (some (fn [^Cookie c] (when (= k (.getName c))
-                                                              c))
-                                            cookies)
-                                      (throw (ex-info "Did not find cookie"
-                                                      {:looking-for k
-                                                       :found (into []
-                                                                    (map Cookie/.getName)
-                                                                    cookies)})))]
-    (.setValue cookie v)
-    (.addCookie cs cookie)))
-
-(deftest setting-settings-include-timestamp
-  (mt/discard-setting-changes [site-name]
-    (let [^CookieStore cs (cookies/cookie-store)]
-      (testing "it sets the cookie when updating settings"
-        (mt/user-real-request  :crowberto :put 204 "setting/site-name"
-                               {:request-options {:cookie-store cs}}
-                               {:value "foo"})
-        (let [setting-cookie (get (cookies/get-cookies cs) cookie-name)]
-          (is (some? setting-cookie) "No cookie set")
-          (is (= (setting/cache-last-updated-at)
-                 (-> setting-cookie :value codec/form-decode))
-              "Cookie value is not most recent cache updated at timestamp")))
-
-      (testing "And when that timestamp is outdated it restores the setting cache"
-        (let [calls (atom 0)]
-          ;; value in 2042 to simulate client has more recent settings
-          (update-cookie cs cookie-name "2042-12-02+19%3A57%3A49.775909%2B00")
-          (with-redefs [setting/restore-cache! (fn [] (swap! calls inc))]
-            (mt/user-real-request :crowberto :get 200 "user/current"
-                                  {:request-options {:cookie-store cs}})
-            (is (= 1 @calls) "Cache was not restored based on cookie value")
-            (testing "And that header resets the settings last updated at so we don't keep updating the cache"
-              (let [setting-cookie (get (cookies/get-cookies cs) cookie-name)]
-                (is (some? setting-cookie) "No cookie set")
-                (is (= (setting/cache-last-updated-at)
-                       (-> setting-cookie :value codec/form-decode))
-                    "The most recent updated at was not set in the header")))))))))

test/metabase/settings/models/setting/cache_test.clj

@@ -1,7 +1,6 @@
 (ns metabase.settings.models.setting.cache-test
   (:require
    [clojure.test :refer :all]
-   [metabase.app-db.core :as mdb]
    [metabase.settings.models.setting-test :as setting-test]
    [metabase.settings.models.setting.cache :as setting.cache]
    [metabase.system.core :as system]
@@ -18,128 +17,42 @@
 (defn clear-cache! []
   (reset! (#'setting.cache/cache*) nil))
 
-(defn- settings-last-updated-value-in-cache []
-  (get (setting.cache/cache) setting.cache/settings-last-updated-key))
-
-(defn- update-settings-last-updated-value-in-db!
-  "Simulate a different instance updating the value of `settings-last-updated` in the DB by updating its value without
-  updating our locally cached value.."
-  []
-  (t2/update! :model/Setting {:key setting.cache/settings-last-updated-key}
-              {:value [:raw (case (mdb/db-type)
-                              ;; make it one second in the future so we don't end up getting an exact match when we try to test
-                              ;; to see if things update below
-                              :h2       "cast(dateadd('second', 1, current_timestamp) AS text)"
-                              :mysql    "cast((current_timestamp + interval 1 second) AS char)"
-                              :postgres "cast((current_timestamp + interval '1 second') AS text)")]}))
-
 (defn- simulate-another-instance-updating-setting! [setting-name new-value]
   (if new-value
     (t2/update! :model/Setting {:key (name setting-name)} {:value new-value})
-    (t2/delete! (t2/table-name :model/Setting) {:key (name setting-name)}))
-  (update-settings-last-updated-value-in-db!))
+    (t2/delete! (t2/table-name :model/Setting) {:key (name setting-name)})))
 
-(defn reset-last-update-check!
-  "Reset the value of `last-update-check` so the next cache access will check for updates."
-  []
-  (.set ^java.util.concurrent.atomic.AtomicLong (var-get #'setting.cache/last-update-check) 0))
-
-(deftest update-settings-last-updated-test
-  (testing "When I update a Setting, does it set/update `settings-last-updated`?"
-    (setting-test/clear-settings-last-updated-value-in-db!)
-    (setting-test/toucan-name! "Bird Can")
-    (is (string? (setting-test/settings-last-updated-value-in-db)))
-
-    (testing "...and is the value updated in the cache as well?"
-      (is (string? (settings-last-updated-value-in-cache))))
-
-    (testing "..and if I update it again, will the value be updated?"
-      (let [first-value (setting-test/settings-last-updated-value-in-db)]
-        ;; MySQL only has the resolution of one second on the timestamps here so we should wait that long to make sure
-        ;; the second-value actually ends up being greater than the first
-        (Thread/sleep (if (= (mdb/db-type) :mysql) 1200 50))
-        (setting-test/toucan-name! "Bird Can")
-        (let [second-value (setting-test/settings-last-updated-value-in-db)]
-          ;; first & second values should be different, and first value should be "less than" the second value
-          (is (not= first-value second-value))
-          (is (neg? (compare first-value second-value))))))))
-
-(deftest cache-out-of-date-test
-  (testing "If there is no cache, it should be considered out of date!"
+(deftest restore-cache-test
+  (testing "restore-cache! populates the cache from the DB"
     (clear-cache!)
-    (#'setting.cache/cache-out-of-date?))
+    (is (nil? (setting.cache/cache)))
+    (setting.cache/restore-cache!)
+    (is (map? (setting.cache/cache)))))
 
-  (testing "But if I set a setting, it should cause the cache to be populated, and be up-to-date"
-    (clear-cache!)
-    (setting-test/toucan-name! "Reggae Toucan")
-    (is (= false
-           (#'setting.cache/cache-out-of-date?))))
+(deftest broadcast-cache-invalidation-test
+  (testing "broadcast-cache-invalidation! does not throw"
+    (setting.cache/broadcast-cache-invalidation!)))
 
-  (testing "If another instance updates a Setting, `cache-out-of-date?` should return `true` based on DB comparisons..."
+(deftest sync-via-restore-cache-test
+  (testing "When another instance changes a setting, restore-cache! picks up the change"
     (clear-cache!)
     (setting-test/toucan-name! "Reggae Toucan")
     (simulate-another-instance-updating-setting! :toucan-name "Bird Can")
-    (is (true?
-         (#'setting.cache/cache-out-of-date?)))))
-
-(deftest restore-cache-if-needed-test
-  (testing (str "of course, `restore-cache-if-needed!` should use TTL memoization, and the cache should not get "
-                "updated right away even if another instance updates a value...")
-    (reset-last-update-check!)
-    (clear-cache!)
-    (setting-test/toucan-name! "Sam")
-    ;; should restore cache, and put in {"setting-test/toucan-name" "Sam"}
-    (is (= "Sam"
+    (is (= "Bird Can"
+           (t2/select-one-fn :value :model/Setting :key "toucan-name")))
+    ;; Our cache still has the old value
+    (is (= "Reggae Toucan"
            (setting-test/toucan-name)))
-    ;; now have another instance change the value
-    (simulate-another-instance-updating-setting! :toucan-name "Bird Can")
-    ;; our cache should not be updated yet because it's on a TTL
-    (is (= "Sam"
+    ;; After restoring, we get the new value
+    (setting.cache/restore-cache!)
+    (is (= "Bird Can"
            (setting-test/toucan-name)))))
 
-;; ...and when it comes time to check our cache for updating (when calling `restore-cache-if-needed!`, it should get
-;; the updated value. (we're not actually going to wait a minute for the memoized values of `should-restore-cache?` to
-;; be invalidated, so we will manually flush the memoization cache to simulate it happening)
-(deftest sync-test-1
-  (clear-cache!)
-  (setting-test/toucan-name! "Reggae Toucan")
-  (simulate-another-instance-updating-setting! :toucan-name "Bird Can")
-  (is (= "Bird Can"
-         (t2/select-one-fn :value :model/Setting :key "toucan-name")))
-  (reset-last-update-check!)
-  ;; calling `setting-test/toucan-name` will call `restore-cache-if-needed!`, which will in turn call `should-restore-cache?`.
-  ;; Since memoized value is no longer present, this should call `cache-out-of-date?`, which checks the DB; it will
-  ;; detect a cache out-of-date situation and flush the cache as appropriate, giving us the updated value when we
-  ;; call! :wow:
-  (is (= "Bird Can"
-         (setting-test/toucan-name))))
-
-;; Simulate experience where:
-;; 1. User writes a setting on Server 1
-;; 2. User reads settings, served from Server 1, memoizing theresult of should-restore-cache?
-;; 3. User writes setting :toucan-name on Server 2
-;; 4. User writes setting on Server 1
-;; 5. User reads setting :toucan-name from Server 1
-;;
-;; This process was causing the updated `:toucan-name` to never be read on Server 1 because Server 1 "thought" it had
-;; the latest values and didn't restore the cache from the db
-(deftest sync-test-2
-  (let [external-cache (constantly (atom nil))]
-    (clear-cache!)
-    (reset-last-update-check!)
-    (setting-test/test-setting-1! "Starfish")
-    ;; 1. User writes
-    (with-redefs [setting.cache/cache* external-cache]
-      (setting-test/toucan-name! "Batman Toucan"))
-    (setting-test/test-setting-1! "Batman")
-    (is (= "Batman Toucan"
-           (setting-test/toucan-name)))))
-
-(deftest sync-test-3
+(deftest sync-locale-test
   (mt/discard-setting-changes [site-locale]
     (clear-cache!)
     (system/site-locale! "en")
     (simulate-another-instance-updating-setting! :site-locale "fr")
-    (reset-last-update-check!)
+    (setting.cache/restore-cache!)
     (is (= "fr"
            (system/site-locale)))))